- pip install fake-useragent - 在spiders同級目錄下建立一個MidWare文件價里面寫一個user_agent_middlewares.py文件內容為 ``` # -*- coding: utf-8 -*- from fake_useragent import UserAgent class RandomUserAgentMiddlware(object): #隨機跟換user-agent def __init__(self,crawler): super(RandomUserAgentMiddlware,self).__init__() self.ua = UserAgent() self.ua_type = crawler.settings.get('RANDOM_UA_TYPE','random')#從setting文件中讀取RANDOM_UA_TYPE值 @classmethod def from_crawler(cls,crawler): return cls(crawler) def process_request(self,request,spider): ###系統電泳函數 def get_ua(): return getattr(self.ua,self.ua_type) # user_agent_random=get_ua() request.headers.setdefault('User_Agent',get_ua()) pass ``` - 在setting.py中添加 ``` # meituan是項目名稱 RANDOM_UA_TYPE = 'random' DOWNLOADER_MIDDLEWARES = { 'meituan.MidWare.user_agent_middlewares.RandomUserAgentMiddlware': 543, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':None, } ``` - 測試 ``` def parse(self, response): print("----------------------------------------------") print(response.request.headers['User_Agent']) print("----------------------------------------------") ```