用selenium下載數據，所以是下載中間件 ~~~ from selenium import webdriver from scrapy.http import HtmlResponse class JSPageMiddleware(object): #通過chrome請求動態網頁 def process_request(self, request, spider): if spider.name == "jobbole": # browser = webdriver.Chrome(executable_path="D:/Temp/chromedriver.exe") spider.browser.get(request.url) import time time.sleep(3) print ("訪問:{0}".format(request.url)) return HtmlResponse(url=spider.browser.current_url, body=spider.browser.page_source, encoding="utf-8", request=request) ~~~ 建立spider的時候建立webdriver ~~~ def __init__(self): self.browser = webdriver.Chrome(executable_path="D:/Temp/chromedriver.exe") super(JobboleSpider, self).__init__() //信號映射，當signals.spider_closed信號發生時，調用spider_closed dispatcher.connect(self.spider_closed, signals.spider_closed) // 自定義函數 def spider_closed(self, spider): #當爬蟲退出的時候關閉chrome print ("spider closed") self.browser.quit() ~~~