## 1. LinkExtractor作用 > 1. 從follow=true的response（經由下載器下載回來的數據）中，以正則的方式提取URL，供爬蟲爬取。 > 2. 每個 LinkExtractor 有唯一的公共方法是 extract_links，其接收 一個 Response 對象， 并返回 scrapy.link.Link 對象? > 3. LikExtractors 只實例化一次，其 extract_links 方法會根據不同的 response 被調用多次來提取鏈接? > 4. LikExtractor通常用于 CrawlSpider和RedisCrawlSpider類中，但是也可以用于Spider中，就是提取連接 ## 2. LinkExtractor源碼 1. 如何使用 ~~~ from scrapy.linkextractors import LinkExtractor rules = ( # 只提取復合規則的頁面鏈接，不做分析，所以跟頁面但是沒有，follow是對網易深一層的爬取，false表示不提取連接，也不請求頁面上的連接 Rule(LinkExtractor(allow=r'categories/\d*?-j\d*?.html'), follow=True), Rule(LinkExtractor(allow=r'list/.*?.html'),follow=True), Rule(LinkExtractor(allow=r'product/\d+?.html'), follow=True), Rule(LinkExtractor(allow=r'store/\d+?/pc_index.shtml'), follow=False, callback='parse_item') ) ~~~ ### 2.1 scrapy.linkextractors包  看出使用的LinkExtractor，其實是LxmlLinkExtractor  ~~~ """ scrapy.linkextractors This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re from six.moves.urllib.parse import urlparse from parsel.csstranslator import HTMLTranslator from w3lib.url import canonicalize_url from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( url_is_from_any_domain, url_has_any_extension, ) # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ # images 'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif', 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', # audio 'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff', # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', 'm4a', 'm4v', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', 'odp', # other 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', ] _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any(r.search(url) for r in regexs) # 用于查找符合規則的URL的lamda表達式（匿名函數） _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} #判斷URL協議，是否是這三種協議 class FilteringLinkExtractor(object): _csstranslator = HTMLTranslator() def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css): self.link_extractor = link_extractor self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] self.allow_domains = set(arg_to_iter(allow_domains)) self.deny_domains = set(arg_to_iter(deny_domains)) self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css))) self.canonicalize = canonicalize if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} def _link_allowed(self, link): if not _is_valid_url(link.url): return False if self.allow_res and not _matches(link.url, self.allow_res): return False if self.deny_res and _matches(link.url, self.deny_res): return False parsed_url = urlparse(link.url) if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): return False if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): return False return True def matches(self, url): if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): return False allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) def _process_links(self, links): links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: link.url = canonicalize_url(link.url) links = self.link_extractor._process_links(links) return links def _extract_links(self, *args, **kwargs): return self.link_extractor._extract_links(*args, **kwargs) # Top-level imports from .lxmlhtml import LxmlLinkExtractor as LinkExtractor ~~~ selector中包含Element對象