class MySpider(CrawlSpider): start_urls = [ "http://www.example.com", ] rules = ( Rule( LxmlLinkExtractor( allow=[r'\w+\/\d+$', r'\w+\/\d+-p\d+$'], ), follow=True ), Rule( LxmlLinkExtractor( allow=(r'\d+.html$'), ), 'parse_action', ), ) def parse_action(self, response): yield WebdriverRequest(response.url, callback=self.parse_item) def parse_item(self, response): self.log('received for %s' % response.url, level=log.WARNING)
class MySpider(CrawlSpider):
start_urls = [
"http://www.example.com",
]
rules = (
Rule(
LxmlLinkExtractor(
allow=[r'\w+/\d+$', r'\w+/\d+-p\d+$'],
),
follow=True
),
Rule(
LxmlLinkExtractor(
allow=(r'\d+.html$'),
),
'parse_action',
),
)
def parse_action(self, response):
yield WebdriverRequest(response.url,
callback=self.parse_item)
def parse_item(self, response):
self.log('received for %s' % response.url, level=log.WARNING)