From 9f0149f1353ede3071604605f1ddde17ce8e498b Mon Sep 17 00:00:00 2001 From: laols574 <31547222+laols574@users.noreply.github.com> Date: Tue, 4 Feb 2020 14:45:17 -0700 Subject: [PATCH 1/3] Update items.py added additional items to gather more information about the profile of the user and the specific reactions --- fbcrawl/items.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/fbcrawl/items.py b/fbcrawl/items.py index 360f41f..1689443 100644 --- a/fbcrawl/items.py +++ b/fbcrawl/items.py @@ -564,7 +564,7 @@ def id_strip(post_id): d = json.loads(post_id[::-1][0]) #nested dict of features return str(d['top_level_post_id']) - +'''Added additional fields''' class FbcrawlItem(scrapy.Item): source = scrapy.Field() date = scrapy.Field() @@ -606,6 +606,7 @@ class FbcrawlItem(scrapy.Item): class CommentsItem(scrapy.Item): source = scrapy.Field() + source_url = scrapy.Field() reply_to=scrapy.Field() date = scrapy.Field( # when was the post published output_processor=parse_date2 @@ -619,14 +620,30 @@ class CommentsItem(scrapy.Item): likes = scrapy.Field( output_processor=reactions_strip ) - source_url = scrapy.Field() url = scrapy.Field() - ahah = scrapy.Field() - love = scrapy.Field() - wow = scrapy.Field() - sigh = scrapy.Field() - grrr = scrapy.Field() - share = scrapy.Field() # num of shares + ahah = scrapy.Field( + output_processor=reactions_strip + ) + love = scrapy.Field( + output_processor=reactions_strip + ) + wow = scrapy.Field( + output_processor=reactions_strip + ) + sigh = scrapy.Field( + output_processor=reactions_strip + ) + grrr = scrapy.Field( + output_processor=reactions_strip + ) + name = scrapy.Field() + gender = scrapy.Field() + birthday = scrapy.Field() + current_city = scrapy.Field() + hometown = scrapy.Field() + work = scrapy.Field() + education = scrapy.Field() + interested_in = scrapy.Field() class ProfileItem(scrapy.Item): name = scrapy.Field() @@ -639,6 +656,7 @@ class ProfileItem(scrapy.Item): interested_in = scrapy.Field() page = scrapy.Field() + """end""" class EventsItem(scrapy.Item): name = scrapy.Field() location = scrapy.Field() From 25cf067cc26a1163590e29420713b8b9e5b14057 Mon Sep 17 00:00:00 2001 From: laols574 <31547222+laols574@users.noreply.github.com> Date: Tue, 4 Feb 2020 15:11:02 -0700 Subject: [PATCH 2/3] Update comments.py see comments for details on changes, but I add random time pauses, crawled specific reaction and profile data --- fbcrawl/spiders/comments.py | 396 ++++++++++++++++++++++++++++-------- 1 file changed, 310 insertions(+), 86 deletions(-) diff --git a/fbcrawl/spiders/comments.py b/fbcrawl/spiders/comments.py index 2b6331c..c920ceb 100644 --- a/fbcrawl/spiders/comments.py +++ b/fbcrawl/spiders/comments.py @@ -1,4 +1,7 @@ import scrapy +#added time and random to prevent getting blocked by Facebook +import time +from random import randrange from scrapy.loader import ItemLoader from scrapy.exceptions import CloseSpider @@ -10,11 +13,15 @@ class CommentsSpider(FacebookSpider): """ Parse FB comments, given a post (needs credentials) - """ + """ name = "comments" + #added the additional fields for specific reactions and profile info custom_settings = { - 'FEED_EXPORT_FIELDS': ['source','reply_to','date','reactions','text', \ - 'source_url','url'], + 'FEED_EXPORT_FIELDS': ['source','reply_to','date','text', \ + 'reactions','likes','url', 'ahah','love','wow', \ + 'sigh','grrr','name', \ + 'gender', 'birthday', 'current_city',\ + 'hometown', 'work', 'education', 'interested_in'], 'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', 'CONCURRENT_REQUESTS' : 1 } @@ -27,7 +34,7 @@ def __init__(self, *args, **kwargs): self.type = 'post' elif 'page' in kwargs: self.type = 'page' - + super().__init__(*args,**kwargs) def parse_page(self, response): @@ -40,17 +47,17 @@ def parse_page(self, response): meta={'index':1}) elif self.type == 'page': #select all posts - for post in response.xpath("//div[contains(@data-ft,'top_level_post_id')]"): + for post in response.xpath("//div[contains(@data-ft,'top_level_post_id')]"): many_features = post.xpath('./@data-ft').get() date = [] date.append(many_features) date = parse_date(date,{'lang':self.lang}) current_date = datetime.strptime(date,'%Y-%m-%d %H:%M:%S') if date is not None else date - + if current_date is None: date_string = post.xpath('.//abbr/text()').get() date = parse_date2([date_string],{'lang':self.lang}) - current_date = datetime(date.year,date.month,date.day) if date is not None else date + current_date = datetime(date.year,date.month,date.day) if date is not None else date date = str(date) if abs(self.count) + 1 > self.max: @@ -58,39 +65,39 @@ def parse_page(self, response): self.logger.info('Parsing post n = {}, post_date = {}'.format(abs(self.count)+1,date)) #returns full post-link in a list - post = post.xpath(".//a[contains(@href,'footer')]/@href").extract() + post = post.xpath(".//a[contains(@href,'footer')]/@href").extract() temp_post = response.urljoin(post[0]) self.count -= 1 - yield scrapy.Request(temp_post, - self.parse_post, + yield scrapy.Request(temp_post, + self.parse_post, priority = self.count, meta={'index':1}) - + #load following page, try to click on "more" - #after few pages have been scraped, the "more" link might disappears + #after few pages have been scraped, the "more" link might disappears #if not present look for the highest year not parsed yet #click once on the year and go back to clicking "more" #new_page is different for groups if self.group == 1: - new_page = response.xpath("//div[contains(@id,'stories_container')]/div[2]/a/@href").extract() + new_page = response.xpath("//div[contains(@id,'stories_container')]/div[2]/a/@href").extract() else: - new_page = response.xpath("//div[2]/a[contains(@href,'timestart=') and not(contains(text(),'ent')) and not(contains(text(),number()))]/@href").extract() - #this is why lang is needed + new_page = response.xpath("//div[2]/a[contains(@href,'timestart=') and not(contains(text(),'ent')) and not(contains(text(),number()))]/@href").extract() + #this is why lang is needed - if not new_page: + if not new_page: self.logger.info('[!] "more" link not found, will look for a "year" link') - #self.k is the year link that we look for - if response.meta['flag'] == self.k and self.k >= self.year: + #self.k is the year link that we look for + if response.meta['flag'] == self.k and self.k >= self.year: xpath = "//div/a[contains(@href,'time') and contains(text(),'" + str(self.k) + "')]/@href" new_page = response.xpath(xpath).extract() if new_page: new_page = response.urljoin(new_page[0]) self.k -= 1 self.logger.info('Found a link for year "{}", new_page = {}'.format(self.k,new_page)) - yield scrapy.Request(new_page, - callback=self.parse_page, - priority = -1000, + yield scrapy.Request(new_page, + callback=self.parse_page, + priority = -1000, meta={'flag':self.k}) else: while not new_page: #sometimes the years are skipped this handles small year gaps @@ -103,25 +110,25 @@ def parse_page(self, response): self.logger.info('Found a link for year "{}", new_page = {}'.format(self.k,new_page)) new_page = response.urljoin(new_page[0]) self.k -= 1 - yield scrapy.Request(new_page, + yield scrapy.Request(new_page, callback=self.parse_page, priority = -1000, - meta={'flag':self.k}) + meta={'flag':self.k}) else: self.logger.info('Crawling has finished with no errors!') else: new_page = response.urljoin(new_page[0]) if 'flag' in response.meta: self.logger.info('Page scraped, clicking on "more"! new_page = {}'.format(new_page)) - yield scrapy.Request(new_page, - callback=self.parse_page, - priority = -1000, + yield scrapy.Request(new_page, + callback=self.parse_page, + priority = -1000, meta={'flag':response.meta['flag']}) else: self.logger.info('First page scraped, clicking on "more"! new_page = {}'.format(new_page)) - yield scrapy.Request(new_page, - callback=self.parse_page, - priority = -1000, + yield scrapy.Request(new_page, + callback=self.parse_page, + priority = -1000, meta={'flag':self.k}) def parse_post(self, response): @@ -132,6 +139,7 @@ def parse_post(self, response): 3) adds simple (not-replied-to) comments 4) follows to new comment page ''' + #load replied-to comments pages #select nested comment one-by-one matching with the index: response.meta['index'] path = './/div[string-length(@class) = 2 and count(@id)=1 and contains("0123456789", substring(@id,1,1)) and .//div[contains(@id,"comment_replies")]]' + '['+ str(response.meta['index']) + ']' @@ -150,21 +158,61 @@ def parse_post(self, response): 'index':response.meta['index'], 'flag':'init', 'group':group_flag}) - #load regular comments + + + #load regular comments if not response.xpath(path): #prevents from exec path2 = './/div[string-length(@class) = 2 and count(@id)=1 and contains("0123456789", substring(@id,1,1)) and not(.//div[contains(@id,"comment_replies")])]' for i,reply in enumerate(response.xpath(path2)): self.logger.info('{} regular comment'.format(i+1)) new = ItemLoader(item=CommentsItem(),selector=reply) - new.context['lang'] = self.lang - new.add_xpath('source','.//h3/a/text()') - new.add_xpath('source_url','.//h3/a/@href') + new.context['lang'] = self.lang + new.add_xpath('source','.//h3/a/text()') + new.add_xpath('source_url','.//h3/a/@href') new.add_xpath('text','.//div[h3]/div[1]//text()') new.add_xpath('date','.//abbr/text()') new.add_xpath('reactions','.//a[contains(@href,"reaction/profile")]//text()') new.add_value('url',response.url) - yield new.load_item() - + + """ + PROFILE REACTIONS SECTION + adds functionality for adding profile and specific reaction data + gets the profile url, creates a new item + if the profile exists, add info to new item and increment 'check' + to signal that new information has been added to the item + and it's already been yielded + repeat this process for reactions + """ + + #profile = response.xpath(".//h3/a/@href") + #profile = response.urljoin(profile[0].extract()) + + profile = "https://mbasic.facebook.com" + new.get_collected_values('source_url')[0] + #print('profile', profile) + #print('new item', new.get_collected_values('name')) + + item = new.load_item() + check = 0 + if profile: + check += 1 + yield scrapy.Request(profile, callback=self.parse_profile, meta={'item':item}) + + temp = ItemLoader(item=CommentsItem(),selector=reply) + temp.context['lang'] = self.lang + + temp.add_xpath('reactions', './/a[contains(@href,"reaction/profile")]/@href') + reactions = temp.get_collected_values('reactions') + if reactions: + check +=1 + reactions = "https://mbasic.facebook.com" + temp.get_collected_values('reactions')[0] + temp = 0 + yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':item}) + + if check == 0: + yield item + + + #new comment page if not response.xpath(path): #for groups @@ -178,7 +226,7 @@ def parse_post(self, response): yield scrapy.Request(new_page, callback=self.parse_post, meta={'index':1, - 'group':1}) + 'group':1}) else: for next_page in response.xpath(next_xpath): new_page = next_page.xpath('.//@href').extract() @@ -187,46 +235,110 @@ def parse_post(self, response): yield scrapy.Request(new_page, callback=self.parse_post, meta={'index':1, - 'group':group_flag}) - + 'group':group_flag}) + def parse_reply(self,response): ''' parse reply to comments, root comment is added if flag ''' # from scrapy.utils.response import open_in_browser # open_in_browser(response) - + if response.meta['flag'] == 'init': #parse root comment - for root in response.xpath('//div[contains(@id,"root")]/div/div/div[count(@id)!=1 and contains("0123456789", substring(@id,1,1))]'): + for root in response.xpath('//div[contains(@id,"root")]/div/div/div[count(@id)!=1 and contains("0123456789", substring(@id,1,1))]'): new = ItemLoader(item=CommentsItem(),selector=root) - new.context['lang'] = self.lang - new.add_xpath('source','.//h3/a/text()') - new.add_xpath('source_url','.//h3/a/@href') + new.context['lang'] = self.lang + new.add_xpath('source','.//h3/a/text()') + new.add_xpath('source_url','.//h3/a/@href') new.add_value('reply_to','ROOT') new.add_xpath('text','.//div[1]//text()') new.add_xpath('date','.//abbr/text()') new.add_xpath('reactions','.//a[contains(@href,"reaction/profile")]//text()') new.add_value('url',response.url) - yield new.load_item() + #response --> reply/root + """ + PROFILE REACTIONS SECTION (REPEAT SEE LINE 176 ) + the only difference is that, when getting the item temporarily + the selector is the root instead of the reply, (it matches the for loop) + """ + #profile = response.xpath(".//h3/a/@href") + #profile = response.urljoin(profile[0].extract()) + profile = "https://mbasic.facebook.com" + new.get_collected_values('source_url')[0] + print('profile', profile) + #print('new item', new.get_collected_values('name')) + item = new.load_item() + check = 0 + if profile: + check += 1 + yield scrapy.Request(profile, callback=self.parse_profile, meta={'item':item}) + + #reactions = new.get_value('reactions') + #print("reactions",reactions) + + temp = ItemLoader(item=CommentsItem(),selector=root) + temp.context['lang'] = self.lang + + temp.add_xpath('reactions', './/a[contains(@href,"reaction/profile")]/@href') + reactions = temp.get_collected_values('reactions') + if reactions: + check += 1 + reactions = "https://mbasic.facebook.com" + temp.get_collected_values('reactions')[0] + temp = 0 + yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':item}) + + if check == 0: + yield item + + + #parse all replies in the page - for reply in response.xpath('//div[contains(@id,"root")]/div/div/div[count(@id)=1 and contains("0123456789", substring(@id,1,1))]'): + for reply in response.xpath('//div[contains(@id,"root")]/div/div/div[count(@id)=1 and contains("0123456789", substring(@id,1,1))]'): new = ItemLoader(item=CommentsItem(),selector=reply) - new.context['lang'] = self.lang - new.add_xpath('source','.//h3/a/text()') - new.add_xpath('source_url','.//h3/a/@href') + new.context['lang'] = self.lang + new.add_xpath('source','.//h3/a/text()') + new.add_xpath('source_url','.//h3/a/@href') new.add_value('reply_to',response.meta['reply_to']) new.add_xpath('text','.//div[h3]/div[1]//text()') new.add_xpath('date','.//abbr/text()') new.add_xpath('reactions','.//a[contains(@href,"reaction/profile")]//text()') - new.add_value('url',response.url) - yield new.load_item() - + new.add_value('url',response.url) + + """ + PROFILE REACTIONS SECTION SECTION (REPEAT SEE LINE 176) + """ + #profile = response.xpath(".//h3/a/@href") + #profile = response.urljoin(profile[0].extract()) + profile = "https://mbasic.facebook.com" + new.get_collected_values('source_url')[0] + + + #print('new item', new.get_collected_values('name')) + item = new.load_item() + check = 0 + if profile: + check += 1 + yield scrapy.Request(profile, callback=self.parse_profile, meta={'item':item}) + + temp = ItemLoader(item=CommentsItem(),selector=reply) + temp.context['lang'] = self.lang + + temp.add_xpath('reactions', './/a[contains(@href,"reaction/profile")]/@href') + reactions = temp.get_collected_values('reactions') + if reactions: + check += 1 + reactions = "https://mbasic.facebook.com" + temp.get_collected_values('reactions')[0] + temp = 0 + yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':item}) + + if check == 0: + yield item + + back = response.xpath('//div[contains(@id,"comment_replies_more_1")]/a/@href').extract() if back: self.logger.info('Back found, more nested comments') back_page = response.urljoin(back[0]) - yield scrapy.Request(back_page, + yield scrapy.Request(back_page, callback=self.parse_reply, priority = 1000, meta={'reply_to':response.meta['reply_to'], @@ -242,26 +354,77 @@ def parse_reply(self,response): callback=self.parse_post, meta={'index':response.meta['index']+1, 'group':response.meta['group']}) - + elif response.meta['flag'] == 'back': + """ + adds random time pauses to prevent blocking + DOWNSIDE: the algorithm will go slower, but still + runs pretty quickly + the greater the length of time, the more + likely you'll go undetected, but if you're using a large amount + of data, this may be unreasonable + """ + #print("did we make it") + r = randrange(0,20) + time.sleep(r) #parse all comments - for reply in response.xpath('//div[contains(@id,"root")]/div/div/div[count(@id)=1 and contains("0123456789", substring(@id,1,1))]'): + for reply in response.xpath('//div[contains(@id,"root")]/div/div/div[count(@id)=1 and contains("0123456789", substring(@id,1,1))]'): + #print("reply") new = ItemLoader(item=CommentsItem(),selector=reply) - new.context['lang'] = self.lang - new.add_xpath('source','.//h3/a/text()') - new.add_xpath('source_url','.//h3/a/@href') + new.context['lang'] = self.lang + new.add_xpath('source','.//h3/a/text()') + new.add_xpath('source_url','.//h3/a/@href') new.add_value('reply_to',response.meta['reply_to']) new.add_xpath('text','.//div[h3]/div[1]//text()') new.add_xpath('date','.//abbr/text()') new.add_xpath('reactions','.//a[contains(@href,"reaction/profile")]//text()') - new.add_value('url',response.url) - yield new.load_item() + new.add_value('url',response.url) + + """ + SECTION (REPEAT SEE LINE 176) + """ + + profile = "https://mbasic.facebook.com" + new.get_collected_values('source_url')[0] + + #profile = response.xpath(".//h3/a/@href") + #profile = response.urljoin(profile[0].extract()) + #print('profile', profile) + #print('new item', new.get_collected_values('name')) + check = 0 + item = new.load_item() + if profile: + check += 1 + print(1) + yield scrapy.Request(profile, callback=self.parse_profile, meta={'item':item}) + + #response --> reply/root + #print("before ", item) + temp = ItemLoader(item=CommentsItem(),selector=reply) + temp.context['lang'] = self.lang + + temp.add_xpath('reactions', './/a[contains(@href,"reaction/profile")]/@href') + reactions = temp.get_collected_values('reactions') + if reactions: + check += 1 + reactions = "https://mbasic.facebook.com" + temp.get_collected_values('reactions')[0] + temp = 0 + print(2) + yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':item}) + + if check == 0: + print(3) + yield item + #print("after ", item) + + + + #keep going backwards back = response.xpath('//div[contains(@id,"comment_replies_more_1")]/a/@href').extract() self.logger.info('Back found, more nested comments') if back: back_page = response.urljoin(back[0]) - yield scrapy.Request(back_page, + yield scrapy.Request(back_page, callback=self.parse_reply, priority=1000, meta={'reply_to':response.meta['reply_to'], @@ -277,29 +440,90 @@ def parse_reply(self,response): callback=self.parse_post, meta={'index':response.meta['index']+1, 'group':response.meta['group']}) - -# ============================================================================= -# CRAWL REACTIONS -# ============================================================================= -# def parse_reactions(self,response): -# new = ItemLoader(item=CommentsItem(),response=response, parent=response.meta['item']) -# new.context['lang'] = self.lang -# new.add_xpath('likes',"//a[contains(@href,'reaction_type=1')]/span/text()") -# new.add_xpath('ahah',"//a[contains(@href,'reaction_type=4')]/span/text()") -# new.add_xpath('love',"//a[contains(@href,'reaction_type=2')]/span/text()") -# new.add_xpath('wow',"//a[contains(@href,'reaction_type=3')]/span/text()") -# new.add_xpath('sigh',"//a[contains(@href,'reaction_type=7')]/span/text()") -# new.add_xpath('grrr',"//a[contains(@href,'reaction_type=8')]/span/text()") -# yield new.load_item() -# -# #substitute -# yield new.load_item() -# ‾‾‾‾‾‾‾‾‾|‾‾‾‾‾‾‾‾‾‾‾ -# _________v___ -# #response --> reply/root -# reactions = response.xpath(".//a[contains(@href,'reaction/profile')]/@href") -# reactions = response.urljoin(reactions[0].extract()) -# if reactions: -# yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':new}) -# else: -# yield new.load_item() \ No newline at end of file + """ + PARSE_PROFILE + This function parses the profile information from the user associated + with the current comment. It first does a random time pause, then adds + profile fields to the item that is associated with a specific comment. + This is done explicitly. I left the other way I tried, feeding in the + item and adding xpaths, but it wouldn't return correctly. (the other + method mirrors the old way of getting reaction information) + """ + def parse_profile(self,response): + '''because visiting profiles causes blocking the quickest, + its especially useful to have a randomized time pause + at the start of parsing each profile. + the if statement ensures that infrequently there will an even + longer time pause, to create greater randomness and infrequently + increase the wait time + ''' + r = randrange(0,20) + if(r == 0): + r2 = randrange(0,60) + time.sleep(60 + r2) + + time.sleep(r) + #new = ItemLoader(item=response.meta['item'],response=response ) + self.logger.info('Crawling profile info') + #new.context['lang'] = self.lang + + item = response.meta['item'] + item['name'] = response.xpath('//span/div/span/strong/text()').extract() + item['gender'] = response.xpath("//div[@id='basic-info']//div[@title='Gender']//div/text()").extract() + item['birthday'] = response.xpath("//div[@id='basic-info']//div[@title='Birthday']//div/text()").extract() + item['current_city'] = response.xpath("//div[@id='living']//div[@title='Current City']//a/text()").extract() + item['hometown'] = response.xpath("//div[@id='living']//div[@title='Hometown']//a/text()").extract() + item['work'] = response.xpath("//div[@id='work']//a/text()").extract() + item['education'] = response.xpath("//div[@id='education']//a/text()").extract() + item['interested_in'] = response.xpath("//div[@id='interested-in']//div[not(contains(text(),'Interested In'))]/text()").extract() + + '''new.add_xpath('name','//span/div/span/strong/text()').extract() + new.add_xpath('gender',"//div[@id='basic-info']//div[@title='Gender']//div/text()").extract() + new.add_xpath('birthday',"//div[@id='basic-info']//div[@title='Birthday']//div/text()").extract() + new.add_xpath('current_city',"//div[@id='living']//div[@title='Current City']//a/text()").extract() + new.add_xpath('hometown',"//div[@id='living']//div[@title='Hometown']//a/text()").extract() + new.add_xpath('work',"//div[@id='work']//a/text()").extract() + new.add_xpath('education',"//div[@id='education']//a/text()").extract() + new.add_xpath('interested_in',"//div[@id='interested-in']//div[not(contains(text(),'Interested In'))]/text()")''' + #print(item['name']) + #print(item) + #self.logger.info('its making it to the end of the function') + #item = new.load_item() + yield item + #yield new.load_item() + + + """ + PARSE_REACTIONS + This function parses the reaction information associated with a + particular comment. It is, other than the difference in field + names, identical to the code in parse reactions. Again, there + is a random time pause. I think it helped to have a pause when + visiting new pages, which is why it was included here, as the + reactions page is a separate URL. This intends to mimic what an + user might do. + """ + def parse_reactions(self,response): + r = randrange(0,20) + time.sleep(r) + #new = ItemLoader(item=CommentsItem(),response=response, parent=response.meta['item']) + #print("reactions",new.item) + #new.context['lang'] = self.lang + item = response.meta['item'] + item['likes'] = response.xpath("//a[contains(@href,'reaction_type=1')]/span/text()").extract() + item['ahah'] = response.xpath("//a[contains(@href,'reaction_type=4')]/span/text()").extract() + item['love'] = response.xpath("//a[contains(@href,'reaction_type=2')]/span/text()").extract() + item['wow'] = response.xpath("//a[contains(@href,'reaction_type=3')]/span/text()").extract() + item['sigh'] = response.xpath("//a[contains(@href,'reaction_type=7')]/span/text()").extract() + item['grrr'] = response.xpath("//a[contains(@href,'reaction_type=8')]/span/text()").extract() + #print(item) + + + yield item + '''new.add_xpath('likes',"//a[contains(@href,'reaction_type=1')]/span/text()") + new.add_xpath('ahah',"//a[contains(@href,'reaction_type=4')]/span/text()") + new.add_xpath('love',"//a[contains(@href,'reaction_type=2')]/span/text()") + new.add_xpath('wow',"//a[contains(@href,'reaction_type=3')]/span/text()") + new.add_xpath('sigh',"//a[contains(@href,'reaction_type=7')]/span/text()") + new.add_xpath('grrr',"//a[contains(@href,'reaction_type=8')]/span/text()")''' + #yield new.load_item() From d7ba4bc0b291578a951fbbfd38a16eedfe2eb43b Mon Sep 17 00:00:00 2001 From: laols574 <31547222+laols574@users.noreply.github.com> Date: Tue, 4 Feb 2020 15:15:42 -0700 Subject: [PATCH 3/3] Update fbcrawl.py updated error handling --- fbcrawl/spiders/fbcrawl.py | 107 +++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 45 deletions(-) diff --git a/fbcrawl/spiders/fbcrawl.py b/fbcrawl/spiders/fbcrawl.py index f4f07ad..26c3d22 100644 --- a/fbcrawl/spiders/fbcrawl.py +++ b/fbcrawl/spiders/fbcrawl.py @@ -10,7 +10,7 @@ class FacebookSpider(scrapy.Spider): ''' Parse FB pages (needs credentials) - ''' + ''' name = 'fb' custom_settings = { 'FEED_EXPORT_FIELDS': ['source','shared_from','date','text', \ @@ -18,14 +18,14 @@ class FacebookSpider(scrapy.Spider): 'sigh','grrr','comments','post_id','url'], 'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', } - + def __init__(self, *args, **kwargs): #turn off annoying logging, set LOG_LEVEL=DEBUG in settings.py to see more logs logger = logging.getLogger('scrapy.middleware') logger.setLevel(logging.WARNING) - + super().__init__(*args,**kwargs) - + #email & pass need to be passed as attributes! if 'email' not in kwargs or 'password' not in kwargs: raise AttributeError('You need to provide valid email and password:\n' @@ -61,27 +61,27 @@ def __init__(self, *args, **kwargs): self.logger.info('Language attribute not provided, fbcrawl will try to guess it from the fb interface') self.logger.info('To specify, add the lang parameter: scrapy fb -a lang="LANGUAGE"') self.logger.info('Currently choices for "LANGUAGE" are: "en", "es", "fr", "it", "pt"') - self.lang = '_' + self.lang = '_' elif self.lang == 'en' or self.lang == 'es' or self.lang == 'fr' or self.lang == 'it' or self.lang == 'pt': self.logger.info('Language attribute recognized, using "{}" for the facebook interface'.format(self.lang)) else: - self.logger.info('Lang "{}" not currently supported'.format(self.lang)) - self.logger.info('Currently supported languages are: "en", "es", "fr", "it", "pt"') + self.logger.info('Lang "{}" not currently supported'.format(self.lang)) + self.logger.info('Currently supported languages are: "en", "es", "fr", "it", "pt"') self.logger.info('Change your interface lang from facebook settings and try again') raise AttributeError('Language provided not currently supported') - + #max num of posts to crawl if 'max' not in kwargs: self.max = int(10e5) else: self.max = int(kwargs['max']) - + #current year, this variable is needed for proper parse_page recursion self.k = datetime.now().year #count number of posts, used to enforce DFS and insert posts orderly in the csv self.count = 0 - - self.start_urls = ['https://mbasic.facebook.com'] + + self.start_urls = ['https://mbasic.facebook.com'] def parse(self, response): ''' @@ -93,15 +93,31 @@ def parse(self, response): formdata={'email': self.email,'pass': self.password}, callback=self.parse_home ) - + def parse_home(self, response): ''' This method has multiple purposes: 1) Handle failed logins due to facebook 'save-device' redirection 2) Set language interface, if not already provided - 3) Navigate to given page + 3) Navigate to given page ''' + """ + I added these three Attribute errors: Using an old password, entering an + incorrect password, and getting blocked + """ #handle 'save-device' redirection + if "This is because you tried to do something too many times. Please try again later" in str(response.body): + raise AttributeError("You got blocked. Use a different login.") + + if "You used an old password." in str(response.body): + raise AttributeError("You used an old password.") + + if "The password you entered is incorrect." in str(response.body): + raise AttributeError("The password you entered is incorrect.") + + """ + end change + """ if response.xpath("//div/a[contains(@href,'save-device')]"): self.logger.info('Going through the "save-device" checkpoint') return FormRequest.from_response( @@ -109,9 +125,10 @@ def parse_home(self, response): formdata={'name_action_selected': 'dont_save'}, callback=self.parse_home ) - + #set language interface if self.lang == '_': + print("RRRRRRR ", response, "\n\n") if response.xpath("//input[@placeholder='Search Facebook']"): self.logger.info('Language recognized: lang="en"') self.lang = 'en' @@ -129,9 +146,9 @@ def parse_home(self, response): self.lang = 'pt' else: raise AttributeError('Language not recognized\n' - 'Change your interface lang from facebook ' + 'Change your interface lang from facebook ' 'and try again') - + #navigate to provided page href = response.urljoin(self.page) self.logger.info('Scraping facebook page {}'.format(href)) @@ -145,22 +162,22 @@ def parse_page(self, response): # #open page in browser for debug # from scrapy.utils.response import open_in_browser # open_in_browser(response) - + #select all posts - for post in response.xpath("//div[contains(@data-ft,'top_level_post_id')]"): - + for post in response.xpath("//div[contains(@data-ft,'top_level_post_id')]"): + many_features = post.xpath('./@data-ft').get() date = [] date.append(many_features) date = parse_date(date,{'lang':self.lang}) current_date = datetime.strptime(date,'%Y-%m-%d %H:%M:%S') if date is not None else date - + if current_date is None: date_string = post.xpath('.//abbr/text()').get() date = parse_date2([date_string],{'lang':self.lang}) - current_date = datetime(date.year,date.month,date.day) if date is not None else date + current_date = datetime(date.year,date.month,date.day) if date is not None else date date = str(date) - + #if 'date' argument is reached stop crawling if self.date > current_date: raise CloseSpider('Reached date: {}'.format(self.date)) @@ -169,34 +186,34 @@ def parse_page(self, response): if abs(self.count) + 1 > self.max: raise CloseSpider('Reached max num of post: {}. Crawling finished'.format(abs(self.count))) self.logger.info('Parsing post n = {}, post_date = {}'.format(abs(self.count)+1,date)) - new.add_xpath('comments', './div[2]/div[2]/a[1]/text()') + new.add_xpath('comments', './div[2]/div[2]/a[1]/text()') new.add_value('date',date) new.add_xpath('post_id','./@data-ft') new.add_xpath('url', ".//a[contains(@href,'footer')]/@href") #page_url #new.add_value('url',response.url) - + #returns full post-link in a list - post = post.xpath(".//a[contains(@href,'footer')]/@href").extract() + post = post.xpath(".//a[contains(@href,'footer')]/@href").extract() temp_post = response.urljoin(post[0]) self.count -= 1 - yield scrapy.Request(temp_post, self.parse_post, priority = self.count, meta={'item':new}) + yield scrapy.Request(temp_post, self.parse_post, priority = self.count, meta={'item':new}) #load following page, try to click on "more" - #after few pages have been scraped, the "more" link might disappears + #after few pages have been scraped, the "more" link might disappears #if not present look for the highest year not parsed yet #click once on the year and go back to clicking "more" - + #new_page is different for groups if self.group == 1: - new_page = response.xpath("//div[contains(@id,'stories_container')]/div[2]/a/@href").extract() + new_page = response.xpath("//div[contains(@id,'stories_container')]/div[2]/a/@href").extract() else: - new_page = response.xpath("//div[2]/a[contains(@href,'timestart=') and not(contains(text(),'ent')) and not(contains(text(),number()))]/@href").extract() - #this is why lang is needed ^^^^^^^^^^^^^^^^^^^^^^^^^^ - - if not new_page: + new_page = response.xpath("//div[2]/a[contains(@href,'timestart=') and not(contains(text(),'ent')) and not(contains(text(),number()))]/@href").extract() + #this is why lang is needed ^^^^^^^^^^^^^^^^^^^^^^^^^^ + + if not new_page: self.logger.info('[!] "more" link not found, will look for a "year" link') - #self.k is the year link that we look for - if response.meta['flag'] == self.k and self.k >= self.year: + #self.k is the year link that we look for + if response.meta['flag'] == self.k and self.k >= self.year: xpath = "//div/a[contains(@href,'time') and contains(text(),'" + str(self.k) + "')]/@href" new_page = response.xpath(xpath).extract() if new_page: @@ -215,7 +232,7 @@ def parse_page(self, response): self.logger.info('Found a link for year "{}", new_page = {}'.format(self.k,new_page)) new_page = response.urljoin(new_page[0]) self.k -= 1 - yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':self.k}) + yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':self.k}) else: self.logger.info('Crawling has finished with no errors!') else: @@ -226,32 +243,32 @@ def parse_page(self, response): else: self.logger.info('First page scraped, clicking on "more"! new_page = {}'.format(new_page)) yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':self.k}) - + def parse_post(self,response): new = ItemLoader(item=FbcrawlItem(),response=response,parent=response.meta['item']) - new.context['lang'] = self.lang + new.context['lang'] = self.lang new.add_xpath('source', "//td/div/h3/strong/a/text() | //span/strong/a/text() | //div/div/div/a[contains(@href,'post_id')]/strong/text()") new.add_xpath('shared_from','//div[contains(@data-ft,"top_level_post_id") and contains(@data-ft,\'"isShare":1\')]/div/div[3]//strong/a/text()') # new.add_xpath('date','//div/div/abbr/text()') new.add_xpath('text','//div[@data-ft]//p//text() | //div[@data-ft]/div[@class]/div[@class]/text()') - + #check reactions for old posts check_reactions = response.xpath("//a[contains(@href,'reaction/profile')]/div/div/text()").get() if not check_reactions: - yield new.load_item() + yield new.load_item() else: - new.add_xpath('reactions',"//a[contains(@href,'reaction/profile')]/div/div/text()") + new.add_xpath('reactions',"//a[contains(@href,'reaction/profile')]/div/div/text()") reactions = response.xpath("//div[contains(@id,'sentence')]/a[contains(@href,'reaction/profile')]/@href") reactions = response.urljoin(reactions[0].extract()) yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':new}) - + def parse_reactions(self,response): new = ItemLoader(item=FbcrawlItem(),response=response, parent=response.meta['item']) - new.context['lang'] = self.lang + new.context['lang'] = self.lang new.add_xpath('likes',"//a[contains(@href,'reaction_type=1')]/span/text()") new.add_xpath('ahah',"//a[contains(@href,'reaction_type=4')]/span/text()") new.add_xpath('love',"//a[contains(@href,'reaction_type=2')]/span/text()") new.add_xpath('wow',"//a[contains(@href,'reaction_type=3')]/span/text()") new.add_xpath('sigh',"//a[contains(@href,'reaction_type=7')]/span/text()") - new.add_xpath('grrr',"//a[contains(@href,'reaction_type=8')]/span/text()") - yield new.load_item() \ No newline at end of file + new.add_xpath('grrr',"//a[contains(@href,'reaction_type=8')]/span/text()") + yield new.load_item()