Update README.md

README.md

@@ -162,13 +162,13 @@ A new spider is now dedicated to crawl all the comments from a post (not a page!
 You can try it out with:
 
 ```
-scrapy crawl comments -a email="EMAILTOLOGIN" -a password="PASSWORDTOLOGIN" -a page="LINKOFTHEPOSTTOCRAWL" -o DUMPFILE.csv
+scrapy crawl comments -a email="EMAILTOLOGIN" -a password="PASSWORDTOLOGIN" -a post="LINKOFTHEPOSTTOCRAWL" -o DUMPFILE.csv
 ```
 
-The use is similar to fb spider, the only difference being the -a page parameter, which now is the link to a post. Make sure that the `page` option is a proper post link, for example:
+The use is similar to fb spider, the only difference being the -a post parameter, which now is the link to a post. Make sure that the `page` option is a proper post link, for example:
 
 ```
-rm trump_comments.csv; scrapy crawl comments -a email="obama@gmail.com" -a password="cm380jixke" -a page="https://mbasic.facebook.com/story.php?story_fbid=10162169751605725&id=153080620724" -o trump_comments.csv
+rm trump_comments.csv; scrapy crawl comments -a email="obama@gmail.com" -a password="cm380jixke" -a post="https://mbasic.facebook.com/story.php?story_fbid=10162169751605725&id=153080620724" -o trump_comments.csv
 ```
 
 

fbcrawl/items.py

@@ -21,6 +21,31 @@ def comments_strip(string,loader_context):
         if(string[0] == 'Share'):
             return '0'
         new_string = string[0].rstrip(' Comments')
+        new_string = string[0].rstrip(' Comments')
+        while new_string.rfind(',') != -1:
+            new_string = new_string[0:new_string.rfind(',')] + new_string[new_string.rfind(',')+1:]
+        return new_string
+    elif lang == 'en':
+        if(string[0] == 'Share'):
+            return '0'
+        new_string = string[0].rstrip(' Comments')
+        new_string = string[0].rstrip(' Comments')
+        while new_string.rfind(',') != -1:
+            new_string = new_string[0:new_string.rfind(',')] + new_string[new_string.rfind(',')+1:]
+        return new_string
+    elif lang == 'kr':
+        if(string[0] == 'Share'):
+            return '0'
+
+        #print("kr:comments:input[", string[0], "]");
+        new_string = string[0].replace(' ', '')
+        new_string = new_string.replace('댓글', '')
+        new_string = new_string.replace('달기', '')
+        new_string = new_string.replace('개', '')
+        if(len(new_string) == 0):
+            new_string = '0'
+        #print("kr:comments:output:[", new_string, "]");
+
         while new_string.rfind(',') != -1:
             new_string = new_string[0:new_string.rfind(',')] + new_string[new_string.rfind(',')+1:]
         return new_string
@@ -580,6 +605,12 @@ class FbcrawlItem(scrapy.Item):
     likes = scrapy.Field(
         output_processor=reactions_strip
     )
+    likers = scrapy.Field(
+        #output_processor=reactions_strip
+    )
+    likers_url = scrapy.Field(
+        #output_processor= reactions_strip
+    )
     ahah = scrapy.Field(
         output_processor=reactions_strip
     )
@@ -603,7 +634,8 @@ class FbcrawlItem(scrapy.Item):
         output_processor=id_strip
     )
     shared_from = scrapy.Field()
-
+    more_likes_url = scrapy.Field()
+
 class CommentsItem(scrapy.Item):
     source = scrapy.Field()
     reply_to=scrapy.Field()

fbcrawl/settings.py

@@ -20,12 +20,12 @@
 ROBOTSTXT_OBEY = False
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-CONCURRENT_REQUESTS = 16
+CONCURRENT_REQUESTS = 1 #16
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-DOWNLOAD_DELAY = 3
+DOWNLOAD_DELAY = 1 #3
 
 # The download delay setting will honor only one of:
 #CONCURRENT_REQUESTS_PER_DOMAIN = 1

fbcrawl/spiders/fbcrawl.py

@@ -11,12 +11,15 @@ class FacebookSpider(scrapy.Spider):
     '''
     Parse FB pages (needs credentials)
     '''    
-    name = 'fb'
+    name = 'facebook'
+    #self.start_urls = ['https://mbasic.facebook.com'] in __init__ function
     custom_settings = {
         'FEED_EXPORT_FIELDS': ['source','shared_from','date','text', \
-                               'reactions','likes','ahah','love','wow', \
+                               'reactions' \
+                               ,'likes','likers', 'likers_url' \
+                               ,'ahah','love','wow', \
                                'sigh','grrr','comments','post_id','url'],
-        'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter',
+        'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', # disable filtering of duplicate requests
     }
 
     def __init__(self, *args, **kwargs):
@@ -35,18 +38,19 @@ def __init__(self, *args, **kwargs):
 
         #page name parsing (added support for full urls)
         if 'page' in kwargs:
+            # check is groups
             if self.page.find('/groups/') != -1:
                 self.group = 1
             else:
                 self.group = 0
+            # remove prepend url
             if self.page.find('https://www.facebook.com/') != -1:
                 self.page = self.page[25:]
             elif self.page.find('https://mbasic.facebook.com/') != -1:
                 self.page = self.page[28:]
             elif self.page.find('https://m.facebook.com/') != -1:
                 self.page = self.page[23:]
 
-
         #parse date
         if 'date' not in kwargs:
             self.logger.info('Date attribute not provided, scraping date set to 2004-02-04 (fb launch date)')
@@ -62,7 +66,7 @@ def __init__(self, *args, **kwargs):
             self.logger.info('To specify, add the lang parameter: scrapy fb -a lang="LANGUAGE"')
             self.logger.info('Currently choices for "LANGUAGE" are: "en", "es", "fr", "it", "pt"')
             self.lang = '_'                       
-        elif self.lang == 'en'  or self.lang == 'es' or self.lang == 'fr' or self.lang == 'it' or self.lang == 'pt':
+        elif self.lang == 'en'  or self.lang == 'es' or self.lang == 'fr' or self.lang == 'it' or self.lang == 'pt' or self.lang == 'kr':
             self.logger.info('Language attribute recognized, using "{}" for the facebook interface'.format(self.lang))
         else:
             self.logger.info('Lang "{}" not currently supported'.format(self.lang))                             
@@ -178,6 +182,7 @@ def parse_page(self, response):
             #returns full post-link in a list
             post = post.xpath(".//a[contains(@href,'footer')]/@href").extract() 
             temp_post = response.urljoin(post[0])
+            #print('temp_post:url:', temp_post);
             self.count -= 1
             yield scrapy.Request(temp_post, self.parse_post, priority = self.count, meta={'item':new})       
 
@@ -243,15 +248,18 @@ def parse_post(self,response):
             new.add_xpath('reactions',"//a[contains(@href,'reaction/profile')]/div/div/text()")              
             reactions = response.xpath("//div[contains(@id,'sentence')]/a[contains(@href,'reaction/profile')]/@href")
             reactions = response.urljoin(reactions[0].extract())
+            print('reactions:url:', reactions);
             yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':new})
 
     def parse_reactions(self,response):
         new = ItemLoader(item=FbcrawlItem(),response=response, parent=response.meta['item'])
         new.context['lang'] = self.lang           
         new.add_xpath('likes',"//a[contains(@href,'reaction_type=1')]/span/text()")
+        new.add_xpath('likers',"//h3//a/text()")
+        new.add_xpath('likers_url',"//h3//a/@href")
         new.add_xpath('ahah',"//a[contains(@href,'reaction_type=4')]/span/text()")
         new.add_xpath('love',"//a[contains(@href,'reaction_type=2')]/span/text()")
         new.add_xpath('wow',"//a[contains(@href,'reaction_type=3')]/span/text()")
         new.add_xpath('sigh',"//a[contains(@href,'reaction_type=7')]/span/text()")
         new.add_xpath('grrr',"//a[contains(@href,'reaction_type=8')]/span/text()")     
-        yield new.load_item()       
+        yield new.load_item()