From 7be4015f1357bf054fc6bd0c32b07dbf4c38ae8b Mon Sep 17 00:00:00 2001 From: jaewnoha Date: Wed, 12 Feb 2020 02:57:00 +0900 Subject: [PATCH 1/2] likers - press the more button and collect all reactions (currently includes angry reactions) --- fbcrawl/__pycache__/__init__.cpython-37.pyc | Bin 139 -> 135 bytes fbcrawl/__pycache__/items.cpython-37.pyc | Bin 11514 -> 12254 bytes fbcrawl/__pycache__/settings.cpython-37.pyc | Bin 551 -> 542 bytes fbcrawl/items.py | 34 +- fbcrawl/settings.py | 4 +- .../__pycache__/__init__.cpython-37.pyc | Bin 147 -> 143 bytes .../__pycache__/comments.cpython-37.pyc | Bin 7765 -> 7761 bytes .../__pycache__/fbcrawl.cpython-37.pyc | Bin 8791 -> 8941 bytes fbcrawl/spiders/fbcrawl.py | 20 +- fbcrawl/spiders/likers.py | 292 ++++++++++++++++++ run.sh | 9 + 11 files changed, 350 insertions(+), 9 deletions(-) create mode 100644 fbcrawl/spiders/likers.py create mode 100755 run.sh diff --git a/fbcrawl/__pycache__/__init__.cpython-37.pyc b/fbcrawl/__pycache__/__init__.cpython-37.pyc index 2b546852d3ac74dd3c2c3730cbc864a0e0331c2c..318a827da7a7a66502c300f0be27295f4105015b 100644 GIT binary patch delta 51 zcmeBXY-i+g;^pOH0D{Bz7I71K%ta0KLyJ?3iuJP+Q_J)7G7|Nj^HWkM#%QsA2B`u7 DMz9U8 delta 55 zcmZo?>}KR~;^pOH0D{#^gJLG~m`j@IXXNLm>KB!!C+3x8=IguUm*?f=C#DonjL>2& H2B`!9n)45P diff --git a/fbcrawl/__pycache__/items.cpython-37.pyc b/fbcrawl/__pycache__/items.cpython-37.pyc index 40067241962d71b41348a113080a21b0a3a4e57f..4b21741b577a7d86dcc2c452dc9a7f7f4847c303 100644 GIT binary patch delta 2390 zcmbVNU2IfE6uxus?)G+^W_p_;ULmX?lbBRd{C|Eq#V zHGYmXS6v||&ds@@^eGhn#!CJL>D?#PAvQ}Ig^~|JJ?>xR_imjok{7PuyLBV40r%dG z_l$rDvVJ>DMTmJmvvZ<^Svki{TS9ZJEU#A+^|i2dwzKP)XECp9(6ol#?10&|+ocxo zR}%;KfLI|qj2H>1F;yokNPv{Xh>{MV!t?9c+`60uv?y?KARQwK&}8wM&p>^gRe?Pb z^m2~vvr0*)-F(&;nhj85d(=K|`$mvegN)T6lmH}4v(_+%kUgF(>1Vc;r5;1qi%t?pE=_Q-c1F*3g zCsnA4jjRQw4`yXH+~e4upLR339W(r9!`W?fYHOq#+Ygf0s{mRey?!Jg&pe8=xJP9}DubQi*Igku0g8_Z`B9HE){qEY}sh;fNTbnxfVlS~r5wKP) zjj$O(wmY=7>Es4X5p>*Pi^-BPtfvSJx^`9sd^=;oyYng7tKnZawUEvH*QO1L3~mWh z%kfPSpyue3mX9oS&8CT&Jd?y>E9-l#0hguow;r-<4_TfF;lw-U&S-< z+Q1b(1=C@ph82)EhVUA~IKpX!*8zl%Z}Vu%ciBng%?Hokz$FIDCJ^2P5Mj^Fv$Q4s zlrfz{(RqZo5ZW#vUPQQruu$xUST2KLod3R|lU(Kv?Gx3@y2o#|C*#X$UjyxTczeex zat*ZOa1pk(?;e+*epQ6`p2QMiMGqS*{yA_CMuhITX%ioA5gZ<{n3W9X23+4whCSb8 z@OZ+TD0F!7i&TVKUK+k-sIVj~`OHp&(f6@Nx47 Lp;#hT7mNG_Zw)Nb delta 1739 zcmb7ETW=dh6yDigZ`MxY#*TAuoRGw6HWxdD6cB_IN-3ljMI@B~gK)RrO|!|`Yi8Ft z0hcIH(kh~YXheh%5+L!y8^VH@KJf#10C5rXfDk``ACQN_Ib)KbU>>lw-=3Ls&d&Fp zIWxcJ_FsseZEcN6@EdyIx0mi*-HmpWz1>sqYZh6OjLzL3789R=vq~(|BVCuPvL&y{ z^nUgb8R;#q!mtU3A5|qw>5(iQ>`PX7Nn*E1KiOkR*@(UToK#hoq^h(eS@0OG?5=z< zX2?85mz|=;HR=b{DY18zbTSSr?cbYO@d|c^mamz`!0|HYyh_paOv`6KE9t>rq)K%7 z$B8Pp$vSBvDv6RdG6`dvs6C^`e$3D(XeSi$tw)7%7arj9(4&TkP zyWu4=$`&K37h|~M1TIFbthVf$LC&IMIBDRdLhV2hGKff}f>)$uQs2&5W?)~2C?6Go zbU8{ixg`=OElQHmncDY}KN935`(~h>Bx^qoTqHBd1HROQFPY2fX*vXEUGX8u{m7i_ z_(9$)l;%HVi}!4w{yPmTY{>K+yf|4umPMa{=`Y~%WdKd$tZis^^52>CDR4K+{gRc2 z_R^1_cOKyk!Xf}yR?9^Mmn&xZ8hujCdHH~zWwXN@2W!+t33iQC34Q_lW%%g^a{iOl z4ml9?f91Tx?u_)3N7+v!Q&+lB$`nEf;R-?qp%Z{NIlk$ZR!kmpy^2l6?obUq>Z<1y zxf(bHn_}^KQ^mH|EOWzX%Z8nz1HA$*0KJ<}jgk!8NsXshF)U=+ZeVg1BK>2f=FLaF zGqYCvJ@txAe0F)veHNRLP9h*vL*Y#&&ku5rCEAltqet|J0MUoKW7V5|2KL-X%6I^~ z2`z(_-;H&Uui0bie)jQ%#x5AsiB;8Zwv^i>8?}`Kn{69o4}YS5DrMWnWI_aV2)k3* zUA7|p-IfZsM2I(^znZ6J#a#&O0$pI?@ln#lrpM<_i;^m^G?dI<4`>2S9s0On7XzR6 z;V^u7q@mo-n(ec%##7G^gNqI#Aafc3@MeF-q_&k?re5J6!VlHcS0U;_6g~?E-a9(p zH?j%`@!(juiO)CJae)N@IENK@T{1f92F|^P@H#>j;X1+_09>bb$=%2W z9`$kVP$k{OA*!NV2yX)Li0_qY-p;{2R2TFXx^5u6jWD*2coX3k!qJW%3EKfjj(s;d zMRwSqlQ%jV(qkV?WqTTB--hf8YfksIpk>5_oGi;^?knSl)Cr4%&)`sK!m4vsY$#Th uA_I!)jrehWuHac^*PhoQ!aoV1lLUbqKdz_rnBJvtO8CE^Yw=U@?!N%a6m$>( diff --git a/fbcrawl/__pycache__/settings.cpython-37.pyc b/fbcrawl/__pycache__/settings.cpython-37.pyc index b18e5e5b05fe32edd9054e54cadf56abc4198ff3..fbb257d2be6e02d430253c706281dc8a4b7d15bf 100644 GIT binary patch delta 91 zcmZ3^GLMDViItl$5yh3l8O)%`H8E}i qBjdy`XXSZ;%8Nk6F9ZG1;?$yI{j9{)^8CDvM1AM{l+?*j7@YwC6&UdV delta 100 zcmbQovYdt2iIz9>tNu5yhFp8O4>t70jT?Juz+q yn*dPdOXi91&noi)Sw$e?mx+Exer~FMQE7T&UP)%YzDs_2UQT{uO7Y~|jLrbBs~c

Ecg{~q-CWOnT7WTX Xv$x0@cJ2V6@*-al5wuxJ#+V5J{(lrp delta 76 zcmca;bJd2&iI@1A?^xp&U} zo!`BC``Df1#%LrGRPYzRHTv50@Q21f!5DS%XP*^GD4t|-mRN|fB-{3>TJm01=7Dvv z)D^wtW82w|D@w`F(yS9(oprID*aldJWw8yi9Lr-Hx~h1TQuvfo*d?yPk$^sL&RMhd z`m*>HhLXN&W!d7k8*I1bH+WS%fEzH>x}na(#p_LFDb6y>>MGb@N;|vB&TX>uo9tbi z?A>JZA8@Y!rq1@b zf)y@9Qz<8FhLdQjtXF*E>5B9%kZ(IvaL@g%pFDr4o|3xETc7zq2FQv-fx%D@66|V~ zn!^Xgxxj(K0qhsR4V24sRnxZ1<*$^#hl?{dQd3+qt#kF-qFEfPGfVs)=pG-$=romR z520<;X>1qsg)rXjX_8Zf52t!OwKnf!;Rz5mBE5?Gqz)lef9X-4gm*E)% zH`p+_ZIu`6D^@|}N6Dpg%Wl+bn9FerlQ2#LUKv|zy&w8q4LwI=CkXb5gT@5(iua5u zD2Us}4D^YC=nK#<-i}VgesL>$b~=xN1&z}tC#?13KfMaN0Vi+dff zz;-d2>hbNMxoL4BRZN^lv!J?qgIDUj;=IYHTHmK$1)tpBl*sJ3xBo@@h_+xkuFq^V ztQzA!nlIDS$vQ;fCwNL6PUpHu$W#fu2(D(%+WZK02MF5x6_?U`hNr22ir^)LhXrZW zZHFHu?-aq)1TzSQkSr3P5N`TZnzGuuX^L7Ej5qYC_PlaE+j-!>TICFv`3qmW6G6eFAdi6>)(r6A@d-l^4@^)@WS@Xl zaVgsv^RgI=uOT;0^`@64SaRL=ttpR~+4g^Ccomm#%OENqH6f$l%EiU~T(9^B+u?k#xRZ-R2aX(slpHVTGhTlwXSbdzwpMcSu}v?8hE6zfLTucJAuS}m71mMZ{1 zMv>zL0R;LBvFQ@gH4k|3!coD8PZ)G9d{htV{rVX_DI-x0vWkykup1HI?pld$WGSt2 IF}Sz>% delta 1582 zcmZWpO>7%g5Z>8!?DbFlm&A#k#C8)m$tDG>O-h1j)e-{z3u#EyR<(+(jo-#`;@IZx zwghSspcTDP1bX5EJtRno!U^GW?STub1VUVjv|PBrEjJFRaA0QbHX*FF-^`o$?Rfs) z+j;Sqnc#GLyHCPr{Qko$`q7VqfBS;8#K+{K1G1-!K^9@rJ94h=mb474gLU3ib8gnf zVmGCnhjp_alq!p}1WGUKWql}pte+)O`fo`tDHm9jw3PS&PAKy7l6VLcA}N0iBaM6V z5^TO#msTUJe@*@n%sY;r+@q)V=;=NB0n+(5$oFiiY+x%yUv9%TfK-`iSqJx8Teb`_ zTKYlKXDv4yx&iM38?MW19(4MT6=fqgpf2S*DnTn+m)US!YObxoR_C^~8pgdY=0l7z ztw_vnCSi`FjUCsoa^nZjCjfn-=pFZGu#UD;s#ttXeC|E^)Htp+u)Xg$zqRcEG9e3_RH;}x#JVF95FFd{C6&Vwet2ra;2#;KAM1)< zK%*wxYK@nwykxz>=NiAqu7KM~bWWV=`SZ|u`a7j)ShicQ)r<<`Zt~;=s**NI;30Tc z6yvGP6sa;n8-lIqOC~=~%Q1px=HjdP;N(25Um;jPc-)#=)wK9CWTo!&V+4x`n%`*` zKPOy?#W+Q^KYe1oFKUTN0C6|*8r&5>CeI4i+XpAbJH4Y*tAV-;k3lD%D2`~B*p2NsaEddHo!*~d6UU`l6^bRen_x3cXXrhJYQ-`(ES{(6oDeUg zW_xM0^Cinz=T6Q#ZSN)?YR)7#^a7rRimA1=G9_9OAr~sFh=pWF{J#UrmvHY7e9a#5 zRcqakN$W`Z3tuyeAVExA zI&f{14m_VFa1H@A(n*dJ+iq{&90j9X&gXX>YQU$7IYZz@piABCK~NPJxMUURHc*wo mN!728sW~<3u(VLK_J@#Yx0?m7i}`_4c(=CA8$di5$ova9%4G%s diff --git a/fbcrawl/spiders/fbcrawl.py b/fbcrawl/spiders/fbcrawl.py index f4f07ad..4046df8 100644 --- a/fbcrawl/spiders/fbcrawl.py +++ b/fbcrawl/spiders/fbcrawl.py @@ -11,12 +11,15 @@ class FacebookSpider(scrapy.Spider): ''' Parse FB pages (needs credentials) ''' - name = 'fb' + name = 'facebook' + #self.start_urls = ['https://mbasic.facebook.com'] in __init__ function custom_settings = { 'FEED_EXPORT_FIELDS': ['source','shared_from','date','text', \ - 'reactions','likes','ahah','love','wow', \ + 'reactions' \ + ,'likes','likers', 'likers_url' \ + ,'ahah','love','wow', \ 'sigh','grrr','comments','post_id','url'], - 'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', + 'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', # disable filtering of duplicate requests } def __init__(self, *args, **kwargs): @@ -35,10 +38,12 @@ def __init__(self, *args, **kwargs): #page name parsing (added support for full urls) if 'page' in kwargs: + # check is groups if self.page.find('/groups/') != -1: self.group = 1 else: self.group = 0 + # remove prepend url if self.page.find('https://www.facebook.com/') != -1: self.page = self.page[25:] elif self.page.find('https://mbasic.facebook.com/') != -1: @@ -46,7 +51,6 @@ def __init__(self, *args, **kwargs): elif self.page.find('https://m.facebook.com/') != -1: self.page = self.page[23:] - #parse date if 'date' not in kwargs: self.logger.info('Date attribute not provided, scraping date set to 2004-02-04 (fb launch date)') @@ -62,7 +66,7 @@ def __init__(self, *args, **kwargs): self.logger.info('To specify, add the lang parameter: scrapy fb -a lang="LANGUAGE"') self.logger.info('Currently choices for "LANGUAGE" are: "en", "es", "fr", "it", "pt"') self.lang = '_' - elif self.lang == 'en' or self.lang == 'es' or self.lang == 'fr' or self.lang == 'it' or self.lang == 'pt': + elif self.lang == 'en' or self.lang == 'es' or self.lang == 'fr' or self.lang == 'it' or self.lang == 'pt' or self.lang == 'kr': self.logger.info('Language attribute recognized, using "{}" for the facebook interface'.format(self.lang)) else: self.logger.info('Lang "{}" not currently supported'.format(self.lang)) @@ -178,6 +182,7 @@ def parse_page(self, response): #returns full post-link in a list post = post.xpath(".//a[contains(@href,'footer')]/@href").extract() temp_post = response.urljoin(post[0]) + #print('temp_post:url:', temp_post); self.count -= 1 yield scrapy.Request(temp_post, self.parse_post, priority = self.count, meta={'item':new}) @@ -243,15 +248,18 @@ def parse_post(self,response): new.add_xpath('reactions',"//a[contains(@href,'reaction/profile')]/div/div/text()") reactions = response.xpath("//div[contains(@id,'sentence')]/a[contains(@href,'reaction/profile')]/@href") reactions = response.urljoin(reactions[0].extract()) + print('reactions:url:', reactions); yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':new}) def parse_reactions(self,response): new = ItemLoader(item=FbcrawlItem(),response=response, parent=response.meta['item']) new.context['lang'] = self.lang new.add_xpath('likes',"//a[contains(@href,'reaction_type=1')]/span/text()") + new.add_xpath('likers',"//h3//a/text()") + new.add_xpath('likers_url',"//h3//a/@href") new.add_xpath('ahah',"//a[contains(@href,'reaction_type=4')]/span/text()") new.add_xpath('love',"//a[contains(@href,'reaction_type=2')]/span/text()") new.add_xpath('wow',"//a[contains(@href,'reaction_type=3')]/span/text()") new.add_xpath('sigh',"//a[contains(@href,'reaction_type=7')]/span/text()") new.add_xpath('grrr',"//a[contains(@href,'reaction_type=8')]/span/text()") - yield new.load_item() \ No newline at end of file + yield new.load_item() diff --git a/fbcrawl/spiders/likers.py b/fbcrawl/spiders/likers.py new file mode 100644 index 0000000..6fecf6f --- /dev/null +++ b/fbcrawl/spiders/likers.py @@ -0,0 +1,292 @@ +import scrapy +import logging + +from scrapy.loader import ItemLoader +from scrapy.http import FormRequest +from scrapy.exceptions import CloseSpider +from fbcrawl.items import FbcrawlItem, parse_date, parse_date2 +from datetime import datetime + +class FacebookSpider(scrapy.Spider): + ''' + Parse FB pages (needs credentials) + ''' + name = 'likers' + # custom_settings = { + # 'FEED_EXPORT_FIELDS': ['source','shared_from','date','text', \ + # 'reactions' \ + # ,'likes','likers', 'likers_url' \ + # ,'ahah','love','wow', \ + # 'sigh','grrr','comments','post_id','url'], + # 'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', # disable filtering of duplicate requests + # } + + custom_settings = { + # 'FEED_EXPORT_FIELDS': ['url','post_id','date','text','likes' #base \ + # ,'likers', 'likers_url', 'more_likes', 'likes_url' #added:likers \ + # ], + 'FEED_EXPORT_FIELDS': ['post_id' + ,'reactions' + #,'likes' , 'ahah', 'love', 'wow' + #'more_likes_url', + ,'likers', 'likers_url' \ + ], + 'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', # disable filtering of duplicate requests + } + + def __init__(self, *args, **kwargs): + #turn off annoying logging, set LOG_LEVEL=DEBUG in settings.py to see more logs + logger = logging.getLogger('scrapy.middleware') + logger.setLevel(logging.WARNING) + + super().__init__(*args,**kwargs) + + #email & pass need to be passed as attributes! + if 'email' not in kwargs or 'password' not in kwargs: + raise AttributeError('You need to provide valid email and password:\n' + 'scrapy fb -a email="EMAIL" -a password="PASSWORD"') + else: + self.logger.info('Email and password provided, will be used to log in') + + #page name parsing (added support for full urls) + if 'page' in kwargs: + # check is groups + if self.page.find('/groups/') != -1: + self.group = 1 + else: + self.group = 0 + # remove prepend url + if self.page.find('https://www.facebook.com/') != -1: + self.page = self.page[25:] + elif self.page.find('https://mbasic.facebook.com/') != -1: + self.page = self.page[28:] + elif self.page.find('https://m.facebook.com/') != -1: + self.page = self.page[23:] + + #parse date + if 'date' not in kwargs: + self.logger.info('Date attribute not provided, scraping date set to 2004-02-04 (fb launch date)') + self.date = datetime(2004,2,4) + else: + self.date = datetime.strptime(kwargs['date'],'%Y-%m-%d') + self.logger.info('Date attribute provided, fbcrawl will stop crawling at {}'.format(kwargs['date'])) + self.year = self.date.year + + #parse lang, if not provided (but is supported) it will be guessed in parse_home + if 'lang' not in kwargs: + self.logger.info('Language attribute not provided, fbcrawl will try to guess it from the fb interface') + self.logger.info('To specify, add the lang parameter: scrapy fb -a lang="LANGUAGE"') + self.logger.info('Currently choices for "LANGUAGE" are: "en", "es", "fr", "it", "pt"') + self.lang = '_' + elif self.lang == 'en' or self.lang == 'es' or self.lang == 'fr' or self.lang == 'it' or self.lang == 'pt' or self.lang == 'kr': + self.logger.info('Language attribute recognized, using "{}" for the facebook interface'.format(self.lang)) + else: + self.logger.info('Lang "{}" not currently supported'.format(self.lang)) + self.logger.info('Currently supported languages are: "en", "es", "fr", "it", "pt"') + self.logger.info('Change your interface lang from facebook settings and try again') + raise AttributeError('Language provided not currently supported') + + #max num of posts to crawl + if 'max' not in kwargs: + self.max = int(10e5) + else: + self.max = int(kwargs['max']) + + #current year, this variable is needed for proper parse_page recursion + self.k = datetime.now().year + #count number of posts, used to enforce DFS and insert posts orderly in the csv + self.count = 0 + + self.start_urls = ['https://mbasic.facebook.com'] + + def parse(self, response): + ''' + Handle login with provided credentials + ''' + return FormRequest.from_response( + response, + formxpath='//form[contains(@action, "login")]', + formdata={'email': self.email,'pass': self.password}, + callback=self.parse_home + ) + + def parse_home(self, response): + ''' + This method has multiple purposes: + 1) Handle failed logins due to facebook 'save-device' redirection + 2) Set language interface, if not already provided + 3) Navigate to given page + ''' + #handle 'save-device' redirection + if response.xpath("//div/a[contains(@href,'save-device')]"): + self.logger.info('Going through the "save-device" checkpoint') + return FormRequest.from_response( + response, + formdata={'name_action_selected': 'dont_save'}, + callback=self.parse_home + ) + + #set language interface + if self.lang == '_': + if response.xpath("//input[@placeholder='Search Facebook']"): + self.logger.info('Language recognized: lang="en"') + self.lang = 'en' + elif response.xpath("//input[@placeholder='Buscar en Facebook']"): + self.logger.info('Language recognized: lang="es"') + self.lang = 'es' + elif response.xpath("//input[@placeholder='Rechercher sur Facebook']"): + self.logger.info('Language recognized: lang="fr"') + self.lang = 'fr' + elif response.xpath("//input[@placeholder='Cerca su Facebook']"): + self.logger.info('Language recognized: lang="it"') + self.lang = 'it' + elif response.xpath("//input[@placeholder='Pesquisa no Facebook']"): + self.logger.info('Language recognized: lang="pt"') + self.lang = 'pt' + else: + raise AttributeError('Language not recognized\n' + 'Change your interface lang from facebook ' + 'and try again') + + #navigate to provided page + href = response.urljoin(self.page) + self.logger.info('Scraping facebook page {}'.format(href)) + return scrapy.Request(url=href,callback=self.parse_page,meta={'index':1}) + + def parse_page(self, response): + ''' + Parse the given page selecting the posts. + Then ask recursively for another page. + ''' +# #open page in browser for debug +# from scrapy.utils.response import open_in_browser +# open_in_browser(response) + + #select all posts + for post in response.xpath("//div[contains(@data-ft,'top_level_post_id')]"): + + many_features = post.xpath('./@data-ft').get() + date = [] + date.append(many_features) + date = parse_date(date,{'lang':self.lang}) + current_date = datetime.strptime(date,'%Y-%m-%d %H:%M:%S') if date is not None else date + + if current_date is None: + date_string = post.xpath('.//abbr/text()').get() + date = parse_date2([date_string],{'lang':self.lang}) + current_date = datetime(date.year,date.month,date.day) if date is not None else date + date = str(date) + + #if 'date' argument is reached stop crawling + if self.date > current_date: + raise CloseSpider('Reached date: {}'.format(self.date)) + + new = ItemLoader(item=FbcrawlItem(),selector=post) + if abs(self.count) + 1 > self.max: + raise CloseSpider('Reached max num of post: {}. Crawling finished'.format(abs(self.count))) + self.logger.info('Parsing post n = {}, post_date = {}'.format(abs(self.count)+1,date)) + new.add_xpath('comments', './div[2]/div[2]/a[1]/text()') + new.add_value('date',date) + new.add_xpath('post_id','./@data-ft') + new.add_xpath('url', ".//a[contains(@href,'footer')]/@href") + #page_url #new.add_value('url',response.url) + + #returns full post-link in a list + post = post.xpath(".//a[contains(@href,'footer')]/@href").extract() + temp_post = response.urljoin(post[0]) + #print('temp_post:url:', temp_post); + self.count -= 1 + yield scrapy.Request(temp_post, self.parse_post, priority = self.count, meta={'item':new}) + + #load following page, try to click on "more" + #after few pages have been scraped, the "more" link might disappears + #if not present look for the highest year not parsed yet + #click once on the year and go back to clicking "more" + + #new_page is different for groups + if self.group == 1: + new_page = response.xpath("//div[contains(@id,'stories_container')]/div[2]/a/@href").extract() + else: + new_page = response.xpath("//div[2]/a[contains(@href,'timestart=') and not(contains(text(),'ent')) and not(contains(text(),number()))]/@href").extract() + #this is why lang is needed ^^^^^^^^^^^^^^^^^^^^^^^^^^ + + if not new_page: + self.logger.info('[!] "more" link not found, will look for a "year" link') + #self.k is the year link that we look for + if response.meta['flag'] == self.k and self.k >= self.year: + xpath = "//div/a[contains(@href,'time') and contains(text(),'" + str(self.k) + "')]/@href" + new_page = response.xpath(xpath).extract() + if new_page: + new_page = response.urljoin(new_page[0]) + self.k -= 1 #decrease year + self.logger.info('Found a link for year "{}", new_page = {}'.format(self.k,new_page)) + yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':self.k}) + else: + while not new_page: #sometimes the years are skipped this handles small year gaps + self.logger.info('Link not found for year {}, trying with previous year {}'.format(self.k,self.k-1)) + self.k -= 1 + if self.k < self.year: + raise CloseSpider('Reached date: {}. Crawling finished'.format(self.date)) + xpath = "//div/a[contains(@href,'time') and contains(text(),'" + str(self.k) + "')]/@href" + new_page = response.xpath(xpath).extract() + self.logger.info('Found a link for year "{}", new_page = {}'.format(self.k,new_page)) + new_page = response.urljoin(new_page[0]) + self.k -= 1 #decrease year + yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':self.k}) + else: + self.logger.info('Crawling has finished with no errors!') + else: + new_page = response.urljoin(new_page[0]) + if 'flag' in response.meta: + self.logger.info('Page scraped, clicking on "more"! new_page = {}'.format(new_page)) + yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':response.meta['flag']}) + else: + self.logger.info('First page scraped, clicking on "more"! new_page = {}'.format(new_page)) + yield scrapy.Request(new_page, callback=self.parse_page, meta={'flag':self.k}) + + def parse_post(self,response): + print(">>parse_post") + new = ItemLoader(item=FbcrawlItem(),response=response,parent=response.meta['item']) + new.context['lang'] = self.lang + new.add_xpath('source', "//td/div/h3/strong/a/text() | //span/strong/a/text() | //div/div/div/a[contains(@href,'post_id')]/strong/text()") + new.add_xpath('shared_from','//div[contains(@data-ft,"top_level_post_id") and contains(@data-ft,\'"isShare":1\')]/div/div[3]//strong/a/text()') + # new.add_xpath('date','//div/div/abbr/text()') + new.add_xpath('text','//div[@data-ft]//p//text() | //div[@data-ft]/div[@class]/div[@class]/text()') + + #check reactions for old posts + check_reactions = response.xpath("//a[contains(@href,'reaction/profile')]/div/div/text()").get() + if not check_reactions: + yield new.load_item() + else: + new.add_xpath('reactions',"//a[contains(@href,'reaction/profile')]/div/div/text()") + reactions = response.xpath("//div[contains(@id,'sentence')]/a[contains(@href,'reaction/profile')]/@href") + reactions = response.urljoin(reactions[0].extract()) + print('reactions:url:', reactions); + yield scrapy.Request(reactions, callback=self.parse_reactions, meta={'item':new}) + + def parse_reactions(self,response): + new = ItemLoader(item=FbcrawlItem(),response=response, parent=response.meta['item']) + new.context['lang'] = self.lang + + new.replace_xpath('likes',"//a[contains(@href,'reaction_type=1')]/span/text()") + new.replace_xpath('likers',"//h3//a/text()") + new.replace_xpath('likers_url',"//h3//a/@href") + + #recusively click 'more' => and then post process... + likes_url = response.xpath('//td/div/a[contains(@href,"/ufi/reaction/profile/browser/fetch/")]/@href') + likes_url = response.urljoin(likes_url[0].extract()) if len(likes_url) > 0 else '' + + # debug + # print("###dbg:more_likes_url", likes_url) + # new.replace_value('more_likes_url',likes_url) + + yield new.load_item() + + # self.logger.info('parse_reactions:new_likers={}, reactions={}'.format(new_likers, reactions)) + # #self.logger.info('parse_reactions:post_id={},new_likers={}'.format(new.post_id, new_likers)) + + if likes_url != '': + print("###dbg:recursive:", likes_url) + yield scrapy.Request(likes_url, callback=self.parse_reactions, meta={'item':new}) + + diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..a8cd041 --- /dev/null +++ b/run.sh @@ -0,0 +1,9 @@ +#TYPE="fb" +TYPE="likers" +EMAIL="nicejaewon@gmail.com" +PASSWORD="qwcv135A" +PAGE="SBS8news" +DATE="2020-02-11" +OUT=$PAGE-$DATE.csv +LANG=kr +scrapy crawl $TYPE -a email="$EMAIL" -a password="$PASSWORD" -a page="$PAGE" -a date="$DATE" -a lang="kr" -o $OUT From 5fefe2fe281621df043bb6c546502062c4de475f Mon Sep 17 00:00:00 2001 From: padraic Date: Sat, 15 Feb 2020 15:44:51 +0000 Subject: [PATCH 2/2] Update README.md For instructions on scraping comments I changed 'page' to 'post', which is the actual working command variable. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 85b0bfe..9f0d86a 100644 --- a/README.md +++ b/README.md @@ -162,13 +162,13 @@ A new spider is now dedicated to crawl all the comments from a post (not a page! You can try it out with: ``` -scrapy crawl comments -a email="EMAILTOLOGIN" -a password="PASSWORDTOLOGIN" -a page="LINKOFTHEPOSTTOCRAWL" -o DUMPFILE.csv +scrapy crawl comments -a email="EMAILTOLOGIN" -a password="PASSWORDTOLOGIN" -a post="LINKOFTHEPOSTTOCRAWL" -o DUMPFILE.csv ``` -The use is similar to fb spider, the only difference being the -a page parameter, which now is the link to a post. Make sure that the `page` option is a proper post link, for example: +The use is similar to fb spider, the only difference being the -a post parameter, which now is the link to a post. Make sure that the `page` option is a proper post link, for example: ``` -rm trump_comments.csv; scrapy crawl comments -a email="obama@gmail.com" -a password="cm380jixke" -a page="https://mbasic.facebook.com/story.php?story_fbid=10162169751605725&id=153080620724" -o trump_comments.csv +rm trump_comments.csv; scrapy crawl comments -a email="obama@gmail.com" -a password="cm380jixke" -a post="https://mbasic.facebook.com/story.php?story_fbid=10162169751605725&id=153080620724" -o trump_comments.csv ```