From 388da7507568ece8dd2b90d88d7800aa68906f26 Mon Sep 17 00:00:00 2001 From: Eliezer Pearl Date: Thu, 18 Jul 2019 12:28:44 -0400 Subject: [PATCH 1/2] Add going, interested, and shared counts --- fbcrawl/spiders/events.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fbcrawl/spiders/events.py b/fbcrawl/spiders/events.py index a1152ed..da381e6 100644 --- a/fbcrawl/spiders/events.py +++ b/fbcrawl/spiders/events.py @@ -42,6 +42,9 @@ def parse_event(self, response): DATE='/html/body/div/div/div[2]/div/table/tbody/tr/td/div[3]/div/div[1]/table/tbody/tr/td[2]/dt/div/text()' EVENT_DESCRIPTION='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div[2]/div[2]/text()' EVENT_COVER='/html/body/div/div/div[2]/div/table/tbody/tr/td/div[2]/div[1]/a/img/@src' + GOING='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div/div/div/div[2]/a/text()' + INTERESTED='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div/div/div[2]/div[2]/a/text()' + SHARED='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div/div/div[3]/div[2]/a/text()' date = response.xpath(DATE).extract_first() start_date = date.split('–')[0] or None end_date = date.split('–')[1] or None @@ -55,4 +58,7 @@ def parse_event(self, response): start_date=start_date, end_date=end_date, description=response.xpath(EVENT_DESCRIPTION).extract_first() + going=response.xpath(GOING).extract_first() + interested=response.xpath(INTERESTED).extract_first() + shared=response.xpath(SHARED).extract_first() ) From 03e65d5fcc3f50b2dbcaebf103fd5b44fcd41281 Mon Sep 17 00:00:00 2001 From: Eliezer Pearl Date: Thu, 18 Jul 2019 12:30:51 -0400 Subject: [PATCH 2/2] add going, interested, shared fields to eventsitem --- fbcrawl/__pycache__/__init__.cpython-37.pyc | Bin 139 -> 168 bytes fbcrawl/__pycache__/items.cpython-37.pyc | Bin 11514 -> 11878 bytes fbcrawl/__pycache__/settings.cpython-37.pyc | Bin 551 -> 580 bytes fbcrawl/items.py | 3 +++ .../__pycache__/__init__.cpython-37.pyc | Bin 147 -> 176 bytes .../__pycache__/comments.cpython-37.pyc | Bin 7765 -> 7794 bytes .../__pycache__/fbcrawl.cpython-37.pyc | Bin 8791 -> 8820 bytes fbcrawl/spiders/events.py | 11 ++++++----- 8 files changed, 9 insertions(+), 5 deletions(-) diff --git a/fbcrawl/__pycache__/__init__.cpython-37.pyc b/fbcrawl/__pycache__/__init__.cpython-37.pyc index 2b546852d3ac74dd3c2c3730cbc864a0e0331c2c..79fc6072b377990028d62222aa0b2ea013235b50 100644 GIT binary patch delta 101 zcmeBXT*1iW#LLUY00dX&8^liJF?SDiwu%WYPAw{qNzKVjt%`9;EzT~WHa^HWN5QtcRlMiqn10{~Y29V`F< diff --git a/fbcrawl/__pycache__/items.cpython-37.pyc b/fbcrawl/__pycache__/items.cpython-37.pyc index 40067241962d71b41348a113080a21b0a3a4e57f..af6c8348e7b7c0582444f7a5de41d8ecd83fd88f 100644 GIT binary patch delta 323 zcmewr`7DOdiI+%AJCz`v^ce>I3_hGGqozlCAB!aB)=fWH&P)f pGhHD+N1-6GAhjr_q^L+2%+gIsEJ=(>OG++EEYE?{lfN<=0RR!uB542s delta 68 zcmX@YvYds-iIIlbVy6S{37xTAW>yUl8LPsgRVJ zu8^OjP>@)VS`<@KRHO@L>82!>B*vsAB^M=@=fLUWg3J_WHa R^HWN5QtcRl_I(Cf3IHwUEO-C_ delta 101 zcmdnMIGK^hiI1-7fTAW%`9Fv-pnOYU&l3JWyl3x(x8>x_# znXZtZqfn4okXjT|QdFc1X6dFRmL$fcB_$Unmgm6f;)2W+kRh9AFmDoIjN0ria)zBd R0BBK>FNg@*tR!R11ORnUE71S| delta 101 zcmexlbJd2&iII3_hGGqozlCAB!aB)=fWH&P)f wGhHD+N1-6GAhjr_q^L+2%+gIsEJ=(>OG++EEYE?{#RZuuAVW5P;!KqQ0J;??8vphT0bK{H&wr=G(9n|Br{*%CBHl`CqFTzSU)W( YxhSzb2Tm6kWTvDR73**2=Sr0T0Qu({+W-In diff --git a/fbcrawl/spiders/events.py b/fbcrawl/spiders/events.py index da381e6..b3a2d3a 100644 --- a/fbcrawl/spiders/events.py +++ b/fbcrawl/spiders/events.py @@ -14,7 +14,8 @@ class EventsSpider(FacebookSpider): name = "events" custom_settings = { 'FEED_EXPORT_FIELDS': ['name','where','location','photo','start_date', \ - 'end_date','description'], + 'end_date','description', 'going', 'interested', \ + 'shared'], 'DUPEFILTER_CLASS' : 'scrapy.dupefilters.BaseDupeFilter', 'CONCURRENT_REQUESTS' : 1 } @@ -44,7 +45,7 @@ def parse_event(self, response): EVENT_COVER='/html/body/div/div/div[2]/div/table/tbody/tr/td/div[2]/div[1]/a/img/@src' GOING='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div/div/div/div[2]/a/text()' INTERESTED='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div/div/div[2]/div[2]/a/text()' - SHARED='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div/div/div[3]/div[2]/a/text()' + SHARED='/html/body/div/div/div[2]/div/table/tbody/tr/td/table/tbody/tr/td/div[2]/div[2]/div/div/div[3]/div[2]/div/text()' date = response.xpath(DATE).extract_first() start_date = date.split('–')[0] or None end_date = date.split('–')[1] or None @@ -57,8 +58,8 @@ def parse_event(self, response): photo=response.xpath(EVENT_COVER).extract_first(), start_date=start_date, end_date=end_date, - description=response.xpath(EVENT_DESCRIPTION).extract_first() - going=response.xpath(GOING).extract_first() - interested=response.xpath(INTERESTED).extract_first() + description=response.xpath(EVENT_DESCRIPTION).extract_first(), + going=response.xpath(GOING).extract_first(), + interested=response.xpath(INTERESTED).extract_first(), shared=response.xpath(SHARED).extract_first() )