From d36a4a04ceca74769fc6c8b0fe7584ab3ead71ae Mon Sep 17 00:00:00 2001 From: Serdar Tumgoren Date: Fri, 2 Dec 2016 20:49:51 -0800 Subject: [PATCH 1/5] DC: Add debugger script that dumps raw bill json --- scripts/dc/dump_bill_json.py | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 scripts/dc/dump_bill_json.py diff --git a/scripts/dc/dump_bill_json.py b/scripts/dc/dump_bill_json.py new file mode 100644 index 0000000000..e53c1d5071 --- /dev/null +++ b/scripts/dc/dump_bill_json.py @@ -0,0 +1,43 @@ +'''Dump JSON for a DC bill + +USAGE: + + $ python dump_bill_json.py PR21-0316 + +''' +import json +import pprint +import sys + +import requests + +def main(): + headers = {"Content-Type":"application/json"} + bill_url = "http://lims.dccouncil.us/_layouts/15/uploader/AdminProxy.aspx/GetPublicData" + try: + bill_id = sys.argv[1] + except IndexError: + msg = "\nERROR: You must supply a bill id! Example:\n\n\tpython {} PR21-0316\n".format(__file__) + sys.exit(msg) + bill_params = { "legislationId" : bill_id } + bill_info = requests.post(bill_url, headers=headers, data=json.dumps(bill_params)) + bill_info = decode_json(bill_info.json()["d"])["data"] + leg_info = bill_info["Legislation"][0] + pprint.pprint(bill_info) + +def decode_json(stringy_json): + #the "json" they send is recursively string-encoded. + if type(stringy_json) == dict: + for key in stringy_json: + stringy_json[key] = decode_json(stringy_json[key]) + elif type(stringy_json) == list: + for i in range(len(stringy_json)): + stringy_json[i] = decode_json(stringy_json[i]) + elif type(stringy_json) in (str,unicode): + if len(stringy_json) > 0 and stringy_json[0] in ["[","{",u"[",u"{"]: + return decode_json(json.loads(stringy_json)) + return stringy_json + + +if __name__ == '__main__': + main() From 1e9d78672fd6b38b62df48553c464664f6393e70 Mon Sep 17 00:00:00 2001 From: Serdar Tumgoren Date: Fri, 2 Dec 2016 20:58:27 -0800 Subject: [PATCH 2/5] DC: remove unused var in bill dump script --- scripts/dc/dump_bill_json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/dc/dump_bill_json.py b/scripts/dc/dump_bill_json.py index e53c1d5071..2c2552633d 100644 --- a/scripts/dc/dump_bill_json.py +++ b/scripts/dc/dump_bill_json.py @@ -22,7 +22,6 @@ def main(): bill_params = { "legislationId" : bill_id } bill_info = requests.post(bill_url, headers=headers, data=json.dumps(bill_params)) bill_info = decode_json(bill_info.json()["d"])["data"] - leg_info = bill_info["Legislation"][0] pprint.pprint(bill_info) def decode_json(stringy_json): From b83870f237d047af260513b149d3cda273f63943 Mon Sep 17 00:00:00 2001 From: Serdar Tumgoren Date: Fri, 2 Dec 2016 21:47:47 -0800 Subject: [PATCH 3/5] Add mkdir_p utility and update DC helper script to dump raw json to data dir --- openstates/utils/__init__.py | 1 + openstates/utils/dir.py | 11 +++++++++++ scripts/dc/dump_bill_json.py | 8 ++++++++ 3 files changed, 20 insertions(+) create mode 100644 openstates/utils/dir.py diff --git a/openstates/utils/__init__.py b/openstates/utils/__init__.py index 58fd21e5d7..68029c5388 100644 --- a/openstates/utils/__init__.py +++ b/openstates/utils/__init__.py @@ -1,4 +1,5 @@ from .lxmlize import LXMLMixin +from .dir import mkdir_p import re diff --git a/openstates/utils/dir.py b/openstates/utils/dir.py new file mode 100644 index 0000000000..00e48809b3 --- /dev/null +++ b/openstates/utils/dir.py @@ -0,0 +1,11 @@ +import errno +import os + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise diff --git a/scripts/dc/dump_bill_json.py b/scripts/dc/dump_bill_json.py index 2c2552633d..03db2f7ffa 100644 --- a/scripts/dc/dump_bill_json.py +++ b/scripts/dc/dump_bill_json.py @@ -11,6 +11,8 @@ import requests +from openstates.utils import mkdir_p + def main(): headers = {"Content-Type":"application/json"} bill_url = "http://lims.dccouncil.us/_layouts/15/uploader/AdminProxy.aspx/GetPublicData" @@ -22,7 +24,13 @@ def main(): bill_params = { "legislationId" : bill_id } bill_info = requests.post(bill_url, headers=headers, data=json.dumps(bill_params)) bill_info = decode_json(bill_info.json()["d"])["data"] + output_dir = 'data/dc/bills_raw/' + outfile = "".join([output_dir, "{}.json".format(bill_id)]) + mkdir_p(output_dir) + with open(outfile, 'w') as out: + json.dump(bill_info, out, sort_keys=True, indent=4, separators=(',', ': ')) pprint.pprint(bill_info) + print("\nRaw bill data saved: {}\n".format(outfile)) def decode_json(stringy_json): #the "json" they send is recursively string-encoded. From bd4fa6cebd26f7aaea52767af9378228ef29f145 Mon Sep 17 00:00:00 2001 From: Serdar Tumgoren Date: Wed, 4 Jan 2017 16:41:29 -0800 Subject: [PATCH 4/5] CO: Add targeted bill scrape --- openstates/co/bills.py | 341 +++++++++++++++++++------------------- scripts/co/scrape_bill.py | 98 +++++++++++ 2 files changed, 269 insertions(+), 170 deletions(-) create mode 100644 scripts/co/scrape_bill.py diff --git a/openstates/co/bills.py b/openstates/co/bills.py index 2d8f04ae07..7710f8df71 100644 --- a/openstates/co/bills.py +++ b/openstates/co/bills.py @@ -267,14 +267,24 @@ def parse_history(self, bill_history_url): return ret + def get_bill_sheet_rows(self, session, chamber): + sheet_url = self.get_bill_folder(session, chamber) + sheet_html = self.get(sheet_url).text + sheet_page = lxml.html.fromstring(sheet_html) + sheet_page.make_links_absolute(sheet_url) + bills = sheet_page.xpath('//table/tr') + return [sheet_url, bills] + def scrape_bill_sheet(self, session, chamber): """ Scrape the bill sheet (the page full of bills and other small bits of data) """ - sheet_url = self.get_bill_folder(session, chamber) - - bill_chamber = {"Senate": "upper", "House": "lower"}[chamber] + sheet_url, bills = self.get_bill_sheet_rows(session, chamber) + upper_or_lower = {"Senate": "upper", "House": "lower"}[chamber] + for bill in bills: + self.process_bill_sheet_row(bill, session, chamber, upper_lower, sheet_url) + def process_bill_sheet_row(self, bill, session, chamber, upper_or_lower, sheet_url): index = { "id": 0, "title_sponsor": 1, @@ -282,180 +292,171 @@ def scrape_bill_sheet(self, session, chamber): "history": 3, "votes": 7 } + bill_id = self.read_td(bill[index["id"]][0]) + if bill_id == None: + # Every other entry is null for some reason + return - sheet_html = self.get(sheet_url).text - sheet_page = lxml.html.fromstring(sheet_html) - sheet_page.make_links_absolute(sheet_url) + dot_loc = bill_id.find('.') + if dot_loc != -1: + # budget bills are missing the .pdf, don't truncate + bill_id = bill_id[:dot_loc] + title_and_sponsor = bill[index["title_sponsor"]][0] + + bill_title = title_and_sponsor.text + bill_title_and_sponsor = title_and_sponsor.text_content() + if bill_title is None: + return # Odd ... + + sponsors = bill_title_and_sponsor.replace(bill_title, "").\ + replace(" & ...", "").split("--") + + cats = { + "SB": "bill", + "HB": "bill", + "HR": "resolution", + "SR": "resolution", + "SCR": "concurrent resolution", + "HCR": "concurrent resolution", + "SJR": "joint resolution", + "HJR": "joint resolution", + "SM": "memorial", + "HM": "memorial" + } - bills = sheet_page.xpath('//table/tr') + bill_type = None - for bill in bills: - bill_id = self.read_td(bill[index["id"]][0]) + for cat in cats: + if bill_id[:len(cat)] == cat: + bill_type = cats[cat] - if bill_id == None: - # Every other entry is null for some reason - continue + b = Bill(session, upper_or_lower, bill_id, bill_title, + type=bill_type) - dot_loc = bill_id.find('.') - if dot_loc != -1: - # budget bills are missing the .pdf, don't truncate - bill_id = bill_id[:dot_loc] - title_and_sponsor = bill[index["title_sponsor"]][0] - - bill_title = title_and_sponsor.text - bill_title_and_sponsor = title_and_sponsor.text_content() - if bill_title is None: - continue # Odd ... - - sponsors = bill_title_and_sponsor.replace(bill_title, "").\ - replace(" & ...", "").split("--") - - cats = { - "SB": "bill", - "HB": "bill", - "HR": "resolution", - "SR": "resolution", - "SCR": "concurrent resolution", - "HCR": "concurrent resolution", - "SJR": "joint resolution", - "HJR": "joint resolution", - "SM": "memorial", - "HM": "memorial" - } - - bill_type = None - - for cat in cats: - if bill_id[:len(cat)] == cat: - bill_type = cats[cat] - - b = Bill(session, bill_chamber, bill_id, bill_title, - type=bill_type) - - b.add_source(sheet_url) - - versions_url = \ - bill[index["version"]].xpath('font/a')[0].attrib["href"] - versions_url = versions_url - versions = self.parse_versions(versions_url) - - for version in versions: - b.add_version(version['name'], version['link'], - mimetype=version['mimetype']) - - bill_history_href = bill[index["history"]][0][0].attrib['href'] - - history = self.parse_history(bill_history_href) - if history is None: - self.logger.warning("Bill history for %s is not correctly formatted" % bill_id) - continue - b.add_source(bill_history_href) - - chamber_map = dict(Senate='upper', House='lower') - for action, date in history: - action_actor = chamber_map.get(chamber, chamber) - attrs = dict(actor=action_actor, action=action, date=date) - attrs.update(self.categorizer.categorize(action)) - b.add_action(**attrs) - - for sponsor in sponsors: - if sponsor != None and sponsor != "(NONE)" and \ - sponsor != "": - if "&" in sponsor: - for sponsor in [x.strip() for x in sponsor.split("&")]: - b.add_sponsor("primary", sponsor) - else: - b.add_sponsor("primary", sponsor) + b.add_source(sheet_url) - # Now that we have history, let's see if we can't grab some - # votes - - bill_vote_href, = bill.xpath(".//a[contains(text(), 'Votes')]") - bill_vote_href = bill_vote_href.attrib['href'] - #bill_vote_href = self.get_vote_url(bill_id, session) - votes = self.parse_votes(bill_vote_href) - - if (votes['sanity-check'] == 'This site only supports frames ' - 'compatible browsers!'): - votes['votes'] = [] - elif votes['sanity-check'] != bill_id: - self.warning("XXX: READ ME! Sanity check failed!") - self.warning(" -> Scraped ID: " + votes['sanity-check']) - self.warning(" -> 'Real' ID: " + bill_id) - assert votes['sanity-check'] == bill_id - - for vote in votes['votes']: - filed_votes = vote['votes'] - passage = vote['meta'] - result = vote['result'] - - composite_time = "%s %s" % ( - passage['x-parent-date'], - passage['TIME'] - ) - # It's now like: 04/01/2011 02:10:14 PM - pydate = dt.datetime.strptime(composite_time, - "%m/%d/%Y %I:%M:%S %p") - hasHouse = "House" in passage['x-parent-ctty'] - hasSenate = "Senate" in passage['x-parent-ctty'] - - if hasHouse and hasSenate: - actor = "joint" - elif hasHouse: - actor = "lower" + versions_url = \ + bill[index["version"]].xpath('font/a')[0].attrib["href"] + versions_url = versions_url + versions = self.parse_versions(versions_url) + + for version in versions: + b.add_version(version['name'], version['link'], + mimetype=version['mimetype']) + + bill_history_href = bill[index["history"]][0][0].attrib['href'] + + history = self.parse_history(bill_history_href) + if history is None: + self.logger.warning("Bill history for %s is not correctly formatted" % bill_id) + return + b.add_source(bill_history_href) + + chamber_map = dict(Senate='upper', House='lower') + for action, date in history: + action_actor = chamber_map.get(chamber, chamber) + attrs = dict(actor=action_actor, action=action, date=date) + attrs.update(self.categorizer.categorize(action)) + b.add_action(**attrs) + + for sponsor in sponsors: + if sponsor != None and sponsor != "(NONE)" and \ + sponsor != "": + if "&" in sponsor: + for sponsor in [x.strip() for x in sponsor.split("&")]: + b.add_sponsor("primary", sponsor) + else: + b.add_sponsor("primary", sponsor) + + # Now that we have history, let's see if we can't grab some + # votes + + bill_vote_href, = bill.xpath(".//a[contains(text(), 'Votes')]") + bill_vote_href = bill_vote_href.attrib['href'] + #bill_vote_href = self.get_vote_url(bill_id, session) + votes = self.parse_votes(bill_vote_href) + + if (votes['sanity-check'] == 'This site only supports frames ' + 'compatible browsers!'): + votes['votes'] = [] + elif votes['sanity-check'] != bill_id: + self.warning("XXX: READ ME! Sanity check failed!") + self.warning(" -> Scraped ID: " + votes['sanity-check']) + self.warning(" -> 'Real' ID: " + bill_id) + assert votes['sanity-check'] == bill_id + + for vote in votes['votes']: + filed_votes = vote['votes'] + passage = vote['meta'] + result = vote['result'] + + composite_time = "%s %s" % ( + passage['x-parent-date'], + passage['TIME'] + ) + # It's now like: 04/01/2011 02:10:14 PM + pydate = dt.datetime.strptime(composite_time, + "%m/%d/%Y %I:%M:%S %p") + hasHouse = "House" in passage['x-parent-ctty'] + hasSenate = "Senate" in passage['x-parent-ctty'] + + if hasHouse and hasSenate: + actor = "joint" + elif hasHouse: + actor = "lower" + else: + actor = "upper" + + other = (int(result['EXC']) + int(result['ABS'])) + # OK, sometimes the Other count is wrong. + local_other = 0 + for voter in filed_votes: + l_vote = filed_votes[voter].lower().strip() + if l_vote != "yes" and l_vote != "no": + local_other = local_other + 1 + + if local_other != other: + self.warning( \ + "XXX: !!!WARNING!!! - resetting the 'OTHER' VOTES") + self.warning(" -> Old: %s // New: %s" % ( + other, local_other + )) + other = local_other + + passed = (result['FINAL_ACTION'] == "PASS") + if passage['MOTION'].strip() == "": + return + + if "without objection" in passage['MOTION'].lower(): + passed = True + + v = Vote(actor, pydate, passage['MOTION'], + passed, + int(result['YES']), int(result['NO']), + other, + moved=passage['MOVED'], + seconded=passage['SECONDED']) + + v.add_source(vote['meta']['url']) + # v.add_source( bill_vote_href ) + + # XXX: Add more stuff to kwargs, we have a ton of data + seen = set([]) + for voter in filed_votes: + who = voter + if who in seen: + raise Exception("Seeing the double-thing. - bug #702") + seen.add(who) + + vote = filed_votes[who] + if vote.lower() == "yes": + v.yes(who) + elif vote.lower() == "no": + v.no(who) else: - actor = "upper" - - other = (int(result['EXC']) + int(result['ABS'])) - # OK, sometimes the Other count is wrong. - local_other = 0 - for voter in filed_votes: - l_vote = filed_votes[voter].lower().strip() - if l_vote != "yes" and l_vote != "no": - local_other = local_other + 1 - - if local_other != other: - self.warning( \ - "XXX: !!!WARNING!!! - resetting the 'OTHER' VOTES") - self.warning(" -> Old: %s // New: %s" % ( - other, local_other - )) - other = local_other - - passed = (result['FINAL_ACTION'] == "PASS") - if passage['MOTION'].strip() == "": - continue - - if "without objection" in passage['MOTION'].lower(): - passed = True - - v = Vote(actor, pydate, passage['MOTION'], - passed, - int(result['YES']), int(result['NO']), - other, - moved=passage['MOVED'], - seconded=passage['SECONDED']) - - v.add_source(vote['meta']['url']) - # v.add_source( bill_vote_href ) - - # XXX: Add more stuff to kwargs, we have a ton of data - seen = set([]) - for voter in filed_votes: - who = voter - if who in seen: - raise Exception("Seeing the double-thing. - bug #702") - seen.add(who) - - vote = filed_votes[who] - if vote.lower() == "yes": - v.yes(who) - elif vote.lower() == "no": - v.no(who) - else: - v.other(who) - b.add_vote(v) - self.save_bill(b) + v.other(who) + b.add_vote(v) + self.save_bill(b) def scrape(self, chamber, session): """ diff --git a/scripts/co/scrape_bill.py b/scripts/co/scrape_bill.py new file mode 100644 index 0000000000..e7e396e9ba --- /dev/null +++ b/scripts/co/scrape_bill.py @@ -0,0 +1,98 @@ +'''Targeted scrape of one or more CO bills + +USAGE: + + $ python scrape_bill.py HB16-1359 + +''' +import re +import sys +import itertools + +from openstates.utils import mkdir_p +from openstates.co import COBillScraper, metadata, session_list + + +class COSoloBillScraper(object): + + def __init__(self, bill_ids=[]): + self.bill_ids = bill_ids + self.output_dir = 'data/co/' + mkdir_p(self.output_dir) + self.scraper = COBillScraper(metadata, output_dir=self.output_dir) + + def run(self): + target_bills = [self._parse_bill_id(bill_id) for bill_id in self.bill_ids] + self._build_raw_bill_lookup(target_bills) + for bill_data in target_bills: + bill_id = bill_data['id'] + chamber = bill_data['chamber'] + upper_lower = {"Senate": "upper", "House": "lower"}[chamber] + # Below data has some extra bits that are also needed... + data = self._bill_sheet_row_lkup[bill_id] + self.scraper.process_bill_sheet_row(data['row'], data['session'], chamber, upper_lower, data['url']) + + def _parse_bill_id(self, bill_id): + bill_type, session_year, bill_num = re.match(r'([A-Z]{2,3})(\d{2})-(.+)', bill_id).groups() + return { + 'id': bill_id, + 'chamber': 'House' if bill_type.startswith('H') else 'Senate', + 'session_year': session_year, + 'bill_num': bill_num.strip() + } + + def _build_raw_bill_lookup(self, bills): + pairs = self._get_session_chamber_pairs(bills) + index_page_meta = self._get_index_page_meta(pairs) + #self._raw_pages_lkup = {} + self._bill_sheet_row_lkup = {} + for session, chamber, sheet_url in index_page_meta: + raw_index_page_rows = self.scraper.get_bill_sheet_rows(session, chamber)[1] + #self._raw_pages_lkup[(session, chamber)] = raw_index_page_rows + for row in raw_index_page_rows: + # if it has a bill id, store it + try: + # bill id, if it exists, is in first column and may have a .pdf file extension + bill_id = row.text_content().split('\n')[0].split('.')[0] + if bill_id: + self._bill_sheet_row_lkup[bill_id] = { 'row': row, 'url': sheet_url, 'session': session } + except TypeError: + continue + + def _get_session_chamber_pairs(self, bill_data): + ''' + Get all target pairings of session and chamber + + NOTE: Leg. term has multiple sessions spanning multiple years + and each combo has its own bill index page. + ''' + payload = [] + for bill in bill_data: + chamber = bill['chamber'] + session_year = bill['session_year'] + for term in self.scraper.metadata['terms']: + if session_year in term['name']: + # Pairs of chambers/sessions e.g. (House, 2016A) + pairings = [chamber_session_pair for chamber_session_pair in itertools.product([chamber], term['sessions'])] + payload.extend(pairings) + break + return payload + + def _get_index_page_meta(self, session_chamber_pairs): + index_page_urls = [] + for chamber, session in session_chamber_pairs: + url = self.scraper.get_bill_folder(session, chamber) + index_page_urls.append((session, chamber, url)) + return index_page_urls + +def main(): + try: + bill_ids = [bill_id.strip() for bill_id in sys.argv[1:]] + except IndexError: + msg = "\nERROR: You must supply one or more bill ids! Example:\n\n\tpython {} HB16-1359\n".format(__file__) + sys.exit(msg) + scraper = COSoloBillScraper(bill_ids) + scraper.run() + +if __name__ == '__main__': + main() From 56ee2f22f1d732f9ac90878452aacff37263705a Mon Sep 17 00:00:00 2001 From: Serdar Tumgoren Date: Wed, 4 Jan 2017 16:51:12 -0800 Subject: [PATCH 5/5] CO: Fix minor bug related to addition of solo bill scraper --- openstates/co/bills.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstates/co/bills.py b/openstates/co/bills.py index 7710f8df71..75b3823211 100644 --- a/openstates/co/bills.py +++ b/openstates/co/bills.py @@ -282,7 +282,7 @@ def scrape_bill_sheet(self, session, chamber): sheet_url, bills = self.get_bill_sheet_rows(session, chamber) upper_or_lower = {"Senate": "upper", "House": "lower"}[chamber] for bill in bills: - self.process_bill_sheet_row(bill, session, chamber, upper_lower, sheet_url) + self.process_bill_sheet_row(bill, session, chamber, upper_or_lower, sheet_url) def process_bill_sheet_row(self, bill, session, chamber, upper_or_lower, sheet_url): index = {