diff --git a/test_ystockquote.py b/test_ystockquote.py index dcd039c..356a1a3 100644 --- a/test_ystockquote.py +++ b/test_ystockquote.py @@ -11,9 +11,7 @@ # # Requires: Python 2.7/3.3+ - import unittest - import ystockquote @@ -50,6 +48,26 @@ def test_get_historical_prices(self): self.assertGreater(float(prices[end_date]['Volume']), 0.0) self.assertGreater(float(prices[end_date]['Adj Close']), 0.0) + def test_get_historical_prices_1wk(self): + symbol = 'GOOG' + start_date = '2013-01-02' + end_date = '2013-01-15' + prices = ystockquote.get_historical_prices( + symbol, start_date, end_date, '1wk') + + self.assertIsInstance(prices, dict) + self.assertEqual(len(prices), 3) + + def test_get_historical_prices_1mo(self): + symbol = 'GOOG' + start_date = '2013-01-02' + end_date = '2013-04-01' + prices = ystockquote.get_historical_prices( + symbol, start_date, end_date, '1mo') + + self.assertIsInstance(prices, dict) + self.assertEqual(len(prices), 4) + if __name__ == '__main__': unittest.main() diff --git a/ystockquote.py b/ystockquote.py index 905b362..5f9040a 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -15,15 +15,21 @@ __version__ = '0.2.6dev' # NOQA +import time +import re try: # py3 from urllib.request import Request, urlopen from urllib.parse import urlencode + from imp import reload except ImportError: # py2 from urllib2 import Request, urlopen from urllib import urlencode + import sys + reload(sys) + sys.setdefaultencoding('utf8') def _request(symbol, stat): @@ -462,29 +468,78 @@ def get_short_ratio(symbol): return _request(symbol, 's7') -def get_historical_prices(symbol, start_date, end_date): +def _get_headers(): + headers = { + 'Connection': 'keep-alive', + 'Expires': str(-1), + 'Upgrade-Insecure-Requests': str(1), + # Google Chrome: + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) \ + AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36' # noqa + } + + return headers + + +def _get_crumb_cookie(symbol): + """ + Get a valid crumb id y scraping the page. + + Returns a dictionary. + """ + tu = "https://finance.yahoo.com/quote/{}/history".format(symbol) + req = Request(tu, headers=_get_headers()) + resp = urlopen(req) + cookies = resp.info()['Set-Cookie'] + + content = str(resp.read().decode('utf-8').strip()) + + # Matches: {'crumb':'AlphaNumeric'} + rpat = '"CrumbStore":{"crumb":"([^"]+)"}' + crumb = re.findall(rpat, content)[0] + return {'crumb': crumb.encode('ascii').decode('unicode-escape'), + 'cookie': cookies} + + +def get_historical_prices(symbol, start_date, end_date, interval='1d'): """ Get historical prices for the given ticker symbol. Date format is 'YYYY-MM-DD' + interval determines the rollup or aggregate of the + time series to return. interval can take the following values: + + 1d - data returned on a daily basis. This is the default + 1wk - data rolled up and returned on a weekly basis + 1mo - data rolled up and returend on a monthly basis + Returns a nested dictionary (dict of dicts). outer dict keys are dates ('YYYY-MM-DD') """ + + pattern = '%Y-%m-%d' + + unix_start = int(time.mktime(time.strptime(start_date, pattern))) + unix_end = int(time.mktime(time.strptime(end_date, pattern))) + + crumb_cookie = _get_crumb_cookie(symbol) + params = urlencode({ - 's': symbol, - 'a': int(start_date[5:7]) - 1, - 'b': int(start_date[8:10]), - 'c': int(start_date[0:4]), - 'd': int(end_date[5:7]) - 1, - 'e': int(end_date[8:10]), - 'f': int(end_date[0:4]), - 'g': 'd', - 'ignore': '.csv', - }) - url = 'http://real-chart.finance.yahoo.com/table.csv?%s' % params - req = Request(url) + 'period1': unix_start, + 'period2': unix_end, + 'interval': interval, + 'events': 'history', + 'crumb': crumb_cookie['crumb'] + }) + + url = 'https://query1.finance.yahoo.com/v7/finance/download/{}?{}'.\ + format(symbol, params) + + req = Request(url, headers=_get_headers()) + req.add_header("Cookie", crumb_cookie['cookie']) resp = urlopen(req) content = str(resp.read().decode('utf-8').strip()) + daily_data = content.splitlines() hist_dict = dict() keys = daily_data[0].split(',')