citibike-publicspace/pull_data.py at master · CUSP2017/citibike-publicspace · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
import os
import requests
import zipfile

# Hide InsecureRequestWarning
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

project_dir = os.path.dirname(__file__)
dl_path = os.path.join(project_dir, "data/external")

# Create data directories if needed
for d in ["data", "data/external", "data/interim", "data/processed", "data/map"]:
    if not os.path.exists(d):
        os.makedirs(d)

files = [
    {
        "name": "citibike-stations.json",
        "url": "https://feeds.citibikenyc.com/stations/stations.json",
        "params": {},
        "zip": False
    },
    {
        "name": "tree-canopy.geojson",
        "url": "https://data.cityofnewyork.us/api/geospatial/pi5s-9p35",
        "params": {"method": "export", "format": "GeoJSON"},
        "zip": False
    },
    {
        "name": "street-assessment.zip",
        "url": "http://www.nyc.gov/html/dot/downloads/misc/street-assessment-rating.zip",
        "params": {},
        "zip": "street-assessment"
    },
    {
        "name": "subway-entrances.geojson",
        "url": "https://data.cityofnewyork.us/api/geospatial/drex-xx56",
        "params": {"method": "export", "format": "GeoJSON"},
        "zip": False
    },
    {
        "name": "bike-lanes.zip",
        "url": "http://www.nyc.gov/html/dot/downloads/misc/nyc-bike-routes.zip",
        "params": {},
        "zip": "bike-lanes"
    },
    {
        "name": "nyc-boroughs.zip",
        "url": "http://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/nybb_16c.zip",
        "params": {},
        "zip": "nyc-boroughs"
    },
    {
        "name": "nyc-traffic.zip",
        "url": "https://www.dot.ny.gov/divisions/engineering/applications/traffic-data-viewer/traffic-data-viewer-repository/TDV_Shapefile_AADT_2015.zip",
        "params": {},
        "zip": "nyc-traffic"
    },
    {
        "name": "parks.geojson",
        "url": "https://data.cityofnewyork.us/api/geospatial/rjaj-zgq7",
        "params": {"method": "export", "format": "GeoJSON"},
        "zip": False
    },
    {
        "name": "subway-entrances.geojson",
        "url": "https://data.cityofnewyork.us/api/geospatial/drex-xx56",
        "params": {"method": "export", "format": "GeoJSON"},
        "zip": False
    },
    {
        "name": "census-tracts.geojson",
        "url": "https://data.cityofnewyork.us/api/geospatial/fxpq-c8ku",
        "params": {"method": "export", "format": "GeoJSON"},
        "zip": False
    },
    {
        "name": "nyc-population-census.csv",
        "url": "https://data.cityofnewyork.us/api/views/37cg-gxjd/rows.csv",
        "params": {},
        "zip": False
    }
]

def download(arg):
    save_path = dl_path + "/" + arg["name"]
    if not os.path.isfile(save_path):
        print("Downloading " + arg["name"])
        r = requests.get(arg["url"], params=arg["params"], stream=True, verify=False)
        with open(save_path, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)

        if arg["zip"]:
            zip = zipfile.ZipFile(save_path)
            zip.extractall(dl_path + '/' + arg["zip"])

# Run the download for the files
for f in files:
    download(f)

# Download citibike ridership for 2015
citibike_months = ["201501", "201502", "201503", "201504", "201505", "201506",
                   "201507", "201508", "201509", "201510", "201511", "201512"]
citibike_data_files = [month + "-citibike-tripdata.zip" for month in citibike_months]

for citi in citibike_data_files:
    download({
        "name": citi,
        "url": "https://s3.amazonaws.com/tripdata/" + citi,
        "params": {},
        "zip": "ridership"
    })