From 62267f8208e3a34a8f22a1c0e09b74a2877978ee Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 27 Dec 2018 11:24:18 -0800 Subject: [PATCH 001/121] Framework for Table class --- urbansim_templates/io/tables.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 urbansim_templates/io/tables.py diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py new file mode 100644 index 0000000..85059da --- /dev/null +++ b/urbansim_templates/io/tables.py @@ -0,0 +1,52 @@ +from __future__ import print_function + +import orca + +from urbansim_templates import modelmanager + + +@modelmanager.template +class Table(): + """ + Class for registering data tables. In the initial implementation, data is expected to + be found locally in a CSV or HDF5 file. + + Parameters + ---------- + + name : str, optional + Name of the table, for Orca. This will also be used as the name of the model step + that generates the table. + + tags : list of str, optional + Tags, passed to ModelManager. + + autorun : bool, optional + Automatically "run" the step with the parameters passed to the constructor. This + will be set to True in dict representations of the object, so that loading it + into ModelManager will automatically register the table with Orca. + + Properties and attributes + ------------------------- + All the parameters listed above can also be get and set as properties of the class + instance. + + """ + def __init__(self): + pass + + + @classmethod + def from_dict(cls, d): + pass + + def to_dict(self): + pass + + + def validate(self): + pass + + + def run(self): + pass \ No newline at end of file From 145c60756ba09c476106296a83f0e70dbee3f520 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 27 Dec 2018 12:10:46 -0800 Subject: [PATCH 002/121] Framework buildout --- urbansim_templates/io/tables.py | 80 +++++++++++++++++++++++++++++- urbansim_templates/modelmanager.py | 2 +- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 85059da..fc8b2c1 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -8,12 +8,88 @@ @modelmanager.template class Table(): """ - Class for registering data tables. In the initial implementation, data is expected to - be found locally in a CSV or HDF5 file. + Class for registering data tables. In the initial implementation, data can come from + local CSV or HDF5 files. + + An instance of this Table() template stores *instructions for loading a data table*, + which can be saved as a yaml file. Running these instructions registers the table + with Orca. Saved tables will be registered automatically when you initialize + ModelManager, replacing the `datasources.py` scripts used in previous versions of + UrbanSim. + + Tables should include a unique index, or a set of columns that jointly represent a + unique index. + + If a column has the same name as the index of another table, ModelManager expects to + be able to use it as a join key. Following these naming conventions eliminates the + need for Orca "broadcasts". + + Usage + ----- + Create an empty class instance: `m = Table()`. + + Give it some properties: `m.name = 'buildings'` etc. (These can also be passed when + you create the object.) + + Register with ModelManager: `modelmanager.register(m)`. This registers the table + loading instructions, runs them, and saves them to disk. They'll automatically be run + the next time you initialize ModelManager. + + You can use all the standard ModelManager commands to get copies of the saved table + loading instructions, modify them, and delete them: + + - `modelmanager.list_steps()` + - `m2 = modelmanager.get_step('name')` + - `modelmanager.register(m2)` + - `modelmanager.remove_step('name')` Parameters ---------- + source_type : 'csv' or 'hdf', optional + This is required to load the table, but does not have to be provided when the + object is created. + + path : str, optional + Local file path, either absolute or relative to the ModelManager config directory. + + TO DO - CROSS PLATFORM SUPPORT? + + url : str, optional + Remote url to download file from, NOT YET IMPLEMENTED. + + csv_index_cols : str or list of str, optional + Required for csv source type. + + csv_settings : dict, optional + Additional parameters to pass to `pd.read_csv()`. + + hdf_key : str, optional + Name of table to read from the HDF5 file, if there are multiple. + + zipped : bool, optional + Whether the source file is zipped, NOT YET IMPLEMENTED. + + path_in_archive : str, optional + NOT YET IMPLEMENTED. + + filters : str or list of str, optional + Filters to apply before registering the table with Orca. + + TO DO - IMPLEMENT + + orca_test_spec : dict, optional + Data characteristics to be tested when the table is validated, NOT YET + IMPLEMENTED. + + cache : bool, optional + Orca table setting (DEFAULT?). + + cache_scope : ??, optional + Orca table setting. + copy_col : ??, optional + Orca table setting. + name : str, optional Name of the table, for Orca. This will also be used as the name of the model step that generates the table. diff --git a/urbansim_templates/modelmanager.py b/urbansim_templates/modelmanager.py index 2b957c2..4c0e89f 100644 --- a/urbansim_templates/modelmanager.py +++ b/urbansim_templates/modelmanager.py @@ -237,7 +237,7 @@ def get_step(name): Returns ------- - RegressionStep or other + instance of a template class """ return copy.deepcopy(_steps[name]) From da3ea590de6c4bee9fad7d40988e9204010a77b9 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 27 Dec 2018 13:59:11 -0800 Subject: [PATCH 003/121] Constructors --- urbansim_templates/io/tables.py | 88 ++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index fc8b2c1..65bd435 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -2,7 +2,7 @@ import orca -from urbansim_templates import modelmanager +from urbansim_templates import modelmanager, __version__ @modelmanager.template @@ -82,13 +82,13 @@ class Table(): IMPLEMENTED. cache : bool, optional - Orca table setting (DEFAULT?). + Passed to `Orca.Table()`. cache_scope : ??, optional - Orca table setting. + Passed to `Orca.Table()`. copy_col : ??, optional - Orca table setting. + Passed to `Orca.Table()`. name : str, optional Name of the table, for Orca. This will also be used as the name of the model step @@ -108,16 +108,88 @@ class Table(): instance. """ - def __init__(self): - pass + def __init__(self, + source_type = None, + path = None, + csv_index_cols = None, + csv_settings = None, + cache = None, + cache_scope = None, + copy_col = None, + name = None, + tags = [], + autorun = None): + + # Template-specific params + self.source_type = source_type + self.path = path + self.csv_index_cols = csv_index_cols + self.csv_settings = csv_settings + self.cache = cache + self.cache_scope = cache_scope + self.copy_col = copy_col + + # Params required by ModelManager + self.name = name + self.tags = tags + self.autorun = autorun + + # Automated params + self.template = type(self).__name__ # class name + self.template_version = __version__ @classmethod def from_dict(cls, d): - pass + """ + Create an object instance from a saved dictionary representation. + + Parameters + ---------- + d : dict + + Returns + ------- + Table + + """ + obj = cls( + source_type = d['source_type'], + path = d['path'], + csv_index_cols = d['csv_index_cols'], + csv_settings = d['csv_settings'], + cache = d['cache'], + cache_scope = d['cache_scope'], + copy_col = d['copy_col'], + name = d['name'], + tags = d['tags'], + autorun = d['autorun'] + ) + return obj def to_dict(self): - pass + """ + Create a dictionary representation of the object. + + Returns + ------- + dict + + """ + d = { + 'template': self.template, + 'template_version': self.template_version, + 'name': self.name, + 'tags': self.tags, + 'autorun': True, + 'source_type': self.source_type, + 'path': self.path, + 'csv_index_cols': self.csv_index_cols, + 'cache': self.cache, + 'cache_scope': self.cache_scope, + 'copy_col': self.copy_col + } + return d def validate(self): From 85e46528ad1a47f225f47c9c324a0436a343a04e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 27 Dec 2018 14:13:41 -0800 Subject: [PATCH 004/121] Registration of csvs --- urbansim_templates/io/tables.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 65bd435..7b83fa7 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -82,13 +82,13 @@ class Table(): IMPLEMENTED. cache : bool, optional - Passed to `Orca.Table()`. + Passed to `orca.add_table()`. - cache_scope : ??, optional - Passed to `Orca.Table()`. + cache_scope : 'step', 'iteration', or 'forever', optional + Passed to `orca.add_table()`. - copy_col : ??, optional - Passed to `Orca.Table()`. + copy_col : bool, optional + Passed to `orca.add_table()`. name : str, optional Name of the table, for Orca. This will also be used as the name of the model step @@ -197,4 +197,22 @@ def validate(self): def run(self): - pass \ No newline at end of file + """ + Register a data table with Orca. + + Returns + ------- + None + + """ + if self.source_type == 'csv': + @orca.table(table_name = self.name, + cache = self.cache, + cache_scope = self.cache_scope, + copy_col=self.copy_col) + def orca_table(): + df = pd.read_csv(self.path).set_index(self.csv_index_cols) + return df + + + \ No newline at end of file From 3dab317929978f1e5b67145f8bf37e3ea53e6b97 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 27 Dec 2018 14:51:55 -0800 Subject: [PATCH 005/121] First tests passing --- tests/test_tables.py | 15 +++++++++++++++ urbansim_templates/io/.gitignore | 1 + urbansim_templates/io/__init__.py | 1 + urbansim_templates/io/tables.py | 5 ++++- urbansim_templates/utils.py | 8 ++++---- 5 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 tests/test_tables.py create mode 100644 urbansim_templates/io/.gitignore create mode 100644 urbansim_templates/io/__init__.py diff --git a/tests/test_tables.py b/tests/test_tables.py new file mode 100644 index 0000000..9760171 --- /dev/null +++ b/tests/test_tables.py @@ -0,0 +1,15 @@ +import orca +import pandas as pd +import pytest + +from urbansim_templates import modelmanager +from urbansim_templates.io import Table +from urbansim_templates.utils import validate_template + + +def test_template_validity(): + """ + Rin the template through the standard validation check. + + """ + assert validate_template(Table) \ No newline at end of file diff --git a/urbansim_templates/io/.gitignore b/urbansim_templates/io/.gitignore new file mode 100644 index 0000000..763624e --- /dev/null +++ b/urbansim_templates/io/.gitignore @@ -0,0 +1 @@ +__pycache__/* \ No newline at end of file diff --git a/urbansim_templates/io/__init__.py b/urbansim_templates/io/__init__.py new file mode 100644 index 0000000..040c109 --- /dev/null +++ b/urbansim_templates/io/__init__.py @@ -0,0 +1 @@ +from .tables import Table \ No newline at end of file diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 7b83fa7..cd12bf3 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -1,6 +1,7 @@ from __future__ import print_function import orca +import pandas as pd from urbansim_templates import modelmanager, __version__ @@ -129,7 +130,7 @@ def __init__(self, self.cache_scope = cache_scope self.copy_col = copy_col - # Params required by ModelManager + # Standard params self.name = name self.tags = tags self.autorun = autorun @@ -167,6 +168,7 @@ def from_dict(cls, d): ) return obj + def to_dict(self): """ Create a dictionary representation of the object. @@ -185,6 +187,7 @@ def to_dict(self): 'source_type': self.source_type, 'path': self.path, 'csv_index_cols': self.csv_index_cols, + 'csv_settings': self.csv_settings, 'cache': self.cache, 'cache_scope': self.cache_scope, 'copy_col': self.copy_col diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index be61b0c..c99b61e 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -40,7 +40,7 @@ def validate_template(cls): m = cls() except: print("Error instantiating object without arguments") - return False + raise methods = ['to_dict', 'from_dict', 'run'] for item in methods: @@ -52,7 +52,7 @@ def validate_template(cls): d = m.to_dict() except: print("Error running 'to_dict()'") - return False + raise params = ['name', 'tags', 'template', 'template_version'] for item in params: @@ -67,8 +67,8 @@ def validate_template(cls): try: cls.from_dict(m.to_dict()) except: - print("Error passing dict to 'from_dict()' method") - return False + print("Error instantiating object with 'from_dict()' method") + raise # TO DO - check supplemental objects? (but nothing there with unconfigured steps) From 0a66adfa65ceeed3a63c5359680e5612c3cf6539 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 27 Dec 2018 16:11:35 -0800 Subject: [PATCH 006/121] Python 2.7 classname fix --- tests/test_tables.py | 8 ++++++-- urbansim_templates/io/tables.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index 9760171..3d6dd00 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -9,7 +9,11 @@ def test_template_validity(): """ - Rin the template through the standard validation check. + Run the template through the standard validation check. """ - assert validate_template(Table) \ No newline at end of file + assert validate_template(Table) + + + + diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index cd12bf3..ec29117 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -136,7 +136,7 @@ def __init__(self, self.autorun = autorun # Automated params - self.template = type(self).__name__ # class name + self.template = self.__class__.__name__ self.template_version = __version__ From 639d8099cf9be26869bcc45673d68e1b576fe3ba Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 28 Dec 2018 11:13:33 -0800 Subject: [PATCH 007/121] Setup, teardown, autorun --- .gitignore | 3 +- tests/data/README.md | 1 + tests/test_tables.py | 94 +++++++++++++++++++++++++++++- urbansim_templates/io/tables.py | 10 ++-- urbansim_templates/modelmanager.py | 11 ++++ 5 files changed, 111 insertions(+), 8 deletions(-) create mode 100644 tests/data/README.md diff --git a/.gitignore b/.gitignore index a81ce57..3fd50f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .cache/* -urbansim_templates.egg-info/* \ No newline at end of file +urbansim_templates.egg-info/* +**/*.pyc \ No newline at end of file diff --git a/tests/data/README.md b/tests/data/README.md new file mode 100644 index 0000000..6b51040 --- /dev/null +++ b/tests/data/README.md @@ -0,0 +1 @@ +This folder stores data that is temporarily generated during tests. \ No newline at end of file diff --git a/tests/test_tables.py b/tests/test_tables.py index 3d6dd00..1be3935 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -1,12 +1,44 @@ -import orca +import os + +import numpy as np import pandas as pd import pytest +import orca + from urbansim_templates import modelmanager from urbansim_templates.io import Table from urbansim_templates.utils import validate_template +@pytest.fixture +def orca_session(): + """ + Set up a clean Orca session and initialize ModelManager. + + """ + orca.clear_all() + modelmanager.initialize() + + +@pytest.fixture +def data(request): + """ + Create data files on disk. + + """ + d1 = {'building_id': np.arange(10), + 'price': 1e6*np.random.random(10)} + + bldg = pd.DataFrame(d1).set_index('building_id') + bldg.to_csv('data/buildings.csv') + + def teardown(): + os.remove('data/buildings.csv') + + request.addfinalizer(teardown) + + def test_template_validity(): """ Run the template through the standard validation check. @@ -15,5 +47,65 @@ def test_template_validity(): assert validate_template(Table) +def test_property_persistence(orca_session): + """ + Test persistence of properties across registration, saving, and reloading. + + """ + pass + + +# test parameters make it through a save + +# test multiple index columns +# test that validation works + +# test loading an h5 file works +# test passing cache settings +# call it TableStep? +# tear down data + +def test_csv(orca_session, data): + """ + Test that loading data from a CSV works. + + """ + s = Table() + s.name = 'buildings' + s.source_type = 'csv' + s.path = 'data/buildings.csv' + s.csv_index_cols = 'building_id' + + assert 'buildings' not in orca.list_tables() + + modelmanager.register(s) + assert 'buildings' in orca.list_tables() + + modelmanager.initialize() + assert 'buildings' in orca.list_tables() + + modelmanager.remove_step('buildings') + + +def test_without_autorun(orca_session, data): + """ + Confirm that disabling autorun works. + + """ + s = Table() + s.name = 'buildings' + s.source_type = 'csv' + s.path = 'data/buildings.csv' + s.csv_index_cols = 'building_id' + s.autorun = False + + modelmanager.register(s) + assert 'buildings' not in orca.list_tables() + + modelmanager.remove_step('buildings') + + + + \ No newline at end of file diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index ec29117..57af671 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -98,10 +98,8 @@ class Table(): tags : list of str, optional Tags, passed to ModelManager. - autorun : bool, optional - Automatically "run" the step with the parameters passed to the constructor. This - will be set to True in dict representations of the object, so that loading it - into ModelManager will automatically register the table with Orca. + autorun : bool, optional (default True) + Automatically run the step whenever it's registered with ModelManager. Properties and attributes ------------------------- @@ -119,7 +117,7 @@ def __init__(self, copy_col = None, name = None, tags = [], - autorun = None): + autorun = True): # Template-specific params self.source_type = source_type @@ -183,7 +181,7 @@ def to_dict(self): 'template_version': self.template_version, 'name': self.name, 'tags': self.tags, - 'autorun': True, + 'autorun': self.autorun, 'source_type': self.source_type, 'path': self.path, 'csv_index_cols': self.csv_index_cols, diff --git a/urbansim_templates/modelmanager.py b/urbansim_templates/modelmanager.py index 4c0e89f..c1f4582 100644 --- a/urbansim_templates/modelmanager.py +++ b/urbansim_templates/modelmanager.py @@ -139,10 +139,17 @@ def register(step, save_to_disk=True): name has not yet been assigned, one will be generated from the template name and a timestamp. + If the model step includes an attribute 'autorun' that's set to True, the step will + run after being registered. + Parameters ---------- step : object + Returns + ------- + None + """ if step.name is None: step.name = update_name(step.template, step.name) # TO DO - test this @@ -159,6 +166,10 @@ def run_step(): return step.run() orca.add_step(step.name, run_step) + + if hasattr(step, 'autorun'): + if step.autorun: + orca.run([step.name]) def list_steps(): From 0a484620f9f9852ceb550661e384103a664f846b Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 28 Dec 2018 16:28:18 -0800 Subject: [PATCH 008/121] Loose ends --- urbansim_templates/io/tables.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 57af671..78a7ae8 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -194,6 +194,22 @@ def to_dict(self): def validate(self): + """ + Check some basic expectations about the data generated by the step: + + - Confirm that the table includes a unique index column (primary key) or set of + columns (composite key). If not, raise a ValueError. + + - If the table contains columns whose names match the index columns of tables + previously registered with Orca, check whether they make sense as join keys. + If the presumptive foreign-key columns include values not found in the + primary key columns, print a warning. + + Returns + ------- + bool + + """ pass From 0bc36668fa8721c60354f111c7602220bb5f522b Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 31 Dec 2018 10:39:15 -0800 Subject: [PATCH 009/121] Index validation and tests --- tests/test_tables.py | 112 +++++++++++++++++++++++++++----- urbansim_templates/io/tables.py | 36 +++++++--- 2 files changed, 121 insertions(+), 27 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index 1be3935..964af71 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -55,16 +55,93 @@ def test_property_persistence(orca_session): pass -# test parameters make it through a save +def test_validation_index_unique(orca_session): + """ + Table validation should pass if the index is unique. + + These tests of the validate() method generate Orca tables directly, which is just a + shortcut for testing -- the intended use is for the method to validate the table + loaded by the TableStep. + + """ + d = {'id': [1,2,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index('id')) + + t = Table(name='tab') + t.validate() + + +def test_validation_index_not_unique(orca_session): + """ + Table validation should raise a ValueError if the index is not unique. + + """ + d = {'id': [1,1,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index('id')) + + t = Table(name='tab') + try: + t.validate() + except ValueError: + return + + pytest.fail() # fail if ValueError wasn't raised + + +def test_validation_multiindex_unique(orca_session): + """ + Table validation should pass with a MultiIndex whose combinations are unique. + + """ + d = {'id': [1,1,1], 'sub_id': [1,2,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) + + t = Table(name='tab') + t.validate() -# test multiple index columns + +def test_validation_multiindex_not_unique(orca_session): + """ + Table validation should raise a ValueError if the MultiIndex combinations are not + unique. + + """ + d = {'id': [1,1,1], 'sub_id': [2,2,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) + + t = Table(name='tab') + try: + t.validate() + except ValueError: + return + + pytest.fail() # fail if ValueError wasn't raised + + +def test_validation_unnamed_index(orca_session): + """ + Table validation should raise a ValueError if index is unnamed. + + """ + d = {'id': [1,1,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name + + t = Table(name='tab') + try: + t.validate() + except ValueError: + return + + pytest.fail() # fail if ValueError wasn't raised + + +# test that parameters make it through a save # test that validation works # test loading an h5 file works # test passing cache settings # call it TableStep? -# tear down data def test_csv(orca_session, data): @@ -72,15 +149,15 @@ def test_csv(orca_session, data): Test that loading data from a CSV works. """ - s = Table() - s.name = 'buildings' - s.source_type = 'csv' - s.path = 'data/buildings.csv' - s.csv_index_cols = 'building_id' + t = Table() + t.name = 'buildings' + t.source_type = 'csv' + t.path = 'data/buildings.csv' + t.csv_index_cols = 'building_id' assert 'buildings' not in orca.list_tables() - modelmanager.register(s) + modelmanager.register(t) assert 'buildings' in orca.list_tables() modelmanager.initialize() @@ -94,18 +171,17 @@ def test_without_autorun(orca_session, data): Confirm that disabling autorun works. """ - s = Table() - s.name = 'buildings' - s.source_type = 'csv' - s.path = 'data/buildings.csv' - s.csv_index_cols = 'building_id' - s.autorun = False - - modelmanager.register(s) + t = Table() + t.name = 'buildings' + t.source_type = 'csv' + t.path = 'data/buildings.csv' + t.csv_index_cols = 'building_id' + t.autorun = False + + modelmanager.register(t) assert 'buildings' not in orca.list_tables() modelmanager.remove_step('buildings') - \ No newline at end of file diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 78a7ae8..7da237c 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -27,12 +27,12 @@ class Table(): Usage ----- - Create an empty class instance: `m = Table()`. + Create an empty class instance: `t = Table()`. - Give it some properties: `m.name = 'buildings'` etc. (These can also be passed when + Give it some properties: `t.name = 'buildings'` etc. (These can also be passed when you create the object.) - Register with ModelManager: `modelmanager.register(m)`. This registers the table + Register with ModelManager: `modelmanager.register(t)`. This registers the table loading instructions, runs them, and saves them to disk. They'll automatically be run the next time you initialize ModelManager. @@ -40,8 +40,8 @@ class Table(): loading instructions, modify them, and delete them: - `modelmanager.list_steps()` - - `m2 = modelmanager.get_step('name')` - - `modelmanager.register(m2)` + - `t2 = modelmanager.get_step('name')` + - `modelmanager.register(t2)` - `modelmanager.remove_step('name')` Parameters @@ -195,22 +195,40 @@ def to_dict(self): def validate(self): """ - Check some basic expectations about the data generated by the step: + Check some basic expectations about the table generated by the step: - - Confirm that the table includes a unique index column (primary key) or set of - columns (composite key). If not, raise a ValueError. + - Confirm that the table includes a unique, named index column (primary key) or + set of columns (composite key). If not, raise a ValueError. - If the table contains columns whose names match the index columns of tables previously registered with Orca, check whether they make sense as join keys. If the presumptive foreign-key columns include values not found in the primary key columns, print a warning. + + - Perform the same check for columns in previously registered tables whose names + match the index of the table generated by this step. + Running this will load all registered Orca tables into memory (if they have not + yet been loaded), which may take a while. + Returns ------- bool """ - pass + # Register table if needed + if not orca.is_table(self.name): + self.run() + + idx = orca.get_table(self.name).index + + # Check index has a name + if list(idx.names) == [None]: + raise ValueError("Index column has no name") + + # Check index is unique + if len(idx.unique()) < len(idx): + raise ValueError("Index not unique") def run(self): From a167800f55a796a44d4ebe5d107ff51a2ab0addc Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 31 Dec 2018 12:52:47 -0800 Subject: [PATCH 010/121] Comparison of implicit join keys --- tests/test_tables.py | 50 ++++++++++++++++++++++++++++++++- urbansim_templates/io/tables.py | 41 ++++++++++++++++++++++++--- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index 964af71..323bd39 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -55,6 +55,10 @@ def test_property_persistence(orca_session): pass +###################################### +### TESTS OF THE VALIDATE() METHOD ### +###################################### + def test_validation_index_unique(orca_session): """ Table validation should pass if the index is unique. @@ -135,8 +139,48 @@ def test_validation_unnamed_index(orca_session): pytest.fail() # fail if ValueError wasn't raised +def test_validation_columns_vs_other_indexes(orca_session): + """ + Table validation should compare the 'households.building_id' column to + 'buildings.build_id'. + + """ + d = {'household_id': [1,2,3], 'building_id': [2,3,4]} + orca.add_table('households', pd.DataFrame(d).set_index('household_id')) + + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) + + t = Table(name='households') + t.validate() + + +def test_validation_index_vs_other_columns(orca_session): + """ + Table validation should compare the 'households.building_id' column to + 'buildings.build_id'. + + """ + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) + + d = {'household_id': [1,2,3], 'building_id': [2,3,5]} + orca.add_table('households', pd.DataFrame(d).set_index('household_id')) + + t = Table(name='buildings') + t.validate() + + +def test_validation_with_multiindexes(orca_session): + """ + Here, table validation should compare 'choice_table.[home_tract,work_tract]' to + 'distances.[home_tract,work_tract]'. + + """ + pass + # test that parameters make it through a save -# test that validation works +# test validation with stand-alone columns # test loading an h5 file works # test passing cache settings @@ -144,6 +188,10 @@ def test_validation_unnamed_index(orca_session): # call it TableStep? +################################# +### TESTS OF THE DATA LOADING ### +################################# + def test_csv(orca_session, data): """ Test that loading data from a CSV works. diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 7da237c..0967a2d 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -202,20 +202,28 @@ def validate(self): - If the table contains columns whose names match the index columns of tables previously registered with Orca, check whether they make sense as join keys. - If the presumptive foreign-key columns include values not found in the - primary key columns, print a warning. + Print a status message with the number of presumptive foreign-key values that + are found in the primary key column. - Perform the same check for columns in previously registered tables whose names match the index of the table generated by this step. - Running this will load all registered Orca tables into memory (if they have not - yet been loaded), which may take a while. + - It doesn't currently compare indexes to indexes. (Maybe it should?) + + Running this will trigger loading all registered Orca tables into memory, which + may take a while if they have not yet been loaded. Stand-alone columns will not + be loaded unless their names match an index column. Returns ------- bool """ + # There are a couple of reasons I'm not using the orca_test library here: + # (a) orca_test doesn't currently support MultiIndexes, and (b) the primary-key/ + # foreign-key comparisons aren't asserting anything, just printing status + # messages. We should update orca_test to support both, probably. + # Register table if needed if not orca.is_table(self.name): self.run() @@ -229,6 +237,31 @@ def validate(self): # Check index is unique if len(idx.unique()) < len(idx): raise ValueError("Index not unique") + + # Compare columns to indexes of other tables, and vice versa + combinations = [(self.name, t) for t in orca.list_tables() if self.name != t] \ + + [(t, self.name) for t in orca.list_tables() if self.name != t] + + for t1, t2 in combinations: + col_names = orca.get_table(t1).columns + idx = orca.get_table(t2).index + + if set(idx.names).issubset(col_names): + vals = orca.get_table(t1).to_frame(idx.names).drop_duplicates() + vals_in_idx = vals.isin(idx).sum() + + if len(idx.names) == 1: + idx_str = idx.names[0] + else: + idx_str = '[{}]'.format(','.join(idx.names)) + + print("'{}.{}': {} of {} unique values are found in '{}.{}' ({}%)"\ + .format(t1, idx_str, + sum(vals_in_idx), len(vals), + t2, idx_str, + round(100*sum(vals_in_idx)/len(vals)))) + + return True def run(self): From 3e857deaa966b4b713f5375312d2411922814d98 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 31 Dec 2018 15:12:18 -0800 Subject: [PATCH 011/121] Cleanup --- urbansim_templates/io/tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 0967a2d..433e907 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -219,7 +219,7 @@ def validate(self): bool """ - # There are a couple of reasons I'm not using the orca_test library here: + # There are a couple of reasons we're not using the orca_test library here: # (a) orca_test doesn't currently support MultiIndexes, and (b) the primary-key/ # foreign-key comparisons aren't asserting anything, just printing status # messages. We should update orca_test to support both, probably. From a5f49d6df6e99d508d94694e3205aa0b6475a410 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 2 Jan 2019 14:49:17 -0800 Subject: [PATCH 012/121] MultiIndex keys working --- tests/test_tables.py | 11 ++++++++++- urbansim_templates/io/tables.py | 10 +++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index 323bd39..ce317d0 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -177,7 +177,16 @@ def test_validation_with_multiindexes(orca_session): 'distances.[home_tract,work_tract]'. """ - pass + d = {'obs_id': [1,1,1,1], 'alt_id': [1,2,3,4], + 'home_tract': [55,55,55,55], 'work_tract': [17,46,19,55]} + orca.add_table('choice_table', pd.DataFrame(d).set_index(['obs_id','alt_id'])) + + d = {'home_tract': [55,55,55], 'work_tract': [17,18,19], 'dist': [1,1,1]} + orca.add_table('distances', pd.DataFrame(d).set_index(['home_tract','work_tract'])) + + t = Table(name='choice_table') + t.validate() + # test that parameters make it through a save # test validation with stand-alone columns diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 433e907..516c9b1 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -248,7 +248,11 @@ def validate(self): if set(idx.names).issubset(col_names): vals = orca.get_table(t1).to_frame(idx.names).drop_duplicates() - vals_in_idx = vals.isin(idx).sum() + + # Easier to compare multi-column values to multi-column index if we + # turn the values into an index as well + vals = vals.reset_index().set_index(idx.names).index + vals_in_idx = sum(vals.isin(idx)) if len(idx.names) == 1: idx_str = idx.names[0] @@ -257,9 +261,9 @@ def validate(self): print("'{}.{}': {} of {} unique values are found in '{}.{}' ({}%)"\ .format(t1, idx_str, - sum(vals_in_idx), len(vals), + vals_in_idx, len(vals), t2, idx_str, - round(100*sum(vals_in_idx)/len(vals)))) + round(100*vals_in_idx/len(vals)))) return True From 1a5f2bff539b72137363c4d8b5b675c520142196 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 13 Feb 2019 15:05:35 -0800 Subject: [PATCH 013/121] Updating docs --- README.md | 3 ++- docs/source/getting-started.rst | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9ecadd2..156acb1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ UrbanSim Templates is a Python library that provides building blocks for Orca-ba The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the [Orca](https://udst.github.io/orca) task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. The package was developed to make it easier to set up new simulation models — model step templates reduce the need for custom code and make settings more portable between models. ### Installation -UrbanSim Templates can be installed using the Pip or Conda package managers: +UrbanSim Templates can be installed using the Pip or Conda package managers. With Conda, you (currently) need to install UrbanSim separately; Pip will handle this automatically. ``` pip install urbansim_templates @@ -17,6 +17,7 @@ pip install urbansim_templates ``` conda install urbansim_templates --channel conda-forge +conda install urbansim --channel udst ``` ### Documentation diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index 2d37be3..8f39858 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -18,7 +18,9 @@ UrbanSim Templates was created in 2018 by Sam Maurer (maurer@urbansim.com), who Installation ------------ -UrbanSim Templates has been tested with Python versions 2.7, 3.5, and 3.6. Installation in Python 3.7 is currently blocked by Orca's PyTables requirement. +UrbanSim Templates is tested with Python versions 2.7, 3.5, 3.6, and 3.7. + +As of Feb. 2019, there is an installation problem in Python 3.7 when using Pip (because of an issue with Orca's PyTables dependency). Conda should work. .. note:: It can be helpful to set up a dedicated Python environment for each project you work on. This lets you use a stable and replicable set of libraries that won't be affected by other projects. Here are some good `environment settings `__ for UrbanSim Templates projects. @@ -26,7 +28,7 @@ UrbanSim Templates has been tested with Python versions 2.7, 3.5, and 3.6. Insta Production releases ~~~~~~~~~~~~~~~~~~~ -UrbanSim Templates can be installed using the Pip or Conda package managers. +UrbanSim Templates can be installed using the Pip or Conda package managers. With Conda, you (currently) need to install UrbanSim separately; Pip will handle this automatically. .. code-block:: python @@ -35,6 +37,7 @@ UrbanSim Templates can be installed using the Pip or Conda package managers. .. code-block:: python conda install urbansim_templates --channel conda-forge + conda install urbansim --channel udst Dependencies include `NumPy `__, `Pandas `__, and `Statsmodels `__, plus two other UDST libraries: `Orca `__ and `ChoiceModels `__. These will be included automatically when you install UrbanSim Templates. From 8f7c1b3e21d3d4965dfe6b66e4085aa6912c44d4 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 14 Feb 2019 11:41:13 -0800 Subject: [PATCH 014/121] Cleanup --- urbansim_templates/io/tables.py | 51 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 516c9b1..8f196ab 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -62,17 +62,12 @@ class Table(): Required for csv source type. csv_settings : dict, optional - Additional parameters to pass to `pd.read_csv()`. + Additional parameters to pass to `pd.read_csv()`. Can include functionality like + extracting data from a zip file. hdf_key : str, optional Name of table to read from the HDF5 file, if there are multiple. - zipped : bool, optional - Whether the source file is zipped, NOT YET IMPLEMENTED. - - path_in_archive : str, optional - NOT YET IMPLEMENTED. - filters : str or list of str, optional Filters to apply before registering the table with Orca. @@ -133,7 +128,7 @@ def __init__(self, self.tags = tags self.autorun = autorun - # Automated params + # Automatic params self.template = self.__class__.__name__ self.template_version = __version__ @@ -193,6 +188,27 @@ def to_dict(self): return d + def run(self): + """ + Register a data table with Orca. + + Returns + ------- + None + + """ + # TO DO - address cache scope issue + + if self.source_type == 'csv': + @orca.table(table_name = self.name, + cache = self.cache, + cache_scope = self.cache_scope, + copy_col=self.copy_col) + def orca_table(): + df = pd.read_csv(self.path).set_index(self.csv_index_cols) + return df + + def validate(self): """ Check some basic expectations about the table generated by the step: @@ -268,23 +284,4 @@ def validate(self): return True - def run(self): - """ - Register a data table with Orca. - - Returns - ------- - None - - """ - if self.source_type == 'csv': - @orca.table(table_name = self.name, - cache = self.cache, - cache_scope = self.cache_scope, - copy_col=self.copy_col) - def orca_table(): - df = pd.read_csv(self.path).set_index(self.csv_index_cols) - return df - - \ No newline at end of file From 3f83d4815300ed6c8d9747f17a219a424cf05382 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 14 Feb 2019 12:37:47 -0800 Subject: [PATCH 015/121] Table -> TableFromDisk --- tests/test_tables.py | 26 ++++++++++++-------------- urbansim_templates/io/__init__.py | 2 +- urbansim_templates/io/tables.py | 4 ++-- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index ce317d0..657c4c8 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -7,7 +7,7 @@ import orca from urbansim_templates import modelmanager -from urbansim_templates.io import Table +from urbansim_templates.io import TableFromDisk from urbansim_templates.utils import validate_template @@ -44,7 +44,7 @@ def test_template_validity(): Run the template through the standard validation check. """ - assert validate_template(Table) + assert validate_template(TableFromDisk) def test_property_persistence(orca_session): @@ -71,7 +71,7 @@ def test_validation_index_unique(orca_session): d = {'id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = Table(name='tab') + t = TableFromDisk(name='tab') t.validate() @@ -83,7 +83,7 @@ def test_validation_index_not_unique(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = Table(name='tab') + t = TableFromDisk(name='tab') try: t.validate() except ValueError: @@ -100,7 +100,7 @@ def test_validation_multiindex_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = Table(name='tab') + t = TableFromDisk(name='tab') t.validate() @@ -113,7 +113,7 @@ def test_validation_multiindex_not_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [2,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = Table(name='tab') + t = TableFromDisk(name='tab') try: t.validate() except ValueError: @@ -130,7 +130,7 @@ def test_validation_unnamed_index(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name - t = Table(name='tab') + t = TableFromDisk(name='tab') try: t.validate() except ValueError: @@ -151,7 +151,7 @@ def test_validation_columns_vs_other_indexes(orca_session): d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) - t = Table(name='households') + t = TableFromDisk(name='households') t.validate() @@ -167,7 +167,7 @@ def test_validation_index_vs_other_columns(orca_session): d = {'household_id': [1,2,3], 'building_id': [2,3,5]} orca.add_table('households', pd.DataFrame(d).set_index('household_id')) - t = Table(name='buildings') + t = TableFromDisk(name='buildings') t.validate() @@ -184,7 +184,7 @@ def test_validation_with_multiindexes(orca_session): d = {'home_tract': [55,55,55], 'work_tract': [17,18,19], 'dist': [1,1,1]} orca.add_table('distances', pd.DataFrame(d).set_index(['home_tract','work_tract'])) - t = Table(name='choice_table') + t = TableFromDisk(name='choice_table') t.validate() @@ -194,8 +194,6 @@ def test_validation_with_multiindexes(orca_session): # test loading an h5 file works # test passing cache settings -# call it TableStep? - ################################# ### TESTS OF THE DATA LOADING ### @@ -206,7 +204,7 @@ def test_csv(orca_session, data): Test that loading data from a CSV works. """ - t = Table() + t = TableFromDisk() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' @@ -228,7 +226,7 @@ def test_without_autorun(orca_session, data): Confirm that disabling autorun works. """ - t = Table() + t = TableFromDisk() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' diff --git a/urbansim_templates/io/__init__.py b/urbansim_templates/io/__init__.py index 040c109..c79aa93 100644 --- a/urbansim_templates/io/__init__.py +++ b/urbansim_templates/io/__init__.py @@ -1 +1 @@ -from .tables import Table \ No newline at end of file +from .tables import TableFromDisk \ No newline at end of file diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 8f196ab..4b0eb5b 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -7,7 +7,7 @@ @modelmanager.template -class Table(): +class TableFromDisk(): """ Class for registering data tables. In the initial implementation, data can come from local CSV or HDF5 files. @@ -27,7 +27,7 @@ class Table(): Usage ----- - Create an empty class instance: `t = Table()`. + Create an empty class instance: `t = TableFromDisk()`. Give it some properties: `t.name = 'buildings'` etc. (These can also be passed when you create the object.) From 385424ae9470610c4f586bf2aaaacb54a30e7cb0 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 14 Feb 2019 14:34:05 -0800 Subject: [PATCH 016/121] Csv settings and cross-platform paths --- tests/test_tables.py | 49 ++++++++++++++++++++++++++++++++- urbansim_templates/io/tables.py | 41 ++++++++++++++++++--------- 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index 657c4c8..a659c3a 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -24,7 +24,7 @@ def orca_session(): @pytest.fixture def data(request): """ - Create data files on disk. + Create some data files on disk. """ d1 = {'building_id': np.arange(10), @@ -32,9 +32,11 @@ def data(request): bldg = pd.DataFrame(d1).set_index('building_id') bldg.to_csv('data/buildings.csv') + bldg.to_csv('data/buildings.csv.gz', compression='gzip') def teardown(): os.remove('data/buildings.csv') + os.remove('data/buildings.csv.gz') request.addfinalizer(teardown) @@ -221,6 +223,51 @@ def test_csv(orca_session, data): modelmanager.remove_step('buildings') +def test_csv_extra_settings(orca_session, data): + """ + Test loading data with extra CSV settings, e.g. for compressed files. + + """ + t = TableFromDisk() + t.name = 'buildings' + t.source_type = 'csv' + t.path = 'data/buildings.csv.gz' + t.csv_index_cols = 'building_id' + t.csv_settings = {'compression': 'gzip'} + + assert 'buildings' not in orca.list_tables() + + modelmanager.register(t) + assert 'buildings' in orca.list_tables() + + modelmanager.initialize() + assert 'buildings' in orca.list_tables() + + modelmanager.remove_step('buildings') + + +def test_csv_windows_path(orca_session, data): + """ + Test loading a file with Windows-formatted path. + + """ + t = TableFromDisk() + t.name = 'buildings' + t.source_type = 'csv' + t.path = 'data\buildings.csv' + t.csv_index_cols = 'building_id' + + assert 'buildings' not in orca.list_tables() + + modelmanager.register(t) + assert 'buildings' in orca.list_tables() + + modelmanager.initialize() + assert 'buildings' in orca.list_tables() + + modelmanager.remove_step('buildings') + + def test_without_autorun(orca_session, data): """ Confirm that disabling autorun works. diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 4b0eb5b..12022fd 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -1,5 +1,7 @@ from __future__ import print_function +import os + import orca import pandas as pd @@ -51,9 +53,10 @@ class TableFromDisk(): object is created. path : str, optional - Local file path, either absolute or relative to the ModelManager config directory. - - TO DO - CROSS PLATFORM SUPPORT? + Local file path to load data from, either absolute or relative to the + ModelManager config directory. Path will be normed at runtime using + `os.path.normpath()`, so either a Unix-style or Windows-style path should work + across platforms. url : str, optional Remote url to download file from, NOT YET IMPLEMENTED. @@ -62,8 +65,9 @@ class TableFromDisk(): Required for csv source type. csv_settings : dict, optional - Additional parameters to pass to `pd.read_csv()`. Can include functionality like - extracting data from a zip file. + Additional arguments to pass to `pd.read_csv()`. For example, you can + automatically extract data from a gzip file using {'compression': 'gzip'}. See + Pandas documentation for additional settings. hdf_key : str, optional Name of table to read from the HDF5 file, if there are multiple. @@ -78,13 +82,15 @@ class TableFromDisk(): IMPLEMENTED. cache : bool, optional - Passed to `orca.add_table()`. + Passed to `orca.table()`. Note that the default is `True`, unlike in the + underlying general-purpose Orca function, because tables read from disk should + not need to be regenerated during the course of a model run. cache_scope : 'step', 'iteration', or 'forever', optional - Passed to `orca.add_table()`. + Passed to `orca.table()`. Default is 'forever', as in Orca. copy_col : bool, optional - Passed to `orca.add_table()`. + Passed to `orca.table()`. Default is `True`, as in Orca. name : str, optional Name of the table, for Orca. This will also be used as the name of the model step @@ -107,9 +113,9 @@ def __init__(self, path = None, csv_index_cols = None, csv_settings = None, - cache = None, - cache_scope = None, - copy_col = None, + cache = True, + cache_scope = 'forever', + copy_col = True, name = None, tags = [], autorun = True): @@ -197,7 +203,14 @@ def run(self): None """ - # TO DO - address cache scope issue + if self.source_type is None: + raise ValueError("Please provide a source type") + + if self.name is None: + raise ValueError("Please provide a table name") + + if self.path is None: + raise ValueError("Please provide a file path") if self.source_type == 'csv': @orca.table(table_name = self.name, @@ -205,7 +218,9 @@ def run(self): cache_scope = self.cache_scope, copy_col=self.copy_col) def orca_table(): - df = pd.read_csv(self.path).set_index(self.csv_index_cols) + path = os.path.normpath(self.path) + kwargs = self.csv_settings + df = pd.read_csv(path, **kwargs).set_index(self.csv_index_cols) return df From 3677e22dc947b7238903b1fe803d8c0b9a9a4098 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Feb 2019 10:23:38 -0800 Subject: [PATCH 017/121] HDF table reading --- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- urbansim_templates/io/tables.py | 61 ++++++++++++++++++++------------- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/setup.py b/setup.py index 5c11e2b..95ece7a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.1.1', + version='0.2.dev0', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index e47f992..2479b9c 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.1.1' +version = __version__ = '0.2.dev0' diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 12022fd..6ffd872 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -58,28 +58,23 @@ class TableFromDisk(): `os.path.normpath()`, so either a Unix-style or Windows-style path should work across platforms. - url : str, optional - Remote url to download file from, NOT YET IMPLEMENTED. + url : str, optional - NOT YET IMPLEMENTED + Remote url to download file from. csv_index_cols : str or list of str, optional Required for csv source type. - csv_settings : dict, optional - Additional arguments to pass to `pd.read_csv()`. For example, you can - automatically extract data from a gzip file using {'compression': 'gzip'}. See - Pandas documentation for additional settings. + extra_settings : dict, optional + Additional arguments to pass to `pd.read_csv()` or `pd.read_hdf()`. For example, + you could automatically extract csv data from a gzip file using {'compression': + 'gzip'}, or specify the table identifier within a multi-object hdf store using + {'key': 'table-name'}. See Pandas documentation for additional settings. - hdf_key : str, optional - Name of table to read from the HDF5 file, if there are multiple. - - filters : str or list of str, optional + filters : str or list of str, optional - NOT YET IMPLEMENTED Filters to apply before registering the table with Orca. - - TO DO - IMPLEMENT - orca_test_spec : dict, optional - Data characteristics to be tested when the table is validated, NOT YET - IMPLEMENTED. + orca_test_spec : dict, optional - NOT YET IMPLEMENTED + Data characteristics to be tested when the table is validated. cache : bool, optional Passed to `orca.table()`. Note that the default is `True`, unlike in the @@ -112,7 +107,7 @@ def __init__(self, source_type = None, path = None, csv_index_cols = None, - csv_settings = None, + extra_settings = None, cache = True, cache_scope = 'forever', copy_col = True, @@ -124,7 +119,7 @@ def __init__(self, self.source_type = source_type self.path = path self.csv_index_cols = csv_index_cols - self.csv_settings = csv_settings + self.extra_settings = extra_settings self.cache = cache self.cache_scope = cache_scope self.copy_col = copy_col @@ -157,7 +152,7 @@ def from_dict(cls, d): source_type = d['source_type'], path = d['path'], csv_index_cols = d['csv_index_cols'], - csv_settings = d['csv_settings'], + extra_settings = d['extra_settings'], cache = d['cache'], cache_scope = d['cache_scope'], copy_col = d['copy_col'], @@ -186,7 +181,7 @@ def to_dict(self): 'source_type': self.source_type, 'path': self.path, 'csv_index_cols': self.csv_index_cols, - 'csv_settings': self.csv_settings, + 'extra_settings': self.extra_settings, 'cache': self.cache, 'cache_scope': self.cache_scope, 'copy_col': self.copy_col @@ -198,13 +193,16 @@ def run(self): """ Register a data table with Orca. + Requires values to be set for 'source_type', 'name', and 'path'. CSV data also + requires 'csv_index_cols'. + Returns ------- None """ - if self.source_type is None: - raise ValueError("Please provide a source type") + if self.source_type not in ['csv', 'hdf']: + raise ValueError("Please provide a source type of 'csv' or 'hdf'") if self.name is None: raise ValueError("Please provide a table name") @@ -212,17 +210,32 @@ def run(self): if self.path is None: raise ValueError("Please provide a file path") + path = os.path.normpath(self.path) + kwargs = self.extra_settings + + # Table from CSV file if self.source_type == 'csv': + if self.csv_index_cols is None: + raise ValueError("Please provide index column name(s) for the csv") + @orca.table(table_name = self.name, cache = self.cache, cache_scope = self.cache_scope, - copy_col=self.copy_col) + copy_col = self.copy_col) def orca_table(): - path = os.path.normpath(self.path) - kwargs = self.csv_settings df = pd.read_csv(path, **kwargs).set_index(self.csv_index_cols) return df + # Table from HDF file + elif self.source_type == 'hdf': + @orca.table(table_name = self.name, + cache = self.cache, + cache_scope = self.cache_scope, + copy_col = self.copy_col) + def orca_table(): + df = pd.read_hdf(path, **kwargs) + return df + def validate(self): """ From 090dadc43750207bc9fcef0a76d1b471067f15f5 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Feb 2019 10:35:22 -0800 Subject: [PATCH 018/121] Test for hdf tables --- tests/test_tables.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index a659c3a..940dcc9 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -33,10 +33,12 @@ def data(request): bldg = pd.DataFrame(d1).set_index('building_id') bldg.to_csv('data/buildings.csv') bldg.to_csv('data/buildings.csv.gz', compression='gzip') + bldg.to_hdf('data/buildings.hdf', key='buildings') def teardown(): os.remove('data/buildings.csv') os.remove('data/buildings.csv.gz') + os.remove('data/buildings.hdf') request.addfinalizer(teardown) @@ -203,7 +205,7 @@ def test_validation_with_multiindexes(orca_session): def test_csv(orca_session, data): """ - Test that loading data from a CSV works. + Test loading data from a CSV file. """ t = TableFromDisk() @@ -223,9 +225,30 @@ def test_csv(orca_session, data): modelmanager.remove_step('buildings') -def test_csv_extra_settings(orca_session, data): +def test_hdf(orca_session, data): """ - Test loading data with extra CSV settings, e.g. for compressed files. + Test loading data from an HDF file. + + """ + t = TableFromDisk() + t.name = 'buildings' + t.source_type = 'hdf' + t.path = 'data/buildings.hdf' + + assert 'buildings' not in orca.list_tables() + + modelmanager.register(t) + assert 'buildings' in orca.list_tables() + + modelmanager.initialize() + assert 'buildings' in orca.list_tables() + + modelmanager.remove_step('buildings') + + +def test_extra_settings(orca_session, data): + """ + Test loading data with extra settings, e.g. for compressed files. """ t = TableFromDisk() @@ -233,7 +256,7 @@ def test_csv_extra_settings(orca_session, data): t.source_type = 'csv' t.path = 'data/buildings.csv.gz' t.csv_index_cols = 'building_id' - t.csv_settings = {'compression': 'gzip'} + t.extra_settings = {'compression': 'gzip'} assert 'buildings' not in orca.list_tables() @@ -246,7 +269,7 @@ def test_csv_extra_settings(orca_session, data): modelmanager.remove_step('buildings') -def test_csv_windows_path(orca_session, data): +def test_windows_path(orca_session, data): """ Test loading a file with Windows-formatted path. From 15e6c76ad698ca180ed2ef2a8323b7d1dc09733e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Feb 2019 14:52:53 -0800 Subject: [PATCH 019/121] Better path normalization --- tests/test_tables.py | 25 +++++++------------- urbansim_templates/io/tables.py | 42 +++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index 940dcc9..e084e74 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -218,6 +218,7 @@ def test_csv(orca_session, data): modelmanager.register(t) assert 'buildings' in orca.list_tables() + _ = orca.get_table('buildings').to_frame() modelmanager.initialize() assert 'buildings' in orca.list_tables() @@ -239,6 +240,7 @@ def test_hdf(orca_session, data): modelmanager.register(t) assert 'buildings' in orca.list_tables() + _ = orca.get_table('buildings').to_frame() modelmanager.initialize() assert 'buildings' in orca.list_tables() @@ -262,6 +264,7 @@ def test_extra_settings(orca_session, data): modelmanager.register(t) assert 'buildings' in orca.list_tables() + _ = orca.get_table('buildings').to_frame() modelmanager.initialize() assert 'buildings' in orca.list_tables() @@ -269,26 +272,14 @@ def test_extra_settings(orca_session, data): modelmanager.remove_step('buildings') -def test_windows_path(orca_session, data): +def test_windows_paths(orca_session, data): """ - Test loading a file with Windows-formatted path. + Test in Windows that a Windows-style path is properly normalized. - """ - t = TableFromDisk() - t.name = 'buildings' - t.source_type = 'csv' - t.path = 'data\buildings.csv' - t.csv_index_cols = 'building_id' - - assert 'buildings' not in orca.list_tables() - - modelmanager.register(t) - assert 'buildings' in orca.list_tables() - - modelmanager.initialize() - assert 'buildings' in orca.list_tables() + TO DO - implement - modelmanager.remove_step('buildings') + """ + pass def test_without_autorun(orca_session, data): diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 6ffd872..87982d8 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -1,6 +1,9 @@ from __future__ import print_function -import os +try: + import pathlib # Python 3.4+ +except: + import os import orca import pandas as pd @@ -54,9 +57,11 @@ class TableFromDisk(): path : str, optional Local file path to load data from, either absolute or relative to the - ModelManager config directory. Path will be normed at runtime using - `os.path.normpath()`, so either a Unix-style or Windows-style path should work - across platforms. + ModelManager config directory. The string you provide will immediately be + normalized to a platform-agnostic format, using `os.path.normpath()` in Python 2 + or `pathlib.Path()` in Python 3. It is always safe to provide a Unix-style path, + and you may provide a Windows-style path if you are creating the model step in + Windows. Saved steps will run on any platform. url : str, optional - NOT YET IMPLEMENTED Remote url to download file from. @@ -76,15 +81,15 @@ class TableFromDisk(): orca_test_spec : dict, optional - NOT YET IMPLEMENTED Data characteristics to be tested when the table is validated. - cache : bool, optional + cache : bool, default True Passed to `orca.table()`. Note that the default is `True`, unlike in the underlying general-purpose Orca function, because tables read from disk should not need to be regenerated during the course of a model run. - cache_scope : 'step', 'iteration', or 'forever', optional + cache_scope : 'step', 'iteration', or 'forever', default 'forever' Passed to `orca.table()`. Default is 'forever', as in Orca. - copy_col : bool, optional + copy_col : bool, default True Passed to `orca.table()`. Default is `True`, as in Orca. name : str, optional @@ -94,7 +99,7 @@ class TableFromDisk(): tags : list of str, optional Tags, passed to ModelManager. - autorun : bool, optional (default True) + autorun : bool, default True Automatically run the step whenever it's registered with ModelManager. Properties and attributes @@ -107,7 +112,7 @@ def __init__(self, source_type = None, path = None, csv_index_cols = None, - extra_settings = None, + extra_settings = {}, cache = True, cache_scope = 'forever', copy_col = True, @@ -189,6 +194,20 @@ def to_dict(self): return d + @property + def path(self): + return self.__path + @path.setter + def path(self, value): + if value is not None: + try: + value = str(pathlib.Path(value)) # Python 3.4+ + except: + value = os.path.normpath(value) + self.__path = value + print(value) + + def run(self): """ Register a data table with Orca. @@ -210,7 +229,6 @@ def run(self): if self.path is None: raise ValueError("Please provide a file path") - path = os.path.normpath(self.path) kwargs = self.extra_settings # Table from CSV file @@ -223,7 +241,7 @@ def run(self): cache_scope = self.cache_scope, copy_col = self.copy_col) def orca_table(): - df = pd.read_csv(path, **kwargs).set_index(self.csv_index_cols) + df = pd.read_csv(self.path, **kwargs).set_index(self.csv_index_cols) return df # Table from HDF file @@ -233,7 +251,7 @@ def orca_table(): cache_scope = self.cache_scope, copy_col = self.copy_col) def orca_table(): - df = pd.read_hdf(path, **kwargs) + df = pd.read_hdf(self.path, **kwargs) return df From 9409d2f44ff6e806b4594696ffdd3f8a01a5865a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Feb 2019 19:09:19 -0800 Subject: [PATCH 020/121] Changelog and documentation --- CHANGELOG.md | 5 ++++ docs/build/.gitignore | 1 + docs/source/data-io.rst | 17 +++++++++++++ docs/source/getting-started.rst | 2 +- docs/source/index.rst | 3 ++- docs/source/model-steps.rst | 5 ++-- urbansim_templates/io/tables.py | 44 +++++++++------------------------ 7 files changed, 39 insertions(+), 38 deletions(-) create mode 100644 docs/build/.gitignore create mode 100644 docs/source/data-io.rst diff --git a/CHANGELOG.md b/CHANGELOG.md index e9c36f1..205acce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # UrbanSim Templates change log +### 0.2.dev0 (2019-02-19) + +- adds first data i/o template: `urbansim_templates.io.TableFromDisk()` +- adds support for `autorun` template property + ### 0.1.1 (2019-02-05) - production release diff --git a/docs/build/.gitignore b/docs/build/.gitignore new file mode 100644 index 0000000..01b7e33 --- /dev/null +++ b/docs/build/.gitignore @@ -0,0 +1 @@ +**/* \ No newline at end of file diff --git a/docs/source/data-io.rst b/docs/source/data-io.rst new file mode 100644 index 0000000..7b726a9 --- /dev/null +++ b/docs/source/data-io.rst @@ -0,0 +1,17 @@ +Data I/O template APIs +====================== + +Data i/o templates let you set up automated model steps for loading data into Orca or saving outputs to disk. + +These templates follow the same principles as the statistical model steps. For example, to set up a data table, create an instance of the ``TableFromDisk`` class and set some properties: the table name, file type, path, and anything else that's needed. + +Registering this object with ModelManager will save it to disk as a yaml file, and create an Orca step with instructions to set up the table. "Running" the object/step registers the table with Orca, but doesn't read the data from disk yet — Orca loads data lazily as it's needed. + +Data registration steps are run automatically when you initialize ModelManager. + + +Table from disk +--------------- + +.. autoclass:: urbansim_templates.io.TableFromDisk + :members: diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index 8f39858..1dce43d 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -105,7 +105,7 @@ The default file location is a ``configs`` folder located in the current working In [2]: import urbansim_templates print(urbansim_templates.__version__) - Out[2]: '0.1.dev12' + Out[2]: '0.2.dev0' Creating a model step diff --git a/docs/source/index.rst b/docs/source/index.rst index 2ba8129..4f765d0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.1.1, released February 5, 2019 +v0.2.dev0, released February 19, 2019 Contents @@ -22,5 +22,6 @@ Contents getting-started modelmanager model-steps + data-io utilities development diff --git a/docs/source/model-steps.rst b/docs/source/model-steps.rst index 54102fa..6f794fc 100644 --- a/docs/source/model-steps.rst +++ b/docs/source/model-steps.rst @@ -1,5 +1,5 @@ -Template APIs -============= +Model step template APIs +======================== The following templates are included in the core package. ModelManager can also work with templates defined elsewhere, as long as they follow the specifications described in the design guidelines. @@ -32,7 +32,6 @@ Large Multinomial Logit :members: - Segmented Large Multinomial Logit --------------------------------- diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 87982d8..1e76cf1 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -3,7 +3,9 @@ try: import pathlib # Python 3.4+ except: - import os + pass + +import os import orca import pandas as pd @@ -14,13 +16,12 @@ @modelmanager.template class TableFromDisk(): """ - Class for registering data tables. In the initial implementation, data can come from - local CSV or HDF5 files. + Class for registering data tables from local CSV or HDF5 files. - An instance of this Table() template stores *instructions for loading a data table*, - which can be saved as a yaml file. Running these instructions registers the table - with Orca. Saved tables will be registered automatically when you initialize - ModelManager, replacing the `datasources.py` scripts used in previous versions of + An instance of this template class stores *instructions for loading a data table*, + packaged into an Orca step. Running the instructions registers the table with Orca. + Saved table registration steps will be run automatically when you initialize + ModelManager, replacing the ``datasources.py`` scripts used in previous versions of UrbanSim. Tables should include a unique index, or a set of columns that jointly represent a @@ -30,24 +31,7 @@ class TableFromDisk(): be able to use it as a join key. Following these naming conventions eliminates the need for Orca "broadcasts". - Usage - ----- - Create an empty class instance: `t = TableFromDisk()`. - - Give it some properties: `t.name = 'buildings'` etc. (These can also be passed when - you create the object.) - - Register with ModelManager: `modelmanager.register(t)`. This registers the table - loading instructions, runs them, and saves them to disk. They'll automatically be run - the next time you initialize ModelManager. - - You can use all the standard ModelManager commands to get copies of the saved table - loading instructions, modify them, and delete them: - - - `modelmanager.list_steps()` - - `t2 = modelmanager.get_step('name')` - - `modelmanager.register(t2)` - - `modelmanager.remove_step('name')` + All the parameters can also be set as properties of the class instance. Parameters ---------- @@ -102,11 +86,6 @@ class TableFromDisk(): autorun : bool, default True Automatically run the step whenever it's registered with ModelManager. - Properties and attributes - ------------------------- - All the parameters listed above can also be get and set as properties of the class - instance. - """ def __init__(self, source_type = None, @@ -205,15 +184,14 @@ def path(self, value): except: value = os.path.normpath(value) self.__path = value - print(value) def run(self): """ Register a data table with Orca. - Requires values to be set for 'source_type', 'name', and 'path'. CSV data also - requires 'csv_index_cols'. + Requires values to be set for ``source_type``, ``name``, and ``path``. CSV data + also requires ``csv_index_cols``. Returns ------- From 9034fa4928beaab4b7ac78fcca69a9bbecccaa81 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 20 Feb 2019 13:44:40 -0800 Subject: [PATCH 021/121] Changing directory structure --- urbansim_templates/data/__init__.py | 1 + urbansim_templates/{io/tables.py => data/io.py} | 0 urbansim_templates/io/.gitignore | 1 - urbansim_templates/io/__init__.py | 1 - 4 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 urbansim_templates/data/__init__.py rename urbansim_templates/{io/tables.py => data/io.py} (100%) delete mode 100644 urbansim_templates/io/.gitignore delete mode 100644 urbansim_templates/io/__init__.py diff --git a/urbansim_templates/data/__init__.py b/urbansim_templates/data/__init__.py new file mode 100644 index 0000000..24651a3 --- /dev/null +++ b/urbansim_templates/data/__init__.py @@ -0,0 +1 @@ +from .io import TableFromDisk \ No newline at end of file diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/data/io.py similarity index 100% rename from urbansim_templates/io/tables.py rename to urbansim_templates/data/io.py diff --git a/urbansim_templates/io/.gitignore b/urbansim_templates/io/.gitignore deleted file mode 100644 index 763624e..0000000 --- a/urbansim_templates/io/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/* \ No newline at end of file diff --git a/urbansim_templates/io/__init__.py b/urbansim_templates/io/__init__.py deleted file mode 100644 index c79aa93..0000000 --- a/urbansim_templates/io/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .tables import TableFromDisk \ No newline at end of file From d59b04f6b620419a6beef3ce4e41db03f895ac4e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 20 Feb 2019 14:25:50 -0800 Subject: [PATCH 022/121] Fixing tests --- tests/test_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index e084e74..cc37fbc 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -7,7 +7,7 @@ import orca from urbansim_templates import modelmanager -from urbansim_templates.io import TableFromDisk +from urbansim_templates.data import TableFromDisk from urbansim_templates.utils import validate_template From 32e52a644da316a0b1e60d311bfc4b767c0e70d9 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 09:57:22 -0800 Subject: [PATCH 023/121] Work in progress --- urbansim_templates/data/io.py | 159 +++++++++++++++++++++++++++++++++- 1 file changed, 158 insertions(+), 1 deletion(-) diff --git a/urbansim_templates/data/io.py b/urbansim_templates/data/io.py index 1e76cf1..90d5bee 100644 --- a/urbansim_templates/data/io.py +++ b/urbansim_templates/data/io.py @@ -308,4 +308,161 @@ def validate(self): return True - \ No newline at end of file +@modelmanager.template +class SaveData(): + """ + Class for saving Orca tables to local CSV or HDF5 files. + + All the parameters can also be set as properties of the class instance. + + Parameters + ---------- + output_type : 'csv' or 'hdf', optional + This is required to save the table, but does not have to be provided when the + object is created. + + path : str, optional + Local file path to save the data to, either absolute or relative to the + ModelManager config directory. The string you provide will immediately be + normalized to a platform-agnostic format, using `os.path.normpath()` in Python 2 + or `pathlib.Path()` in Python 3. It is always safe to provide a Unix-style path, + and you may provide a Windows-style path if you are creating the model step in + Windows. Saved steps will run on any platform. + + extra_settings : dict, optional + Additional arguments to pass to `pd.to_csv()` or `pd.to_hdf()`. For example, you + could automatically compress csv data using {'compression': 'gzip'}, or specify + a custom table name for an hdf store using {'key': 'table-name'}. See Pandas + documentation for additional settings. + + name : str, optional + Name of the Orca table. HOW TO NAME THE MODEL STEP? + + tags : list of str, optional + Tags, passed to ModelManager. + + """ + def __init__(self, + output_type = None, + path = None, + extra_settings = {}, + name = None, + tags = [], + + # Template-specific params + self.output_type = output_type + self.path = path + self.extra_settings = extra_settings + + # Standard params + self.name = name + self.tags = tags + + # Automatic params + self.template = self.__class__.__name__ + self.template_version = __version__ + + + @classmethod + def from_dict(cls, d): + """ + Create an object instance from a saved dictionary representation. + + Parameters + ---------- + d : dict + + Returns + ------- + Table + + """ + obj = cls( + output_type = d['output_type'], + path = d['path'], + extra_settings = d['extra_settings'], + name = d['name'], + tags = d['tags'], + ) + return obj + + + def to_dict(self): + """ + Create a dictionary representation of the object. + + Returns + ------- + dict + + """ + d = { + 'template': self.template, + 'template_version': self.template_version, + 'name': self.name, + 'tags': self.tags, + 'output_type': self.output_type, + 'path': self.path, + 'extra_settings': self.extra_settings, + } + return d + + + @property + def path(self): + return self.__path + @path.setter + def path(self, value): + if value is not None: + try: + value = str(pathlib.Path(value)) # Python 3.4+ + except: + value = os.path.normpath(value) + self.__path = value + + + def run(self): + """ + Register a data table with Orca. + + Requires values to be set for ``source_type``, ``name``, and ``path``. CSV data + also requires ``csv_index_cols``. + + Returns + ------- + None + + """ + if self.source_type not in ['csv', 'hdf']: + raise ValueError("Please provide a source type of 'csv' or 'hdf'") + + if self.name is None: + raise ValueError("Please provide a table name") + + if self.path is None: + raise ValueError("Please provide a file path") + + kwargs = self.extra_settings + + # Table from CSV file + if self.source_type == 'csv': + if self.csv_index_cols is None: + raise ValueError("Please provide index column name(s) for the csv") + + @orca.table(table_name = self.name, + cache = self.cache, + cache_scope = self.cache_scope, + copy_col = self.copy_col) + def orca_table(): + df = pd.read_csv(self.path, **kwargs).set_index(self.csv_index_cols) + return df + + # Table from HDF file + elif self.source_type == 'hdf': + @orca.table(table_name = self.name, + cache = self.cache, + cache_scope = self.cache_scope, + copy_col = self.copy_col) + def orca_table(): + df = pd.read_hdf(self.path, **kwargs) + return df From 9a4a40384bbba6159b01aeed3763d9938611036d Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 10:52:55 -0800 Subject: [PATCH 024/121] Data saving works --- tests/{test_tables.py => test_data_load.py} | 5 +- tests/test_data_save.py | 89 +++++++++++ urbansim_templates/data/__init__.py | 3 +- urbansim_templates/data/io.py | 160 +------------------- urbansim_templates/data/save_data.py | 142 +++++++++++++++++ 5 files changed, 235 insertions(+), 164 deletions(-) rename tests/{test_tables.py => test_data_load.py} (98%) create mode 100644 tests/test_data_save.py create mode 100644 urbansim_templates/data/save_data.py diff --git a/tests/test_tables.py b/tests/test_data_load.py similarity index 98% rename from tests/test_tables.py rename to tests/test_data_load.py index cc37fbc..e26be90 100644 --- a/tests/test_tables.py +++ b/tests/test_data_load.py @@ -45,7 +45,7 @@ def teardown(): def test_template_validity(): """ - Run the template through the standard validation check. + Run the templates through the standard validation check. """ assert validate_template(TableFromDisk) @@ -192,10 +192,7 @@ def test_validation_with_multiindexes(orca_session): t.validate() -# test that parameters make it through a save # test validation with stand-alone columns - -# test loading an h5 file works # test passing cache settings diff --git a/tests/test_data_save.py b/tests/test_data_save.py new file mode 100644 index 0000000..6eb5d9f --- /dev/null +++ b/tests/test_data_save.py @@ -0,0 +1,89 @@ +import os + +import numpy as np +import pandas as pd +import pytest + +import orca + +from urbansim_templates import modelmanager +from urbansim_templates.data import SaveData +from urbansim_templates.utils import validate_template + + +@pytest.fixture +def orca_session(): + """ + Set up a clean Orca session and initialize ModelManager. + + """ + orca.clear_all() + modelmanager.initialize() + + +@pytest.fixture +def data(): + """ + Create a data table. + + """ + d1 = {'building_id': np.arange(10), + 'price': (1e6*np.random.random(10)).astype(int)} + + df = pd.DataFrame(d1).set_index('building_id') + + orca.add_table('buildings', df) + + +def test_template_validity(): + """ + Run the templates through the standard validation check. + + """ + assert validate_template(SaveData) + + +def test_property_persistence(orca_session): + """ + Test persistence of properties across registration, saving, and reloading. + + """ + pass + + +def test_csv(orca_session, data): + """ + Test saving data to a CSV file. + + """ + t = SaveData() + t.name = 'buildings' + t.output_type = 'csv' + t.path = 'data/buildings.csv' + + t.run() + + df = pd.read_csv(t.path).set_index('building_id') + assert(df.equals(orca.get_table(t.name).to_frame())) + + os.remove(t.path) + + +def test_hdf(orca_session, data): + """ + Test saving data to an HDF file. + + """ + t = SaveData() + t.name = 'buildings' + t.output_type = 'hdf' + t.path = 'data/buildings.h5' + + t.run() + + df = pd.read_hdf(t.path) + assert(df.equals(orca.get_table(t.name).to_frame())) + + os.remove(t.path) + + diff --git a/urbansim_templates/data/__init__.py b/urbansim_templates/data/__init__.py index 24651a3..7a01ff6 100644 --- a/urbansim_templates/data/__init__.py +++ b/urbansim_templates/data/__init__.py @@ -1 +1,2 @@ -from .io import TableFromDisk \ No newline at end of file +from .io import TableFromDisk +from .save_data import SaveData \ No newline at end of file diff --git a/urbansim_templates/data/io.py b/urbansim_templates/data/io.py index 90d5bee..070002c 100644 --- a/urbansim_templates/data/io.py +++ b/urbansim_templates/data/io.py @@ -31,7 +31,7 @@ class TableFromDisk(): be able to use it as a join key. Following these naming conventions eliminates the need for Orca "broadcasts". - All the parameters can also be set as properties of the class instance. + All the parameters can also be set as properties after creating the class instance. Parameters ---------- @@ -308,161 +308,3 @@ def validate(self): return True -@modelmanager.template -class SaveData(): - """ - Class for saving Orca tables to local CSV or HDF5 files. - - All the parameters can also be set as properties of the class instance. - - Parameters - ---------- - output_type : 'csv' or 'hdf', optional - This is required to save the table, but does not have to be provided when the - object is created. - - path : str, optional - Local file path to save the data to, either absolute or relative to the - ModelManager config directory. The string you provide will immediately be - normalized to a platform-agnostic format, using `os.path.normpath()` in Python 2 - or `pathlib.Path()` in Python 3. It is always safe to provide a Unix-style path, - and you may provide a Windows-style path if you are creating the model step in - Windows. Saved steps will run on any platform. - - extra_settings : dict, optional - Additional arguments to pass to `pd.to_csv()` or `pd.to_hdf()`. For example, you - could automatically compress csv data using {'compression': 'gzip'}, or specify - a custom table name for an hdf store using {'key': 'table-name'}. See Pandas - documentation for additional settings. - - name : str, optional - Name of the Orca table. HOW TO NAME THE MODEL STEP? - - tags : list of str, optional - Tags, passed to ModelManager. - - """ - def __init__(self, - output_type = None, - path = None, - extra_settings = {}, - name = None, - tags = [], - - # Template-specific params - self.output_type = output_type - self.path = path - self.extra_settings = extra_settings - - # Standard params - self.name = name - self.tags = tags - - # Automatic params - self.template = self.__class__.__name__ - self.template_version = __version__ - - - @classmethod - def from_dict(cls, d): - """ - Create an object instance from a saved dictionary representation. - - Parameters - ---------- - d : dict - - Returns - ------- - Table - - """ - obj = cls( - output_type = d['output_type'], - path = d['path'], - extra_settings = d['extra_settings'], - name = d['name'], - tags = d['tags'], - ) - return obj - - - def to_dict(self): - """ - Create a dictionary representation of the object. - - Returns - ------- - dict - - """ - d = { - 'template': self.template, - 'template_version': self.template_version, - 'name': self.name, - 'tags': self.tags, - 'output_type': self.output_type, - 'path': self.path, - 'extra_settings': self.extra_settings, - } - return d - - - @property - def path(self): - return self.__path - @path.setter - def path(self, value): - if value is not None: - try: - value = str(pathlib.Path(value)) # Python 3.4+ - except: - value = os.path.normpath(value) - self.__path = value - - - def run(self): - """ - Register a data table with Orca. - - Requires values to be set for ``source_type``, ``name``, and ``path``. CSV data - also requires ``csv_index_cols``. - - Returns - ------- - None - - """ - if self.source_type not in ['csv', 'hdf']: - raise ValueError("Please provide a source type of 'csv' or 'hdf'") - - if self.name is None: - raise ValueError("Please provide a table name") - - if self.path is None: - raise ValueError("Please provide a file path") - - kwargs = self.extra_settings - - # Table from CSV file - if self.source_type == 'csv': - if self.csv_index_cols is None: - raise ValueError("Please provide index column name(s) for the csv") - - @orca.table(table_name = self.name, - cache = self.cache, - cache_scope = self.cache_scope, - copy_col = self.copy_col) - def orca_table(): - df = pd.read_csv(self.path, **kwargs).set_index(self.csv_index_cols) - return df - - # Table from HDF file - elif self.source_type == 'hdf': - @orca.table(table_name = self.name, - cache = self.cache, - cache_scope = self.cache_scope, - copy_col = self.copy_col) - def orca_table(): - df = pd.read_hdf(self.path, **kwargs) - return df diff --git a/urbansim_templates/data/save_data.py b/urbansim_templates/data/save_data.py new file mode 100644 index 0000000..959865a --- /dev/null +++ b/urbansim_templates/data/save_data.py @@ -0,0 +1,142 @@ +from __future__ import print_function + +import orca +import pandas as pd + +from urbansim_templates import modelmanager, __version__ + + +@modelmanager.template +class SaveData(): + """ + Class for saving Orca tables to local CSV or HDF5 files. + + All the parameters can also be set as properties after creating the class instance. + + Parameters + ---------- + output_type : 'csv' or 'hdf', optional + This is required to save the table, but does not have to be provided when the + object is created. + + path : str, optional + Local file path to save the data to, either absolute or relative to the + ModelManager config directory. Please provide a Unix-style path (this will work + on any platform, but a Windows-style path won't, and they're hard to normalize + automatically). For dynamic file names, you can include the characters ``%RUN%``, + ``%ITER%``, or ``%TS%``. These will be replaced by the model run number, the + iteration value, or a timestamp when the output file is created. + + extra_settings : dict, optional + Additional arguments to pass to `pd.to_csv()` or `pd.to_hdf()`. For example, you + could automatically compress csv data using {'compression': 'gzip'}, or specify + a custom table name for an hdf store using {'key': 'table-name'}. See Pandas + documentation for additional settings. + + name : str, optional + Name of the Orca table. HOW TO NAME THE MODEL STEP? + + tags : list of str, optional + Tags, passed to ModelManager. + + """ + def __init__(self, + output_type = None, + path = None, + extra_settings = {}, + name = None, + tags = []): + + # Template-specific params + self.output_type = output_type + self.path = path + self.extra_settings = extra_settings + + # Standard params + self.name = name + self.tags = tags + + # Automatic params + self.template = self.__class__.__name__ + self.template_version = __version__ + + + @classmethod + def from_dict(cls, d): + """ + Create an object instance from a saved dictionary representation. + + Parameters + ---------- + d : dict + + Returns + ------- + Table + + """ + obj = cls( + output_type = d['output_type'], + path = d['path'], + extra_settings = d['extra_settings'], + name = d['name'], + tags = d['tags'], + ) + return obj + + + def to_dict(self): + """ + Create a dictionary representation of the object. + + Returns + ------- + dict + + """ + d = { + 'template': self.template, + 'template_version': self.template_version, + 'name': self.name, + 'tags': self.tags, + 'output_type': self.output_type, + 'path': self.path, + 'extra_settings': self.extra_settings, + } + return d + + + def run(self): + """ + + ``pd.to_hdf()`` requires a ``key`` to identify the table in the HDF store. We'll + use the Orca table name for this, unless you provide a different ``key`` in the + ``extra_settings``. + + Returns + ------- + None + + """ + if self.output_type not in ['csv', 'hdf']: + raise ValueError("Please provide an output type of 'csv' or 'hdf'") + + if self.name is None: + raise ValueError("Please provide the table name") + + if self.path is None: + raise ValueError("Please provide a file path") + + kwargs = self.extra_settings + df = orca.get_table(self.name).to_frame() + + if self.output_type == 'csv': + df.to_csv(self.path, **kwargs) + + elif self.output_type == 'hdf': + if 'key' not in kwargs: + kwargs['key'] = self.name + + df.to_hdf(self.path, **kwargs) + + \ No newline at end of file From 051644ba3d2861b846d4010f498f50fda877c0e8 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 10:56:20 -0800 Subject: [PATCH 025/121] TableFromDisk -> LoadData --- tests/test_data_load.py | 28 +++++++++---------- urbansim_templates/data/__init__.py | 2 +- .../data/{io.py => load_data.py} | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) rename urbansim_templates/data/{io.py => load_data.py} (99%) diff --git a/tests/test_data_load.py b/tests/test_data_load.py index e26be90..28b5929 100644 --- a/tests/test_data_load.py +++ b/tests/test_data_load.py @@ -7,7 +7,7 @@ import orca from urbansim_templates import modelmanager -from urbansim_templates.data import TableFromDisk +from urbansim_templates.data import LoadData from urbansim_templates.utils import validate_template @@ -48,7 +48,7 @@ def test_template_validity(): Run the templates through the standard validation check. """ - assert validate_template(TableFromDisk) + assert validate_template(LoadData) def test_property_persistence(orca_session): @@ -75,7 +75,7 @@ def test_validation_index_unique(orca_session): d = {'id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = TableFromDisk(name='tab') + t = LoadData(name='tab') t.validate() @@ -87,7 +87,7 @@ def test_validation_index_not_unique(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = TableFromDisk(name='tab') + t = LoadData(name='tab') try: t.validate() except ValueError: @@ -104,7 +104,7 @@ def test_validation_multiindex_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = TableFromDisk(name='tab') + t = LoadData(name='tab') t.validate() @@ -117,7 +117,7 @@ def test_validation_multiindex_not_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [2,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = TableFromDisk(name='tab') + t = LoadData(name='tab') try: t.validate() except ValueError: @@ -134,7 +134,7 @@ def test_validation_unnamed_index(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name - t = TableFromDisk(name='tab') + t = LoadData(name='tab') try: t.validate() except ValueError: @@ -155,7 +155,7 @@ def test_validation_columns_vs_other_indexes(orca_session): d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) - t = TableFromDisk(name='households') + t = LoadData(name='households') t.validate() @@ -171,7 +171,7 @@ def test_validation_index_vs_other_columns(orca_session): d = {'household_id': [1,2,3], 'building_id': [2,3,5]} orca.add_table('households', pd.DataFrame(d).set_index('household_id')) - t = TableFromDisk(name='buildings') + t = LoadData(name='buildings') t.validate() @@ -188,7 +188,7 @@ def test_validation_with_multiindexes(orca_session): d = {'home_tract': [55,55,55], 'work_tract': [17,18,19], 'dist': [1,1,1]} orca.add_table('distances', pd.DataFrame(d).set_index(['home_tract','work_tract'])) - t = TableFromDisk(name='choice_table') + t = LoadData(name='choice_table') t.validate() @@ -205,7 +205,7 @@ def test_csv(orca_session, data): Test loading data from a CSV file. """ - t = TableFromDisk() + t = LoadData() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' @@ -228,7 +228,7 @@ def test_hdf(orca_session, data): Test loading data from an HDF file. """ - t = TableFromDisk() + t = LoadData() t.name = 'buildings' t.source_type = 'hdf' t.path = 'data/buildings.hdf' @@ -250,7 +250,7 @@ def test_extra_settings(orca_session, data): Test loading data with extra settings, e.g. for compressed files. """ - t = TableFromDisk() + t = LoadData() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv.gz' @@ -284,7 +284,7 @@ def test_without_autorun(orca_session, data): Confirm that disabling autorun works. """ - t = TableFromDisk() + t = LoadData() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' diff --git a/urbansim_templates/data/__init__.py b/urbansim_templates/data/__init__.py index 7a01ff6..479b442 100644 --- a/urbansim_templates/data/__init__.py +++ b/urbansim_templates/data/__init__.py @@ -1,2 +1,2 @@ -from .io import TableFromDisk +from .load_data import LoadData from .save_data import SaveData \ No newline at end of file diff --git a/urbansim_templates/data/io.py b/urbansim_templates/data/load_data.py similarity index 99% rename from urbansim_templates/data/io.py rename to urbansim_templates/data/load_data.py index 070002c..e9b5910 100644 --- a/urbansim_templates/data/io.py +++ b/urbansim_templates/data/load_data.py @@ -14,7 +14,7 @@ @modelmanager.template -class TableFromDisk(): +class LoadData(): """ Class for registering data tables from local CSV or HDF5 files. @@ -307,4 +307,4 @@ def validate(self): return True - + \ No newline at end of file From b75aaf8dbf7ef2ce5674d2bc8f6871c1fb60970e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 11:21:14 -0800 Subject: [PATCH 026/121] More parameters --- tests/test_data_save.py | 8 +++--- urbansim_templates/data/save_data.py | 43 +++++++++++++++++++--------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/test_data_save.py b/tests/test_data_save.py index 6eb5d9f..64243c7 100644 --- a/tests/test_data_save.py +++ b/tests/test_data_save.py @@ -57,14 +57,14 @@ def test_csv(orca_session, data): """ t = SaveData() - t.name = 'buildings' + t.table = 'buildings' t.output_type = 'csv' t.path = 'data/buildings.csv' t.run() df = pd.read_csv(t.path).set_index('building_id') - assert(df.equals(orca.get_table(t.name).to_frame())) + assert(df.equals(orca.get_table(t.table).to_frame())) os.remove(t.path) @@ -75,14 +75,14 @@ def test_hdf(orca_session, data): """ t = SaveData() - t.name = 'buildings' + t.table = 'buildings' t.output_type = 'hdf' t.path = 'data/buildings.h5' t.run() df = pd.read_hdf(t.path) - assert(df.equals(orca.get_table(t.name).to_frame())) + assert(df.equals(orca.get_table(t.table).to_frame())) os.remove(t.path) diff --git a/urbansim_templates/data/save_data.py b/urbansim_templates/data/save_data.py index 959865a..28741e7 100644 --- a/urbansim_templates/data/save_data.py +++ b/urbansim_templates/data/save_data.py @@ -15,9 +15,15 @@ class SaveData(): Parameters ---------- + table : str, optional + Name of the Orca table. Must be provided before running the step. + + columns : str or list of str, optional + Names of columns to include, in addition to indexes. "None" will return all + columns. + output_type : 'csv' or 'hdf', optional - This is required to save the table, but does not have to be provided when the - object is created. + Type of file to be created. Must be provided before running the step. path : str, optional Local file path to save the data to, either absolute or relative to the @@ -34,13 +40,15 @@ class SaveData(): documentation for additional settings. name : str, optional - Name of the Orca table. HOW TO NAME THE MODEL STEP? + Name of the model step. tags : list of str, optional Tags, passed to ModelManager. """ def __init__(self, + table = None, + columns = None, output_type = None, path = None, extra_settings = {}, @@ -48,6 +56,8 @@ def __init__(self, tags = []): # Template-specific params + self.table = table + self.columns = columns self.output_type = output_type self.path = path self.extra_settings = extra_settings @@ -76,11 +86,13 @@ def from_dict(cls, d): """ obj = cls( - output_type = d['output_type'], - path = d['path'], - extra_settings = d['extra_settings'], - name = d['name'], - tags = d['tags'], + table = d['table'], + columns = d['columns'], + output_type = d['output_type'], + path = d['path'], + extra_settings = d['extra_settings'], + name = d['name'], + tags = d['tags'], ) return obj @@ -99,6 +111,8 @@ def to_dict(self): 'template_version': self.template_version, 'name': self.name, 'tags': self.tags, + 'table': self.table, + 'columns': self.columns, 'output_type': self.output_type, 'path': self.path, 'extra_settings': self.extra_settings, @@ -108,10 +122,11 @@ def to_dict(self): def run(self): """ + Save a table to disk. - ``pd.to_hdf()`` requires a ``key`` to identify the table in the HDF store. We'll - use the Orca table name for this, unless you provide a different ``key`` in the - ``extra_settings``. + Adding a table to an HDF store requires providing a ``key`` that will be used to + identify the table in the store. We'll use the Orca table name, unless you + provide a different ``key`` in the ``extra_settings``. Returns ------- @@ -121,21 +136,21 @@ def run(self): if self.output_type not in ['csv', 'hdf']: raise ValueError("Please provide an output type of 'csv' or 'hdf'") - if self.name is None: + if self.table is None: raise ValueError("Please provide the table name") if self.path is None: raise ValueError("Please provide a file path") kwargs = self.extra_settings - df = orca.get_table(self.name).to_frame() + df = orca.get_table(self.table).to_frame(self.columns) if self.output_type == 'csv': df.to_csv(self.path, **kwargs) elif self.output_type == 'hdf': if 'key' not in kwargs: - kwargs['key'] = self.name + kwargs['key'] = self.table df.to_hdf(self.path, **kwargs) From 92ca51bddfa9450db2f23251771c9929b88a4c7e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 12:01:36 -0800 Subject: [PATCH 027/121] Data filters --- tests/test_data_save.py | 29 ++++++++++++++++++++++++++++ urbansim_templates/data/save_data.py | 20 ++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/tests/test_data_save.py b/tests/test_data_save.py index 64243c7..61b7b38 100644 --- a/tests/test_data_save.py +++ b/tests/test_data_save.py @@ -87,3 +87,32 @@ def test_hdf(orca_session, data): os.remove(t.path) +def test_columns(orca_session, data): + """ + """ + pass + + +def test_filters(orca_session, data): + """ + """ + t = SaveData() + t.table = 'buildings' + t.filters = 'price < 200000' + t.output_type = 'csv' + t.path = 'data/buildings.csv' + + t.run() + + df = pd.read_csv(t.path).set_index('building_id') + assert(len(df) < 10) + + os.remove(t.path) + + +def test_extra_settings(orca_session, data): + """ + """ + pass + + diff --git a/urbansim_templates/data/save_data.py b/urbansim_templates/data/save_data.py index 28741e7..1e5b30b 100644 --- a/urbansim_templates/data/save_data.py +++ b/urbansim_templates/data/save_data.py @@ -4,6 +4,7 @@ import pandas as pd from urbansim_templates import modelmanager, __version__ +from urbansim_templates.utils import get_data @modelmanager.template @@ -22,6 +23,10 @@ class SaveData(): Names of columns to include, in addition to indexes. "None" will return all columns. + filters : str or list of str, optional + Filters to apply to the data before saving. Will be passed to + ``pd.DataFrame.query()``. + output_type : 'csv' or 'hdf', optional Type of file to be created. Must be provided before running the step. @@ -49,15 +54,17 @@ class SaveData(): def __init__(self, table = None, columns = None, + filters = None, output_type = None, path = None, - extra_settings = {}, + extra_settings = None, name = None, tags = []): # Template-specific params self.table = table self.columns = columns + self.filters = filters self.output_type = output_type self.path = path self.extra_settings = extra_settings @@ -88,6 +95,7 @@ def from_dict(cls, d): obj = cls( table = d['table'], columns = d['columns'], + filters = d['filters'], output_type = d['output_type'], path = d['path'], extra_settings = d['extra_settings'], @@ -113,6 +121,7 @@ def to_dict(self): 'tags': self.tags, 'table': self.table, 'columns': self.columns, + 'filters': self.filters, 'output_type': self.output_type, 'path': self.path, 'extra_settings': self.extra_settings, @@ -124,7 +133,7 @@ def run(self): """ Save a table to disk. - Adding a table to an HDF store requires providing a ``key`` that will be used to + Saving a table to an HDF store requires providing a ``key`` that will be used to identify the table in the store. We'll use the Orca table name, unless you provide a different ``key`` in the ``extra_settings``. @@ -143,7 +152,12 @@ def run(self): raise ValueError("Please provide a file path") kwargs = self.extra_settings - df = orca.get_table(self.table).to_frame(self.columns) + if kwargs is None: + kwargs = dict() + + df = get_data(tables = self.table, + filters = self.filters, + extra_columns = self.columns) if self.output_type == 'csv': df.to_csv(self.path, **kwargs) From 9aaf73db46d2ec32c7f18629c952a2edc38ca5ac Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 14:54:21 -0800 Subject: [PATCH 028/121] Dynamic filepaths --- tests/test_data_save.py | 22 ++++++++++++++++ urbansim_templates/data/save_data.py | 39 +++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/test_data_save.py b/tests/test_data_save.py index 61b7b38..10aaf4e 100644 --- a/tests/test_data_save.py +++ b/tests/test_data_save.py @@ -95,6 +95,8 @@ def test_columns(orca_session, data): def test_filters(orca_session, data): """ + Test applying data filters before table is saved. + """ t = SaveData() t.table = 'buildings' @@ -116,3 +118,23 @@ def test_extra_settings(orca_session, data): pass +def test_dynamic_paths(orca_session): + """ + Test inserting run id, model iteration, or timestamp into path. + + """ + t = SaveData() + t.path = '%RUN%-%ITER%' + + assert(t.get_dynamic_filepath() == '0-0') + + orca.add_injectable('run_id', 5) + orca.add_injectable('iter_var', 3) + + assert(t.get_dynamic_filepath() == '5-3') + + t.path = '%TS%' + s = t.get_dynamic_filepath() + assert(len(s) == 15) + + diff --git a/urbansim_templates/data/save_data.py b/urbansim_templates/data/save_data.py index 1e5b30b..1c6359d 100644 --- a/urbansim_templates/data/save_data.py +++ b/urbansim_templates/data/save_data.py @@ -1,5 +1,7 @@ from __future__ import print_function +import datetime + import orca import pandas as pd @@ -35,7 +37,7 @@ class SaveData(): ModelManager config directory. Please provide a Unix-style path (this will work on any platform, but a Windows-style path won't, and they're hard to normalize automatically). For dynamic file names, you can include the characters ``%RUN%``, - ``%ITER%``, or ``%TS%``. These will be replaced by the model run number, the + ``%ITER%``, or ``%TS%``. These will be replaced by the run id, the model iteration value, or a timestamp when the output file is created. extra_settings : dict, optional @@ -129,6 +131,41 @@ def to_dict(self): return d + def get_dynamic_filepath(self): + """ + Substitute run id, model iteration, and/or timestamp into the filename. + + For the run id and model iteration, we look for Orca injectables named ``run_id`` + and ``iter_var``, respectively. If none is found, we use ``0``. + + The timestamp is UTC, formatted as ``YYYYMMDD-HHMMSS``. + + Returns + ------- + str + + """ + if self.path is None: + raise ValueError("Please provide a file path") + + run = 0 + if orca.is_injectable('run_id'): + run = orca.get_injectable('run_id') + + iter = 0 + if orca.is_injectable('iter_var'): + iter = orca.get_injectable('iter_var') + + ts = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S') + + s = self.path + s = s.replace('%RUN%', str(run)) + s = s.replace('%ITER%', str(iter)) + s = s.replace('%TS%', ts) + + return s + + def run(self): """ Save a table to disk. From d9b0175c07f97e327aba76e13137ad760e15b8be Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 16:53:21 -0800 Subject: [PATCH 029/121] Work in progress --- urbansim_templates/data/save_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/urbansim_templates/data/save_data.py b/urbansim_templates/data/save_data.py index 1c6359d..ba8fbc3 100644 --- a/urbansim_templates/data/save_data.py +++ b/urbansim_templates/data/save_data.py @@ -195,14 +195,14 @@ def run(self): df = get_data(tables = self.table, filters = self.filters, extra_columns = self.columns) - + if self.output_type == 'csv': - df.to_csv(self.path, **kwargs) + df.to_csv(self.get_dynamic_filepath(), **kwargs) elif self.output_type == 'hdf': if 'key' not in kwargs: kwargs['key'] = self.table - df.to_hdf(self.path, **kwargs) + df.to_hdf(self.get_dynamic_filepath(), **kwargs) \ No newline at end of file From 831c84344c8cacc7cb2ae7a049ed97b3e5cd86aa Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 17:10:37 -0800 Subject: [PATCH 030/121] Reproduces error --- tests/test_small_multinomial_logit.py | 34 +++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/test_small_multinomial_logit.py b/tests/test_small_multinomial_logit.py index 3630bcd..004dad3 100644 --- a/tests/test_small_multinomial_logit.py +++ b/tests/test_small_multinomial_logit.py @@ -11,12 +11,29 @@ @pytest.fixture def orca_session(): - d1 = {'a': np.random.random(100), - 'b': np.random.random(100), + d1 = {'id': np.arange(100), + 'building_id': np.arange(100), + 'a': np.random.random(100), 'choice': np.random.randint(3, size=100)} + + d2 = {'building_id': np.arange(100), + 'b': np.random.random(100)} + + households = pd.DataFrame(d1).set_index('id') + orca.add_table('households', households) + + buildings = pd.DataFrame(d2).set_index('building_id') + orca.add_table('buildings', buildings) - obs = pd.DataFrame(d1) - orca.add_table('obs', obs) + orca.broadcast(cast='buildings', onto='households', + cast_index=True, onto_on='building_id') + +# d1 = {'a': np.random.random(100), +# 'b': np.random.random(100), +# 'choice': np.random.randint(3, size=100)} +# +# obs = pd.DataFrame(d1) +# orca.add_table('obs', obs) def test_template_validity(): @@ -35,7 +52,7 @@ def test_small_mnl(orca_session): modelmanager.initialize() m = SmallMultinomialLogitStep() - m.tables = 'obs' + m.tables = ['households', 'buildings'] m.choice_column = 'choice' m.model_expression = OrderedDict([ ('intercept', [1,2]), ('a', [0,2]), ('b', [0,2])]) @@ -51,6 +68,13 @@ def test_small_mnl(orca_session): print(m.model_expression) + # TEST SIMULATION + m.out_column = 'simulated_choice' + # orca.get_table('households')['simulated_choice'] = 0 + + m.run() + print(orca.get_table('households').to_frame()) + modelmanager.initialize() m = modelmanager.get_step('small-mnl-test') assert(m.model_expression is not None) From 3c90803e5285e93068563b6f2502aab42c7315ba Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 17:22:44 -0800 Subject: [PATCH 031/121] Fixes error --- .../models/small_multinomial_logit.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/urbansim_templates/models/small_multinomial_logit.py b/urbansim_templates/models/small_multinomial_logit.py index c8fefbd..cb9b0dc 100644 --- a/urbansim_templates/models/small_multinomial_logit.py +++ b/urbansim_templates/models/small_multinomial_logit.py @@ -10,9 +10,9 @@ from choicemodels import MultinomialLogit import orca -from ..utils import update_column -from .. import modelmanager -from .shared import TemplateStep +from urbansim_templates import modelmanager +from urbansim_templates.models import TemplateStep +from urbansim_templates.utils import get_data, update_column @modelmanager.template @@ -87,8 +87,6 @@ class SmallMultinomialLogitStep(TemplateStep): in the primary output table, it will be created. If not provided, the `choice_column` will be used. Replaces the `out_fname` argument in UrbanSim. - # TO DO - auto-generation not yet working; column must exist in the primary table - out_filters : str or list of str, optional Filters to apply to the data before simulation. If not provided, no filters will be applied. Replaces the `predict_filters` argument in UrbanSim. @@ -319,7 +317,8 @@ def fit(self): if (self.initial_coefs is None) or (len(self.initial_coefs) != pc): self.initial_coefs = np.zeros(pc).tolist() - model = MultinomialLogit(data=long_df, observation_id_col='_obs_id', + model = MultinomialLogit(data=long_df, + observation_id_col='_obs_id', choice_col='_chosen', model_expression=self.model_expression, model_labels=self.model_labels, @@ -352,7 +351,14 @@ def run(self): model step. """ - df = self._get_data('predict') + expr_cols = [t[0] for t in list(self.model_expression.items()) \ + if t[0] != 'intercept'] + + df = get_data(tables = self.out_tables, + fallback_tables = self.tables, + filters = self.out_filters, + extra_columns = expr_cols) + long_df = self._to_long(df, 'predict') num_obs = len(df) From 1cec92d77f7d87e3a0d9ae4591378b17e8860675 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 17:31:23 -0800 Subject: [PATCH 032/121] Cleanup --- tests/test_small_multinomial_logit.py | 8 -------- urbansim_templates/models/shared.py | 10 ---------- urbansim_templates/models/small_multinomial_logit.py | 9 ++++++++- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/tests/test_small_multinomial_logit.py b/tests/test_small_multinomial_logit.py index 004dad3..4e35511 100644 --- a/tests/test_small_multinomial_logit.py +++ b/tests/test_small_multinomial_logit.py @@ -28,13 +28,6 @@ def orca_session(): orca.broadcast(cast='buildings', onto='households', cast_index=True, onto_on='building_id') -# d1 = {'a': np.random.random(100), -# 'b': np.random.random(100), -# 'choice': np.random.randint(3, size=100)} -# -# obs = pd.DataFrame(d1) -# orca.add_table('obs', obs) - def test_template_validity(): """ @@ -70,7 +63,6 @@ def test_small_mnl(orca_session): # TEST SIMULATION m.out_column = 'simulated_choice' - # orca.get_table('households')['simulated_choice'] = 0 m.run() print(orca.get_table('households').to_frame()) diff --git a/urbansim_templates/models/shared.py b/urbansim_templates/models/shared.py index 52e33d7..a283859 100644 --- a/urbansim_templates/models/shared.py +++ b/urbansim_templates/models/shared.py @@ -204,16 +204,6 @@ def _get_data(self, task='fit'): return df - def _get_filter_columns(self): - """ - THIS METHOD DOES NOT WORK YET. - - Return list of column names referenced in the filters. - - """ - return - - def _get_out_column(self): """ Return name of the column to save data to. This is 'out_column' if it exsits, diff --git a/urbansim_templates/models/small_multinomial_logit.py b/urbansim_templates/models/small_multinomial_logit.py index cb9b0dc..5cfbfa9 100644 --- a/urbansim_templates/models/small_multinomial_logit.py +++ b/urbansim_templates/models/small_multinomial_logit.py @@ -310,7 +310,14 @@ def fit(self): with Orca or ModelManager until the `register()` method is run. """ - long_df = self._to_long(self._get_data()) + expr_cols = [t[0] for t in list(self.model_expression.items()) \ + if t[0] != 'intercept'] + + df = get_data(tables = self.tables, + filters = self.filters, + extra_columns = expr_cols + [self.choice_column]) + + long_df = self._to_long(df) # Set initial coefs to 0 if none provided pc = self._get_param_count() From 72f1b9e31edb2354720dcbd17bb1cb08ed55bcd8 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 21 Feb 2019 17:41:25 -0800 Subject: [PATCH 033/121] Updating version --- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 95ece7a..29f85a0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.2.dev0', + version='0.2.dev1', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index 2479b9c..be7e7bc 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev0' +version = __version__ = '0.2.dev1' From 6e01203160a1e6202682f8cf4d5237424464e493 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 22 Feb 2019 15:54:57 -0800 Subject: [PATCH 034/121] Removing OrderedDict case from TemplateStep._get_data() --- urbansim_templates/models/shared.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/urbansim_templates/models/shared.py b/urbansim_templates/models/shared.py index a283859..f0d27aa 100644 --- a/urbansim_templates/models/shared.py +++ b/urbansim_templates/models/shared.py @@ -168,16 +168,6 @@ def _get_data(self, task='fit'): if isinstance(self.model_expression, str): expr_cols = util.columns_in_formula(self.model_expression) - # This is for PyLogit model expressions - elif isinstance(self.model_expression, OrderedDict): - # TO DO - check that this works in Python 2.7 - expr_cols = [t[0] for t in list(self.model_expression.items()) \ - if t[0] != 'intercept'] - # TO DO - not very general, maybe we should just override the method - # TO DO - and this only applies to the fit condition - if self.choice_column is not None: - expr_cols += [self.choice_column] - if (task == 'fit'): tables = self.tables columns = expr_cols + util.columns_in_filters(self.filters) From 9cfe2730a2387c1aec0e3fb88fc5f932e2624b09 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 26 Feb 2019 08:29:29 -0800 Subject: [PATCH 035/121] Updating contribution guide based on choicemodels --- CONTRIBUTING.md | 91 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 26 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 649a6a2..7039ba7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,55 +1,94 @@ -Thanks for using UrbanSim Templates! This is an open source project that's part of the Urban Data Science Toolkit. Development and maintenance is a collaboration between UrbanSim Inc and U.C. Berkeley's Urban Analytics Lab. You can contact Sam Maurer, the lead developer, at `maurer@urbansim.com`. +Thanks for using UrbanSim Templates! -### If you encounter an error or find a bug: +This is an open source project that's part of the Urban Data Science Toolkit. Development and maintenance is a collaboration between UrbanSim Inc and U.C. Berkeley's Urban Analytics Lab. -- Take a look at the [open issues](https://github.com/UDST/urbansim_templates/issues) and [closed issues](https://github.com/UDST/urbansim_templates/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a discussion of the problem +You can contact Sam Maurer, the lead developer, at `maurer@urbansim.com`. -- Open a new issue describing the problem: circumstances, error messages, operating system you are using, version of python, and version of any libraries that may be relevant -### If you have a feature proposal: +## If you have a problem: -- Take a look at the [open issues](https://github.com/UDST/urbansim_templates/issues) and [closed issues](https://github.com/UDST/urbansim_templates/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a discussion of the topic +- Take a look at the [open issues](https://github.com/UDST/urbansim_templates/issues) and [closed issues](https://github.com/UDST/urbansim_templates/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a related discussion + +- Open a new issue describing the problem -- if possible, include any error messages, the operating system and version of python you're using, and versions of any libraries that may be relevant + + +## Feature proposals: + +- Take a look at the [open issues](https://github.com/UDST/urbansim_templates/issues) and [closed issues](https://github.com/UDST/urbansim_templates/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a related discussion - Post your proposal as a new issue, so we can discuss it (some proposals may not be a good fit for the project) -### Adding a feature or fixing a bug: -- Create a new branch of UDST/urbansim_templates, or fork the repository to your own account +## Contributing code: + +- Create a new branch of `UDST/urbansim_templates`, or fork the repository to your own account + +- Make your changes, following the existing styles for code and inline documentation -- Make your changes, adhering to the existing styles for coding, commenting, and especially the documentation strings at the beginning of functions +- Add [tests](https://github.com/UDST/urbansim_templates/tree/master/tests) if possible! -- Add [tests](https://github.com/UDST/urbansim_templates/tree/master/tests) if possible +- Open a pull request to the `UDST/urbansim_templates` master branch, including a writeup of your changes -- take a look at some of the closed PR's for examples -- When you're ready to begin code review, open a pull request to the UDST/urbansim_templates master branch +- Current maintainers will review the code, suggest changes, and hopefully merge it! -- The pull request writeup should be clear and thorough, to facilitate code review, documentation, and release notes (see [example here](https://github.com/UDST/choicemodels/pull/43)). First, briefly summarize the changes, referencing any associated issue threads. Then describe the changes in more detail: implementation, usage, performance, and anything else that's relevant -- Make note in the pull request writeup of any API changes (class/method/function names, parameters, and behavior), particularly changes that could affect users' existing code +## Updating the version number: -- Each substantial pull request should increment the development version number, e.g. from 0.2.dev7 to 0.2.dev8 +- Each pull request that changes substantive code should increment the development version number, e.g. from `0.2.dev7` to `0.2.dev8`, so that users know exactly which version they're running -- If incrementing the version number: (1) update `setup.py`, (2) update `urbansim_templates/__init__.py`, (3) add a section to `CHANGELOG.md`, and (4) add the version number to the beginning of the pull request name +- It works best to do this just before merging (in case other PR's are merged first, and so you know the release date for the changelog and documentation) -### Preparing a production release: +- There are three places where the version number needs to be changed: + - `setup.py` + - `urbansim_templates/__init__.py` + - `docs/source/index.rst` -- Create a branch for release prep +- Please also add a section to `CHANGELOG.md` describing the changes! -- Make sure all the tests are passing -- Update the version number (e.g. from 0.2.dev8 to 0.2) in `setup.py` and `urbansim_templates/__init__.py` +## Updating the documentation: -- Update `CHANGELOG.md`, collapsing development release sections into a single, reorganized list +- See instructions in `docs/README.md` -- Check if updates are needed to `README.md` and to the documentation source files -- Rebuild the documentation webpages (DETAILS TK) +## Preparing a production release: -- Open a pull request to the master branch +- Make a new branch for release prep -- Merge the pull request +- Update the version number and `CHANGELOG.md` + +- Make sure all the tests are passing, and check if updates are needed to `README.md` or to the documentation + +- Open a pull request to the master branch and merge it - Tag the release on Github -- Update the Python Package Index (DETAILS TK) -- Update the UDST Conda channel (DETAILS TK) +## Distributing a release on PyPI (for pip installation): + +- Register an account at https://pypi.org, ask one of the current maintainers to add you to the project, and `pip install twine` + +- Run `python setup.py sdist bdist_wheel --universal` + +- This should create a `dist` directory containing two package files -- delete any old ones before the next step + +- Run `twine upload dist/*` -- this will prompt you for your pypi.org credentials + +- Check https://pypi.org/project/urbansim-templates/ for the new version + + +## Distributing a release on Conda Forge (for conda installation): + +- Make a fork of the [conda-forge/urbansim_templates-feedstock](https://github.com/conda-forge/urbansim_templates-feedstock) repository -- there may already be a fork in udst + +- Edit `recipe/meta.yaml`: + - update the version number + - paste a new hash matching the tar.gz file that was uploaded to pypi (it's available on the pypi.org project page) + +- Check that the run requirements still match `requirements.txt` + +- Open a pull request to the `conda-forge/urbansim_templates-feedstock` master branch + +- Automated tests will run, and after they pass one of the current project maintainers will be able to merge the PR -- you can add your Github user name to the maintainers list in `meta.yaml` for the next update + +- Check https://anaconda.org/conda-forge/urbansim-templates for the new version (may take a few minutes for it to appear) From d20648b417ba5860e33920f1bd9c5f1e94a78443 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 26 Feb 2019 08:57:56 -0800 Subject: [PATCH 036/121] Updating a link the readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 156acb1..8ec043e 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ conda install urbansim --channel udst ### Documentation -See the online documentation for much more: [http://docs.udst.org/projects/urbansim-templates](https://docs.udst.org/projects/urbansim-templates/en/latest) +See the online documentation for much more: https://urbansim-templates.readthedocs.io Some additional documentation is available within the repo in `CHANGELOG.md`, `CONTRIBUTING.md`, `/docs/README.md`, and `/tests/README.md`. From 6d993992ba593463befb6081c345480e1c5200fa Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 26 Feb 2019 09:22:45 -0800 Subject: [PATCH 037/121] Adding instructions to patch an earlier release --- CONTRIBUTING.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7039ba7..f351706 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,10 +64,27 @@ You can contact Sam Maurer, the lead developer, at `maurer@urbansim.com`. - Tag the release on Github +## Patching an earlier release: + +- We're not maintaining separate code branches for dev/ production/ major releases, but you can easily recreate them from tags if you need to patch an earlier release + +- In Github, create a new branch from the tag for the version you'd like to patch, calling it something like `v1-production` + +- Create a second branch from that one, called something like `v1-patch` + +- Make your changes in the `v1-patch` branch, and open a PR to `v1-production` to finalize it + +- After merging, tag the release on Github and follow the normal distribution procedures + +- After the new release is tagged, you can delete the extra branches -- a branch is just a tag pointing to the latest commit in a chain, and the commits will still be there + + ## Distributing a release on PyPI (for pip installation): - Register an account at https://pypi.org, ask one of the current maintainers to add you to the project, and `pip install twine` +- Check out the copy of the code you'd like to release + - Run `python setup.py sdist bdist_wheel --universal` - This should create a `dist` directory containing two package files -- delete any old ones before the next step From 841bff77a73d96814a8ea7ca4d585551fb325432 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 26 Feb 2019 09:24:30 -0800 Subject: [PATCH 038/121] Tweaking release instructions --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f351706..c4498a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,9 +59,9 @@ You can contact Sam Maurer, the lead developer, at `maurer@urbansim.com`. - Make sure all the tests are passing, and check if updates are needed to `README.md` or to the documentation -- Open a pull request to the master branch and merge it +- Open a pull request to the master branch to finalize it -- Tag the release on Github +- After merging, tag the release on Github and follow the distribution procedures below ## Patching an earlier release: From ab1af8b1bbb84b5fc5f4c33dc3e0cf86704a226f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 27 Feb 2019 18:49:19 -0800 Subject: [PATCH 039/121] Updating changelog and docs version --- CHANGELOG.md | 4 ++++ docs/source/index.rst | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 205acce..0bedd17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # UrbanSim Templates change log +### 0.2.dev1 (2019-02-27) + +- fixes a crash in small MNL simulation + ### 0.2.dev0 (2019-02-19) - adds first data i/o template: `urbansim_templates.io.TableFromDisk()` diff --git a/docs/source/index.rst b/docs/source/index.rst index 4f765d0..1f93665 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev0, released February 19, 2019 +v0.2.dev1, released February 27, 2019 Contents From 6bc22b96024176084e708cd3d28bd227d4f29fb4 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Sat, 2 Mar 2019 10:35:38 -0800 Subject: [PATCH 040/121] Clarifying branch vs tag --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c4498a7..5bcac12 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,7 +76,7 @@ You can contact Sam Maurer, the lead developer, at `maurer@urbansim.com`. - After merging, tag the release on Github and follow the normal distribution procedures -- After the new release is tagged, you can delete the extra branches -- a branch is just a tag pointing to the latest commit in a chain, and the commits will still be there +- After the new release is tagged, you can delete the extra branches -- a branch is just a pointer to the latest commit in a chain, and these commits will still be accessible via the tag ## Distributing a release on PyPI (for pip installation): From 698d8bb01daf2df63c1b1a25c71933f612bc2a62 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Sat, 2 Mar 2019 10:47:54 -0800 Subject: [PATCH 041/121] New format for changelog --- CHANGELOG.md | 60 +++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 205acce..c6372c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,97 +1,99 @@ # UrbanSim Templates change log -### 0.2.dev0 (2019-02-19) + +## 0.2 (not yet released) + +#### 0.2.dev1 (2019-02-27) + +- fixes a crash in small MNL simulation + +#### 0.2.dev0 (2019-02-19) - adds first data i/o template: `urbansim_templates.io.TableFromDisk()` - adds support for `autorun` template property -### 0.1.1 (2019-02-05) -- production release +## 0.1.2 (2019-02-28) -### 0.1.1.dev1 (2019-01-30) +- patch to incorporate the small MNL bug fix from 0.2.dev1 + + +## 0.1.1 (2019-02-05) + +#### 0.1.1.dev1 (2019-01-30) - adds support for passing multiple tables of interaction terms in large MNL - enables on-the-fly creation of output columns in small MNL -### 0.1.1.dev0 (2019-01-20) +#### 0.1.1.dev0 (2019-01-20) - allows join keys to be used as data filters in MNL simulation -### 0.1 (2019-01-16) -- first production release! +## 0.1 (2019-01-16) -### 0.1.dev25 (2019-01-15) +#### 0.1.dev25 (2019-01-15) - fixes an OLS simulation bug that raised an error when the output column didn't exist yet - - implements `out_transform` for OLS simulation -### 0.1.dev24 (2018-12-20) +#### 0.1.dev24 (2018-12-20) - fixes a string comparison bug that caused problems with binary logit output in Windows - - adds `model` as an attribute of large MNL model steps, which provides a `choicemodels.MultinomialLogitResults` object and is available any time after a model step is fitted - - enables on-the-fly creation of output columns in large MNL - - fixes a large MNL simulation bug when there are no valid choosers or alternatives after evaluating the filters - - moves unit tests out of the module directory -### 0.1.dev23 (2018-12-13) +#### 0.1.dev23 (2018-12-13) - fixes a bug with interaction terms passed into `LargeMultinomialLogitStep.run()` -### 0.1.dev22 (2018-12-13) +#### 0.1.dev22 (2018-12-13) - narrows the output of `utils.get_data()` to include only the columns requested (plus the index of the primary table) -- previously Orca had also provided some extra columns such as join keys -### 0.1.dev21 (2018-12-11) +#### 0.1.dev21 (2018-12-11) - adds a new function `utils.get_data()` to assemble data from Orca, automatically detecting columns included in model expressions and filters - implements `SegmentedLargeMultinomialLogit.run_all()` -### 0.1.dev20 (2018-12-11) +#### 0.1.dev20 (2018-12-11) - fixes a model expression persistence bug in the small MNL template -### 0.1.dev19 (2018-12-06) +#### 0.1.dev19 (2018-12-06) - fixes a bug to allow large MNL simulation with multiple chooser tables -### 0.1.dev18 (2018-11-19) +#### 0.1.dev18 (2018-11-19) - improves installation and testing -### 0.1.dev17 (2018-11-15) +#### 0.1.dev17 (2018-11-15) - adds an `interaction_terms` parameter that users can manually pass to `LargeMultinomialLogitStep.run()`, as a temporary solution until interaction terms are fully handled by the templates - - also adds a `chooser_batch_size` parameter in the same place, to reduce memory pressure when there are large numbers of choosers -### 0.1.dev16 (2018-11-06) +#### 0.1.dev16 (2018-11-06) - adds a tool for testing template validity -### 0.1.dev15 (2018-10-15) +#### 0.1.dev15 (2018-10-15) - adds new `LargeMultinomialLogitStep` parameters related to choice simulation: `constrained_choices`, `alt_capacity`, `chooser_size`, and `max_iter` - - updates `LargeMultinomialLogitStep.run()` to use improved simulation utilities from ChoiceModels 0.2.dev4 -### 0.1.dev14 (2018-09-25) +#### 0.1.dev14 (2018-09-25) - adds a template for segmented large MNL models: `SegmentedLargeMultinomialLogitStep`, which can automatically generate a set of large MNL models based on segmentation rules -### 0.1.dev13 (2018-09-24) +#### 0.1.dev13 (2018-09-24) - adds a `@modelmanager.template` decorator that makes a class available to the currently running instance of ModelManager -### 0.1.dev12 (2018-09-19) +#### 0.1.dev12 (2018-09-19) - moves the `register()` operation to `modelmanager` (previously it was a method implemented by the individual templates) - - adds general ModelManager support for supplemental objects like pickled model results \ No newline at end of file From 6b57330b56ddb3610826c37663e5db01487be07a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 11:48:23 -0800 Subject: [PATCH 042/121] Load, SaveData -> Load, SaveTable --- tests/test_data_load.py | 28 +++++++++---------- tests/test_data_save.py | 12 ++++---- urbansim_templates/data/__init__.py | 4 +-- .../data/{load_data.py => load_table.py} | 2 +- .../data/{save_data.py => save_table.py} | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) rename urbansim_templates/data/{load_data.py => load_table.py} (99%) rename urbansim_templates/data/{save_data.py => save_table.py} (99%) diff --git a/tests/test_data_load.py b/tests/test_data_load.py index 28b5929..6498a3f 100644 --- a/tests/test_data_load.py +++ b/tests/test_data_load.py @@ -7,7 +7,7 @@ import orca from urbansim_templates import modelmanager -from urbansim_templates.data import LoadData +from urbansim_templates.data import LoadTable from urbansim_templates.utils import validate_template @@ -48,7 +48,7 @@ def test_template_validity(): Run the templates through the standard validation check. """ - assert validate_template(LoadData) + assert validate_template(LoadTable) def test_property_persistence(orca_session): @@ -75,7 +75,7 @@ def test_validation_index_unique(orca_session): d = {'id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = LoadData(name='tab') + t = LoadTable(name='tab') t.validate() @@ -87,7 +87,7 @@ def test_validation_index_not_unique(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = LoadData(name='tab') + t = LoadTable(name='tab') try: t.validate() except ValueError: @@ -104,7 +104,7 @@ def test_validation_multiindex_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = LoadData(name='tab') + t = LoadTable(name='tab') t.validate() @@ -117,7 +117,7 @@ def test_validation_multiindex_not_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [2,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = LoadData(name='tab') + t = LoadTable(name='tab') try: t.validate() except ValueError: @@ -134,7 +134,7 @@ def test_validation_unnamed_index(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name - t = LoadData(name='tab') + t = LoadTable(name='tab') try: t.validate() except ValueError: @@ -155,7 +155,7 @@ def test_validation_columns_vs_other_indexes(orca_session): d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) - t = LoadData(name='households') + t = LoadTable(name='households') t.validate() @@ -171,7 +171,7 @@ def test_validation_index_vs_other_columns(orca_session): d = {'household_id': [1,2,3], 'building_id': [2,3,5]} orca.add_table('households', pd.DataFrame(d).set_index('household_id')) - t = LoadData(name='buildings') + t = LoadTable(name='buildings') t.validate() @@ -188,7 +188,7 @@ def test_validation_with_multiindexes(orca_session): d = {'home_tract': [55,55,55], 'work_tract': [17,18,19], 'dist': [1,1,1]} orca.add_table('distances', pd.DataFrame(d).set_index(['home_tract','work_tract'])) - t = LoadData(name='choice_table') + t = LoadTable(name='choice_table') t.validate() @@ -205,7 +205,7 @@ def test_csv(orca_session, data): Test loading data from a CSV file. """ - t = LoadData() + t = LoadTable() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' @@ -228,7 +228,7 @@ def test_hdf(orca_session, data): Test loading data from an HDF file. """ - t = LoadData() + t = LoadTable() t.name = 'buildings' t.source_type = 'hdf' t.path = 'data/buildings.hdf' @@ -250,7 +250,7 @@ def test_extra_settings(orca_session, data): Test loading data with extra settings, e.g. for compressed files. """ - t = LoadData() + t = LoadTable() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv.gz' @@ -284,7 +284,7 @@ def test_without_autorun(orca_session, data): Confirm that disabling autorun works. """ - t = LoadData() + t = LoadTable() t.name = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' diff --git a/tests/test_data_save.py b/tests/test_data_save.py index 10aaf4e..4207fc9 100644 --- a/tests/test_data_save.py +++ b/tests/test_data_save.py @@ -7,7 +7,7 @@ import orca from urbansim_templates import modelmanager -from urbansim_templates.data import SaveData +from urbansim_templates.data import SaveTable from urbansim_templates.utils import validate_template @@ -40,7 +40,7 @@ def test_template_validity(): Run the templates through the standard validation check. """ - assert validate_template(SaveData) + assert validate_template(SaveTable) def test_property_persistence(orca_session): @@ -56,7 +56,7 @@ def test_csv(orca_session, data): Test saving data to a CSV file. """ - t = SaveData() + t = SaveTable() t.table = 'buildings' t.output_type = 'csv' t.path = 'data/buildings.csv' @@ -74,7 +74,7 @@ def test_hdf(orca_session, data): Test saving data to an HDF file. """ - t = SaveData() + t = SaveTable() t.table = 'buildings' t.output_type = 'hdf' t.path = 'data/buildings.h5' @@ -98,7 +98,7 @@ def test_filters(orca_session, data): Test applying data filters before table is saved. """ - t = SaveData() + t = SaveTable() t.table = 'buildings' t.filters = 'price < 200000' t.output_type = 'csv' @@ -123,7 +123,7 @@ def test_dynamic_paths(orca_session): Test inserting run id, model iteration, or timestamp into path. """ - t = SaveData() + t = SaveTable() t.path = '%RUN%-%ITER%' assert(t.get_dynamic_filepath() == '0-0') diff --git a/urbansim_templates/data/__init__.py b/urbansim_templates/data/__init__.py index 479b442..ccdcdf3 100644 --- a/urbansim_templates/data/__init__.py +++ b/urbansim_templates/data/__init__.py @@ -1,2 +1,2 @@ -from .load_data import LoadData -from .save_data import SaveData \ No newline at end of file +from .load_table import LoadTable +from .save_table import SaveTable diff --git a/urbansim_templates/data/load_data.py b/urbansim_templates/data/load_table.py similarity index 99% rename from urbansim_templates/data/load_data.py rename to urbansim_templates/data/load_table.py index e9b5910..df61955 100644 --- a/urbansim_templates/data/load_data.py +++ b/urbansim_templates/data/load_table.py @@ -14,7 +14,7 @@ @modelmanager.template -class LoadData(): +class LoadTable(): """ Class for registering data tables from local CSV or HDF5 files. diff --git a/urbansim_templates/data/save_data.py b/urbansim_templates/data/save_table.py similarity index 99% rename from urbansim_templates/data/save_data.py rename to urbansim_templates/data/save_table.py index ba8fbc3..2e2e574 100644 --- a/urbansim_templates/data/save_data.py +++ b/urbansim_templates/data/save_table.py @@ -10,7 +10,7 @@ @modelmanager.template -class SaveData(): +class SaveTable(): """ Class for saving Orca tables to local CSV or HDF5 files. From a9f3dc06b9af01e15867118c93942bb2c8373da1 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 14:00:39 -0800 Subject: [PATCH 043/121] More test cases for column updating --- tests/test_utils.py | 18 +++++++++++++++++- urbansim_templates/data/save_table.py | 4 ++-- urbansim_templates/utils.py | 24 ++++++++++++------------ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1299377..cc2a602 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,9 @@ -import orca +import numpy as np import pandas as pd import pytest +import orca + from urbansim_templates import utils @@ -115,3 +117,17 @@ def test_update_column_incomplete_series(orca_session): assert(orca.get_table(table).to_frame()[column].tolist() == [5,2,10]) +def test_add_column_incomplete_series(orca_session): + """ + Add an incomplete column to confirm that it's aligned based on the index. (The ints + will be cast to floats to accommodate the missing values.) + + """ + table = 'buildings' + column = 'pop2' + data = pd.Series([10,5], index=[3,1]) + + utils.update_column(table, column, data) + stored_data = orca.get_table(table).to_frame()[column].tolist() + + np.testing.assert_array_equal(stored_data, [5.0, np.nan, 10.0]) diff --git a/urbansim_templates/data/save_table.py b/urbansim_templates/data/save_table.py index 2e2e574..9276afe 100644 --- a/urbansim_templates/data/save_table.py +++ b/urbansim_templates/data/save_table.py @@ -22,8 +22,8 @@ class SaveTable(): Name of the Orca table. Must be provided before running the step. columns : str or list of str, optional - Names of columns to include, in addition to indexes. "None" will return all - columns. + Names of columns to include. ``None`` will return all columns. Indexes will + always be included. filters : str or list of str, optional Filters to apply to the data before saving. Will be passed to diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index c99b61e..fbe0458 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -211,29 +211,29 @@ def get_data(tables, fallback_tables=None, filters=None, model_expression=None, def update_column(table, column, data, fallback_table=None, fallback_column=None): """ - Update an Orca column. If it doesn't exist yet, add it to the table. + Update an Orca column. If it doesn't exist yet, add it to the table. Values will be + aligned using the indexes if possible. - If the column already exists, 'data' will be cast to match the column's data type. If - the column needs to be created, it will be given the same data type as 'data'. - - Require an index? + If the column already exists, new values will be cast to match the existing data + type. New columns may be cast to accommodate missing values (e.g. int to float) if + they don't fully align with the table's index. Parameters ---------- table : str or list of str - Name of an Orca table. If list, the first element will be used. + Name of Orca table to update. If list, the first element will be used. column : str - Name of a column in the table. Cannot be an index. + Name of existing column to update, or new column to create. Cannot be an index. data : pd.Series - Should either align with the index of the table, or have the same number of rows. + Column of data to update or add. fallback_table : str or list of str - Name of Orca table to use if 'table' evaluates to None. + Name of Orca table to use if ``table`` evaluates to None. fallback_column : str - Name of Orca column to use if 'column' evaluates to None. + Name of Orca column to use if ``column`` evaluates to None. Returns ------- @@ -252,10 +252,10 @@ def update_column(table, column, data, fallback_table=None, fallback_column=None dfw = orca.get_table(table) if column not in dfw.columns: - dfw.update_col(column, data) + dfw.update_col(column, data) # adds column else: - dfw.update_col_from_series(column, data, cast=True) + dfw.update_col_from_series(column, data, cast=True) # updates existing column ######################## From 92c82fd606e4304ab0f71562a337cc0e39cf2a7e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 14:26:17 -0800 Subject: [PATCH 044/121] Test requesting specific columns --- tests/test_data_save.py | 20 ++++++++++++++++++-- urbansim_templates/utils.py | 14 ++++++++------ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/test_data_save.py b/tests/test_data_save.py index 4207fc9..014d7fb 100644 --- a/tests/test_data_save.py +++ b/tests/test_data_save.py @@ -8,7 +8,7 @@ from urbansim_templates import modelmanager from urbansim_templates.data import SaveTable -from urbansim_templates.utils import validate_template +from urbansim_templates.utils import update_column, validate_template @pytest.fixture @@ -89,8 +89,24 @@ def test_hdf(orca_session, data): def test_columns(orca_session, data): """ + Test requesting specific columns. + """ - pass + update_column(table = 'buildings', + column = 'price2', + data = (1e6*np.random.random(10)).astype(int)) + + t = SaveTable() + t.table = 'buildings' + t.columns = 'price2' + t.output_type = 'csv' + t.path = 'data/buildings.csv' + + t.run() + + df = pd.read_csv(t.path).set_index('building_id') + assert(list(df.columns) == ['price2']) + def test_filters(orca_session, data): diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index fbe0458..45396eb 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -211,12 +211,14 @@ def get_data(tables, fallback_tables=None, filters=None, model_expression=None, def update_column(table, column, data, fallback_table=None, fallback_column=None): """ - Update an Orca column. If it doesn't exist yet, add it to the table. Values will be - aligned using the indexes if possible. - - If the column already exists, new values will be cast to match the existing data - type. New columns may be cast to accommodate missing values (e.g. int to float) if - they don't fully align with the table's index. + Update an Orca column. If it doesn't exist yet, add it to the wrapped DataFrame. + Values will be aligned using the indexes if possible. + + Data types: If the column already exists, new values will be cast to match the + existing data type. If the column is new, it will retain the data type of the + pd.Series that's passed to this function -- unless it doesn't fully align with the + table's index, in which case it may be cast to allow missing values (e.g. from int + to float). Parameters ---------- From 7fa8605d31ea58963d5ff19896322ecdc50b3dc7 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 14:38:19 -0800 Subject: [PATCH 045/121] Requiring unix-style paths for LoadData --- tests/test_data_load.py | 10 ---------- tests/test_data_save.py | 1 - urbansim_templates/data/load_table.py | 21 +++------------------ 3 files changed, 3 insertions(+), 29 deletions(-) diff --git a/tests/test_data_load.py b/tests/test_data_load.py index 6498a3f..7fb8565 100644 --- a/tests/test_data_load.py +++ b/tests/test_data_load.py @@ -269,16 +269,6 @@ def test_extra_settings(orca_session, data): modelmanager.remove_step('buildings') -def test_windows_paths(orca_session, data): - """ - Test in Windows that a Windows-style path is properly normalized. - - TO DO - implement - - """ - pass - - def test_without_autorun(orca_session, data): """ Confirm that disabling autorun works. diff --git a/tests/test_data_save.py b/tests/test_data_save.py index 014d7fb..5b02c19 100644 --- a/tests/test_data_save.py +++ b/tests/test_data_save.py @@ -106,7 +106,6 @@ def test_columns(orca_session, data): df = pd.read_csv(t.path).set_index('building_id') assert(list(df.columns) == ['price2']) - def test_filters(orca_session, data): diff --git a/urbansim_templates/data/load_table.py b/urbansim_templates/data/load_table.py index df61955..f4a9acd 100644 --- a/urbansim_templates/data/load_table.py +++ b/urbansim_templates/data/load_table.py @@ -41,11 +41,9 @@ class LoadTable(): path : str, optional Local file path to load data from, either absolute or relative to the - ModelManager config directory. The string you provide will immediately be - normalized to a platform-agnostic format, using `os.path.normpath()` in Python 2 - or `pathlib.Path()` in Python 3. It is always safe to provide a Unix-style path, - and you may provide a Windows-style path if you are creating the model step in - Windows. Saved steps will run on any platform. + ModelManager config directory. Please provide a Unix-style path (this will work + on any platform, but a Windows-style path won't, and they're hard to normalize + automatically). url : str, optional - NOT YET IMPLEMENTED Remote url to download file from. @@ -173,19 +171,6 @@ def to_dict(self): return d - @property - def path(self): - return self.__path - @path.setter - def path(self, value): - if value is not None: - try: - value = str(pathlib.Path(value)) # Python 3.4+ - except: - value = os.path.normpath(value) - self.__path = value - - def run(self): """ Register a data table with Orca. From 3a1c5fa1c4dc668364069d180934b3834f1a3f02 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 14:50:32 -0800 Subject: [PATCH 046/121] Removing column and filter params from LoadTable --- urbansim_templates/data/load_table.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/urbansim_templates/data/load_table.py b/urbansim_templates/data/load_table.py index f4a9acd..75da684 100644 --- a/urbansim_templates/data/load_table.py +++ b/urbansim_templates/data/load_table.py @@ -57,9 +57,6 @@ class LoadTable(): 'gzip'}, or specify the table identifier within a multi-object hdf store using {'key': 'table-name'}. See Pandas documentation for additional settings. - filters : str or list of str, optional - NOT YET IMPLEMENTED - Filters to apply before registering the table with Orca. - orca_test_spec : dict, optional - NOT YET IMPLEMENTED Data characteristics to be tested when the table is validated. From 73d5e98c3aae71eeaf68809e6550fb604bd1543f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 15:16:44 -0800 Subject: [PATCH 047/121] Adding separate table name for LoadTable --- tests/test_data_load.py | 32 +++++++++++++------------- urbansim_templates/data/load_table.py | 33 +++++++++++++++------------ 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/tests/test_data_load.py b/tests/test_data_load.py index 7fb8565..4c6d52d 100644 --- a/tests/test_data_load.py +++ b/tests/test_data_load.py @@ -75,7 +75,7 @@ def test_validation_index_unique(orca_session): d = {'id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = LoadTable(name='tab') + t = LoadTable(table='tab') t.validate() @@ -87,7 +87,7 @@ def test_validation_index_not_unique(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index('id')) - t = LoadTable(name='tab') + t = LoadTable(table='tab') try: t.validate() except ValueError: @@ -104,7 +104,7 @@ def test_validation_multiindex_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [1,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = LoadTable(name='tab') + t = LoadTable(table='tab') t.validate() @@ -117,7 +117,7 @@ def test_validation_multiindex_not_unique(orca_session): d = {'id': [1,1,1], 'sub_id': [2,2,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - t = LoadTable(name='tab') + t = LoadTable(table='tab') try: t.validate() except ValueError: @@ -134,7 +134,7 @@ def test_validation_unnamed_index(orca_session): d = {'id': [1,1,3], 'value': [4,4,4]} orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name - t = LoadTable(name='tab') + t = LoadTable(table='tab') try: t.validate() except ValueError: @@ -155,7 +155,7 @@ def test_validation_columns_vs_other_indexes(orca_session): d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) - t = LoadTable(name='households') + t = LoadTable(table='households') t.validate() @@ -171,7 +171,7 @@ def test_validation_index_vs_other_columns(orca_session): d = {'household_id': [1,2,3], 'building_id': [2,3,5]} orca.add_table('households', pd.DataFrame(d).set_index('household_id')) - t = LoadTable(name='buildings') + t = LoadTable(table='buildings') t.validate() @@ -188,7 +188,7 @@ def test_validation_with_multiindexes(orca_session): d = {'home_tract': [55,55,55], 'work_tract': [17,18,19], 'dist': [1,1,1]} orca.add_table('distances', pd.DataFrame(d).set_index(['home_tract','work_tract'])) - t = LoadTable(name='choice_table') + t = LoadTable(table='choice_table') t.validate() @@ -206,7 +206,7 @@ def test_csv(orca_session, data): """ t = LoadTable() - t.name = 'buildings' + t.table = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' t.csv_index_cols = 'building_id' @@ -220,7 +220,7 @@ def test_csv(orca_session, data): modelmanager.initialize() assert 'buildings' in orca.list_tables() - modelmanager.remove_step('buildings') + modelmanager.remove_step(t.name) def test_hdf(orca_session, data): @@ -229,7 +229,7 @@ def test_hdf(orca_session, data): """ t = LoadTable() - t.name = 'buildings' + t.table = 'buildings' t.source_type = 'hdf' t.path = 'data/buildings.hdf' @@ -242,7 +242,7 @@ def test_hdf(orca_session, data): modelmanager.initialize() assert 'buildings' in orca.list_tables() - modelmanager.remove_step('buildings') + modelmanager.remove_step(t.name) def test_extra_settings(orca_session, data): @@ -251,7 +251,7 @@ def test_extra_settings(orca_session, data): """ t = LoadTable() - t.name = 'buildings' + t.table = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv.gz' t.csv_index_cols = 'building_id' @@ -266,7 +266,7 @@ def test_extra_settings(orca_session, data): modelmanager.initialize() assert 'buildings' in orca.list_tables() - modelmanager.remove_step('buildings') + modelmanager.remove_step(t.name) def test_without_autorun(orca_session, data): @@ -275,7 +275,7 @@ def test_without_autorun(orca_session, data): """ t = LoadTable() - t.name = 'buildings' + t.table = 'buildings' t.source_type = 'csv' t.path = 'data/buildings.csv' t.csv_index_cols = 'building_id' @@ -284,7 +284,7 @@ def test_without_autorun(orca_session, data): modelmanager.register(t) assert 'buildings' not in orca.list_tables() - modelmanager.remove_step('buildings') + modelmanager.remove_step(t.name) \ No newline at end of file diff --git a/urbansim_templates/data/load_table.py b/urbansim_templates/data/load_table.py index 75da684..4d276d8 100644 --- a/urbansim_templates/data/load_table.py +++ b/urbansim_templates/data/load_table.py @@ -35,9 +35,11 @@ class LoadTable(): Parameters ---------- + table : str, optional + Name of the Orca table to be created. Must be provided before running the step. + source_type : 'csv' or 'hdf', optional - This is required to load the table, but does not have to be provided when the - object is created. + Source type. Must be provided before running the step. path : str, optional Local file path to load data from, either absolute or relative to the @@ -72,8 +74,7 @@ class LoadTable(): Passed to `orca.table()`. Default is `True`, as in Orca. name : str, optional - Name of the table, for Orca. This will also be used as the name of the model step - that generates the table. + Name of the model step. tags : list of str, optional Tags, passed to ModelManager. @@ -83,6 +84,7 @@ class LoadTable(): """ def __init__(self, + table = None, source_type = None, path = None, csv_index_cols = None, @@ -95,6 +97,7 @@ def __init__(self, autorun = True): # Template-specific params + self.table = table self.source_type = source_type self.path = path self.csv_index_cols = csv_index_cols @@ -128,6 +131,7 @@ def from_dict(cls, d): """ obj = cls( + table = d['table'], source_type = d['source_type'], path = d['path'], csv_index_cols = d['csv_index_cols'], @@ -157,6 +161,7 @@ def to_dict(self): 'name': self.name, 'tags': self.tags, 'autorun': self.autorun, + 'table': self.table, 'source_type': self.source_type, 'path': self.path, 'csv_index_cols': self.csv_index_cols, @@ -172,7 +177,7 @@ def run(self): """ Register a data table with Orca. - Requires values to be set for ``source_type``, ``name``, and ``path``. CSV data + Requires values to be set for ``table``, ``source_type``, and ``path``. CSV data also requires ``csv_index_cols``. Returns @@ -180,12 +185,12 @@ def run(self): None """ + if self.table is None: + raise ValueError("Please provide a table name") + if self.source_type not in ['csv', 'hdf']: raise ValueError("Please provide a source type of 'csv' or 'hdf'") - if self.name is None: - raise ValueError("Please provide a table name") - if self.path is None: raise ValueError("Please provide a file path") @@ -196,7 +201,7 @@ def run(self): if self.csv_index_cols is None: raise ValueError("Please provide index column name(s) for the csv") - @orca.table(table_name = self.name, + @orca.table(table_name = self.table, cache = self.cache, cache_scope = self.cache_scope, copy_col = self.copy_col) @@ -206,7 +211,7 @@ def orca_table(): # Table from HDF file elif self.source_type == 'hdf': - @orca.table(table_name = self.name, + @orca.table(table_name = self.table, cache = self.cache, cache_scope = self.cache_scope, copy_col = self.copy_col) @@ -247,10 +252,10 @@ def validate(self): # messages. We should update orca_test to support both, probably. # Register table if needed - if not orca.is_table(self.name): + if not orca.is_table(self.table): self.run() - idx = orca.get_table(self.name).index + idx = orca.get_table(self.table).index # Check index has a name if list(idx.names) == [None]: @@ -261,8 +266,8 @@ def validate(self): raise ValueError("Index not unique") # Compare columns to indexes of other tables, and vice versa - combinations = [(self.name, t) for t in orca.list_tables() if self.name != t] \ - + [(t, self.name) for t in orca.list_tables() if self.name != t] + combinations = [(self.table, t) for t in orca.list_tables() if self.table != t] \ + + [(t, self.table) for t in orca.list_tables() if self.table != t] for t1, t2 in combinations: col_names = orca.get_table(t1).columns From b5fb0834bb8b9d7b35d696f65c22b11e8da79f46 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 15:23:33 -0800 Subject: [PATCH 048/121] Updating version --- docs/source/index.rst | 2 +- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1f93665..653a4bb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev1, released February 27, 2019 +v0.2.dev2, released March 4, 2019 Contents diff --git a/setup.py b/setup.py index 29f85a0..9d3792f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.2.dev1', + version='0.2.dev2', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index be7e7bc..8184599 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev1' +version = __version__ = '0.2.dev2' From 84eeee6a4b5a738d9cc33f2967ee2c266fd1ef2f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 15:44:44 -0800 Subject: [PATCH 049/121] Tests for property persistence --- tests/test_data_load.py | 21 ++++++++++++++++++++- tests/test_data_save.py | 18 +++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_data_load.py b/tests/test_data_load.py index 4c6d52d..ee01b57 100644 --- a/tests/test_data_load.py +++ b/tests/test_data_load.py @@ -56,7 +56,26 @@ def test_property_persistence(orca_session): Test persistence of properties across registration, saving, and reloading. """ - pass + t = LoadTable() + t.table = 'buildings' + t.source_type = 'csv' + t.path = 'data/buildings.csv' + t.csv_index_cols = 'building_id' + t.extra_settings = {'make_data_awesome': True} # unfortunately not a valid setting + t.cache = False + t.cache_scope = 'iteration' + t.copy_col = False + t.name = 'buildings-csv' + t.tags = ['awesome', 'data'] + t.autorun = False + + d1 = t.to_dict() + modelmanager.register(t) + modelmanager.initialize() + d2 = modelmanager.get_step(t.name).to_dict() + + assert d1 == d2 + modelmanager.remove_step(t.name) ###################################### diff --git a/tests/test_data_save.py b/tests/test_data_save.py index 5b02c19..52a2c5f 100644 --- a/tests/test_data_save.py +++ b/tests/test_data_save.py @@ -48,7 +48,23 @@ def test_property_persistence(orca_session): Test persistence of properties across registration, saving, and reloading. """ - pass + t = SaveTable() + t.table = 'buildings' + t.columns = ['window_panes', 'number_of_chimneys'] + t.filters = 'number_of_chimneys > 15' + t.output_type = 'csv' + t.path = 'data/buildings.csv' + t.extra_settings = {'make_data_awesome': True} + t.name = 'save-buildings-csv' + t.tags = ['awesome', 'chimneys'] + + d1 = t.to_dict() + modelmanager.register(t) + modelmanager.initialize() + d2 = modelmanager.get_step(t.name).to_dict() + + assert d1 == d2 + modelmanager.remove_step(t.name) def test_csv(orca_session, data): From 7caeb6e6c0ade007ac17a18b2d4fefa46956e9fa Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 16:20:14 -0800 Subject: [PATCH 050/121] Updating documentation --- .../{data-io.rst => data-templates.rst} | 21 ++++++++++++------- docs/source/index.rst | 2 +- urbansim_templates/data/load_table.py | 17 ++++++++------- urbansim_templates/data/save_table.py | 14 ++++++------- 4 files changed, 31 insertions(+), 23 deletions(-) rename docs/source/{data-io.rst => data-templates.rst} (52%) diff --git a/docs/source/data-io.rst b/docs/source/data-templates.rst similarity index 52% rename from docs/source/data-io.rst rename to docs/source/data-templates.rst index 7b726a9..c0a94a1 100644 --- a/docs/source/data-io.rst +++ b/docs/source/data-templates.rst @@ -1,17 +1,24 @@ -Data I/O template APIs -====================== +Data template APIs +================== -Data i/o templates let you set up automated model steps for loading data into Orca or saving outputs to disk. +Data templates help you set up model steps for loading data into `Orca `__ or saving outputs to disk. -These templates follow the same principles as the statistical model steps. For example, to set up a data table, create an instance of the ``TableFromDisk`` class and set some properties: the table name, file type, path, and anything else that's needed. +These templates follow the same principles as the statistical model steps. For example, to set up a data table, create an instance of the ``LoadTable`` class and set some properties: the table name, file type, path, and anything else that's needed. Registering this object with ModelManager will save it to disk as a yaml file, and create an Orca step with instructions to set up the table. "Running" the object/step registers the table with Orca, but doesn't read the data from disk yet — Orca loads data lazily as it's needed. Data registration steps are run automatically when you initialize ModelManager. -Table from disk ---------------- +Loading data +------------ -.. autoclass:: urbansim_templates.io.TableFromDisk +.. autoclass:: urbansim_templates.data.LoadTable + :members: + + +Saving data +----------- + +.. autoclass:: urbansim_templates.data.SaveTable :members: diff --git a/docs/source/index.rst b/docs/source/index.rst index 653a4bb..1da0545 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,6 +22,6 @@ Contents getting-started modelmanager model-steps - data-io + data-templates utilities development diff --git a/urbansim_templates/data/load_table.py b/urbansim_templates/data/load_table.py index 4d276d8..e5bc9de 100644 --- a/urbansim_templates/data/load_table.py +++ b/urbansim_templates/data/load_table.py @@ -16,7 +16,7 @@ @modelmanager.template class LoadTable(): """ - Class for registering data tables from local CSV or HDF5 files. + Class for registering data tables from local CSV or HDF files. An instance of this template class stores *instructions for loading a data table*, packaged into an Orca step. Running the instructions registers the table with Orca. @@ -54,24 +54,25 @@ class LoadTable(): Required for csv source type. extra_settings : dict, optional - Additional arguments to pass to `pd.read_csv()` or `pd.read_hdf()`. For example, - you could automatically extract csv data from a gzip file using {'compression': - 'gzip'}, or specify the table identifier within a multi-object hdf store using - {'key': 'table-name'}. See Pandas documentation for additional settings. + Additional arguments to pass to ``pd.read_csv()`` or ``pd.read_hdf()``. For + example, you could automatically extract csv data from a gzip file using + {'compression': 'gzip'}, or specify the table identifier within a multi-object + hdf store using {'key': 'table-name'}. See Pandas documentation for additional + settings. orca_test_spec : dict, optional - NOT YET IMPLEMENTED Data characteristics to be tested when the table is validated. cache : bool, default True - Passed to `orca.table()`. Note that the default is `True`, unlike in the + Passed to ``orca.table()``. Note that the default is True, unlike in the underlying general-purpose Orca function, because tables read from disk should not need to be regenerated during the course of a model run. cache_scope : 'step', 'iteration', or 'forever', default 'forever' - Passed to `orca.table()`. Default is 'forever', as in Orca. + Passed to ``orca.table()``. Default is 'forever', as in Orca. copy_col : bool, default True - Passed to `orca.table()`. Default is `True`, as in Orca. + Passed to ``orca.table()``. Default is True, as in Orca. name : str, optional Name of the model step. diff --git a/urbansim_templates/data/save_table.py b/urbansim_templates/data/save_table.py index 9276afe..c05813f 100644 --- a/urbansim_templates/data/save_table.py +++ b/urbansim_templates/data/save_table.py @@ -36,15 +36,15 @@ class SaveTable(): Local file path to save the data to, either absolute or relative to the ModelManager config directory. Please provide a Unix-style path (this will work on any platform, but a Windows-style path won't, and they're hard to normalize - automatically). For dynamic file names, you can include the characters ``%RUN%``, - ``%ITER%``, or ``%TS%``. These will be replaced by the run id, the model - iteration value, or a timestamp when the output file is created. + automatically). For dynamic file names, you can include the characters "%RUN%", + "%ITER%", or "%TS%". These will be replaced by the run id, the model iteration + value, or a timestamp when the output file is created. extra_settings : dict, optional - Additional arguments to pass to `pd.to_csv()` or `pd.to_hdf()`. For example, you - could automatically compress csv data using {'compression': 'gzip'}, or specify - a custom table name for an hdf store using {'key': 'table-name'}. See Pandas - documentation for additional settings. + Additional arguments to pass to ``pd.to_csv()`` or ``pd.to_hdf()``. For example, + you could automatically compress csv data using {'compression': 'gzip'}, or + specify a custom table name for an hdf store using {'key': 'table-name'}. See + Pandas documentation for additional settings. name : str, optional Name of the model step. From da826b3e877f9f8b4db157705d115ebdb7cc83ae Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 Mar 2019 16:23:30 -0800 Subject: [PATCH 051/121] Updating changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccfe0b8..7a8cbfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## 0.2 (not yet released) +#### 0.2.dev2 (2019-03-04) + +- adds template for saving data: `urbansim_templates.data.SaveTable()` +- renames `TableFromDisk()` to `urbansim_templates.data.LoadTable()` + #### 0.2.dev1 (2019-02-27) - fixes a crash in small MNL simulation From 25044e18adf6cf021d1d43ae0aaa9af3cc941f69 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 Mar 2019 12:30:38 -0800 Subject: [PATCH 052/121] Moving table validation to utilities --- tests/test_data_load.py | 141 -------------------------- tests/test_utils_broadcasts.py | 137 +++++++++++++++++++++++++ urbansim_templates/data/load_table.py | 75 -------------- urbansim_templates/utils.py | 107 +++++++++++++++++++ 4 files changed, 244 insertions(+), 216 deletions(-) create mode 100644 tests/test_utils_broadcasts.py diff --git a/tests/test_data_load.py b/tests/test_data_load.py index ee01b57..07e3059 100644 --- a/tests/test_data_load.py +++ b/tests/test_data_load.py @@ -78,147 +78,6 @@ def test_property_persistence(orca_session): modelmanager.remove_step(t.name) -###################################### -### TESTS OF THE VALIDATE() METHOD ### -###################################### - -def test_validation_index_unique(orca_session): - """ - Table validation should pass if the index is unique. - - These tests of the validate() method generate Orca tables directly, which is just a - shortcut for testing -- the intended use is for the method to validate the table - loaded by the TableStep. - - """ - d = {'id': [1,2,3], 'value': [4,4,4]} - orca.add_table('tab', pd.DataFrame(d).set_index('id')) - - t = LoadTable(table='tab') - t.validate() - - -def test_validation_index_not_unique(orca_session): - """ - Table validation should raise a ValueError if the index is not unique. - - """ - d = {'id': [1,1,3], 'value': [4,4,4]} - orca.add_table('tab', pd.DataFrame(d).set_index('id')) - - t = LoadTable(table='tab') - try: - t.validate() - except ValueError: - return - - pytest.fail() # fail if ValueError wasn't raised - - -def test_validation_multiindex_unique(orca_session): - """ - Table validation should pass with a MultiIndex whose combinations are unique. - - """ - d = {'id': [1,1,1], 'sub_id': [1,2,3], 'value': [4,4,4]} - orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - - t = LoadTable(table='tab') - t.validate() - - -def test_validation_multiindex_not_unique(orca_session): - """ - Table validation should raise a ValueError if the MultiIndex combinations are not - unique. - - """ - d = {'id': [1,1,1], 'sub_id': [2,2,3], 'value': [4,4,4]} - orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) - - t = LoadTable(table='tab') - try: - t.validate() - except ValueError: - return - - pytest.fail() # fail if ValueError wasn't raised - - -def test_validation_unnamed_index(orca_session): - """ - Table validation should raise a ValueError if index is unnamed. - - """ - d = {'id': [1,1,3], 'value': [4,4,4]} - orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name - - t = LoadTable(table='tab') - try: - t.validate() - except ValueError: - return - - pytest.fail() # fail if ValueError wasn't raised - - -def test_validation_columns_vs_other_indexes(orca_session): - """ - Table validation should compare the 'households.building_id' column to - 'buildings.build_id'. - - """ - d = {'household_id': [1,2,3], 'building_id': [2,3,4]} - orca.add_table('households', pd.DataFrame(d).set_index('household_id')) - - d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} - orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) - - t = LoadTable(table='households') - t.validate() - - -def test_validation_index_vs_other_columns(orca_session): - """ - Table validation should compare the 'households.building_id' column to - 'buildings.build_id'. - - """ - d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} - orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) - - d = {'household_id': [1,2,3], 'building_id': [2,3,5]} - orca.add_table('households', pd.DataFrame(d).set_index('household_id')) - - t = LoadTable(table='buildings') - t.validate() - - -def test_validation_with_multiindexes(orca_session): - """ - Here, table validation should compare 'choice_table.[home_tract,work_tract]' to - 'distances.[home_tract,work_tract]'. - - """ - d = {'obs_id': [1,1,1,1], 'alt_id': [1,2,3,4], - 'home_tract': [55,55,55,55], 'work_tract': [17,46,19,55]} - orca.add_table('choice_table', pd.DataFrame(d).set_index(['obs_id','alt_id'])) - - d = {'home_tract': [55,55,55], 'work_tract': [17,18,19], 'dist': [1,1,1]} - orca.add_table('distances', pd.DataFrame(d).set_index(['home_tract','work_tract'])) - - t = LoadTable(table='choice_table') - t.validate() - - -# test validation with stand-alone columns -# test passing cache settings - - -################################# -### TESTS OF THE DATA LOADING ### -################################# - def test_csv(orca_session, data): """ Test loading data from a CSV file. diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py new file mode 100644 index 0000000..656126e --- /dev/null +++ b/tests/test_utils_broadcasts.py @@ -0,0 +1,137 @@ +import pandas as pd +import pytest + +import orca + +from urbansim_templates.utils import validate_table + + +@pytest.fixture +def orca_session(): + """ + Set up a clean Orca session. + + """ + orca.clear_all() + + +def test_validation_index_unique(orca_session): + """ + Table validation should pass if the index is unique. + + These tests of the validate() method generate Orca tables directly, which is just a + shortcut for testing -- the intended use is for the method to validate the table + loaded by the TableStep. + + """ + d = {'id': [1,2,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index('id')) + + validate_table('tab') + + +def test_validation_index_not_unique(orca_session): + """ + Table validation should raise a ValueError if the index is not unique. + + """ + d = {'id': [1,1,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index('id')) + + try: + validate_table('tab') + except ValueError: + return + + pytest.fail() # fail if ValueError wasn't raised + + +def test_validation_multiindex_unique(orca_session): + """ + Table validation should pass with a MultiIndex whose combinations are unique. + + """ + d = {'id': [1,1,1], 'sub_id': [1,2,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) + + validate_table('tab') + + +def test_validation_multiindex_not_unique(orca_session): + """ + Table validation should raise a ValueError if the MultiIndex combinations are not + unique. + + """ + d = {'id': [1,1,1], 'sub_id': [2,2,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d).set_index(['id', 'sub_id'])) + + try: + validate_table('tab') + except ValueError: + return + + pytest.fail() # fail if ValueError wasn't raised + + +def test_validation_unnamed_index(orca_session): + """ + Table validation should raise a ValueError if index is unnamed. + + """ + d = {'id': [1,1,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name + + try: + validate_table('tab') + except ValueError: + return + + pytest.fail() # fail if ValueError wasn't raised + + +def test_validation_columns_vs_other_indexes(orca_session): + """ + Table validation should compare the 'households.building_id' column to + 'buildings.build_id'. + + """ + d = {'household_id': [1,2,3], 'building_id': [2,3,4]} + orca.add_table('households', pd.DataFrame(d).set_index('household_id')) + + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) + + validate_table('households') + + +def test_validation_index_vs_other_columns(orca_session): + """ + Table validation should compare the 'households.building_id' column to + 'buildings.build_id'. + + """ + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) + + d = {'household_id': [1,2,3], 'building_id': [2,3,5]} + orca.add_table('households', pd.DataFrame(d).set_index('household_id')) + + validate_table('buildings') + + +def test_validation_with_multiindexes(orca_session): + """ + Here, table validation should compare 'choice_table.[home_tract,work_tract]' to + 'distances.[home_tract,work_tract]'. + + """ + d = {'obs_id': [1,1,1,1], 'alt_id': [1,2,3,4], + 'home_tract': [55,55,55,55], 'work_tract': [17,46,19,55]} + orca.add_table('choice_table', pd.DataFrame(d).set_index(['obs_id','alt_id'])) + + d = {'home_tract': [55,55,55], 'work_tract': [17,18,19], 'dist': [1,1,1]} + orca.add_table('distances', pd.DataFrame(d).set_index(['home_tract','work_tract'])) + + validate_table('choice_table') + diff --git a/urbansim_templates/data/load_table.py b/urbansim_templates/data/load_table.py index e5bc9de..7b97c32 100644 --- a/urbansim_templates/data/load_table.py +++ b/urbansim_templates/data/load_table.py @@ -221,78 +221,3 @@ def orca_table(): return df - def validate(self): - """ - Check some basic expectations about the table generated by the step: - - - Confirm that the table includes a unique, named index column (primary key) or - set of columns (composite key). If not, raise a ValueError. - - - If the table contains columns whose names match the index columns of tables - previously registered with Orca, check whether they make sense as join keys. - Print a status message with the number of presumptive foreign-key values that - are found in the primary key column. - - - Perform the same check for columns in previously registered tables whose names - match the index of the table generated by this step. - - - It doesn't currently compare indexes to indexes. (Maybe it should?) - - Running this will trigger loading all registered Orca tables into memory, which - may take a while if they have not yet been loaded. Stand-alone columns will not - be loaded unless their names match an index column. - - Returns - ------- - bool - - """ - # There are a couple of reasons we're not using the orca_test library here: - # (a) orca_test doesn't currently support MultiIndexes, and (b) the primary-key/ - # foreign-key comparisons aren't asserting anything, just printing status - # messages. We should update orca_test to support both, probably. - - # Register table if needed - if not orca.is_table(self.table): - self.run() - - idx = orca.get_table(self.table).index - - # Check index has a name - if list(idx.names) == [None]: - raise ValueError("Index column has no name") - - # Check index is unique - if len(idx.unique()) < len(idx): - raise ValueError("Index not unique") - - # Compare columns to indexes of other tables, and vice versa - combinations = [(self.table, t) for t in orca.list_tables() if self.table != t] \ - + [(t, self.table) for t in orca.list_tables() if self.table != t] - - for t1, t2 in combinations: - col_names = orca.get_table(t1).columns - idx = orca.get_table(t2).index - - if set(idx.names).issubset(col_names): - vals = orca.get_table(t1).to_frame(idx.names).drop_duplicates() - - # Easier to compare multi-column values to multi-column index if we - # turn the values into an index as well - vals = vals.reset_index().set_index(idx.names).index - vals_in_idx = sum(vals.isin(idx)) - - if len(idx.names) == 1: - idx_str = idx.names[0] - else: - idx_str = '[{}]'.format(','.join(idx.names)) - - print("'{}.{}': {} of {} unique values are found in '{}.{}' ({}%)"\ - .format(t1, idx_str, - vals_in_idx, len(vals), - t2, idx_str, - round(100*vals_in_idx/len(vals)))) - - return True - - \ No newline at end of file diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 45396eb..8740753 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -75,6 +75,113 @@ def validate_template(cls): return True +##################################### +## REPLACEMENT FOR ORCA BROADCASTS ## +##################################### + +""" +These utilities provide functionality for merging tables using implicit join keys instead +of Orca broadcasts. See Github issue #78 for discussion of the rationale. + +""" + +def validate_table(table): + """ + Check some basic expectations about an Orca table: + + - Confirm that it includes a unique, named index column (a.k.a. primary key) or set + of columns (multi-index, a.k.a. composite key). If not, raise a ValueError. + + - Confirm that none of the other columns in the table share names with the index(es). + If they do, raise a ValueError. + + - If the table contains columns whose names match the index columns of other tables + registered with Orca, check whether they make sense as join keys. This prints a + status message with the number of presumptive foreign-key values that are found in + the primary/composite key, for evaluation by the user. + + - Perform the same check for columns in _other_ tables whose names match the index + column(s) of _this_ table. + + - It doesn't currently compare indexes to indexes. (Maybe it should?) + + Running this will trigger loading all registered Orca tables, which may take a while. + Stand-alone columns will not be loaded unless their names match an index column. + + Doesn't currently incorporate ``orca_test`` validation, but it might be added. + + Parameters + ---------- + table : str + Name of Orca table to validate. + + Returns + ------- + bool + + """ + # There are a couple of reasons we're not using the orca_test library here: + # (a) orca_test doesn't currently support MultiIndexes, and (b) the primary-key/ + # foreign-key comparisons aren't asserting anything, just printing status + # messages. We should update orca_test to support both, probably. + + if not orca.is_table(table): + raise ValueError("Table not yet registered with Orca") + + idx = orca.get_table(table).index + + # Check index has a name + if list(idx.names) == [None]: + raise ValueError("Index column has no name") + + # CHECK INDEX NAME DISTINCT FROM COLUMN NAMES + + # Check index is unique + if len(idx.unique()) < len(idx): + raise ValueError("Index not unique") + + # Compare columns to indexes of other tables, and vice versa + combinations = [(table, t) for t in orca.list_tables() if table != t] \ + + [(t, table) for t in orca.list_tables() if table != t] + + for t1, t2 in combinations: + col_names = orca.get_table(t1).columns + idx = orca.get_table(t2).index + + if set(idx.names).issubset(col_names): + vals = orca.get_table(t1).to_frame(idx.names).drop_duplicates() + + # Easier to compare multi-column values to multi-column index if we + # turn the values into an index as well + vals = vals.reset_index().set_index(idx.names).index + vals_in_idx = sum(vals.isin(idx)) + + if len(idx.names) == 1: + idx_str = idx.names[0] + else: + idx_str = '[{}]'.format(','.join(idx.names)) + + print("'{}.{}': {} of {} unique values are found in '{}.{}' ({}%)"\ + .format(t1, idx_str, + vals_in_idx, len(vals), + t2, idx_str, + round(100*vals_in_idx/len(vals)))) + + return True + + +def validate_all_tables(): + """ + """ + pass + + +def merge_tables(): + """ + """ + pass + + ############################### ## TEMPLATE HELPER FUNCTIONS ## ############################### From a0e15ef60d19c8fd21dd7dad66d037269c76613a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 Mar 2019 12:47:09 -0800 Subject: [PATCH 053/121] More tests --- tests/test_utils_broadcasts.py | 72 +++++++++++++++++++++++++--------- urbansim_templates/utils.py | 9 +++-- 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index 656126e..6d18ea6 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -15,6 +15,56 @@ def orca_session(): orca.clear_all() +def test_validation_table_not_registered(orca_session): + """ + Table validation should raise a ValueError if the table isn't registered. + + """ + try: + validate_table('tab') + except ValueError as e: + print(e) + return + + pytest.fail() # fail is ValueError wasn't raised + + +def test_validation_index_unnamed(orca_session): + """ + Table validation should raise a ValueError if index is unnamed. + + """ + d = {'id': [1,1,3], 'value': [4,4,4]} + orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name + + try: + validate_table('tab') + except ValueError as e: + print(e) + return + + pytest.fail() # fail if ValueError wasn't raised + + +def test_validation_duplicate_colnames(orca_session): + """ + Table validation should raise a ValueError if columns share a name with index. + + """ + d = {'id1': [1,1,3], 'id2': [3,3,9], 'value': [4,4,4]} + df = pd.DataFrame(d).set_index(['id1', 'id2']) + df['id2'] = [10,10,10] # column with same name as one of the multi-index levels + orca.add_table('tab', df) + + try: + validate_table('tab') + except ValueError as e: + print(e) + return + + pytest.fail() # fail if ValueError wasn't raised + + def test_validation_index_unique(orca_session): """ Table validation should pass if the index is unique. @@ -40,7 +90,8 @@ def test_validation_index_not_unique(orca_session): try: validate_table('tab') - except ValueError: + except ValueError as e: + print(e) return pytest.fail() # fail if ValueError wasn't raised @@ -68,28 +119,13 @@ def test_validation_multiindex_not_unique(orca_session): try: validate_table('tab') - except ValueError: + except ValueError as e: + print(e) return pytest.fail() # fail if ValueError wasn't raised -def test_validation_unnamed_index(orca_session): - """ - Table validation should raise a ValueError if index is unnamed. - - """ - d = {'id': [1,1,3], 'value': [4,4,4]} - orca.add_table('tab', pd.DataFrame(d)) # generates auto index without a name - - try: - validate_table('tab') - except ValueError: - return - - pytest.fail() # fail if ValueError wasn't raised - - def test_validation_columns_vs_other_indexes(orca_session): """ Table validation should compare the 'households.building_id' column to diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 8740753..8dfc358 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -126,7 +126,7 @@ def validate_table(table): # messages. We should update orca_test to support both, probably. if not orca.is_table(table): - raise ValueError("Table not yet registered with Orca") + raise ValueError("Table not registered with Orca: '{}'".format(table)) idx = orca.get_table(table).index @@ -134,9 +134,12 @@ def validate_table(table): if list(idx.names) == [None]: raise ValueError("Index column has no name") - # CHECK INDEX NAME DISTINCT FROM COLUMN NAMES + # Check for unique column names + for name in list(idx.names): + if name in list(orca.get_table(table).columns): + raise ValueError("Index names and column names overlap: '{}'".format(name)) - # Check index is unique + # Check for unique index values if len(idx.unique()) < len(idx): raise ValueError("Index not unique") From 35a98a0533f7883357ffed0d4c130663e179b4d3 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 6 Mar 2019 10:39:54 -0800 Subject: [PATCH 054/121] Validate all tables --- tests/test_utils_broadcasts.py | 31 ++++++++++++++++++++++++++++++- urbansim_templates/utils.py | 21 +++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index 6d18ea6..b91fb73 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -3,7 +3,7 @@ import orca -from urbansim_templates.utils import validate_table +from urbansim_templates.utils import validate_table, validate_all_tables @pytest.fixture @@ -156,6 +156,22 @@ def test_validation_index_vs_other_columns(orca_session): validate_table('buildings') +def test_validation_reciprocal_false(orca_session): + """ + This combination should not produce any column comparisons. + + """ + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) + + d = {'household_id': [1,2,3], 'building_id': [2,3,5]} + orca.add_table('households', pd.DataFrame(d).set_index('household_id')) + + print("Begin reciprocal test") + validate_table('buildings', reciprocal=False) + print("End reciprocal test") + + def test_validation_with_multiindexes(orca_session): """ Here, table validation should compare 'choice_table.[home_tract,work_tract]' to @@ -171,3 +187,16 @@ def test_validation_with_multiindexes(orca_session): validate_table('choice_table') + +def test_validate_all_tables(orca_session): + """ + + """ + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + orca.add_table('buildings', pd.DataFrame(d).set_index('building_id')) + + d = {'household_id': [1,2,3], 'building_id': [2,3,5]} + orca.add_table('households', pd.DataFrame(d).set_index('household_id')) + + validate_all_tables() + diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 8dfc358..3c93651 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -85,7 +85,7 @@ def validate_template(cls): """ -def validate_table(table): +def validate_table(table, reciprocal=True): """ Check some basic expectations about an Orca table: @@ -115,6 +115,10 @@ def validate_table(table): table : str Name of Orca table to validate. + reciprocal : bool, default True + Whether to also check how columns of other tables align with this one's index. + If False, only check this table's columns against other tables' indexes. + Returns ------- bool @@ -144,8 +148,10 @@ def validate_table(table): raise ValueError("Index not unique") # Compare columns to indexes of other tables, and vice versa - combinations = [(table, t) for t in orca.list_tables() if table != t] \ - + [(t, table) for t in orca.list_tables() if table != t] + combinations = [(table, t) for t in orca.list_tables() if table != t] + + if reciprocal: + combinations += [(t, table) for t in orca.list_tables() if table != t] for t1, t2 in combinations: col_names = orca.get_table(t1).columns @@ -175,8 +181,15 @@ def validate_table(table): def validate_all_tables(): """ + Validate all tables registered with Orca. See ``validate_table()`` above. + + Returns + ------- + bool + """ - pass + for t in orca.list_tables(): + validate_table(t, reciprocal=False) def merge_tables(): From 309fe0fba97df9d871b5edcbd6917ea14dabd6ea Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 6 Mar 2019 15:05:24 -0800 Subject: [PATCH 055/121] Work in progress --- urbansim_templates/utils.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 3c93651..5053cfc 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -192,9 +192,34 @@ def validate_all_tables(): validate_table(t, reciprocal=False) -def merge_tables(): +def merge_tables(tables, coumns=None): """ + + accept either orca table names or dataframes? + + Parameters + ---------- + tables : list of str, or list of pd.DataFrame + Two or more tables to merge. + + columns : list of str, optional + + """ + + + # given df1 and df2, merge them + + # TO DO - filter for necessary columns + join_keys = list(df1.index.names) + # TO DO - check that join keys exist in second table + merged = pd.merge(df1, df2, on=join_keys) # indexes by name requires Pandas 0.23 + + + + + while len(tables) < 1: + pass From 84f9de07c23e5c0d7c22728f8baf5b7612667286 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 11 Mar 2019 21:30:58 -0700 Subject: [PATCH 056/121] Sketch of column from expression --- .../data/column_from_expression.py | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 urbansim_templates/data/column_from_expression.py diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py new file mode 100644 index 0000000..099b2ce --- /dev/null +++ b/urbansim_templates/data/column_from_expression.py @@ -0,0 +1,156 @@ +from __future__ import print_function + +import orca +import pandas as pd + +from urbansim_templates import modelmanager, __version__ + + +@modelmanager.template +class ColumnFromExpression(): + """ + Template to register a column of derived data with Orca, based on an expression. The + column will be associated with an existing table. Values will be calculated lazily, + only when the column is requested for a specific operation. + + The expression will be passed to ``pd.eval()`` and can refer to other columns in the + table. See the Pandas documentation for further details. + + All the parameters can also be set as properties after creating the template + instance. + + Parameters + ---------- + column_name : str, optional + Name of the Orca column to be registered. Must be provided to run the template. + + table : str, optional + Name of the Orca table the column will be associated with. Must be provided to + run the template. + + expression : str, optional + Expression for calculating values of the column. Must be provided to run the + template. + + cache : bool, default False + Passed to ``orca.column()``. + + cache_scope : 'step', 'iteration', or 'forever', default 'forever' + Passed to ``orca.table()``. + + name : str, optional + Name of the template instance and associated model step. + + tags : list of str, optional + Tags, passed to ModelManager. + + autorun : bool, default True + Automatically run the template whenever it's registered with ModelManager. + + """ + def __init__(self, + column_name = None, + table = None, + expression = None, + cache = False, + cache_scope = 'forever', + name = None, + tags = [], + autorun = True): + + # Template-specific params + self.column_name = column_name + self.table = table + self.expression = expression + self.cache = cache + self.cache_scope = cache_scope + + # Standard params + self.name = name + self.tags = tags + self.autorun = autorun + + # Automatic params + self.template = self.__class__.__name__ + self.template_version = __version__ + + + @classmethod + def from_dict(cls, d): + """ + Create an object instance from a saved dictionary representation. + + Parameters + ---------- + d : dict + + Returns + ------- + Table + + """ + obj = cls( + column_name = d['column_name'], + table = d['table'], + expression = d['expression'], + cache = d['cache'], + cache_scope = d['cache_scope'], + name = d['name'], + tags = d['tags'], + autorun = d['autorun'] + ) + return obj + + + def to_dict(self): + """ + Create a dictionary representation of the object. + + Returns + ------- + dict + + """ + d = { + 'template': self.template, + 'template_version': self.template_version, + 'name': self.name, + 'tags': self.tags, + 'autorun': self.autorun, + 'column_name': self.column_name, + 'table': self.table, + 'expression': self.expression, + 'cache': self.cache, + 'cache_scope': self.cache_scope, + } + return d + + + def run(self): + """ + Run the template, registering a column of derived data with Orca. + + Requires values to be set for ``column_name``, ``table``, and ``expression``. + + Returns + ------- + None + + """ + if self.column_name is None: + raise ValueError("Please provide a column name") + + if self.table not in ['csv', 'hdf']: + raise ValueError("Please provide a table") + + if self.expression is None: + raise ValueError("Please provide an expression") + + @orca.column(table_name = self.table, + column_name = self.column_name, + cache = self.cache, + cache_scope = self.cache_scope) + def orca_column(): + pass + + \ No newline at end of file From ee67fe2b11361fc29573b4d7dd708ded8c75940f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 11 Mar 2019 21:41:29 -0700 Subject: [PATCH 057/121] Better docstrings --- .../data/column_from_expression.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 099b2ce..9bbbd3a 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -14,7 +14,7 @@ class ColumnFromExpression(): only when the column is requested for a specific operation. The expression will be passed to ``pd.eval()`` and can refer to other columns in the - table. See the Pandas documentation for further details. + same table. See the Pandas documentation for further details. All the parameters can also be set as properties after creating the template instance. @@ -22,30 +22,30 @@ class ColumnFromExpression(): Parameters ---------- column_name : str, optional - Name of the Orca column to be registered. Must be provided to run the template. + Name of the Orca column to be registered. Required before running. table : str, optional - Name of the Orca table the column will be associated with. Must be provided to - run the template. + Name of the Orca table the column will be associated with. Required before + running. expression : str, optional - Expression for calculating values of the column. Must be provided to run the - template. + Expression for calculating values of the column. Required before running. cache : bool, default False - Passed to ``orca.column()``. + Whether to cache column values after they are calculated. cache_scope : 'step', 'iteration', or 'forever', default 'forever' - Passed to ``orca.table()``. + How long to cache column values for (ignored if ``cache`` is False). name : str, optional Name of the template instance and associated model step. tags : list of str, optional - Tags, passed to ModelManager. + Tags to associate with the template instance. autorun : bool, default True - Automatically run the template whenever it's registered with ModelManager. + Whether to run automatically when the template instance is registered with + ModelManager. """ def __init__(self, From 1d6b22a6fb6342707f4445e5a47d831bd5ca1c72 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 12 Mar 2019 12:02:31 -0700 Subject: [PATCH 058/121] Implementing column generation logic --- .../data/column_from_expression.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 9bbbd3a..f137d8e 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -1,5 +1,7 @@ from __future__ import print_function +import re + import orca import pandas as pd @@ -13,7 +15,7 @@ class ColumnFromExpression(): column will be associated with an existing table. Values will be calculated lazily, only when the column is requested for a specific operation. - The expression will be passed to ``pd.eval()`` and can refer to other columns in the + The expression will be passed to ``df.eval()`` and can refer to other columns in the same table. See the Pandas documentation for further details. All the parameters can also be set as properties after creating the template @@ -29,7 +31,16 @@ class ColumnFromExpression(): running. expression : str, optional - Expression for calculating values of the column. Required before running. + String describing operations on existing columns of the table, for example + "a/log(b+c)". Required before running. Supports arithmetic and math functions + including sqrt, abs, log, log1p, exp, and expm1 -- see Pandas ``df.eval()`` + documentation for further details. + + data_type : str, optional + Python type or ``numpy.dtype`` to cast the column's values into. + + missing_values : str or numeric, optional + Value to use for rows that would otherwise be missing. cache : bool, default False Whether to cache column values after they are calculated. @@ -52,6 +63,8 @@ def __init__(self, column_name = None, table = None, expression = None, + data_type = None, + missing_values = None, cache = False, cache_scope = 'forever', name = None, @@ -62,6 +75,8 @@ def __init__(self, self.column_name = column_name self.table = table self.expression = expression + self.data_type = data_type + self.missing_values = missing_values self.cache = cache self.cache_scope = cache_scope @@ -93,6 +108,8 @@ def from_dict(cls, d): column_name = d['column_name'], table = d['table'], expression = d['expression'], + data_type = d['data_type'], + missing_values = d['missing_values'], cache = d['cache'], cache_scope = d['cache_scope'], name = d['name'], @@ -120,6 +137,8 @@ def to_dict(self): 'column_name': self.column_name, 'table': self.table, 'expression': self.expression, + 'data_type': self.data_type, + 'missing_values': self.missing_values, 'cache': self.cache, 'cache_scope': self.cache_scope, } @@ -146,11 +165,26 @@ def run(self): if self.expression is None: raise ValueError("Please provide an expression") + # Some column names in the expression may not be part of the core DataFrame, so + # we'll need to request them from Orca explicitly. Identify tokens that begin + # with a letter and contain any number of alphanumerics or underscores, but do + # not end with an opening parenthesis. + cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression) + @orca.column(table_name = self.table, column_name = self.column_name, cache = self.cache, cache_scope = self.cache_scope) def orca_column(): - pass + df = orca.get_table(self.table).to_frame(columns=cols) + series = df.eval(self.expression) + + if self.missing_values is not None: + series = series.fillna(self.missing_values) + + if self.data_type is not None: + series = series.astype(self.data_type) + + return series \ No newline at end of file From 2d31bb8575bdddc1293d0fdc151210e8afd04aaa Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 12 Mar 2019 12:38:14 -0700 Subject: [PATCH 059/121] Adding to do --- urbansim_templates/data/column_from_expression.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index f137d8e..f4c5f55 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -171,6 +171,10 @@ def run(self): # not end with an opening parenthesis. cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression) + # TO DO - make sure requesting indexes by name doesn't raise an error from Orca + # - probably should just check which of the elements in the list Orca thinks are + # valid columns, and only request those + @orca.column(table_name = self.table, column_name = self.column_name, cache = self.cache, From 94616458d084653889ea9b28c9b337716fdf8760 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 12 Mar 2019 14:14:24 -0700 Subject: [PATCH 060/121] Initial tests --- tests/test_column_expression.py | 65 +++++++++++++++++++++++++++++ urbansim_templates/data/__init__.py | 1 + 2 files changed, 66 insertions(+) create mode 100644 tests/test_column_expression.py diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py new file mode 100644 index 0000000..aad7da9 --- /dev/null +++ b/tests/test_column_expression.py @@ -0,0 +1,65 @@ +import numpy as np +import pandas as pd +import pytest + +import orca + +from urbansim_templates import modelmanager +from urbansim_templates.data import ColumnFromExpression +from urbansim_templates.utils import validate_template + + +@pytest.fixture +def orca_session(): + """ + Set up a clean Orca and ModelManager session, with a data table. + + """ + orca.clear_all() + modelmanager.initialize() + + d1 = {'id': np.arange(10), + 'a': np.random.random(10), + 'b': np.random.choice(np.arange(20), size=10)} + + df = pd.DataFrame(d1).set_index('id') + orca.add_table('obs', df) + + +def test_template_validity(): + """ + Check template conforms to basic spec. + + """ + assert validate_template(ColumnFromExpression) + + +def test_run_requirements(orca_session): + """ + + + """ + pass + + +def test_expression(orca_session): + """ + Check that column is created correctly. + + """ + c = ColumnFromExpression() + c.column_name = 'c' + c.table = 'obs' + c.expression = 'a + sqrt(b)' + + c.run() + series = orca.get_column('obs', 'c') + print(series) + + +def test_modelmanager_registration(orca_session): + """ + Check that modelmanager registration and auto-run work as expected. + + """ + pass \ No newline at end of file diff --git a/urbansim_templates/data/__init__.py b/urbansim_templates/data/__init__.py index ccdcdf3..90dc264 100644 --- a/urbansim_templates/data/__init__.py +++ b/urbansim_templates/data/__init__.py @@ -1,2 +1,3 @@ +from .column_from_expression import ColumnFromExpression from .load_table import LoadTable from .save_table import SaveTable From 2ea8ac74b01c0d9440e4654b8a246151bbe2ce22 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 12 Mar 2019 14:48:24 -0700 Subject: [PATCH 061/121] Fixing failures --- tests/test_column_expression.py | 19 ++++++++++++++++--- .../data/column_from_expression.py | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index aad7da9..f523935 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -50,16 +50,29 @@ def test_expression(orca_session): c = ColumnFromExpression() c.column_name = 'c' c.table = 'obs' - c.expression = 'a + sqrt(b)' + c.expression = 'a * 5 + sqrt(b)' c.run() - series = orca.get_column('obs', 'c') + series = orca.get_table('obs').get_column('c') print(series) +def test_data_type_and_missing_values(orca_session): + """ + """ + pass + + def test_modelmanager_registration(orca_session): """ Check that modelmanager registration and auto-run work as expected. """ - pass \ No newline at end of file + pass + + +def test_expression_with_standalone_columns(orca_session): + """ + """ + pass + diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index f4c5f55..4e384af 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -159,7 +159,7 @@ def run(self): if self.column_name is None: raise ValueError("Please provide a column name") - if self.table not in ['csv', 'hdf']: + if self.table is None: raise ValueError("Please provide a table") if self.expression is None: From 952c1f3d03b90da92bafdaba4e5814361c865787 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 13 Mar 2019 12:03:11 -0700 Subject: [PATCH 062/121] More tests --- tests/test_column_expression.py | 109 +++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 10 deletions(-) diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index f523935..edf6e24 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -18,9 +18,9 @@ def orca_session(): orca.clear_all() modelmanager.initialize() - d1 = {'id': np.arange(10), - 'a': np.random.random(10), - 'b': np.random.choice(np.arange(20), size=10)} + d1 = {'id': np.arange(5), + 'a': np.random.random(5), + 'b': np.random.choice(np.arange(20), size=5)} df = pd.DataFrame(d1).set_index('id') orca.add_table('obs', df) @@ -34,17 +34,63 @@ def test_template_validity(): assert validate_template(ColumnFromExpression) -def test_run_requirements(orca_session): +def test_missing_colname(orca_session): """ + Missing column_name should raise a ValueError. + """ + c = ColumnFromExpression() + c.table = 'tab' + c.expression = 'a' + + try: + c.run() + except ValueError as e: + print(e) + return + pytest.fail() + + +def test_missing_table(orca_session): """ - pass + Missing table should raise a ValueError. + + """ + c = ColumnFromExpression() + c.column_name = 'col' + c.expression = 'a' + + try: + c.run() + except ValueError as e: + print(e) + return + + pytest.fail() + + +def test_missing_expression(orca_session): + """ + Missing expression should raise a ValueError. + + """ + c = ColumnFromExpression() + c.column_name = 'col' + c.table = 'tab' + + try: + c.run() + except ValueError as e: + print(e) + return + + pytest.fail() def test_expression(orca_session): """ - Check that column is created correctly. + Check that column is created and expression evaluated correctly. """ c = ColumnFromExpression() @@ -53,14 +99,57 @@ def test_expression(orca_session): c.expression = 'a * 5 + sqrt(b)' c.run() - series = orca.get_table('obs').get_column('c') - print(series) + + val1 = orca.get_table('obs').get_column('c') + df = orca.get_table('obs').to_frame() + val2 = df.a * 5 + np.sqrt(df.b) + assert(val1.equals(val2)) -def test_data_type_and_missing_values(orca_session): +def test_data_type(orca_session): """ + Check that casting data type works. + """ - pass + orca.add_table('tab', pd.DataFrame({'a': [0.1, 1.33, 2.4]})) + + c = ColumnFromExpression() + c.column_name = 'b' + c.table = 'tab' + c.expression = 'a' + c.run() + + v1 = orca.get_table('tab').get_column('b').values + np.testing.assert_equal(v1, [0.1, 1.33, 2.4]) + + c.data_type = 'int' + c.run() + + v1 = orca.get_table('tab').get_column('b').values + np.testing.assert_equal(v1, [0, 1, 2]) + + +def test_missing_values(orca_session): + """ + Check that filling in missing values works. + + """ + orca.add_table('tab', pd.DataFrame({'a': [0.1, np.nan, 2.4]})) + + c = ColumnFromExpression() + c.column_name = 'b' + c.table = 'tab' + c.expression = 'a' + c.run() + + v1 = orca.get_table('tab').get_column('b').values + np.testing.assert_equal(v1, [0.1, np.nan, 2.4]) + + c.missing_values = 5 + c.run() + + v1 = orca.get_table('tab').get_column('b').values + np.testing.assert_equal(v1, [0.1, 5.0, 2.4]) def test_modelmanager_registration(orca_session): From 4ac546b0a6b79dcaee02017f335400ec6b54fbcf Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Mar 2019 11:37:29 -0700 Subject: [PATCH 063/121] Raising pandas requirement --- requirements.txt | 2 +- urbansim_templates/utils.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 286508c..c3584b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ choicemodels >= 0.2.dev4 numpy >= 1.14 orca >= 1.4 -pandas >= 0.22 +pandas >= 0.23 patsy >= 0.4 statsmodels >= 0.8 urbansim >= 3.1 diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 5053cfc..89175ba 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -217,10 +217,6 @@ def merge_tables(tables, coumns=None): - - while len(tables) < 1: - - pass ############################### From 1592d276e4449839677791c07732849e0c6ecb77 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Mar 2019 16:15:53 -0700 Subject: [PATCH 064/121] Docstrings for merge_tables() --- urbansim_templates/utils.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 89175ba..8828288 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -192,18 +192,40 @@ def validate_all_tables(): validate_table(t, reciprocal=False) -def merge_tables(tables, coumns=None): +def merge_tables(tables, columns=None): """ - - accept either orca table names or dataframes? + Merge multiple tables into a single DataFrame. Tables will be merged from right to + left following ModelManager table schema rules -- so they should generally be listed + from finer-grained to coarser-grained. + + For example, suppose we merge ``[buildings, zones]`` where ``zones`` has an index + named ``zone_id``. The algorithm will look for a column or index with the same name + in the ``buildings`` table, and use it to merge the ``zones`` columns onto the + ``buildings`` table. Multi-indexes require all the index columns to be present in the + target table. + + The input tables are expected to be DataFrame-like: ``pd.DataFrame``, + ``orca.DataFrameWrapper``, etc (what operations?). We don't currently support + accessing Orca tables by name gere, although it might be added. The function will + return a new ``pd.DataFrame``. + + If you provide a list of ``columns``, only these will be returned. The index(es) of + the left-most table will always be included. + + If column names overlap, other than for join keys, the tables can't be automatically + merged. If the duplicate column names are incidental and not needed in the final + output, you can merge the tables by providing a ``columns`` list that excludes them. Parameters ---------- - tables : list of str, or list of pd.DataFrame + tables : list of pd.DataFrame or similar Two or more tables to merge. columns : list of str, optional - + + Returns + ------- + pd.DataFrame """ From 24b90271d89a3a1cf648b893a338762995a527a1 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Mar 2019 16:35:43 -0700 Subject: [PATCH 065/121] Basic working example of merge --- urbansim_templates/utils.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 8828288..b6c247e 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -229,15 +229,18 @@ def merge_tables(tables, columns=None): """ + source = tables[1] # TO DO: adapt to handle multiple tables + target = tables[0] - # given df1 and df2, merge them - - # TO DO - filter for necessary columns - join_keys = list(df1.index.names) - # TO DO - check that join keys exist in second table - merged = pd.merge(df1, df2, on=join_keys) # indexes by name requires Pandas 0.23 - + keys = list(source.index.names) + + # TO DO: filter for appropriate columns + # TO DO: check that join keys exist in the target table + + # pandas 0.23+ required to merge on index or column names interchangeably + merged = pd.merge(df1, df2, how='left', on=keys) + return merged From f922d8aa39a23a7efc73ab115936c08da733e26d Mon Sep 17 00:00:00 2001 From: Liming Wang Date: Tue, 19 Mar 2019 16:43:57 -0700 Subject: [PATCH 066/121] added mct argument for SegmentedLargeMultinomialLogitStep.fit_all() and interaction_terms argument for SegmentedLargeMultinomialLogitStep.run_all() --- .../segmented_large_multinomial_logit.py | 178 ++++++++++-------- 1 file changed, 98 insertions(+), 80 deletions(-) diff --git a/urbansim_templates/models/segmented_large_multinomial_logit.py b/urbansim_templates/models/segmented_large_multinomial_logit.py index 96cc9d1..1d0b19a 100644 --- a/urbansim_templates/models/segmented_large_multinomial_logit.py +++ b/urbansim_templates/models/segmented_large_multinomial_logit.py @@ -19,94 +19,94 @@ class SegmentedLargeMultinomialLogitStep(TemplateStep): """ This template automatically generates a set of LargeMultinomialLogitStep submodels - corresponding to "segments" or categories of choosers. The submodels can be directly - accessed and edited. - + corresponding to "segments" or categories of choosers. The submodels can be directly + accessed and edited. + Running 'build_submodels()' will create a submodel for each category of choosers identified in the segmentation column. The submodels are implemented using filter - queries. - + queries. + Once they are generated, the 'submodels' property contains a dict of - LargeMultinomialLogitStep objects, identified by category name. You can edit their - properties as needed, fit them individually, etc. - - Editing a property in the 'defaults' object will update all the submodels at once, + LargeMultinomialLogitStep objects, identified by category name. You can edit their + properties as needed, fit them individually, etc. + + Editing a property in the 'defaults' object will update all the submodels at once, while leaving customizations to other properties intact. - + Parameters ---------- defaults : LargeMultinomialLogitStep, optional Object containing initial parameter values for the submodels. Values for - 'choosers', 'alternatives', and 'choice_column' are required to generate + 'choosers', 'alternatives', and 'choice_column' are required to generate submodels, but do not have to be provided when the object is created. - + segmentation_column : str, optional - Name of a column of categorical values in the 'defaults.choosers' table. Any data - that can be interpreted by Pandas as categorical is valid. This is required to + Name of a column of categorical values in the 'defaults.choosers' table. Any data + that can be interpreted by Pandas as categorical is valid. This is required to generate submodels, but does not have to be provided when the object is created. - + name : str, optional Name of the model step. - + tags : list of str, optional - Tags associated with the model step. - + Tags associated with the model step. + """ def __init__(self, defaults=None, segmentation_column=None, name=None, tags=[]): - + if defaults is None: defaults = LargeMultinomialLogitStep() - + self.defaults = defaults self.defaults.bind_to(self.update_submodels) - + self.segmentation_column = segmentation_column - + self.name = name self.tags = tags - + self.template = self.__class__.__name__ self.template_version = __version__ - + # Properties to be filled in by build_submodels() or from_dict() self.submodels = {} - - + + @classmethod def from_dict(cls, d): """ Create an object instance from a saved dictionary representation. - + Parameters ---------- d : dict - + Returns ------- SegmentedLargeMultinomialLogitStep - + """ mnl_step = LargeMultinomialLogitStep.from_dict - + obj = cls( defaults = mnl_step(d['defaults']), segmentation_column = d['segmentation_column'], name = d['name'], tags = d['tags']) - + obj.submodels = {k: mnl_step(m) for k, m in d['submodels'].items()} - + return obj - + def to_dict(self): """ Create a dictionary representation of the object. - + Returns ------- dict - + """ d = { 'template': self.template, @@ -118,40 +118,40 @@ def to_dict(self): 'submodels': {k: m.to_dict() for k, m in self.submodels.items()} } return d - - + + def get_segmentation_column(self): """ - Get the column of segmentation values from Orca. Chooser and alternative filters + Get the column of segmentation values from Orca. Chooser and alternative filters are applied to identify valid observations. - + Returns ------- pd.Series - + """ obs = get_data(tables = self.defaults.choosers, filters = self.defaults.chooser_filters, - extra_columns = [self.defaults.choice_column, + extra_columns = [self.defaults.choice_column, self.segmentation_column]) - + alts = get_data(tables = self.defaults.alternatives, filters = self.defaults.alt_filters) - + df = pd.merge(obs, alts, how='inner', left_on=self.defaults.choice_column, right_index=True) - + return df[self.segmentation_column] - - + + def build_submodels(self): """ Create a submodel for each category of choosers identified in the segmentation column. Only categories with at least one observation remaining after applying - chooser and alternative filters will be included. - + chooser and alternative filters will be included. + Running this method will overwrite any previous submodels. - + """ self.submodels = {} submodel = LargeMultinomialLogitStep.from_dict(self.defaults.to_dict()) @@ -162,44 +162,44 @@ def build_submodels(self): print("Warning: No valid observations after applying the chooser and "+ "alternative filters") return - + cats = col.astype('category').cat.categories.values print("Building submodels for {} categories: {}".format(len(cats), cats)) - + for cat in cats: m = copy.deepcopy(submodel) seg_filter = "{} == '{}'".format(self.segmentation_column, cat) - + if isinstance(m.chooser_filters, list): m.chooser_filters += [seg_filter] - + elif isinstance(m.chooser_filters, str): m.chooser_filters = [m.chooser_filters, seg_filter] - + else: m.chooser_filters = seg_filter - + # TO DO - same for out_chooser_filters, once we handle simulation self.submodels[cat] = m - - + + def update_submodels(self, param, value): """ - Updates a property across all the submodels. This method is bound to the + Updates a property across all the submodels. This method is bound to the `defaults` object and runs automatically when one of its properties is changed. - - Note that the `chooser_filters` and `alt_filters` properties cannot currently be - updated this way, because they can affect the model segmentation. If you are + + Note that the `chooser_filters` and `alt_filters` properties cannot currently be + updated this way, because they can affect the model segmentation. If you are confident the changes are valid, you can edit the submodels directly. Otherwise, you can regenerate them using updated defaults by running `build_submodels()`. - + Parameters ---------- param : str Property name. value : anything - + """ if (param in ['chooser_filters', 'alt_filters']) & (len(self.submodels) > 0): print("Warning: Changing '{}' can affect the model segmentation. Changes " + @@ -207,42 +207,60 @@ def update_submodels(self, param, value): "regenerate them using the new defaults, run 'build_submodels()'."\ .format(param)) return - + for k, m in self.submodels.items(): setattr(m, param, value) - - - def fit_all(self): + + + def fit_all(self, mct=None): """ - Fit all the submodels. Build the subomdels first, if they don't exist yet. This + Fit all the submodels. Build the subomdels first, if they don't exist yet. This method can be run as many times as desired. - + + Parameters + ---------- + mct : choicemodels.tools.MergedChoiceTable + This parameter is a temporary backdoor allowing us to pass in a more + complicated choice table than can be generated within the template, for + example including sampling weights or interaction terms. + + """ if (len(self.submodels) == 0): self.build_submodels() - + for k, m in self.submodels.items(): print(' SEGMENT: {0} = {1} '.format( self.segmentation_column, str(k)).center(70, '#')) - m.fit() - + m.fit(mct=mct) + self.name = update_name(self.template, self.name) - - + + def run(self): """ Convenience method (requied by template spec) that invokes `run_all()`. - + """ self.run_all() - - - def run_all(self): + + + def run_all(self, interaction_terms=None): """ Run all the submodels. - + + Parameters + ---------- + interaction_terms : pandas.Series, pandas.DataFrame, or list of either, optional + Additional column(s) of interaction terms whose values depend on the + combination of observation and alternative, to be merged onto the final data + table. If passed as a Series or DataFrame, it should include a two-level + MultiIndex. One level's name and values should match an index or column from + the observations table, and the other should match an index or column from the + alternatives table. + """ for k, m in self.submodels.items(): print(' SEGMENT: {0} = {1} '.format( self.segmentation_column, str(k)).center(70, '#')) - m.run() + m.run(interaction_terms=interaction_terms) From 812e194b556477756653504525ceeb686c3dc53a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 20 Mar 2019 13:30:48 -0700 Subject: [PATCH 067/121] Progress on merging --- tests/test_utils_broadcasts.py | 44 +++++++++++++++++++++++++++++++++- urbansim_templates/utils.py | 26 ++++++++++++++++---- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index b91fb73..3dc5aca 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -1,9 +1,14 @@ +""" +Tests for the utilities for merging tables using implicit join keys instead of Orca +broadcasts. + +""" import pandas as pd import pytest import orca -from urbansim_templates.utils import validate_table, validate_all_tables +from urbansim_templates.utils import validate_table, validate_all_tables, merge_tables @pytest.fixture @@ -200,3 +205,40 @@ def test_validate_all_tables(orca_session): validate_all_tables() + +def test_merge_tables(): + """ + Merge tables. + + """ + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + buildings = pd.DataFrame(d).set_index('building_id') + + d = {'household_id': [1,2,3], 'building_id': [2,3,4]} + households = pd.DataFrame(d).set_index('household_id') + + merged = merge_tables([households, buildings]) + print(merged) + + +def test_merge_tables_limit_columns(): + """ + Merge tables and remove some of the columns. + + """ + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + buildings = pd.DataFrame(d).set_index('building_id') + + d = {'household_id': [1,2,3], 'building_id': [2,3,4]} + households = pd.DataFrame(d).set_index('household_id') + + merged = merge_tables([households, buildings], columns=['value']) + print(merged) + + +# Merge tables and remove columns that otherwise would cause merge to fail + + + + + diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index b6c247e..4e188c2 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -206,7 +206,7 @@ def merge_tables(tables, columns=None): The input tables are expected to be DataFrame-like: ``pd.DataFrame``, ``orca.DataFrameWrapper``, etc (what operations?). We don't currently support - accessing Orca tables by name gere, although it might be added. The function will + accessing Orca tables by name here, although it might be added. The function will return a new ``pd.DataFrame``. If you provide a list of ``columns``, only these will be returned. The index(es) of @@ -229,16 +229,32 @@ def merge_tables(tables, columns=None): """ + def trim_columns(df, columns): + """ + columns may contain duplicate names or names not in df. + + """ + overlap = set(columns) & set(df.columns) + return df[list(overlap)] + source = tables[1] # TO DO: adapt to handle multiple tables target = tables[0] keys = list(source.index.names) - # TO DO: filter for appropriate columns - # TO DO: check that join keys exist in the target table + if columns is not None: + source = trim_columns(source, columns) + target = trim_columns(target, columns + keys) + + # TO DO: check that join keys exist in the target table, to provide helpful error - # pandas 0.23+ required to merge on index or column names interchangeably - merged = pd.merge(df1, df2, how='left', on=keys) + # pandas 0.23+ required to join on index or column names interchangeably + merged = target.join(source, on=keys, how='left') + + + # final filter in case last set of join keys is not needed + if columns is not None: + merged = trim_columns(merged, columns) return merged From 306cab7558a16db5b51afb6a6ee948bee6c2922c Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 20 Mar 2019 14:03:55 -0700 Subject: [PATCH 068/121] Refactoring trim_columns() --- tests/test_utils_broadcasts.py | 23 ++++++++++++++++++++++- urbansim_templates/utils.py | 28 +++++++++++++++++++--------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index 3dc5aca..df270cb 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -236,8 +236,29 @@ def test_merge_tables_limit_columns(): print(merged) -# Merge tables and remove columns that otherwise would cause merge to fail +def test_merge_tables_duplicate_column_names(): + """ + Confirm tables can be merged with overlapping column names, as long as they're not + included in the list of columns to retain. + + """ + d = {'building_id': [1,2,3,4], 'value': [4,4,4,4], 'dupe': [1,1,1,1]} + buildings = pd.DataFrame(d).set_index('building_id') + + d = {'household_id': [1,2,3], 'building_id': [2,3,4], 'dupe': [1,1,1]} + households = pd.DataFrame(d).set_index('household_id') + + # Duplicate columns should raise a ValueError + try: + merged = merge_tables([households, buildings]) + pytest.fail() + except ValueError as e: + print(e) + + # Excluding the duplicated name should make things ok + merged = merge_tables([households, buildings], columns=['value']) + print(merged) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 4e188c2..1eb0855 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -228,15 +228,6 @@ def merge_tables(tables, columns=None): pd.DataFrame """ - - def trim_columns(df, columns): - """ - columns may contain duplicate names or names not in df. - - """ - overlap = set(columns) & set(df.columns) - return df[list(overlap)] - source = tables[1] # TO DO: adapt to handle multiple tables target = tables[0] @@ -264,6 +255,25 @@ def trim_columns(df, columns): ## TEMPLATE HELPER FUNCTIONS ## ############################### +def trim_columns(df, columns): + """ + Limit a DataFrame to columns that appear in a list of strings. List may contain + duplicates or names not in the DataFrame. Index(es) of the DataFrame will be retained. + + Parameters + ---------- + df : pd.DataFrame + columns : list of str + + Returns + ------- + pd.DataFrame + + """ + overlap = set(columns) & set(df.columns) + return df[list(overlap)] + + def to_list(items): """ In many places we accept either a single string or a list of strings. This function From aecac43dd860dbdf71230ada82ab7a388bc012bb Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 20 Mar 2019 14:13:52 -0700 Subject: [PATCH 069/121] Better docstrings --- tests/test_utils_broadcasts.py | 3 ++- urbansim_templates/utils.py | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index df270cb..117cc14 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -261,5 +261,6 @@ def test_merge_tables_duplicate_column_names(): print(merged) - +# test multiple tables +# test multi-indexes diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 1eb0855..2c77552 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -204,24 +204,25 @@ def merge_tables(tables, columns=None): ``buildings`` table. Multi-indexes require all the index columns to be present in the target table. - The input tables are expected to be DataFrame-like: ``pd.DataFrame``, - ``orca.DataFrameWrapper``, etc (what operations?). We don't currently support - accessing Orca tables by name here, although it might be added. The function will - return a new ``pd.DataFrame``. + The input tables must be provided as DataFrames; we don't currently support accessing + Orca tables by name here, although it might be added. The function will return a new + ``pd.DataFrame``. - If you provide a list of ``columns``, only these will be returned. The index(es) of - the left-most table will always be included. + If you provide a list of ``columns``, the output table will be limited to columns in + this list, plus the index(es) of the left-most table. - If column names overlap, other than for join keys, the tables can't be automatically - merged. If the duplicate column names are incidental and not needed in the final - output, you can merge the tables by providing a ``columns`` list that excludes them. + If tables contain columns with identical names (other than the join keys), the tables + can't be automatically merged. If these columns are just incidental and not needed in + the final output, you can merge the tables by providing a ``columns`` list that + excludes them. Parameters ---------- - tables : list of pd.DataFrame or similar + tables : list of pd.DataFrame Two or more tables to merge. columns : list of str, optional + Names of columns to retain in the final output. Returns ------- @@ -237,10 +238,8 @@ def merge_tables(tables, columns=None): source = trim_columns(source, columns) target = trim_columns(target, columns + keys) - # TO DO: check that join keys exist in the target table, to provide helpful error - - # pandas 0.23+ required to join on index or column names interchangeably - merged = target.join(source, on=keys, how='left') + # TO DO: check that join keys exist in the target table + merged = target.join(source, on=keys, how='left') # pandas 0.23+ for on=keys # final filter in case last set of join keys is not needed From c7fe5005eb1ca8e55584298b9c166768ae72d461 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 20 Mar 2019 16:19:16 -0700 Subject: [PATCH 070/121] More merging tests --- tests/test_utils_broadcasts.py | 31 ++++++++++++++++++++++++++----- urbansim_templates/utils.py | 21 +++++++++++---------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index 117cc14..df6d117 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -206,9 +206,9 @@ def test_validate_all_tables(orca_session): validate_all_tables() -def test_merge_tables(): +def test_merge_two_tables(): """ - Merge tables. + Merge two tables. """ d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} @@ -221,18 +221,40 @@ def test_merge_tables(): print(merged) +def test_merge_three_tables(): + """ + Merge three tables. + + """ + d = {'zone_id': [1], 'size': [1]} + zones = pd.DataFrame(d).set_index('zone_id') + + d = {'building_id': [1,2,3,4], 'zone_id': [1,1,1,1], 'height': [4,4,4,4]} + buildings = pd.DataFrame(d).set_index('building_id') + + d = {'household_id': [1,2,3], 'building_id': [2,3,4]} + households = pd.DataFrame(d).set_index('household_id') + + merged = merge_tables([households, buildings, zones]) + print(merged) + + def test_merge_tables_limit_columns(): """ Merge tables and remove some of the columns. """ - d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]} + d = {'zone_id': [1], 'size': [1]} + zones = pd.DataFrame(d).set_index('zone_id') + + d = {'building_id': [1,2,3,4], 'zone_id': [1,1,1,1], 'height': [4,4,4,4]} buildings = pd.DataFrame(d).set_index('building_id') d = {'household_id': [1,2,3], 'building_id': [2,3,4]} households = pd.DataFrame(d).set_index('household_id') - merged = merge_tables([households, buildings], columns=['value']) + merged = merge_tables([households, buildings, zones], + columns=['zone_id', 'height', 'size']) print(merged) @@ -261,6 +283,5 @@ def test_merge_tables_duplicate_column_names(): print(merged) -# test multiple tables # test multi-indexes diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 2c77552..9896604 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -229,20 +229,21 @@ def merge_tables(tables, columns=None): pd.DataFrame """ - source = tables[1] # TO DO: adapt to handle multiple tables - target = tables[0] + while len(tables) > 1: + source = tables[-1] + target = tables[-2] - keys = list(source.index.names) + keys = list(source.index.names) - if columns is not None: - source = trim_columns(source, columns) - target = trim_columns(target, columns + keys) + if columns is not None: + source = trim_columns(source, columns) + target = trim_columns(target, columns + keys) - # TO DO: check that join keys exist in the target table - merged = target.join(source, on=keys, how='left') # pandas 0.23+ for on=keys + # TO DO: confirm join keys exist in the target table + merged = target.join(source, on=keys, how='left') # pandas 0.23+ for on=keys + tables = tables[:-2] + [merged] - # final filter in case last set of join keys is not needed if columns is not None: merged = trim_columns(merged, columns) @@ -256,7 +257,7 @@ def merge_tables(tables, columns=None): def trim_columns(df, columns): """ - Limit a DataFrame to columns that appear in a list of strings. List may contain + Limit a DataFrame to columns that appear in a list of names. List may contain duplicates or names not in the DataFrame. Index(es) of the DataFrame will be retained. Parameters From e32e66ae89fe3546bc7c6b5f40f66b3b9e1a7624 Mon Sep 17 00:00:00 2001 From: Liming Wang Date: Thu, 21 Mar 2019 09:34:05 -0700 Subject: [PATCH 071/121] updated version to 0.2dev3; added change log entry --- CHANGELOG.md | 8 +++++++- docs/source/index.rst | 2 +- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a8cbfc..675dc65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## 0.2 (not yet released) +#### 0.2.dev2 (2019-03-21) + +- adds an mct argment to models.segmented_large_multinomial_logit.fit_all() +- adds an interaction_terms argment to models.segmented_large_multinomial_logit.run_all() + + #### 0.2.dev2 (2019-03-04) - adds template for saving data: `urbansim_templates.data.SaveTable()` @@ -100,4 +106,4 @@ #### 0.1.dev12 (2018-09-19) - moves the `register()` operation to `modelmanager` (previously it was a method implemented by the individual templates) -- adds general ModelManager support for supplemental objects like pickled model results \ No newline at end of file +- adds general ModelManager support for supplemental objects like pickled model results diff --git a/docs/source/index.rst b/docs/source/index.rst index 1da0545..acb4ef9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev2, released March 4, 2019 +v0.2.dev3, released March 21, 2019 Contents diff --git a/setup.py b/setup.py index 9d3792f..d79599a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.2.dev2', + version='0.2.dev3', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index 8184599..9991a92 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev2' +version = __version__ = '0.2.dev3' From 44f986d34235b89c003962c9e5731e9597e55cb0 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Sun, 24 Mar 2019 17:17:02 -0700 Subject: [PATCH 072/121] Better docstrings --- urbansim_templates/utils.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 9896604..f9367f7 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -194,27 +194,28 @@ def validate_all_tables(): def merge_tables(tables, columns=None): """ - Merge multiple tables into a single DataFrame. Tables will be merged from right to - left following ModelManager table schema rules -- so they should generally be listed - from finer-grained to coarser-grained. + Merge multiple tables into a single DataFrame. - For example, suppose we merge ``[buildings, zones]`` where ``zones`` has an index - named ``zone_id``. The algorithm will look for a column or index with the same name - in the ``buildings`` table, and use it to merge the ``zones`` columns onto the - ``buildings`` table. Multi-indexes require all the index columns to be present in the - target table. + Tables should be listed in order from finer-grained to coarser-grained. If there are + more than two tables, the last one will be merged onto the next-to-last, continuing + until all the data is merged into the first table. In each merge stage, we'll refer + to the right-hand table as the "source" and the left-hand one as the "target". - The input tables must be provided as DataFrames; we don't currently support accessing - Orca tables by name here, although it might be added. The function will return a new - ``pd.DataFrame``. + Tables are merged using ModelManager schema rules. The source table must have a + unique index, and the target table must have a column with a matching name, which + will be used as the join key. Multi-indexes are fine, but all of the index columns + need to be present in the target table. + For now, this function only accepts DataFrames. In the future we might support + accessing Orca tables by name here. The function will return a new ``pd.DataFrame``. + If you provide a list of ``columns``, the output table will be limited to columns in this list, plus the index(es) of the left-most table. - If tables contain columns with identical names (other than the join keys), the tables - can't be automatically merged. If these columns are just incidental and not needed in - the final output, you can merge the tables by providing a ``columns`` list that - excludes them. + If two tables contain columns with identical names (other than join keys), they can't + be automatically merged. If the columns are just incidental and not needed in the + final output, you can perform the merge by providing a ``columns`` list that excludes + them. Parameters ---------- From a6665b151db56708c5760d65546a0f5e2484f649 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Sun, 24 Mar 2019 17:32:22 -0700 Subject: [PATCH 073/121] Final unit tests --- tests/test_utils_broadcasts.py | 31 ++++++++++++++++++++++++++++++- urbansim_templates/utils.py | 5 +++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index df6d117..54783a4 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -283,5 +283,34 @@ def test_merge_tables_duplicate_column_names(): print(merged) -# test multi-indexes +def test_merge_tables_multiindex(): + """ + Merge tables where the source table has a multi-index. + + """ + d = {'building_id': [1,1,2,2], 'unit_id': [1,2,1,2], 'value': [4,4,4,4]} + units = pd.DataFrame(d).set_index(['building_id', 'unit_id']) + + d = {'household_id': [1,2,3], 'building_id': [1,1,2], 'unit_id': [1,2,1]} + households = pd.DataFrame(d).set_index('household_id') + merged = merge_tables([households, units]) + print(merged) + + +def test_merge_tables_missing_values(): + """ + If the target table includes identifiers not found in the source table, missing + values should be inserted.. + + """ + d = {'building_id': [1,1,2,2], 'unit_id': [1,2,1,2], 'value': [4,4,4,4]} + units = pd.DataFrame(d).set_index(['building_id', 'unit_id']) + + d = {'household_id': [1,2,3], 'building_id': [1,1,3], 'unit_id': [1,2,1]} + households = pd.DataFrame(d).set_index('household_id') + + merged = merge_tables([households, units]) + print(merged) + + diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index f9367f7..420ef29 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -217,6 +217,11 @@ def merge_tables(tables, columns=None): final output, you can perform the merge by providing a ``columns`` list that excludes them. + A note about data types: They will be retained, but if NaN values need to be added + (e.g. if some identifiers from the target table aren't found in the source table), + data may be cast to a type that allows missing values. For better control over this, + see ``urbansim_templates.data.ColumnFromBroadcast()``. + Parameters ---------- tables : list of pd.DataFrame From ff03f5309d96d23104a17f6add80a930fc5279b4 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Sun, 24 Mar 2019 17:58:19 -0700 Subject: [PATCH 074/121] Work in progress --- urbansim_templates/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 420ef29..0677b1e 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -210,7 +210,8 @@ def merge_tables(tables, columns=None): accessing Orca tables by name here. The function will return a new ``pd.DataFrame``. If you provide a list of ``columns``, the output table will be limited to columns in - this list, plus the index(es) of the left-most table. + this list, plus the index(es) of the left-most table. Column names not found will be + ignored. If two tables contain columns with identical names (other than join keys), they can't be automatically merged. If the columns are just incidental and not needed in the @@ -222,6 +223,12 @@ def merge_tables(tables, columns=None): data may be cast to a type that allows missing values. For better control over this, see ``urbansim_templates.data.ColumnFromBroadcast()``. + TO DO: The merged + + TO DO: We should add a case where if tables are merged index-to-index, it's an outer + join rather than a left join. This is what people would expect if they were merging + something like two nodes tables that contained different subsets of nodes, i think. + Parameters ---------- tables : list of pd.DataFrame @@ -235,6 +242,9 @@ def merge_tables(tables, columns=None): pd.DataFrame """ + # TO DO: the merges should not be strictly in order. We should search left until we + # find a table with a matching identifier. + while len(tables) > 1: source = tables[-1] target = tables[-2] From 3679f527c18e356fbfa30fce472712ce9c281c2e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 25 Mar 2019 13:10:32 -0700 Subject: [PATCH 075/121] Utility to get anything as dataframe --- tests/test_utils.py | 79 +++++++++++++++++++++++++++++++++++++ urbansim_templates/utils.py | 68 +++++++++++++++++++++++++++---- 2 files changed, 139 insertions(+), 8 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index cc2a602..7f27dd3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -25,6 +25,85 @@ def test_version_greater_or_equal(): assert utils.version_greater_or_equal('1.1.3.dev0', '1.1.3') == False +############################### +## get_df + +@pytest.fixture +def df(): + d = {'id': [1,2,3], 'val1': [4,5,6], 'val2': [7,8,9]} + return pd.DataFrame(d).set_index('id') + + +def test_get_df_dataframe(df): + """ + Confirm that get_df() works when passed a DataFrame. + + """ + df_out = utils.get_df(df) + pd.testing.assert_frame_equal(df, df_out) + + +def test_get_df_str(df): + """ + Confirm that get_df() works with str input. + + """ + orca.add_table('df', df) + df_out = utils.get_df('df') + pd.testing.assert_frame_equal(df, df_out) + + +def test_get_df_dataframewrapper(df): + """ + Confirm that get_df() works with orca.DataFrameWrapper input. + + """ + dfw = orca.DataFrameWrapper('df', df) + df_out = utils.get_df(dfw) + pd.testing.assert_frame_equal(df, df_out) + + +def test_get_df_tablefuncwrapper(df): + """ + Confirm that get_df() works with orca.TableFuncWrapper input. + + """ + def df_callable(): + return df + + tfw = orca.TableFuncWrapper('df', df_callable) + df_out = utils.get_df(tfw) + pd.testing.assert_frame_equal(df, df_out) + + +def test_get_df_columns(df): + """ + Confirm that get_df() limits columns, and filters out duplicates and invalid ones. + + """ + dfw = orca.DataFrameWrapper('df', df) + df_out = utils.get_df(dfw, ['id', 'val1', 'val1', 'val3']) + pd.testing.assert_frame_equal(df[['val1']], df_out) + + +def test_get_df_unsupported_type(df): + """ + Confirm that get_df() raises an error for an unsupported type. + + """ + try: + df_out = utils.get_df([df]) + except ValueError as e: + print(e) + return + + pytest.fail() + + + +############################### +## get_data + @pytest.fixture def orca_session(): d1 = {'id': [1, 2, 3], diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 0677b1e..4493cd6 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -2,6 +2,8 @@ from datetime import datetime as dt +import pandas as pd + import orca from urbansim.models.util import (apply_filter_query, columns_in_filters, columns_in_formula) @@ -252,8 +254,8 @@ def merge_tables(tables, columns=None): keys = list(source.index.names) if columns is not None: - source = trim_columns(source, columns) - target = trim_columns(target, columns + keys) + source = trim_cols(source, columns) + target = trim_cols(target, columns + keys) # TO DO: confirm join keys exist in the target table @@ -261,7 +263,7 @@ def merge_tables(tables, columns=None): tables = tables[:-2] + [merged] if columns is not None: - merged = trim_columns(merged, columns) + merged = trim_cols(merged, columns) return merged @@ -271,23 +273,73 @@ def merge_tables(tables, columns=None): ## TEMPLATE HELPER FUNCTIONS ## ############################### -def trim_columns(df, columns): +def get_df(table, columns=None): + """ + Returns a table as a ``pd.DataFrame``. Input can be an Orca table name, + ``orca.DataFrameWrapper``, ``orca.TableFuncWrapper``, or ``pd.DataFrame``. + + Optionally, columns can be limited to those that appear in a list of names. The list + may contain duplicates or columns not in the table. Index(es) will always be + retained, but it's a good practice to list them anyway. + + Parameters + ---------- + table : str, orca.DataFrameWrapper, orca.TableFuncWrapper, or pd.DataFrame + columns : list of str, optional + + Returns + ------- + pd.DataFrame + + """ + if type(table) not in [str, + orca.DataFrameWrapper, + orca.TableFuncWrapper, + pd.DataFrame]: + raise ValueError("Table has unsupported type: {}".format(type(table))) + + if type(table) == pd.DataFrame: + return trim_cols(table, columns) + + elif type(table) == str: + table = orca.get_table(table) + + if columns is not None: + # Orca requires column list to be unique and existing, or None + columns = list(set(columns) & set(table.columns)) + + return table.to_frame(columns=columns) + + +def all_cols(table): + """ + """ + # all_cols += list(dfw.index.names) + list(dfw.columns) + pass + + +def trim_cols(df, columns=None): """ Limit a DataFrame to columns that appear in a list of names. List may contain - duplicates or names not in the DataFrame. Index(es) of the DataFrame will be retained. + duplicates or names not in the DataFrame. Index(es) of the DataFrame will always be + retained, but it's a good practice to list them anyway. If ``columns`` is None, all + columns are retained. Returns the original DataFrame, not a copy. Parameters ---------- df : pd.DataFrame - columns : list of str + columns : list of str, optional Returns ------- pd.DataFrame """ - overlap = set(columns) & set(df.columns) - return df[list(overlap)] + if columns is None: + return df + + cols = set(columns) & set(df.columns) # unique, existing columns + return df[list(cols)] def to_list(items): From c62fce5570fcf721ee40ed47d54014f0fc6a38ad Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 25 Mar 2019 13:43:31 -0700 Subject: [PATCH 076/121] Utility to list all columns in a table --- tests/test_utils.py | 49 +++++++++++++++++++++++++++++++++++++ urbansim_templates/utils.py | 39 ++++++++++++++++++----------- 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7f27dd3..07a4576 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -101,6 +101,55 @@ def test_get_df_unsupported_type(df): +############################### +## all_cols + +def test_all_cols_dataframe(df): + """ + Confirm that all_cols() works with DataFrame input. + + """ + cols = utils.all_cols(df) + assert sorted(cols) == sorted(['id', 'val1', 'val2']) + + +def test_all_cols_orca(df): + """ + Confirm that all_cols() works with Orca input. + + """ + orca.add_table('df', df) + cols = utils.all_cols('df') + assert sorted(cols) == sorted(['id', 'val1', 'val2']) + + +def test_all_cols_extras(df): + """ + Confirm that all_cols() includes columns not part of the Orca core table. + + """ + orca.add_table('df', df) + orca.add_column('df', 'newcol', pd.Series()) + cols = utils.all_cols('df') + assert sorted(cols) == sorted(['id', 'val1', 'val2', 'newcol']) + + +def test_all_cols_unsupported_type(df): + """ + Confirm that all_cols() raises an error for an unsupported type. + + """ + try: + cols = utils.all_cols([df]) + except ValueError as e: + print(e) + return + + pytest.fail() + + + + ############################### ## get_data diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 4493cd6..a6577a3 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -208,9 +208,6 @@ def merge_tables(tables, columns=None): will be used as the join key. Multi-indexes are fine, but all of the index columns need to be present in the target table. - For now, this function only accepts DataFrames. In the future we might support - accessing Orca tables by name here. The function will return a new ``pd.DataFrame``. - If you provide a list of ``columns``, the output table will be limited to columns in this list, plus the index(es) of the left-most table. Column names not found will be ignored. @@ -224,17 +221,11 @@ def merge_tables(tables, columns=None): (e.g. if some identifiers from the target table aren't found in the source table), data may be cast to a type that allows missing values. For better control over this, see ``urbansim_templates.data.ColumnFromBroadcast()``. - - TO DO: The merged - - TO DO: We should add a case where if tables are merged index-to-index, it's an outer - join rather than a left join. This is what people would expect if they were merging - something like two nodes tables that contained different subsets of nodes, i think. - + Parameters ---------- - tables : list of pd.DataFrame - Two or more tables to merge. + tables : list of str, orca.DataFrameWrapper, orca.TableFuncWrapper, or pd.DataFrame + Two or more tables to merge. Types can be mixed and matched. columns : list of str, optional Names of columns to retain in the final output. @@ -313,9 +304,29 @@ def get_df(table, columns=None): def all_cols(table): """ + Returns a list of all column names in a table, including index(es). Input can be an + Orca table name, ``orca.DataFrameWrapper``, ``orca.TableFuncWrapper``, or + ``pd.DataFrame``. + + Parameters + ---------- + table : str, orca.DataFrameWrapper, orca.TableFuncWrapper, or pd.DataFrame + + Returns + ------- + list of str + """ - # all_cols += list(dfw.index.names) + list(dfw.columns) - pass + if type(table) not in [str, + orca.DataFrameWrapper, + orca.TableFuncWrapper, + pd.DataFrame]: + raise ValueError("Table has unsupported type: {}".format(type(table))) + + if type(table) == str: + table = orca.get_table(table) + + return list(table.index.names) + list(table.columns) def trim_cols(df, columns=None): From f4958bc0516275ee0dca70f293e54473ba8e8524 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 25 Mar 2019 15:35:22 -0700 Subject: [PATCH 077/121] Improved merge_tables() --- tests/test_utils_broadcasts.py | 45 +++++++++++++++++++----- urbansim_templates/utils.py | 63 +++++++++++++++++++++------------- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/tests/test_utils_broadcasts.py b/tests/test_utils_broadcasts.py index 54783a4..3d0a4a2 100644 --- a/tests/test_utils_broadcasts.py +++ b/tests/test_utils_broadcasts.py @@ -9,6 +9,7 @@ import orca from urbansim_templates.utils import validate_table, validate_all_tables, merge_tables +from urbansim_templates.utils import all_cols @pytest.fixture @@ -20,6 +21,9 @@ def orca_session(): orca.clear_all() +############################### +## validate_tables() + def test_validation_table_not_registered(orca_session): """ Table validation should raise a ValueError if the table isn't registered. @@ -206,6 +210,9 @@ def test_validate_all_tables(orca_session): validate_all_tables() +############################### +## merge_tables() + def test_merge_two_tables(): """ Merge two tables. @@ -218,7 +225,7 @@ def test_merge_two_tables(): households = pd.DataFrame(d).set_index('household_id') merged = merge_tables([households, buildings]) - print(merged) + assert sorted(all_cols(merged)) == sorted(['household_id', 'building_id', 'value']) def test_merge_three_tables(): @@ -236,7 +243,27 @@ def test_merge_three_tables(): households = pd.DataFrame(d).set_index('household_id') merged = merge_tables([households, buildings, zones]) - print(merged) + assert sorted(all_cols(merged)) == sorted( + ['household_id', 'building_id', 'zone_id', 'height', 'size']) + + +def test_merge_three_tables_out_of_order(): + """ + Merge three tables, where the second and third are each merged onto the first. + + """ + d = {'zone_id': [1], 'size': [1]} + zones = pd.DataFrame(d).set_index('zone_id') + + d = {'building_id': [1,2,3,4], 'height': [4,4,4,4]} + buildings = pd.DataFrame(d).set_index('building_id') + + d = {'household_id': [1,2,3], 'building_id': [2,3,4], 'zone_id': [1,1,1]} + households = pd.DataFrame(d).set_index('household_id') + + merged = merge_tables([households, buildings, zones]) + assert sorted(all_cols(merged)) == sorted( + ['household_id', 'building_id', 'zone_id', 'height', 'size']) def test_merge_tables_limit_columns(): @@ -255,7 +282,8 @@ def test_merge_tables_limit_columns(): merged = merge_tables([households, buildings, zones], columns=['zone_id', 'height', 'size']) - print(merged) + assert sorted(all_cols(merged)) == sorted( + ['household_id', 'zone_id', 'height', 'size']) def test_merge_tables_duplicate_column_names(): @@ -279,8 +307,7 @@ def test_merge_tables_duplicate_column_names(): # Excluding the duplicated name should make things ok merged = merge_tables([households, buildings], columns=['value']) - - print(merged) + assert sorted(all_cols(merged)) == sorted(['household_id', 'value']) def test_merge_tables_multiindex(): @@ -295,13 +322,14 @@ def test_merge_tables_multiindex(): households = pd.DataFrame(d).set_index('household_id') merged = merge_tables([households, units]) - print(merged) + assert sorted(all_cols(merged)) == sorted( + ['household_id', 'building_id', 'unit_id', 'value']) def test_merge_tables_missing_values(): """ If the target table includes identifiers not found in the source table, missing - values should be inserted.. + values should be inserted, changing the data type. """ d = {'building_id': [1,1,2,2], 'unit_id': [1,2,1,2], 'value': [4,4,4,4]} @@ -311,6 +339,7 @@ def test_merge_tables_missing_values(): households = pd.DataFrame(d).set_index('household_id') merged = merge_tables([households, units]) - print(merged) + assert units.value.dtype == 'int64' + assert merged.values.dtype == 'float64' diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index a6577a3..7cab6c9 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -198,19 +198,30 @@ def merge_tables(tables, columns=None): """ Merge multiple tables into a single DataFrame. - Tables should be listed in order from finer-grained to coarser-grained. If there are - more than two tables, the last one will be merged onto the next-to-last, continuing - until all the data is merged into the first table. In each merge stage, we'll refer - to the right-hand table as the "source" and the left-hand one as the "target". + All the data will eventually be merged onto the first table in the list. In each + merge stage, we'll refer to the right-hand table as the "source" and the left-hand + one as the "target". - Tables are merged using ModelManager schema rules. The source table must have a + Tables are merged using ModelManager schema rules: The source table must have a unique index, and the target table must have a column with a matching name, which will be used as the join key. Multi-indexes are fine, but all of the index columns need to be present in the target table. + The last table in the list is the initial source. The algorithm searches backward + through the list for a table that qualifies as a target. The source table is left- + merged onto the target, and then the algorithm continues with the second-to-last + table as the new source. + + Example 1: Tables A and B share join keys. Tables B and C share join keys. Merging + [A, B, C] will left-join C onto B, and then left-join the result onto A. + + Example 2: Tables A and B share join keys. Tables A and C also share join keys, but + tables B and C don't. Merging [A, B, C] will left-join C onto A, and then left-join + B onto the result of the first join. + If you provide a list of ``columns``, the output table will be limited to columns in - this list, plus the index(es) of the left-most table. Column names not found will be - ignored. + this list. The index(es) of the left-most table will always be retained, but it's a + good practice to list them anyway. Column names not found will be ignored. If two tables contain columns with identical names (other than join keys), they can't be automatically merged. If the columns are just incidental and not needed in the @@ -235,26 +246,32 @@ def merge_tables(tables, columns=None): pd.DataFrame """ - # TO DO: the merges should not be strictly in order. We should search left until we - # find a table with a matching identifier. - while len(tables) > 1: - source = tables[-1] - target = tables[-2] - + # last table becomes the source + source = get_df(tables[-1], columns) keys = list(source.index.names) + + # search for target table + target_position = None + for i in range(len(tables)-2, -1, -1): + if set(keys).issubset(set(all_cols(tables[i]))): + target_position = i + target_columns = columns + keys if columns is not None else None + target = get_df(tables[i], target_columns) + break + + if target_position is None: + msg = "Could not find a target to merge table {} onto".format(len(tables)) + raise ValueError(msg) - if columns is not None: - source = trim_cols(source, columns) - target = trim_cols(target, columns + keys) - - # TO DO: confirm join keys exist in the target table - + # merge source onto target merged = target.join(source, on=keys, how='left') # pandas 0.23+ for on=keys - tables = tables[:-2] + [merged] - - if columns is not None: - merged = trim_cols(merged, columns) + + tables = tables[:-1] + tables[target_position] = merged + + # drop final merge keys if not needed + merged = trim_cols(merged, columns) return merged From 4375350cc963abf5fd1bb318069dee3e32759a95 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 25 Mar 2019 15:41:29 -0700 Subject: [PATCH 078/121] Better docstrings --- urbansim_templates/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 7cab6c9..77d25dc 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -196,7 +196,7 @@ def validate_all_tables(): def merge_tables(tables, columns=None): """ - Merge multiple tables into a single DataFrame. + Merge two or more tables into a single DataFrame. All the data will eventually be merged onto the first table in the list. In each merge stage, we'll refer to the right-hand table as the "source" and the left-hand @@ -209,7 +209,7 @@ def merge_tables(tables, columns=None): The last table in the list is the initial source. The algorithm searches backward through the list for a table that qualifies as a target. The source table is left- - merged onto the target, and then the algorithm continues with the second-to-last + joined onto the target, and then the algorithm continues with the second-to-last table as the new source. Example 1: Tables A and B share join keys. Tables B and C share join keys. Merging @@ -230,8 +230,8 @@ def merge_tables(tables, columns=None): A note about data types: They will be retained, but if NaN values need to be added (e.g. if some identifiers from the target table aren't found in the source table), - data may be cast to a type that allows missing values. For better control over this, - see ``urbansim_templates.data.ColumnFromBroadcast()``. + data may need to be cast to a type that allows missing values. For better control + over this, see ``urbansim_templates.data.ColumnFromBroadcast()``. Parameters ---------- From 33be09b0d42f91d5bdbe4ef35e05acdc1a5b8929 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 25 Mar 2019 17:04:27 -0700 Subject: [PATCH 079/121] Refactoring get_data() --- urbansim_templates/utils.py | 51 ++++++++----------------------------- 1 file changed, 11 insertions(+), 40 deletions(-) diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 77d25dc..cdb1d56 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -418,28 +418,15 @@ def update_name(template, name=None): def get_data(tables, fallback_tables=None, filters=None, model_expression=None, extra_columns=None): """ - Generate a pd.DataFrame from one or more tables registered with Orca. Templates should - call this function immediately before the data is needed, so that it's as up-to-date - as possible. + Generate a ``pd.DataFrame`` for model estimation or simulation. Automatically loads + tables from Orca, merges them, and removes columns not referenced in a model + expression or data filter. Additional columns can be requested. If filters are provided, the output will include only rows that match the filter criteria. - Default behavior is for the output to inclue all columns. If a model_expression and/or - extra_columns is provided, non-relevant columns will be dropped from the output. - Relevant columns include any mentioned in the model expression, filters, or list of - extras. Join keys will *not* be included in the final output even if the data is drawn - from multiple tables, unless they appear in the model expression or filters as well. - - If a named column is not found in the source tables, it will just be skipped. This is - to support use cases where data is assembled separately for choosers and alternatives - and then merged together -- the model expression would include terms from both sets - of tables. - - Duplicate column names are not recommended -- columns are expected to be unique within - the set of tables they're being drawn from, with the exception of join keys. If column - names are repeated, current behavior is to follow the Orca default and keep the - left-most copy of the column. This may change later and should not be relied on. + See ``urbansim_templates.utils.merge_tables()`` for a detailed description of how + the merges are performed. Parameters ---------- @@ -469,33 +456,17 @@ def get_data(tables, fallback_tables=None, filters=None, model_expression=None, if tables is None: tables = fallback_tables - tables = to_list(tables) - colnames = None # this will get all columns from Orca utilities - + colnames = None # this will get all columns if (model_expression is not None) or (extra_columns is not None): colnames = set(columns_in_formula(model_expression) + \ columns_in_filters(filters) + to_list(extra_columns)) - - # skip cols not found in any of the source tables - have to check for this - # explicitly because the orca utilities will raise an error if we request column - # names that aren't there - all_cols = [] - for t in tables: - dfw = orca.get_table(t) - all_cols += list(dfw.index.names) + list(dfw.columns) - - colnames = [c for c in colnames if c in all_cols] - - if len(tables) == 1: - df = orca.get_table(table_name=tables[0]).to_frame(columns=colnames) + + if not isinstance(tables, list): + df = get_df(tables, colnames) else: - df = orca.merge_tables(target=tables[0], tables=tables, columns=colnames) - - if colnames is not None: - if len(df.columns) > len(colnames): - df = df[colnames] - + df = merge_tables(tables, colnames) + df = apply_filter_query(df, filters) return df From ffe0ac2ce4bc9ba8e0fea8da9ba14f1a3a7e4261 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 25 Mar 2019 17:31:40 -0700 Subject: [PATCH 080/121] Removing _get_df() --- urbansim_templates/models/binary_logit.py | 14 ++++-- urbansim_templates/models/regression.py | 15 +++++-- urbansim_templates/models/shared.py | 52 ----------------------- urbansim_templates/utils.py | 4 +- 4 files changed, 25 insertions(+), 60 deletions(-) diff --git a/urbansim_templates/models/binary_logit.py b/urbansim_templates/models/binary_logit.py index 4857d32..9253c24 100644 --- a/urbansim_templates/models/binary_logit.py +++ b/urbansim_templates/models/binary_logit.py @@ -9,6 +9,7 @@ import orca from .. import modelmanager +from ..utils import get_data from .shared import TemplateStep @@ -180,8 +181,12 @@ def fit(self): # https://github.com/statsmodels/statsmodels/issues/3931 from scipy import stats stats.chisqprob = lambda chisq, df: stats.chi2.sf(chisq, df) + + df = get_data(tables = self.tables, + filters = self.filters, + model_expression = self.model_expression) - m = Logit.from_formula(data=self._get_data(), formula=self.model_expression) + m = Logit.from_formula(data=df, formula=self.model_expression) results = m.fit() self.name = self._generate_name() @@ -219,8 +224,11 @@ def run(self): """ # TO DO - verify that params are in place for prediction - df = self._get_data('predict') - + df = get_data(tables = self.out_tables, + fallback_tables = self.tables, + filters = self.out_filters, + model_expression = self.model_expression) + dm = patsy.dmatrices(data=df, formula_like=self.model_expression, return_type='dataframe')[1] # right-hand-side design matrix diff --git a/urbansim_templates/models/regression.py b/urbansim_templates/models/regression.py index ce2ed49..7683498 100644 --- a/urbansim_templates/models/regression.py +++ b/urbansim_templates/models/regression.py @@ -9,7 +9,7 @@ from urbansim.utils import yamlio from .. import modelmanager -from ..utils import update_column +from ..utils import get_data, update_column from .shared import TemplateStep @@ -170,7 +170,11 @@ def fit(self): fit_filters=self.filters, predict_filters=self.out_filters, ytransform=self.out_transform, name=self.name) - results = self.model.fit(self._get_data()) + df = get_data(tables = self.tables, + filters = self.filters, + model_expression = self.model_expression) + + results = self.model.fit(df) self.name = self._generate_name() self.summary_table = str(results.summary()) @@ -194,7 +198,12 @@ def run(self): predicted values are written to Orca. """ - values = self.model.predict(self._get_data('predict')) + df = get_data(tables = self.out_tables, + fallback_tables = self.tables, + filters = self.out_filters, + model_expression = self.model_expression) + + values = self.model.predict(df) self.predicted_values = values if self.out_transform is not None: diff --git a/urbansim_templates/models/shared.py b/urbansim_templates/models/shared.py index f0d27aa..3238dff 100644 --- a/urbansim_templates/models/shared.py +++ b/urbansim_templates/models/shared.py @@ -142,58 +142,6 @@ def out_tables(self, out_tables): self.__out_tables = self._normalize_table_param(out_tables) - def _get_data(self, task='fit'): - """ - DEPRECATED - this should be replaced by the more general utils.get_data() - - Generate a data table for estimation or prediction, relying on functionality from - Orca and UrbanSim.models.util. This should be performed immediately before - estimation or prediction so that it reflects the current data state. - - The output includes only the necessary columns: those mentioned in the model - expression or filters, plus (it appears) the index of each merged table. Relevant - filter queries are applied. - - Parameters - ---------- - task : 'fit' or 'predict' - - Returns - ------- - DataFrame - - """ - # TO DO - verify input data - - if isinstance(self.model_expression, str): - expr_cols = util.columns_in_formula(self.model_expression) - - if (task == 'fit'): - tables = self.tables - columns = expr_cols + util.columns_in_filters(self.filters) - filters = self.filters - - elif (task == 'predict'): - if self.out_tables is not None: - tables = self.out_tables - else: - tables = self.tables - - columns = expr_cols + util.columns_in_filters(self.out_filters) - if self.out_column is not None: - columns += [self.out_column] - - filters = self.out_filters - - if isinstance(tables, list): - df = orca.merge_tables(target=tables[0], tables=tables, columns=columns) - else: - df = orca.get_table(tables).to_frame(columns) - - df = util.apply_filter_query(df, filters) - return df - - def _get_out_column(self): """ Return name of the column to save data to. This is 'out_column' if it exsits, diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index cdb1d56..73dc6d7 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -458,8 +458,8 @@ def get_data(tables, fallback_tables=None, filters=None, model_expression=None, colnames = None # this will get all columns if (model_expression is not None) or (extra_columns is not None): - colnames = set(columns_in_formula(model_expression) + \ - columns_in_filters(filters) + to_list(extra_columns)) + colnames = list(set(columns_in_formula(model_expression) + \ + columns_in_filters(filters) + to_list(extra_columns))) if not isinstance(tables, list): df = get_df(tables, colnames) From 2aef53ed149e9bf7b535c635d915e48b53f415a6 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 25 Mar 2019 18:14:08 -0700 Subject: [PATCH 081/121] Updating versioning and changelog --- CHANGELOG.md | 12 +++++++++--- docs/source/index.rst | 2 +- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 675dc65..68650e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,17 @@ ## 0.2 (not yet released) -#### 0.2.dev2 (2019-03-21) +#### 0.2.dev4 (2019-03-26) -- adds an mct argment to models.segmented_large_multinomial_logit.fit_all() -- adds an interaction_terms argment to models.segmented_large_multinomial_logit.run_all() +- adds new data management utilities: `utils.validate_table()`, `utils.validate_all_tables()`, `utils.merge_tables()` +- updates `utils.get_data()` to use the new merge tool +- updates `BinaryLogitStep` and `OLSRegressionStep` to use the shared to use `utils.get_data()`, removing any reliance on Orca broadcasts +- raises the `pandas` requirement to 0.23 +#### 0.2.dev3 (2019-03-21) + +- adds an `mct` argment to `SegmentedLargeMultinomialLogitStep.fit_all()` +- adds an `interaction_terms` argument to `SegmentedLargeMultinomialLogitStep.run_all()` #### 0.2.dev2 (2019-03-04) diff --git a/docs/source/index.rst b/docs/source/index.rst index acb4ef9..0b8a812 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev3, released March 21, 2019 +v0.2.dev4, released March 26, 2019 Contents diff --git a/setup.py b/setup.py index d79599a..635e8e4 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.2.dev3', + version='0.2.dev4', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index 9991a92..c8a0bc7 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev3' +version = __version__ = '0.2.dev4' From 3f8bcb5c80fde12de206db2e2b9ab01d6ce71450 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 26 Mar 2019 17:27:02 -0700 Subject: [PATCH 082/121] Documentation updates --- docs/README.md | 2 +- docs/source/data-templates.rst | 81 ++++++++++++++++++++++++--- docs/source/utilities.rst | 22 ++++++-- urbansim_templates/data/load_table.py | 12 +--- 4 files changed, 93 insertions(+), 24 deletions(-) diff --git a/docs/README.md b/docs/README.md index b705a7b..eaa350e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ Sphinx reads from the source files, plus the docstrings in the code, and renders For now we're building the docs manually, which gives us maximum control: ``` -sphinx-build -b html source_dir build_dir +sphinx-build -b html source build ``` Building the docs requires the python libraries `sphinx`, `numpydoc`, and `sphinx_rtd_theme`, plus the baseline requirements for `urbansim_templates`. You should rebuild the docs for each release. Rendered files won't appear online until they're merged into the master branch, but you can preview them locally. diff --git a/docs/source/data-templates.rst b/docs/source/data-templates.rst index c0a94a1..fc0d3e1 100644 --- a/docs/source/data-templates.rst +++ b/docs/source/data-templates.rst @@ -1,24 +1,89 @@ Data template APIs ================== -Data templates help you set up model steps for loading data into `Orca `__ or saving outputs to disk. +Discussion +---------- -These templates follow the same principles as the statistical model steps. For example, to set up a data table, create an instance of the ``LoadTable`` class and set some properties: the table name, file type, path, and anything else that's needed. +Data templates help you load tables into `Orca `__ or save tables or subsets of tables to disk. -Registering this object with ModelManager will save it to disk as a yaml file, and create an Orca step with instructions to set up the table. "Running" the object/step registers the table with Orca, but doesn't read the data from disk yet — Orca loads data lazily as it's needed. +Example +~~~~~~~ -Data registration steps are run automatically when you initialize ModelManager. +.. code-block:: python + + from urbansim_templates.data import LoadTable + + t = LoadTable() + t.table = 'buildings' # a name for the Orca table + t.source_type = 'csv' + t.path = 'buildings.csv' + t.csv_index_cols = 'building_id' + t.name = 'load_buildings' # a name for the model step that sets up the table +You can run this directly using ``t.run()``, or register the configured template to be part of a larger workflow: -Loading data ------------- +.. code-block:: python + + from urbansim_templates import modelmanager + + modelmanager.register(t) + +Registration does two things: (a) it saves the configured template to disk as a yaml file, and (b) it creates a model step with logic for loading the table. Running the model step is equivalent to running the configured template object: + +.. code-block:: python + + t.run() + + # equivalent: + import orca + orca.run(['load_buildings']) + +Strictly speaking, running the model step doesn't load the data, it just sets up an Orca table with instructions for loading the data when it's needed. (This is called lazy evaluation.) + +.. code-block:: python + + orca.run(['load_buildings']) # now an Orca table named 'buildings' is registered + + orca.get_table('buildings').to_frame() # now the data is read from disk + +Because "running" the table-loading step is costless, it's done automatically when you register a configured template. It's also done automatically when you initialize a ModelManager session and table-loading configs are read from yaml. (If you'd like to disable this for a particular table, you can set ``t.autorun == False``.) + + +Recommended data schemas +~~~~~~~~~~~~~~~~~~~~~~~~ + +The :mod:`~urbansim_templates.data.LoadTable` template will work with any data that can be loaded into a Pandas DataFrame. But we highly recommend following stricter data schema rules: + +1. Each table should include a unique, named index column (a.k.a. primary key) or set of columns (multi-index, a.k.a composite key). + +2. If a column is meant to be a join key for another table, it should have the same name as the index of that table. + +3. Duplication of column names across tables (except for the join keys) is discouraged, for clarity. + +If you follow these rules, tables can be automatically merged on the fly, for example to assemble estimation data or calculate indicators. + +You can use :func:`~urbansim_templates.utils.validate_table()` or :func:`~urbansim_templates.utils.validate_all_tables()` to check whether these expectations are met. When templates merge tables on the fly, they use :func:`~urbansim_templates.utils.merge_tables()`. + +These utility functions work with any Orca table that meets the schema expectations, whether or not it was created with a template. + + +Compatibility with Orca +~~~~~~~~~~~~~~~~~~~~~~~ + +From Orca's perspective, tables set up using the :mod:`~urbansim_templates.data.LoadTable` template are equivalent to tables that are registered using ``orca.add_table()`` or the ``@orca.table`` decorator. Technically, they are ``orca.TableFuncWrapper`` objects. + +Unlike the templates, Orca relies on user-specified "`broadcast `__" relationships to perform automatic merging of tables. :mod:`~urbansim_templates.data.LoadTable` does not register any broadcasts, because they're not needed if tables follow the schema rules above. So if you use these tables in non-template model steps, you may need to add broadcasts separately. + + +LoadTable API +------------- .. autoclass:: urbansim_templates.data.LoadTable :members: -Saving data ------------ +SaveTable API +------------- .. autoclass:: urbansim_templates.data.SaveTable :members: diff --git a/docs/source/utilities.rst b/docs/source/utilities.rst index 1493c10..0e3f76e 100644 --- a/docs/source/utilities.rst +++ b/docs/source/utilities.rst @@ -4,15 +4,29 @@ Utilities API The utilities are mainly helper functions for templates. -Spec validation ---------------- +Template validation +------------------- .. automodule:: urbansim_templates.utils :members: validate_template -Template helper functions +Table schemas and merging ------------------------- .. automodule:: urbansim_templates.utils - :members: get_data, update_column, update_name, to_list + :members: validate_table, validate_all_tables, merge_tables + + +Other helper functions +---------------------- + +.. automodule:: urbansim_templates.utils + :members: all_cols, get_data, get_df, trim_cols, update_column, to_list, update_column, update_name + + +Version management +------------------ + +.. automodule:: urbansim_templates.utils + :members: parse_version, version_greater_or_equal diff --git a/urbansim_templates/data/load_table.py b/urbansim_templates/data/load_table.py index 7b97c32..344ecf6 100644 --- a/urbansim_templates/data/load_table.py +++ b/urbansim_templates/data/load_table.py @@ -20,16 +20,6 @@ class LoadTable(): An instance of this template class stores *instructions for loading a data table*, packaged into an Orca step. Running the instructions registers the table with Orca. - Saved table registration steps will be run automatically when you initialize - ModelManager, replacing the ``datasources.py`` scripts used in previous versions of - UrbanSim. - - Tables should include a unique index, or a set of columns that jointly represent a - unique index. - - If a column has the same name as the index of another table, ModelManager expects to - be able to use it as a join key. Following these naming conventions eliminates the - need for Orca "broadcasts". All the parameters can also be set as properties after creating the class instance. @@ -51,7 +41,7 @@ class LoadTable(): Remote url to download file from. csv_index_cols : str or list of str, optional - Required for csv source type. + Required for tables loaded from csv. extra_settings : dict, optional Additional arguments to pass to ``pd.read_csv()`` or ``pd.read_hdf()``. For From a4edb1b2ddd61412205602d45ca7396563629c21 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 28 Mar 2019 20:52:05 -0700 Subject: [PATCH 083/121] Use new get_df utility --- urbansim_templates/data/column_from_expression.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 4e384af..a844c27 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -6,6 +6,7 @@ import pandas as pd from urbansim_templates import modelmanager, __version__ +from urbansim_templates.utils import get_df @modelmanager.template @@ -166,21 +167,18 @@ def run(self): raise ValueError("Please provide an expression") # Some column names in the expression may not be part of the core DataFrame, so - # we'll need to request them from Orca explicitly. Identify tokens that begin - # with a letter and contain any number of alphanumerics or underscores, but do - # not end with an opening parenthesis. + # we'll need to request them from Orca explicitly. This regex pulls out column + # names into a list, by identifying tokens in the expression that begin with a + # letter and contain any number of alphanumerics or underscores, but do not end + # with an opening parenthesis. cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression) - # TO DO - make sure requesting indexes by name doesn't raise an error from Orca - # - probably should just check which of the elements in the list Orca thinks are - # valid columns, and only request those - @orca.column(table_name = self.table, column_name = self.column_name, cache = self.cache, cache_scope = self.cache_scope) def orca_column(): - df = orca.get_table(self.table).to_frame(columns=cols) + df = get_df(self.table, columns=cols) series = df.eval(self.expression) if self.missing_values is not None: From d98f023c62137302e429a831fed97de6d1919139 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 28 Mar 2019 21:06:39 -0700 Subject: [PATCH 084/121] Final tests --- tests/test_column_expression.py | 28 +++++++++++++++++-- .../data/column_from_expression.py | 3 +- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index edf6e24..4af02df 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -157,11 +157,35 @@ def test_modelmanager_registration(orca_session): Check that modelmanager registration and auto-run work as expected. """ - pass + c = ColumnFromExpression() + c.column_name = 'c' + c.table = 'obs' + c.expression = 'a + b' + + modelmanager.register(c) + modelmanager.remove_step(c.name) + assert('c' in orca.get_table('obs').columns) def test_expression_with_standalone_columns(orca_session): """ + Check that expression can assemble data from stand-alone columns that are not part + of the core DataFrame wrapped by a table. + """ - pass + c = ColumnFromExpression() + c.column_name = 'c' + c.table = 'obs' + c.expression = 'a + b' + + modelmanager.register(c) + modelmanager.remove_step(c.name) + + d = ColumnFromExpression() + d.column_name = 'd' + d.table = 'obs' + d.expression = 'a + c' + + d.run() + assert('d' in orca.get_table('obs').columns) diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index a844c27..8c6c40d 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -170,7 +170,8 @@ def run(self): # we'll need to request them from Orca explicitly. This regex pulls out column # names into a list, by identifying tokens in the expression that begin with a # letter and contain any number of alphanumerics or underscores, but do not end - # with an opening parenthesis. + # with an opening parenthesis. This will also pick up constants, like "pi", but + # invalid column names will be ignored when we request them from get_df(). cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression) @orca.column(table_name = self.table, From 45ecb47750d5be0dac807644b9018b151baa84cf Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 28 Mar 2019 21:08:01 -0700 Subject: [PATCH 085/121] Updating version --- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 635e8e4..b77543a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.2.dev4', + version='0.2.dev5', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index c8a0bc7..aba1d24 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev4' +version = __version__ = '0.2.dev5' From b5843587aed84e54fefdbeec0bd2342bfb4a2531 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 29 Mar 2019 11:46:35 -0700 Subject: [PATCH 086/121] Docs and changelog --- CHANGELOG.md | 4 ++++ docs/source/data-templates.rst | 19 +++++++++++++------ docs/source/index.rst | 2 +- .../data/column_from_expression.py | 4 ++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68650e3..c196666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 0.2 (not yet released) +#### 0.2.dev5 (2019-03-29) + +- adds new template: `urbansim_templates.data.ColumnFromExpression` + #### 0.2.dev4 (2019-03-26) - adds new data management utilities: `utils.validate_table()`, `utils.validate_all_tables()`, `utils.merge_tables()` diff --git a/docs/source/data-templates.rst b/docs/source/data-templates.rst index fc0d3e1..9317a3e 100644 --- a/docs/source/data-templates.rst +++ b/docs/source/data-templates.rst @@ -1,8 +1,8 @@ Data template APIs ================== -Discussion ----------- +Usage +----- Data templates help you load tables into `Orca `__ or save tables or subsets of tables to disk. @@ -75,15 +75,22 @@ From Orca's perspective, tables set up using the :mod:`~urbansim_templates.data. Unlike the templates, Orca relies on user-specified "`broadcast `__" relationships to perform automatic merging of tables. :mod:`~urbansim_templates.data.LoadTable` does not register any broadcasts, because they're not needed if tables follow the schema rules above. So if you use these tables in non-template model steps, you may need to add broadcasts separately. -LoadTable API -------------- +LoadTable() +----------- .. autoclass:: urbansim_templates.data.LoadTable :members: -SaveTable API -------------- +SaveTable() +----------- .. autoclass:: urbansim_templates.data.SaveTable :members: + + +ColumnFromExpression() +---------------------- + +.. autoclass:: urbansim_templates.data.ColumnFromExpression + :members: diff --git a/docs/source/index.rst b/docs/source/index.rst index 0b8a812..1e5ce05 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev4, released March 26, 2019 +v0.2.dev5, released March 29, 2019 Contents diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 8c6c40d..a7ce796 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -14,9 +14,9 @@ class ColumnFromExpression(): """ Template to register a column of derived data with Orca, based on an expression. The column will be associated with an existing table. Values will be calculated lazily, - only when the column is requested for a specific operation. + only when the column is needed for a specific operation. - The expression will be passed to ``df.eval()`` and can refer to other columns in the + The expression will be passed to ``df.eval()`` and can refer to any columns in the same table. See the Pandas documentation for further details. All the parameters can also be set as properties after creating the template From b6dced80c7f3337e27560d2e020d9ca2cfe8eaf7 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 1 Apr 2019 17:12:17 -0700 Subject: [PATCH 087/121] CoreTemplateSettings --- urbansim_templates/shared/core.py | 101 ++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 urbansim_templates/shared/core.py diff --git a/urbansim_templates/shared/core.py b/urbansim_templates/shared/core.py new file mode 100644 index 0000000..5880afd --- /dev/null +++ b/urbansim_templates/shared/core.py @@ -0,0 +1,101 @@ +from __future__ import print_function + +from urbansim_templates import __version__ + + +class CoreTemplateSettings(): + """ + Stores standard parameters and logic used by all templates. Parameters can be passed + to the constructor or set as attributes. + + Parameters + ---------- + name : str, optional + Name of the configured template instance. + + tags : list of str, optional + Tags associated with the configured template instance. + + notes : str, optional + Notes associates with the configured template instance. + + autorun : bool, optional + Whether to run the configured template instance automatically when it's + registered or loaded by ModelManager. The overall default is False, but the + default can be overriden at the template level. + + template : str + Name of the template class associated with a configured instance. + + template_version : str + Version of the template class package. + + Attributes + ---------- + modelmanager_version : str + Version of the ModelManager package that created the CoreTemplateSettings. + + """ + def __init__(self, + name = None, + tags = [], + notes = None, + autorun = False, + template = None, + template_version = None): + + self.name = name + self.tags = tags + self.notes = notes + self.autorun = autorun + self.template = template + self.template_version = template_version + + # automatic attributes + self.modelmanager_version = __version__ + + + @classmethod + def from_dict(cls, d): + """ + Create a class instance from a saved dictionary representation. + + Parameters + ---------- + d : dict + + Returns + ------- + meta : CoreTemplateSettings + + """ + obj = cls( + name = d['name'], + tags = d['tags'], + notes = d['notes'], + autorun = d['autorun'], + template = d['template'], + template_version = d['template_version'], + ) + return d + + + def to_dict(self): + """ + Create a dictionary representation of the object. + + Returns + ------- + d : dict + + """ + d = { + 'name': self.name, + 'tags': self.tags, + 'notes': self.notes, + 'autorun': self.autorun, + 'template': self.template, + 'template_version': self.template_version, + 'modelmanager_version': self.modelmanager_version, + } + From 942ff7abec7eeecc02cd881723d23ade88f88e1b Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 1 Apr 2019 18:45:34 -0700 Subject: [PATCH 088/121] Work in progress --- .../data/column_from_expression.py | 89 +++++++++++-------- urbansim_templates/shared/__init__.py | 1 + 2 files changed, 51 insertions(+), 39 deletions(-) create mode 100644 urbansim_templates/shared/__init__.py diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index a7ce796..45ae175 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -6,6 +6,7 @@ import pandas as pd from urbansim_templates import modelmanager, __version__ +from urbansim_templates.shared import CoreTemplateSettings from urbansim_templates.utils import get_df @@ -19,11 +20,13 @@ class ColumnFromExpression(): The expression will be passed to ``df.eval()`` and can refer to any columns in the same table. See the Pandas documentation for further details. - All the parameters can also be set as properties after creating the template - instance. + Parameters can be passed to the constructor or set as attributes. Parameters ---------- + meta : :mod:`~urbansim_templates.shared.CoreTemplateSettings`, optional + Stores a name for the configured template and other standard settings. + column_name : str, optional Name of the Orca column to be registered. Required before running. @@ -48,31 +51,25 @@ class ColumnFromExpression(): cache_scope : 'step', 'iteration', or 'forever', default 'forever' How long to cache column values for (ignored if ``cache`` is False). - - name : str, optional - Name of the template instance and associated model step. - - tags : list of str, optional - Tags to associate with the template instance. - - autorun : bool, default True - Whether to run automatically when the template instance is registered with - ModelManager. - + """ - def __init__(self, - column_name = None, - table = None, - expression = None, - data_type = None, - missing_values = None, - cache = False, - cache_scope = 'forever', - name = None, - tags = [], - autorun = True): - - # Template-specific params + def __init__(self, + meta = None, + column_name = None, + table = None, + expression = None, + data_type = None, + missing_values = None, + cache = False, + cache_scope = 'forever'): + + if meta is None: + self.meta = CoreTemplateSettings() + + self.meta.template = self.__class__.__name__ + self.meta.template_version = __version__ + + # Template-specific settings self.column_name = column_name self.table = table self.expression = expression @@ -80,19 +77,37 @@ def __init__(self, self.missing_values = missing_values self.cache = cache self.cache_scope = cache_scope + + + @classmethod + def from_dict(cls, d): + """ + Create an object instance from a saved dictionary representation. - # Standard params - self.name = name - self.tags = tags - self.autorun = autorun + Parameters + ---------- + d : dict - # Automatic params - self.template = self.__class__.__name__ - self.template_version = __version__ + Returns + ------- + Table + + """ + obj = cls( + meta = d['meta'], + column_name = d['column_name'], + table = d['table'], + expression = d['expression'], + data_type = d['data_type'], + missing_values = d['missing_values'], + cache = d['cache'], + cache_scope = d['cache_scope'], + ) + return obj @classmethod - def from_dict(cls, d): + def from_dict_0_2_dev5(cls, d): """ Create an object instance from a saved dictionary representation. @@ -130,11 +145,7 @@ def to_dict(self): """ d = { - 'template': self.template, - 'template_version': self.template_version, - 'name': self.name, - 'tags': self.tags, - 'autorun': self.autorun, + 'meta': self.meta.to_dict(), 'column_name': self.column_name, 'table': self.table, 'expression': self.expression, diff --git a/urbansim_templates/shared/__init__.py b/urbansim_templates/shared/__init__.py new file mode 100644 index 0000000..7cdc673 --- /dev/null +++ b/urbansim_templates/shared/__init__.py @@ -0,0 +1 @@ +from .core import CoreTemplateSettings From 7516717be579fe14d3f5546ae75ed27cf18d8308 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 2 Apr 2019 09:58:00 -0700 Subject: [PATCH 089/121] Updating ModelManager to support CoreTemplateSettings --- tests/test_column_expression.py | 16 +++--- .../data/column_from_expression.py | 5 +- urbansim_templates/modelmanager.py | 52 ++++++++++++++----- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index 4af02df..810434a 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -26,12 +26,12 @@ def orca_session(): orca.add_table('obs', df) -def test_template_validity(): - """ - Check template conforms to basic spec. - - """ - assert validate_template(ColumnFromExpression) +# def test_template_validity(): +# """ +# Check template conforms to basic spec. +# +# """ +# assert validate_template(ColumnFromExpression) def test_missing_colname(orca_session): @@ -163,7 +163,7 @@ def test_modelmanager_registration(orca_session): c.expression = 'a + b' modelmanager.register(c) - modelmanager.remove_step(c.name) + modelmanager.remove_step(c.meta.name) assert('c' in orca.get_table('obs').columns) @@ -179,7 +179,7 @@ def test_expression_with_standalone_columns(orca_session): c.expression = 'a + b' modelmanager.register(c) - modelmanager.remove_step(c.name) + modelmanager.remove_step(c.meta.name) d = ColumnFromExpression() d.column_name = 'd' diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 45ae175..46b2791 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -25,7 +25,8 @@ class ColumnFromExpression(): Parameters ---------- meta : :mod:`~urbansim_templates.shared.CoreTemplateSettings`, optional - Stores a name for the configured template and other standard settings. + Stores a name for the configured template and other standard settings. For + column templates, the default for 'autorun' is True. column_name : str, optional Name of the Orca column to be registered. Required before running. @@ -64,7 +65,7 @@ def __init__(self, cache_scope = 'forever'): if meta is None: - self.meta = CoreTemplateSettings() + self.meta = CoreTemplateSettings(autorun=True) self.meta.template = self.__class__.__name__ self.meta.template_version = __version__ diff --git a/urbansim_templates/modelmanager.py b/urbansim_templates/modelmanager.py index c1f4582..da064f2 100644 --- a/urbansim_templates/modelmanager.py +++ b/urbansim_templates/modelmanager.py @@ -97,12 +97,14 @@ def build_step(d): object """ + template = d['meta']['template'] if 'meta' in d else d['template'] + if 'supplemental_objects' in d: for i, item in enumerate(d['supplemental_objects']): content = load_supplemental_object(d['name'], **item) d['supplemental_objects'][i]['content'] = content - return _templates[d['template']].from_dict(d) + return _templates[template].from_dict(d) def load_supplemental_object(step_name, name, content_type, required=True): @@ -151,25 +153,36 @@ def register(step, save_to_disk=True): None """ - if step.name is None: - step.name = update_name(step.template, step.name) # TO DO - test this + # Currently supporting both step.name and step.meta.name + if hasattr(step, 'meta'): + # TO DO: move the name updating to CoreTemplateSettings? + step.meta.name = update_name(step.meta.template, step.meta.name) + name = step.meta.name + + else: + step.name = update_name(step.template, step.name) + name = step.name if save_to_disk: save_step_to_disk(step) - print("Registering model step '{}'".format(step.name)) + print("Registering model step '{}'".format(name)) - _steps[step.name] = step + _steps[name] = step # Create a callable that runs the model step, and register it with orca def run_step(): return step.run() - orca.add_step(step.name, run_step) + orca.add_step(name, run_step) + + if hasattr(step, 'meta'): + if step.meta.autorun: + orca.run([name]) - if hasattr(step, 'autorun'): + elif hasattr(step, 'autorun'): if step.autorun: - orca.run([step.name]) + orca.run([name]) def list_steps(): @@ -181,9 +194,18 @@ def list_steps(): list of dicts, ordered by name """ - return [{'name': _steps[k].name, - 'template': type(_steps[k]).__name__, - 'tags': _steps[k].tags} for k in sorted(_steps.keys())] + steps = [] + for k in sorted(_steps.keys()): + if hasattr(_steps[k], 'meta'): + steps += [{'name': _steps[k].meta.name, + 'template': _steps[k].meta.template, + 'tags': _steps[k].meta.tags, + 'notes': _steps[k].meta.notes}] + else: + steps += [{'name': _steps[k].name, + 'template': _steps[k].template, + 'tags': _steps[k].tags}] + return steps def save_step_to_disk(step): @@ -192,11 +214,13 @@ def save_step_to_disk(step): 'model-name.yaml' and will be saved to the initialization directory. """ + name = step.meta.name if hasattr(step, 'meta') else step.name + if _disk_store is None: print("Please run 'modelmanager.initialize()' before registering new model steps") return - print("Saving '{}.yaml': {}".format(step.name, + print("Saving '{}.yaml': {}".format(name, os.path.join(os.getcwd(), _disk_store))) d = step.to_dict() @@ -204,7 +228,7 @@ def save_step_to_disk(step): # Save supplemental objects if 'supplemental_objects' in d: for item in filter(None, d['supplemental_objects']): - save_supplemental_object(step.name, **item) + save_supplemental_object(name, **item) del item['content'] # Save main yaml file @@ -213,7 +237,7 @@ def save_step_to_disk(step): content = OrderedDict(headers) content.update({'saved_object': d}) - yamlio.convert_to_yaml(content, os.path.join(_disk_store, step.name+'.yaml')) + yamlio.convert_to_yaml(content, os.path.join(_disk_store, name+'.yaml')) def save_supplemental_object(step_name, name, content, content_type, required=True): From e6380ca43011f212587d243018c22a613e1f3f41 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 2 Apr 2019 11:40:54 -0700 Subject: [PATCH 090/121] Tests for CoreTemplateSettings --- tests/test_shared_core.py | 26 ++++++++++++++++++++++++++ urbansim_templates/shared/core.py | 5 ++--- 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 tests/test_shared_core.py diff --git a/tests/test_shared_core.py b/tests/test_shared_core.py new file mode 100644 index 0000000..855e762 --- /dev/null +++ b/tests/test_shared_core.py @@ -0,0 +1,26 @@ +from __future__ import print_function + +import pytest + +from urbansim_templates.shared import CoreTemplateSettings + + +def test_property_persistence(): + """ + Confirm properties persist through to_dict() and from_dict(). + + """ + obj = CoreTemplateSettings() + obj.name = 'name' + obj.tags = ['tag1', 'tag2'] + obj.notes = 'notes' + obj.autorun = True + obj.template = 'CoolNewTemplate' + obj.template_version = '0.1.dev0' + + d = obj.to_dict() + print(d) + + obj2 = CoreTemplateSettings.from_dict(d) + assert(obj2.to_dict() == d) + diff --git a/urbansim_templates/shared/core.py b/urbansim_templates/shared/core.py index 5880afd..7c226ca 100644 --- a/urbansim_templates/shared/core.py +++ b/urbansim_templates/shared/core.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from urbansim_templates import __version__ @@ -77,7 +75,7 @@ def from_dict(cls, d): template = d['template'], template_version = d['template_version'], ) - return d + return obj def to_dict(self): @@ -98,4 +96,5 @@ def to_dict(self): 'template_version': self.template_version, 'modelmanager_version': self.modelmanager_version, } + return d From cea79097e85eddfb3180537f24f5b0ee67d2ddc6 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 2 Apr 2019 13:43:26 -0700 Subject: [PATCH 091/121] Adding OutputColumnSettings --- tests/test_column_expression.py | 20 ++-- .../data/column_from_expression.py | 70 +++++-------- urbansim_templates/shared/__init__.py | 1 + urbansim_templates/shared/output_column.py | 97 +++++++++++++++++++ 4 files changed, 130 insertions(+), 58 deletions(-) create mode 100644 urbansim_templates/shared/output_column.py diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index 810434a..b5d8bbd 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -58,7 +58,7 @@ def test_missing_table(orca_session): """ c = ColumnFromExpression() - c.column_name = 'col' + c.output.column_name = 'col' c.expression = 'a' try: @@ -76,7 +76,7 @@ def test_missing_expression(orca_session): """ c = ColumnFromExpression() - c.column_name = 'col' + c.output.column_name = 'col' c.table = 'tab' try: @@ -94,7 +94,7 @@ def test_expression(orca_session): """ c = ColumnFromExpression() - c.column_name = 'c' + c.output.column_name = 'c' c.table = 'obs' c.expression = 'a * 5 + sqrt(b)' @@ -114,7 +114,7 @@ def test_data_type(orca_session): orca.add_table('tab', pd.DataFrame({'a': [0.1, 1.33, 2.4]})) c = ColumnFromExpression() - c.column_name = 'b' + c.output.column_name = 'b' c.table = 'tab' c.expression = 'a' c.run() @@ -122,7 +122,7 @@ def test_data_type(orca_session): v1 = orca.get_table('tab').get_column('b').values np.testing.assert_equal(v1, [0.1, 1.33, 2.4]) - c.data_type = 'int' + c.output.data_type = 'int' c.run() v1 = orca.get_table('tab').get_column('b').values @@ -137,7 +137,7 @@ def test_missing_values(orca_session): orca.add_table('tab', pd.DataFrame({'a': [0.1, np.nan, 2.4]})) c = ColumnFromExpression() - c.column_name = 'b' + c.output.column_name = 'b' c.table = 'tab' c.expression = 'a' c.run() @@ -145,7 +145,7 @@ def test_missing_values(orca_session): v1 = orca.get_table('tab').get_column('b').values np.testing.assert_equal(v1, [0.1, np.nan, 2.4]) - c.missing_values = 5 + c.output.missing_values = 5 c.run() v1 = orca.get_table('tab').get_column('b').values @@ -158,7 +158,7 @@ def test_modelmanager_registration(orca_session): """ c = ColumnFromExpression() - c.column_name = 'c' + c.output.column_name = 'c' c.table = 'obs' c.expression = 'a + b' @@ -174,7 +174,7 @@ def test_expression_with_standalone_columns(orca_session): """ c = ColumnFromExpression() - c.column_name = 'c' + c.output.column_name = 'c' c.table = 'obs' c.expression = 'a + b' @@ -182,7 +182,7 @@ def test_expression_with_standalone_columns(orca_session): modelmanager.remove_step(c.meta.name) d = ColumnFromExpression() - d.column_name = 'd' + d.output.column_name = 'd' d.table = 'obs' d.expression = 'a + c' diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 46b2791..0a545f6 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -1,12 +1,10 @@ -from __future__ import print_function - import re import orca import pandas as pd from urbansim_templates import modelmanager, __version__ -from urbansim_templates.shared import CoreTemplateSettings +from urbansim_templates.shared import CoreTemplateSettings, OutputColumnSettings from urbansim_templates.utils import get_df @@ -28,9 +26,6 @@ class ColumnFromExpression(): Stores a name for the configured template and other standard settings. For column templates, the default for 'autorun' is True. - column_name : str, optional - Name of the Orca column to be registered. Required before running. - table : str, optional Name of the Orca table the column will be associated with. Required before running. @@ -41,28 +36,15 @@ class ColumnFromExpression(): including sqrt, abs, log, log1p, exp, and expm1 -- see Pandas ``df.eval()`` documentation for further details. - data_type : str, optional - Python type or ``numpy.dtype`` to cast the column's values into. - - missing_values : str or numeric, optional - Value to use for rows that would otherwise be missing. - - cache : bool, default False - Whether to cache column values after they are calculated. - - cache_scope : 'step', 'iteration', or 'forever', default 'forever' - How long to cache column values for (ignored if ``cache`` is False). + output : :mod:`~urbansim_templates.shared.OutputColumnSettings`, optional + Stores settings for the column that will be generated. """ def __init__(self, meta = None, - column_name = None, table = None, expression = None, - data_type = None, - missing_values = None, - cache = False, - cache_scope = 'forever'): + output = None): if meta is None: self.meta = CoreTemplateSettings(autorun=True) @@ -71,13 +53,11 @@ def __init__(self, self.meta.template_version = __version__ # Template-specific settings - self.column_name = column_name self.table = table self.expression = expression - self.data_type = data_type - self.missing_values = missing_values - self.cache = cache - self.cache_scope = cache_scope + + if output is None: + self.output = OutputColumnSettings() @classmethod @@ -96,13 +76,9 @@ def from_dict(cls, d): """ obj = cls( meta = d['meta'], - column_name = d['column_name'], table = d['table'], expression = d['expression'], - data_type = d['data_type'], - missing_values = d['missing_values'], - cache = d['cache'], - cache_scope = d['cache_scope'], + output = d['output'], ) return obj @@ -147,13 +123,9 @@ def to_dict(self): """ d = { 'meta': self.meta.to_dict(), - 'column_name': self.column_name, 'table': self.table, 'expression': self.expression, - 'data_type': self.data_type, - 'missing_values': self.missing_values, - 'cache': self.cache, - 'cache_scope': self.cache_scope, + 'output': self.output.to_dict(), } return d @@ -169,10 +141,12 @@ def run(self): None """ - if self.column_name is None: + if self.output.column_name is None: raise ValueError("Please provide a column name") - if self.table is None: + table = self.table if self.output.table is None else self.output.table + + if table is None: raise ValueError("Please provide a table") if self.expression is None: @@ -186,19 +160,19 @@ def run(self): # invalid column names will be ignored when we request them from get_df(). cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression) - @orca.column(table_name = self.table, - column_name = self.column_name, - cache = self.cache, - cache_scope = self.cache_scope) + @orca.column(table_name = table, + column_name = self.output.column_name, + cache = self.output.cache, + cache_scope = self.output.cache_scope) def orca_column(): - df = get_df(self.table, columns=cols) + df = get_df(table, columns=cols) series = df.eval(self.expression) - if self.missing_values is not None: - series = series.fillna(self.missing_values) + if self.output.missing_values is not None: + series = series.fillna(self.output.missing_values) - if self.data_type is not None: - series = series.astype(self.data_type) + if self.output.data_type is not None: + series = series.astype(self.output.data_type) return series diff --git a/urbansim_templates/shared/__init__.py b/urbansim_templates/shared/__init__.py index 7cdc673..67e76ac 100644 --- a/urbansim_templates/shared/__init__.py +++ b/urbansim_templates/shared/__init__.py @@ -1 +1,2 @@ from .core import CoreTemplateSettings +from .output_column import OutputColumnSettings diff --git a/urbansim_templates/shared/output_column.py b/urbansim_templates/shared/output_column.py new file mode 100644 index 0000000..6ff376f --- /dev/null +++ b/urbansim_templates/shared/output_column.py @@ -0,0 +1,97 @@ +from urbansim_templates import __version__ + + +class OutputColumnSettings(): + """ + Stores standard parameters and logic used by templates that generate or modify + columns. Parameters can be passed to the constructor or set as attributes. + + Parameters + ---------- + column_name : str, optional + Name of the Orca column to be created or modified. Generally required before + running a configured template. + + table : str, optional + Name of Orca table the column will be associated with. Generally required before + running the configured template. + + data_type : str, optional + Python type or ``numpy.dtype`` to case the column's values to. + + missing_values : str or numeric, optional + Value to use for rows that would otherwise be missing. + + cache : bool, default False + Whether to cache column values after they are calculated + + cache_scope : 'step', 'iteration', or 'forever', default 'forever' + How long to cache column values for (ignored if ``cache`` is False). + + """ + # TO DO: say something about Orca defaults and about core vs. computed columns. + + def __init__(self, + column_name = None, + table = None, + data_type = None, + missing_values = None, + cache = False, + cache_scope = 'forever'): + + self.column_name = column_name + self.table = table + self.data_type = data_type + self.missing_values = missing_values + self.cache = cache + self.cache_scope = cache_scope + + # automatic attributes + self.modelmanager_version = __version__ + + + @classmethod + def from_dict(cls, d): + """ + Create a class instance from a saved dictionary representation. + + Parameters + ---------- + d : dict + + Returns + ------- + meta : OutputColumnSettings + + """ + obj = cls( + column_name = d['column_name'], + table = d['table'], + data_type = d['data_type'], + missing_values = d['missing_values'], + cache = d['cache'], + cache_scope = d['cache_scope'], + ) + return obj + + + def to_dict(self): + """ + Create a dictionary representation of the object. + + Returns + ------- + d : dict + + """ + d = { + 'column_name': self.column_name, + 'table': self.table, + 'data_type': self.data_type, + 'missing_values': self.missing_values, + 'cache': self.cache, + 'cache_scope': self.cache_scope, + 'modelmanager_version': self.modelmanager_version, + } + return d + From 60fe04d8ec79fb43273071295f8d8a390b6aa79a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 2 Apr 2019 14:23:07 -0700 Subject: [PATCH 092/121] Adding ExpressionSettings --- tests/test_column_expression.py | 32 ++-- tests/test_shared_output_column.py | 26 +++ .../data/column_from_expression.py | 149 ++++++++---------- urbansim_templates/shared/core.py | 2 +- urbansim_templates/shared/output_column.py | 14 +- 5 files changed, 111 insertions(+), 112 deletions(-) create mode 100644 tests/test_shared_output_column.py diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index b5d8bbd..3204b87 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -40,8 +40,8 @@ def test_missing_colname(orca_session): """ c = ColumnFromExpression() - c.table = 'tab' - c.expression = 'a' + c.data.table = 'tab' + c.data.expression = 'a' try: c.run() @@ -58,8 +58,8 @@ def test_missing_table(orca_session): """ c = ColumnFromExpression() + c.data.expression = 'a' c.output.column_name = 'col' - c.expression = 'a' try: c.run() @@ -76,8 +76,8 @@ def test_missing_expression(orca_session): """ c = ColumnFromExpression() + c.data.table = 'tab' c.output.column_name = 'col' - c.table = 'tab' try: c.run() @@ -94,9 +94,9 @@ def test_expression(orca_session): """ c = ColumnFromExpression() + c.data.table = 'obs' + c.data.expression = 'a * 5 + sqrt(b)' c.output.column_name = 'c' - c.table = 'obs' - c.expression = 'a * 5 + sqrt(b)' c.run() @@ -114,9 +114,9 @@ def test_data_type(orca_session): orca.add_table('tab', pd.DataFrame({'a': [0.1, 1.33, 2.4]})) c = ColumnFromExpression() + c.data.table = 'tab' + c.data.expression = 'a' c.output.column_name = 'b' - c.table = 'tab' - c.expression = 'a' c.run() v1 = orca.get_table('tab').get_column('b').values @@ -137,9 +137,9 @@ def test_missing_values(orca_session): orca.add_table('tab', pd.DataFrame({'a': [0.1, np.nan, 2.4]})) c = ColumnFromExpression() + c.data.table = 'tab' + c.data.expression = 'a' c.output.column_name = 'b' - c.table = 'tab' - c.expression = 'a' c.run() v1 = orca.get_table('tab').get_column('b').values @@ -158,9 +158,9 @@ def test_modelmanager_registration(orca_session): """ c = ColumnFromExpression() + c.data.table = 'obs' + c.data.expression = 'a + b' c.output.column_name = 'c' - c.table = 'obs' - c.expression = 'a + b' modelmanager.register(c) modelmanager.remove_step(c.meta.name) @@ -174,17 +174,17 @@ def test_expression_with_standalone_columns(orca_session): """ c = ColumnFromExpression() + c.data.table = 'obs' + c.data.expression = 'a + b' c.output.column_name = 'c' - c.table = 'obs' - c.expression = 'a + b' modelmanager.register(c) modelmanager.remove_step(c.meta.name) d = ColumnFromExpression() + d.data.table = 'obs' + d.data.expression = 'a + c' d.output.column_name = 'd' - d.table = 'obs' - d.expression = 'a + c' d.run() assert('d' in orca.get_table('obs').columns) diff --git a/tests/test_shared_output_column.py b/tests/test_shared_output_column.py new file mode 100644 index 0000000..c1ef94d --- /dev/null +++ b/tests/test_shared_output_column.py @@ -0,0 +1,26 @@ +from __future__ import print_function + +import pytest + +from urbansim_templates.shared import CoreTemplateSettings + + +def test_property_persistence(): + """ + Confirm properties persist through to_dict() and from_dict(). + + """ + obj = CoreTemplateSettings() + obj.column_name = 'column' + obj.table = 'table' + obj.data_type = 'int32' + obj.missing_values = 5 + obj.cache = True + obj.cache_scope = 'iteration' + + d = obj.to_dict() + print(d) + + obj2 = CoreTemplateSettings.from_dict(d) + assert(obj2.to_dict() == d) + diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 0a545f6..ded27d0 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -8,6 +8,36 @@ from urbansim_templates.utils import get_df +class ExpressionSettings(): + """ + Stores custom parameters used by the ColumnFromExpression template. Parameters can be + passed to the constructor or set as attributes. + + Parameters + ---------- + table : str, optional + Name of Orca table the expression will be evaluated on. Required before running + then template. + + expression : str, optional + String describing operations on existing columns of the table, for example + "a/log(b+c)". Required before running. Supports arithmetic and math functions + including sqrt, abs, log, log1p, exp, and expm1 -- see Pandas ``df.eval()`` + documentation for further details. + + """ + def __init__(self, table = None, expression = None): + self.table = table + self.expression = expression + + @classmethod + def from_dict(cls, d): + return cls(table=d['table'], expression=d['expression']) + + def to_dict(self): + return {'table': self.table, 'expression': self.expression} + + @modelmanager.template class ColumnFromExpression(): """ @@ -18,86 +48,49 @@ class ColumnFromExpression(): The expression will be passed to ``df.eval()`` and can refer to any columns in the same table. See the Pandas documentation for further details. - Parameters can be passed to the constructor or set as attributes. - Parameters ---------- meta : :mod:`~urbansim_templates.shared.CoreTemplateSettings`, optional - Stores a name for the configured template and other standard settings. For - column templates, the default for 'autorun' is True. - - table : str, optional - Name of the Orca table the column will be associated with. Required before - running. - - expression : str, optional - String describing operations on existing columns of the table, for example - "a/log(b+c)". Required before running. Supports arithmetic and math functions - including sqrt, abs, log, log1p, exp, and expm1 -- see Pandas ``df.eval()`` - documentation for further details. + Standard parameters. This template sets the default value of ``meta.autorun`` + to True. + data : :mod:`~urbansim_templates.data.ExpressionSettings`, optional + Special parameters for this template. + output : :mod:`~urbansim_templates.shared.OutputColumnSettings`, optional - Stores settings for the column that will be generated. + Parameters for the column that will be generated. This template uses + ``data.table`` as the default value for ``output.table``. """ - def __init__(self, - meta = None, - table = None, - expression = None, - output = None): - - if meta is None: - self.meta = CoreTemplateSettings(autorun=True) + def __init__(self, meta=None, data=None, output=None): + self.meta = CoreTemplateSettings(autorun=True) if meta is None else meta self.meta.template = self.__class__.__name__ self.meta.template_version = __version__ - # Template-specific settings - self.table = table - self.expression = expression - - if output is None: - self.output = OutputColumnSettings() + self.data = ExpressionSettings() if data is None else data + self.output = OutputColumnSettings() if output is None else output @classmethod def from_dict(cls, d): - """ - Create an object instance from a saved dictionary representation. - - Parameters - ---------- - d : dict - Returns - ------- - Table + if 'meta' not in d: + return ColumnFromExpression.from_dict_0_2_dev5(d) - """ - obj = cls( - meta = d['meta'], - table = d['table'], - expression = d['expression'], - output = d['output'], - ) - return obj + return cls( + meta = CoreTemplateSettings.from_dict(d['meta']), + data = ExpressionSettings.from_dict(d['data']), + output = OutputColumnSettings.from_dict(d['output'])) @classmethod def from_dict_0_2_dev5(cls, d): """ - Create an object instance from a saved dictionary representation. - - Parameters - ---------- - d : dict - - Returns - ------- - Table + Converter to read saved data from 0.2.dev5 or earlier. """ - obj = cls( + return cls( column_name = d['column_name'], table = d['table'], expression = d['expression'], @@ -107,58 +100,42 @@ def from_dict_0_2_dev5(cls, d): cache_scope = d['cache_scope'], name = d['name'], tags = d['tags'], - autorun = d['autorun'] - ) - return obj + autorun = d['autorun']) def to_dict(self): - """ - Create a dictionary representation of the object. - - Returns - ------- - dict - - """ - d = { - 'meta': self.meta.to_dict(), - 'table': self.table, - 'expression': self.expression, - 'output': self.output.to_dict(), - } - return d + return { + 'meta': self.meta.to_dict(), + 'data': self.data.to_dict(), + 'output': self.output.to_dict()} def run(self): """ Run the template, registering a column of derived data with Orca. - Requires values to be set for ``column_name``, ``table``, and ``expression``. - - Returns - ------- - None + Requires values to be set for ``data.table``, ``data.expression``, and + ``output.column_name``. """ - if self.output.column_name is None: - raise ValueError("Please provide a column name") - - table = self.table if self.output.table is None else self.output.table + table = self.data.table if self.output.table is None else self.output.table if table is None: raise ValueError("Please provide a table") - if self.expression is None: + if self.data.expression is None: raise ValueError("Please provide an expression") + if self.output.column_name is None: + raise ValueError("Please provide a column name") + # Some column names in the expression may not be part of the core DataFrame, so # we'll need to request them from Orca explicitly. This regex pulls out column # names into a list, by identifying tokens in the expression that begin with a # letter and contain any number of alphanumerics or underscores, but do not end # with an opening parenthesis. This will also pick up constants, like "pi", but # invalid column names will be ignored when we request them from get_df(). - cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.expression) + cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.data.expression) @orca.column(table_name = table, column_name = self.output.column_name, @@ -166,7 +143,7 @@ def run(self): cache_scope = self.output.cache_scope) def orca_column(): df = get_df(table, columns=cols) - series = df.eval(self.expression) + series = df.eval(self.data.expression) if self.output.missing_values is not None: series = series.fillna(self.output.missing_values) diff --git a/urbansim_templates/shared/core.py b/urbansim_templates/shared/core.py index 7c226ca..02f099b 100644 --- a/urbansim_templates/shared/core.py +++ b/urbansim_templates/shared/core.py @@ -64,7 +64,7 @@ def from_dict(cls, d): Returns ------- - meta : CoreTemplateSettings + obj : CoreTemplateSettings """ obj = cls( diff --git a/urbansim_templates/shared/output_column.py b/urbansim_templates/shared/output_column.py index 6ff376f..d8d5759 100644 --- a/urbansim_templates/shared/output_column.py +++ b/urbansim_templates/shared/output_column.py @@ -61,18 +61,16 @@ def from_dict(cls, d): Returns ------- - meta : OutputColumnSettings + obj : OutputColumnSettings """ - obj = cls( + return cls( column_name = d['column_name'], table = d['table'], data_type = d['data_type'], missing_values = d['missing_values'], cache = d['cache'], - cache_scope = d['cache_scope'], - ) - return obj + cache_scope = d['cache_scope']) def to_dict(self): @@ -84,14 +82,12 @@ def to_dict(self): d : dict """ - d = { + return { 'column_name': self.column_name, 'table': self.table, 'data_type': self.data_type, 'missing_values': self.missing_values, 'cache': self.cache, 'cache_scope': self.cache_scope, - 'modelmanager_version': self.modelmanager_version, - } - return d + 'modelmanager_version': self.modelmanager_version} From 5d1ed62368db2c8ae83cdb82d37131f3daa67fce Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 2 Apr 2019 15:40:20 -0700 Subject: [PATCH 093/121] Converter for older yaml files --- .../data/column_from_expression.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index ded27d0..dbe559a 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -42,12 +42,10 @@ def to_dict(self): class ColumnFromExpression(): """ Template to register a column of derived data with Orca, based on an expression. The - column will be associated with an existing table. Values will be calculated lazily, - only when the column is needed for a specific operation. - - The expression will be passed to ``df.eval()`` and can refer to any columns in the - same table. See the Pandas documentation for further details. - + expression can refer to any columns in the same table, and will be evaluated using + ``df.eval()``. Values will be calculated lazily, only when the column is needed for + a specific operation. + Parameters ---------- meta : :mod:`~urbansim_templates.shared.CoreTemplateSettings`, optional @@ -76,7 +74,7 @@ def __init__(self, meta=None, data=None, output=None): def from_dict(cls, d): if 'meta' not in d: - return ColumnFromExpression.from_dict_0_2_dev5(d) + return cls.from_dict_0_2_dev5(d) return cls( meta = CoreTemplateSettings.from_dict(d['meta']), @@ -91,16 +89,19 @@ def from_dict_0_2_dev5(cls, d): """ return cls( - column_name = d['column_name'], - table = d['table'], - expression = d['expression'], - data_type = d['data_type'], - missing_values = d['missing_values'], - cache = d['cache'], - cache_scope = d['cache_scope'], - name = d['name'], - tags = d['tags'], - autorun = d['autorun']) + meta = CoreTemplateSettings( + name = d['name'], + tags = d['tags'], + autorun = d['autorun']), + data = ExpressionSettings( + table = d['table'], + expression = d['expression']), + output = OutputColumnSettings( + column_name = d['column_name'], + data_type = d['data_type'], + missing_values = d['missing_values'], + cache = d['cache'], + cache_scope = d['cache_scope'])) def to_dict(self): From f159d532d041f8f4963ee21cd57bd57268f45188 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 3 Apr 2019 17:55:58 -0700 Subject: [PATCH 094/121] Refactoring column utilities --- tests/test_column_expression.py | 18 ++++++- urbansim_templates/data/__init__.py | 2 +- .../data/column_from_expression.py | 54 +++++++++++-------- urbansim_templates/shared/__init__.py | 2 +- urbansim_templates/shared/output_column.py | 35 ++++++++++++ urbansim_templates/utils.py | 22 ++++++++ 6 files changed, 109 insertions(+), 24 deletions(-) diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index 3204b87..394c7c9 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -5,10 +5,26 @@ import orca from urbansim_templates import modelmanager -from urbansim_templates.data import ColumnFromExpression +from urbansim_templates.data import ColumnFromExpression, ExpressionSettings from urbansim_templates.utils import validate_template +def test_expression_settings_persistence(): + """ + Confirm ExpressionSettings properties persist through to_dict() and from_dict(). + + """ + obj = ExpressionSettings() + obj.table = 'table' + obj.expression = 'expression' + + d = obj.to_dict() + print(d) + + obj2 = ExpressionSettings.from_dict(d) + assert(obj2.to_dict() == d) + + @pytest.fixture def orca_session(): """ diff --git a/urbansim_templates/data/__init__.py b/urbansim_templates/data/__init__.py index 90dc264..e9c7c54 100644 --- a/urbansim_templates/data/__init__.py +++ b/urbansim_templates/data/__init__.py @@ -1,3 +1,3 @@ -from .column_from_expression import ColumnFromExpression +from .column_from_expression import ColumnFromExpression, ExpressionSettings from .load_table import LoadTable from .save_table import SaveTable diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index dbe559a..9cd547c 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -1,11 +1,8 @@ -import re - import orca import pandas as pd -from urbansim_templates import modelmanager, __version__ +from urbansim_templates import modelmanager, shared, utils, __version__ from urbansim_templates.shared import CoreTemplateSettings, OutputColumnSettings -from urbansim_templates.utils import get_df class ExpressionSettings(): @@ -119,9 +116,10 @@ def run(self): ``output.column_name``. """ - table = self.data.table if self.output.table is None else self.output.table - if table is None: +# table = self.data.table if self.output.table is None else self.output.table + + if self.data.table is None: raise ValueError("Please provide a table") if self.data.expression is None: @@ -130,28 +128,42 @@ def run(self): if self.output.column_name is None: raise ValueError("Please provide a column name") + settings = self.output + + if settings.table is None: + settings.table = self.data.table + # Some column names in the expression may not be part of the core DataFrame, so # we'll need to request them from Orca explicitly. This regex pulls out column # names into a list, by identifying tokens in the expression that begin with a # letter and contain any number of alphanumerics or underscores, but do not end # with an opening parenthesis. This will also pick up constants, like "pi", but # invalid column names will be ignored when we request them from get_df(). - cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.data.expression) - - @orca.column(table_name = table, - column_name = self.output.column_name, - cache = self.output.cache, - cache_scope = self.output.cache_scope) - def orca_column(): - df = get_df(table, columns=cols) +# cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.data.expression) + + cols = utils.cols_in_expression(self.data.expression) + + def build_column(): + df = utils.get_df(self.data.table, columns=cols) series = df.eval(self.data.expression) - - if self.output.missing_values is not None: - series = series.fillna(self.output.missing_values) - - if self.output.data_type is not None: - series = series.astype(self.output.data_type) - return series + + shared.register_column(build_column, settings) + +# @orca.column(table_name = table, +# column_name = self.output.column_name, +# cache = self.output.cache, +# cache_scope = self.output.cache_scope) +# def orca_column(): +# df = get_df(table, columns=cols) +# series = df.eval(self.data.expression) +# +# if self.output.missing_values is not None: +# series = series.fillna(self.output.missing_values) +# +# if self.output.data_type is not None: +# series = series.astype(self.output.data_type) +# +# return series \ No newline at end of file diff --git a/urbansim_templates/shared/__init__.py b/urbansim_templates/shared/__init__.py index 67e76ac..c2c00b1 100644 --- a/urbansim_templates/shared/__init__.py +++ b/urbansim_templates/shared/__init__.py @@ -1,2 +1,2 @@ from .core import CoreTemplateSettings -from .output_column import OutputColumnSettings +from .output_column import OutputColumnSettings, register_column diff --git a/urbansim_templates/shared/output_column.py b/urbansim_templates/shared/output_column.py index d8d5759..7198438 100644 --- a/urbansim_templates/shared/output_column.py +++ b/urbansim_templates/shared/output_column.py @@ -1,3 +1,5 @@ +import orca + from urbansim_templates import __version__ @@ -91,3 +93,36 @@ def to_dict(self): 'cache_scope': self.cache_scope, 'modelmanager_version': self.modelmanager_version} + +###################################### +###################################### + + +def register_column(build_column, settings): + """ + Register a callable as an Orca column. + + Parameters + ---------- + build_column : callable + Callable should return a ``pd.Series``. + + settings : ColumnOutputSettings + + """ + @orca.column(table_name = settings.table, + column_name = settings.column_name, + cache = settings.cache, + cache_scope = settings.cache_scope) + + def orca_column(): + series = build_column() + + if settings.missing_values is not None: + series = series.fillna(settings.missing_values) + + if settings.data_type is not None: + series = series.astype(settings.data_type) + + return series + diff --git a/urbansim_templates/utils.py b/urbansim_templates/utils.py index 73dc6d7..7879ffb 100644 --- a/urbansim_templates/utils.py +++ b/urbansim_templates/utils.py @@ -1,5 +1,6 @@ from __future__ import print_function +import re from datetime import datetime as dt import pandas as pd @@ -346,6 +347,27 @@ def all_cols(table): return list(table.index.names) + list(table.columns) +def cols_in_expression(expression): + """ + Extract all possible column names from a ``df.eval()``-style expression. + + This is achieved using regex to identify tokens in the expression that begin with a + letter and contain any number of alphanumerics or underscores, but do not end with an + opening parenthesis. This excludes function names, but would not exclude constants + (e.g. "pi"), which are semantically indistinguishable from column names. + + Parameters + ---------- + expression : str + + Returns + ------- + cols : list of str + + """ + return re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', expression) + + def trim_cols(df, columns=None): """ Limit a DataFrame to columns that appear in a list of names. List may contain From 1a666f28ca11ae494c323d5e9812b0be08b0f023 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 3 Apr 2019 21:08:31 -0700 Subject: [PATCH 095/121] Remaining tests --- tests/test_column_expression.py | 88 +++++++------------ tests/test_shared_core.py | 2 +- tests/test_shared_output_column.py | 77 +++++++++++++++- .../data/column_from_expression.py | 27 ------ 4 files changed, 108 insertions(+), 86 deletions(-) diff --git a/tests/test_column_expression.py b/tests/test_column_expression.py index 394c7c9..dc288c4 100644 --- a/tests/test_column_expression.py +++ b/tests/test_column_expression.py @@ -11,18 +11,44 @@ def test_expression_settings_persistence(): """ - Confirm ExpressionSettings properties persist through to_dict() and from_dict(). + Confirm ExpressionSettings properties persist through the constructor, to_dict(), + and from_dict(). """ - obj = ExpressionSettings() - obj.table = 'table' - obj.expression = 'expression' + d = {'table': 'tab', 'expression': 'a + b + c'} + obj = ExpressionSettings(table = 'tab', expression = 'a + b + c') - d = obj.to_dict() - print(d) + assert(d == obj.to_dict() == ExpressionSettings.from_dict(d).to_dict()) + + +def test_legacy_data_loader(orca_session): + """ + Check that loading a saved dict with the legacy format works. + + """ + d = { + 'name': 'n', + 'tags': ['a', 'b'], + 'autorun': False, + 'column_name': 'col', + 'table': 'tab', + 'expression': 'abc', + 'data_type': 'int', + 'missing_values': 5, + 'cache': True, + 'cache_scope': 'step'} - obj2 = ExpressionSettings.from_dict(d) - assert(obj2.to_dict() == d) + c = ColumnFromExpression.from_dict(d) + assert(c.meta.name == d['name']) + assert(c.meta.tags == d['tags']) + assert(c.meta.autorun == d['autorun']) + assert(c.data.table == d['table']) + assert(c.data.expression == d['expression']) + assert(c.output.column_name == d['column_name']) + assert(c.output.data_type == d['data_type']) + assert(c.output.missing_values == d['missing_values']) + assert(c.output.cache == d['cache']) + assert(c.output.cache_scope == d['cache_scope']) @pytest.fixture @@ -122,52 +148,6 @@ def test_expression(orca_session): assert(val1.equals(val2)) -def test_data_type(orca_session): - """ - Check that casting data type works. - - """ - orca.add_table('tab', pd.DataFrame({'a': [0.1, 1.33, 2.4]})) - - c = ColumnFromExpression() - c.data.table = 'tab' - c.data.expression = 'a' - c.output.column_name = 'b' - c.run() - - v1 = orca.get_table('tab').get_column('b').values - np.testing.assert_equal(v1, [0.1, 1.33, 2.4]) - - c.output.data_type = 'int' - c.run() - - v1 = orca.get_table('tab').get_column('b').values - np.testing.assert_equal(v1, [0, 1, 2]) - - -def test_missing_values(orca_session): - """ - Check that filling in missing values works. - - """ - orca.add_table('tab', pd.DataFrame({'a': [0.1, np.nan, 2.4]})) - - c = ColumnFromExpression() - c.data.table = 'tab' - c.data.expression = 'a' - c.output.column_name = 'b' - c.run() - - v1 = orca.get_table('tab').get_column('b').values - np.testing.assert_equal(v1, [0.1, np.nan, 2.4]) - - c.output.missing_values = 5 - c.run() - - v1 = orca.get_table('tab').get_column('b').values - np.testing.assert_equal(v1, [0.1, 5.0, 2.4]) - - def test_modelmanager_registration(orca_session): """ Check that modelmanager registration and auto-run work as expected. diff --git a/tests/test_shared_core.py b/tests/test_shared_core.py index 855e762..d0018b3 100644 --- a/tests/test_shared_core.py +++ b/tests/test_shared_core.py @@ -7,7 +7,7 @@ def test_property_persistence(): """ - Confirm properties persist through to_dict() and from_dict(). + Confirm CoreTemplateSettings properties persist through to_dict() and from_dict(). """ obj = CoreTemplateSettings() diff --git a/tests/test_shared_output_column.py b/tests/test_shared_output_column.py index c1ef94d..f2627f9 100644 --- a/tests/test_shared_output_column.py +++ b/tests/test_shared_output_column.py @@ -1,16 +1,20 @@ from __future__ import print_function +import numpy as np +import pandas as pd import pytest -from urbansim_templates.shared import CoreTemplateSettings +import orca + +from urbansim_templates.shared import OutputColumnSettings, register_column def test_property_persistence(): """ - Confirm properties persist through to_dict() and from_dict(). + Confirm OutputColumnSettings properties persist through to_dict() and from_dict(). """ - obj = CoreTemplateSettings() + obj = OutputColumnSettings() obj.column_name = 'column' obj.table = 'table' obj.data_type = 'int32' @@ -21,6 +25,71 @@ def test_property_persistence(): d = obj.to_dict() print(d) - obj2 = CoreTemplateSettings.from_dict(d) + obj2 = OutputColumnSettings.from_dict(d) assert(obj2.to_dict() == d) + +# Tests for register_column().. + +@pytest.fixture +def orca_session(): + """ + Set up a clean Orca session, with a data table. + + """ + orca.clear_all() + + df = pd.DataFrame({'a': [0.1, 1.33, 2.4]}, index=[1,2,3]) + orca.add_table('tab', df) + + +def test_column_registration(orca_session): + """ + Confirm column registration works. + + """ + series = pd.Series([4,5,6], index=[1,2,3]) + + def build_column(): + return series + + settings = OutputColumnSettings(column_name='col', table='tab') + register_column(build_column, settings) + + assert(orca.get_table('tab').get_column('col').equals(series)) + + +def test_filling_missing_values(orca_session): + """ + Confirm that filling missing values works. + + """ + series1 = pd.Series([4.0, np.nan, 6.0], index=[1,2,3]) + series2 = pd.Series([4.0, 5.0, 6.0], index=[1,2,3]) + + def build_column(): + return series1 + + settings = OutputColumnSettings(column_name='col', table='tab', missing_values=5) + register_column(build_column, settings) + + assert(orca.get_table('tab').get_column('col').equals(series2)) + + +def test_casting_data_type(orca_session): + """ + Confirm that filling missing values works. + + """ + series1 = pd.Series([4.0, 5.0, 6.0], index=[1,2,3]) + series2 = pd.Series([4, 5, 6], index=[1,2,3]) + + def build_column(): + return series1 + + settings = OutputColumnSettings(column_name='col', table='tab', data_type='int') + register_column(build_column, settings) + + assert(orca.get_table('tab').get_column('col').equals(series2)) + + diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 9cd547c..437b3eb 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -116,9 +116,6 @@ def run(self): ``output.column_name``. """ - -# table = self.data.table if self.output.table is None else self.output.table - if self.data.table is None: raise ValueError("Please provide a table") @@ -133,14 +130,6 @@ def run(self): if settings.table is None: settings.table = self.data.table - # Some column names in the expression may not be part of the core DataFrame, so - # we'll need to request them from Orca explicitly. This regex pulls out column - # names into a list, by identifying tokens in the expression that begin with a - # letter and contain any number of alphanumerics or underscores, but do not end - # with an opening parenthesis. This will also pick up constants, like "pi", but - # invalid column names will be ignored when we request them from get_df(). -# cols = re.findall('[a-zA-Z_][a-zA-Z0-9_]*(?!\()', self.data.expression) - cols = utils.cols_in_expression(self.data.expression) def build_column(): @@ -149,21 +138,5 @@ def build_column(): return series shared.register_column(build_column, settings) - -# @orca.column(table_name = table, -# column_name = self.output.column_name, -# cache = self.output.cache, -# cache_scope = self.output.cache_scope) -# def orca_column(): -# df = get_df(table, columns=cols) -# series = df.eval(self.data.expression) -# -# if self.output.missing_values is not None: -# series = series.fillna(self.output.missing_values) -# -# if self.output.data_type is not None: -# series = series.astype(self.output.data_type) -# -# return series \ No newline at end of file From daa4c03bb6aceb6c7f3d008b1e2048123ff8b56a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 4 Apr 2019 10:31:14 -0700 Subject: [PATCH 096/121] Versioning and changelog --- CHANGELOG.md | 14 ++++++++++---- docs/source/index.rst | 2 +- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c196666..0fd22e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,15 @@ ## 0.2 (not yet released) +#### 0.2.dev6 (2019-04-04) + +- introduces classes for storing common settings: `shared.CoreTemplateSettings`, `shared.OutputColumnSettings` +- adds new shared functions: `shared.register_column()`, `utils.cols_in_expression()` +- modifies `ColumnFromExpression` template to divide its parameters into three groups + #### 0.2.dev5 (2019-03-29) -- adds new template: `urbansim_templates.data.ColumnFromExpression` +- adds new template: `data.ColumnFromExpression` #### 0.2.dev4 (2019-03-26) @@ -20,8 +26,8 @@ #### 0.2.dev2 (2019-03-04) -- adds template for saving data: `urbansim_templates.data.SaveTable()` -- renames `TableFromDisk()` to `urbansim_templates.data.LoadTable()` +- adds template for saving data: `data.SaveTable()` +- renames `io.TableFromDisk()` to `data.LoadTable()` #### 0.2.dev1 (2019-02-27) @@ -29,7 +35,7 @@ #### 0.2.dev0 (2019-02-19) -- adds first data i/o template: `urbansim_templates.io.TableFromDisk()` +- adds first data i/o template: `io.TableFromDisk()` - adds support for `autorun` template property diff --git a/docs/source/index.rst b/docs/source/index.rst index 1e5ce05..10451ec 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev5, released March 29, 2019 +v0.2.dev6, released April 4, 2019 Contents diff --git a/setup.py b/setup.py index b77543a..e9e99af 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.2.dev5', + version='0.2.dev6', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index aba1d24..8157a0e 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev5' +version = __version__ = '0.2.dev6' From 07d6f70e31496c581e79825c86eacc2bcb741652 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 4 Apr 2019 14:28:39 -0700 Subject: [PATCH 097/121] Documentation updates --- docs/source/conf.py | 1 + docs/source/data-templates.rst | 45 +++++++---- docs/source/utilities.rst | 81 ++++++++++++++++--- .../data/column_from_expression.py | 28 ++++--- urbansim_templates/data/load_table.py | 5 +- urbansim_templates/data/save_table.py | 5 +- urbansim_templates/shared/core.py | 5 -- urbansim_templates/shared/output_column.py | 4 +- 8 files changed, 124 insertions(+), 50 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 9d7541f..1faf741 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -37,6 +37,7 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode'] diff --git a/docs/source/data-templates.rst b/docs/source/data-templates.rst index 9317a3e..ab8b676 100644 --- a/docs/source/data-templates.rst +++ b/docs/source/data-templates.rst @@ -1,13 +1,10 @@ -Data template APIs -================== +Data management templates +========================= Usage ----- -Data templates help you load tables into `Orca `__ or save tables or subsets of tables to disk. - -Example -~~~~~~~ +Data templates help you load tables into `Orca `__, create columns of derived data, or save tables or subsets of tables to disk. .. code-block:: python @@ -75,22 +72,42 @@ From Orca's perspective, tables set up using the :mod:`~urbansim_templates.data. Unlike the templates, Orca relies on user-specified "`broadcast `__" relationships to perform automatic merging of tables. :mod:`~urbansim_templates.data.LoadTable` does not register any broadcasts, because they're not needed if tables follow the schema rules above. So if you use these tables in non-template model steps, you may need to add broadcasts separately. -LoadTable() ------------ +Data loading API +---------------- + +.. currentmodule:: urbansim_templates.data + +.. autosummary:: + LoadTable .. autoclass:: urbansim_templates.data.LoadTable :members: -SaveTable() ------------ +Column creation API +------------------- -.. autoclass:: urbansim_templates.data.SaveTable +.. currentmodule:: urbansim_templates.data + +.. autosummary:: + ColumnFromExpression + ExpressionSettings + +.. autoclass:: urbansim_templates.data.ColumnFromExpression :members: +.. autoclass:: urbansim_templates.data.ExpressionSettings + :members: -ColumnFromExpression() ----------------------- +Data output API +--------------- -.. autoclass:: urbansim_templates.data.ColumnFromExpression +.. currentmodule:: urbansim_templates.data + +.. autosummary:: + SaveTable + +.. autoclass:: urbansim_templates.data.SaveTable :members: + + diff --git a/docs/source/utilities.rst b/docs/source/utilities.rst index 0e3f76e..a86d392 100644 --- a/docs/source/utilities.rst +++ b/docs/source/utilities.rst @@ -1,32 +1,87 @@ -Utilities API -============= +Shared utilities +================ The utilities are mainly helper functions for templates. -Template validation -------------------- +General template tools API +-------------------------- -.. automodule:: urbansim_templates.utils - :members: validate_template +.. currentmodule:: urbansim_templates.shared + +.. autosummary:: + CoreTemplateSettings + +.. automodule:: urbansim_templates.shared + :members: CoreTemplateSettings + + +Column output tools API +----------------------- +.. currentmodule:: urbansim_templates.shared -Table schemas and merging -------------------------- +.. autosummary:: + OutputColumnSettings + register_column + +.. automodule:: urbansim_templates.shared + :members: OutputColumnSettings, register_column + + +Table schemas and merging API +----------------------------- + +.. currentmodule:: urbansim_templates.utils + +.. autosummary:: + validate_table + validate_all_tables + merge_tables .. automodule:: urbansim_templates.utils :members: validate_table, validate_all_tables, merge_tables -Other helper functions ----------------------- +Other helper functions API +-------------------------- + +.. currentmodule:: urbansim_templates.utils + +.. autosummary:: + all_cols + cols_in_expression + get_data + get_df + trim_cols + to_list + update_column + update_name .. automodule:: urbansim_templates.utils - :members: all_cols, get_data, get_df, trim_cols, update_column, to_list, update_column, update_name + :members: all_cols, cols_in_expression, get_data, get_df, trim_cols, to_list, update_column, update_name + + +Spec validation API +------------------- + +.. currentmodule:: urbansim_templates.utils + +.. autosummary:: + validate_template + +.. automodule:: urbansim_templates.utils + :members: validate_template + + +Version management API +---------------------- +.. currentmodule:: urbansim_templates.utils -Version management ------------------- +.. autosummary:: + parse_version + version_greater_or_equal .. automodule:: urbansim_templates.utils :members: parse_version, version_greater_or_equal diff --git a/urbansim_templates/data/column_from_expression.py b/urbansim_templates/data/column_from_expression.py index 437b3eb..bf2ae34 100644 --- a/urbansim_templates/data/column_from_expression.py +++ b/urbansim_templates/data/column_from_expression.py @@ -7,7 +7,8 @@ class ExpressionSettings(): """ - Stores custom parameters used by the ColumnFromExpression template. Parameters can be + Stores custom parameters used by the + :mod:`~urbansim_templates.data.ColumnFromExpression` template. Parameters can be passed to the constructor or set as attributes. Parameters @@ -38,10 +39,11 @@ def to_dict(self): @modelmanager.template class ColumnFromExpression(): """ - Template to register a column of derived data with Orca, based on an expression. The - expression can refer to any columns in the same table, and will be evaluated using - ``df.eval()``. Values will be calculated lazily, only when the column is needed for - a specific operation. + Template to register a column of derived data with Orca, based on an expression. + Parameters may be passed to the constructor, but they are easier to set as + attributes. The expression can refer to any columns in the same table, and will be + evaluated using ``df.eval()``. Values will be calculated lazily, only when the column + is needed for a specific operation. Parameters ---------- @@ -69,7 +71,10 @@ def __init__(self, meta=None, data=None, output=None): @classmethod def from_dict(cls, d): + """ + Create a class instance from a saved dictionary. + """ if 'meta' not in d: return cls.from_dict_0_2_dev5(d) @@ -82,7 +87,8 @@ def from_dict(cls, d): @classmethod def from_dict_0_2_dev5(cls, d): """ - Converter to read saved data from 0.2.dev5 or earlier. + Converter to read saved data from 0.2.dev5 or earlier. Automatically invoked by + ``from_dict()`` as needed. """ return cls( @@ -102,6 +108,10 @@ def from_dict_0_2_dev5(cls, d): def to_dict(self): + """ + Create a dictionary representation of the object. + + """ return { 'meta': self.meta.to_dict(), 'data': self.data.to_dict(), @@ -110,10 +120,8 @@ def to_dict(self): def run(self): """ - Run the template, registering a column of derived data with Orca. - - Requires values to be set for ``data.table``, ``data.expression``, and - ``output.column_name``. + Run the template, registering a column of derived data with Orca. Requires values + to be set for ``data.table``, ``data.expression``, and ``output.column_name``. """ if self.data.table is None: diff --git a/urbansim_templates/data/load_table.py b/urbansim_templates/data/load_table.py index 344ecf6..3aa6f55 100644 --- a/urbansim_templates/data/load_table.py +++ b/urbansim_templates/data/load_table.py @@ -16,13 +16,12 @@ @modelmanager.template class LoadTable(): """ - Class for registering data tables from local CSV or HDF files. + Template for registering data tables from local CSV or HDF files. Parameters can be + passed to the constructor or set as attributes. An instance of this template class stores *instructions for loading a data table*, packaged into an Orca step. Running the instructions registers the table with Orca. - All the parameters can also be set as properties after creating the class instance. - Parameters ---------- table : str, optional diff --git a/urbansim_templates/data/save_table.py b/urbansim_templates/data/save_table.py index c05813f..052f928 100644 --- a/urbansim_templates/data/save_table.py +++ b/urbansim_templates/data/save_table.py @@ -12,9 +12,8 @@ @modelmanager.template class SaveTable(): """ - Class for saving Orca tables to local CSV or HDF5 files. - - All the parameters can also be set as properties after creating the class instance. + Template for saving Orca tables to local CSV or HDF5 files. Parameters can be passed + to the constructor or set as attributes. Parameters ---------- diff --git a/urbansim_templates/shared/core.py b/urbansim_templates/shared/core.py index 02f099b..e5ba71b 100644 --- a/urbansim_templates/shared/core.py +++ b/urbansim_templates/shared/core.py @@ -28,11 +28,6 @@ class CoreTemplateSettings(): template_version : str Version of the template class package. - Attributes - ---------- - modelmanager_version : str - Version of the ModelManager package that created the CoreTemplateSettings. - """ def __init__(self, name = None, diff --git a/urbansim_templates/shared/output_column.py b/urbansim_templates/shared/output_column.py index 7198438..e66566b 100644 --- a/urbansim_templates/shared/output_column.py +++ b/urbansim_templates/shared/output_column.py @@ -5,8 +5,8 @@ class OutputColumnSettings(): """ - Stores standard parameters and logic used by templates that generate or modify - columns. Parameters can be passed to the constructor or set as attributes. + Stores standard parameters used by templates that generate or modify columns. + Parameters can be passed to the constructor or set as attributes. Parameters ---------- From 7744ad4f2af3758ccb189125fd728b1364a2b8cd Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 4 Apr 2019 14:33:12 -0700 Subject: [PATCH 098/121] Updating travis script --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99709b9..e294b14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,8 @@ matrix: - python: '3.7' # temp solution until python 3.7 is more cleanly supported dist: xenial sudo: true - allow_failures: - - python: '3.7' # dependencies are blocking installation - fast_finish: true install: - - pip install git+git://github.com/udst/choicemodels.git - pip install . - pip install -r requirements-extras.txt - pip install -r requirements-dev.txt From eeca0280a131e2f1e87fdf9552286db29bbe7b9c Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 15 Jul 2019 15:31:39 -0700 Subject: [PATCH 099/121] Fixing least squares out-transformation bug --- tests/test_regression.py | 21 +++++++++++++++++++++ urbansim_templates/models/regression.py | 15 +++++++++------ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/tests/test_regression.py b/tests/test_regression.py index 1220395..cd6ea23 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -64,3 +64,24 @@ def test_simulation(orca_session): assert orca.get_table('obs').to_frame()['a_predicted'].equals(m.predicted_values) + +def test_out_transform(orca_session): + """ + Test transformation of the predicted values. + + """ + modelmanager.initialize() + + m = OLSRegressionStep() + m.tables = 'obs' + m.model_expression = 'a ~ b' + m.fit() + + m.out_column = 'a_predicted' + m.out_transform = 'np.exp' + m.run() + + predictions = m.predicted_values.apply(np.exp) + + assert orca.get_table('obs').to_frame()['a_predicted'].equals(predictions) + diff --git a/urbansim_templates/models/regression.py b/urbansim_templates/models/regression.py index 7683498..9d6b843 100644 --- a/urbansim_templates/models/regression.py +++ b/urbansim_templates/models/regression.py @@ -1,5 +1,6 @@ from __future__ import print_function +import math import numpy as np import pandas as pd from datetime import datetime as dt @@ -69,10 +70,12 @@ class OLSRegressionStep(TemplateStep): side variable from the model expression will be used. Replaces the `out_fname` argument in UrbanSim. - out_transform : callable, optional - Transformation to apply to the predicted values, for example to reverse a - transformation of the left-hand-side variable in the model expression. Replaces - the `ytransform` argument in UrbanSim. + out_transform : str, optional + Element-wise transformation to apply to the predicted values, for example to + reverse a transformation of the left-hand-side variable in the model expression. + This should be provided as a string containing a function name. Supports anything + from NumPy or Python's built-in math library, for example 'np.exp' or + 'math.floor'. Replaces the `ytransform` argument in UrbanSim. out_filters : str or list of str, optional Filters to apply to the data before simulation. If not provided, no filters will @@ -168,7 +171,7 @@ def fit(self): """ self.model = RegressionModel(model_expression=self.model_expression, fit_filters=self.filters, predict_filters=self.out_filters, - ytransform=self.out_transform, name=self.name) + ytransform=None, name=self.name) df = get_data(tables = self.tables, filters = self.filters, @@ -207,7 +210,7 @@ def run(self): self.predicted_values = values if self.out_transform is not None: - values = self.out_transform(values) + values = values.apply(eval(self.out_transform)) colname = self._get_out_column() tabname = self._get_out_table() From ed63144f06843c23166f6a8664fd6a0449f08a51 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 15 Jul 2019 18:13:11 -0700 Subject: [PATCH 100/121] Updating versioning and changelog --- CHANGELOG.md | 9 +++++++++ docs/source/index.rst | 2 +- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fd22e4..026f9d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 0.2 (not yet released) +#### 0.2.dev7 (2019-07-15) + +- fixes a bug with the `out_transform` parameter for `OLSRegressionStep` + #### 0.2.dev6 (2019-04-04) - introduces classes for storing common settings: `shared.CoreTemplateSettings`, `shared.OutputColumnSettings` @@ -39,6 +43,11 @@ - adds support for `autorun` template property +## 0.1.3 (2019-07-15) + +- patch to incorporate the `out_transform` bug fix for `OLSRegressionStep`, from 0.2.dev7 + + ## 0.1.2 (2019-02-28) - patch to incorporate the small MNL bug fix from 0.2.dev1 diff --git a/docs/source/index.rst b/docs/source/index.rst index 10451ec..2ad48b2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev6, released April 4, 2019 +v0.2.dev7, released July 15, 2019 Contents diff --git a/setup.py b/setup.py index e9e99af..511b093 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim_templates', - version='0.2.dev6', + version='0.2.dev7', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index 8157a0e..baa94ae 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev6' +version = __version__ = '0.2.dev7' From 825611d5b7acb1fb37bbf22272f0f0fcc3793b5f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 22 Jul 2019 09:25:51 -0700 Subject: [PATCH 101/121] Updating release date --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 2ad48b2..4f888e6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev7, released July 15, 2019 +v0.2.dev7, released July 22, 2019 Contents From 0b55c25bd17eb5d0a9e9887ebae3f74a90de4fbb Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Mon, 13 Apr 2020 17:48:48 +0000 Subject: [PATCH 102/121] add chooser filtering for backdoor mct to accommodate segmented mnl use case --- .../models/large_multinomial_logit.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/urbansim_templates/models/large_multinomial_logit.py b/urbansim_templates/models/large_multinomial_logit.py index 12004cc..a3f1fd2 100644 --- a/urbansim_templates/models/large_multinomial_logit.py +++ b/urbansim_templates/models/large_multinomial_logit.py @@ -1,13 +1,15 @@ from __future__ import print_function import orca -from urbansim.models.util import columns_in_formula +from urbansim.models.util import columns_in_formula, apply_filter_query +from choicemodels.tools import MergedChoiceTable from .. import modelmanager from ..utils import get_data, update_column, to_list, version_greater_or_equal from .shared import TemplateStep + def check_choicemodels_version(): try: import choicemodels @@ -458,8 +460,12 @@ def fit(self, mct=None): from choicemodels.tools import MergedChoiceTable if (mct is not None): - data = mct - + df_from_mct = mct.to_frame() + idx_names = df_from_mct.index.names + df_from_mct = df_from_mct.reset_index() + df_from_mct = apply_filter_query(df_from_mct, self.chooser_filters) + mct = MergedChoiceTable.from_df(df_from_mct).set_index(idx_names) + else: observations = get_data(tables = self.choosers, filters = self.chooser_filters, @@ -606,4 +612,3 @@ def probs(mct): column = self.out_column, fallback_column = self.choice_column, data = choices) - From 6d6f9383046e318091a181ae76ad9fd3ed7a43e7 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Mon, 13 Apr 2020 20:59:33 +0000 Subject: [PATCH 103/121] ensure mct in large mnl is filtered by segmentation column for segmented large mnl use case --- urbansim_templates/models/large_multinomial_logit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/urbansim_templates/models/large_multinomial_logit.py b/urbansim_templates/models/large_multinomial_logit.py index a3f1fd2..fab92ce 100644 --- a/urbansim_templates/models/large_multinomial_logit.py +++ b/urbansim_templates/models/large_multinomial_logit.py @@ -463,8 +463,9 @@ def fit(self, mct=None): df_from_mct = mct.to_frame() idx_names = df_from_mct.index.names df_from_mct = df_from_mct.reset_index() - df_from_mct = apply_filter_query(df_from_mct, self.chooser_filters) - mct = MergedChoiceTable.from_df(df_from_mct).set_index(idx_names) + df_from_mct = apply_filter_query( + df_from_mct, self.chooser_filters).set_index(idx_names) + mct = MergedChoiceTable.from_df(df_from_mct) else: observations = get_data(tables = self.choosers, From 86adbac4021bf1a3eb012a897180b5c0873487af Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Mon, 13 Apr 2020 21:01:44 +0000 Subject: [PATCH 104/121] enable backdoor mct for segmented large mnl --- .../segmented_large_multinomial_logit.py | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/urbansim_templates/models/segmented_large_multinomial_logit.py b/urbansim_templates/models/segmented_large_multinomial_logit.py index 1d0b19a..d290072 100644 --- a/urbansim_templates/models/segmented_large_multinomial_logit.py +++ b/urbansim_templates/models/segmented_large_multinomial_logit.py @@ -120,31 +120,41 @@ def to_dict(self): return d - def get_segmentation_column(self): + def get_segmentation_column(self, mct=None): """ Get the column of segmentation values from Orca. Chooser and alternative filters are applied to identify valid observations. + Parameters + ---------- + mct : choicemodels.tools.MergedChoiceTable + This parameter is a temporary backdoor allowing us to pass in a more + complicated choice table than can be generated within the template, for + example including sampling weights or interaction terms. + Returns ------- pd.Series """ - obs = get_data(tables = self.defaults.choosers, - filters = self.defaults.chooser_filters, - extra_columns = [self.defaults.choice_column, - self.segmentation_column]) + if mct is not None: + df = mct.to_frame() + else: + obs = get_data(tables = self.defaults.choosers, + filters = self.defaults.chooser_filters, + extra_columns = [self.defaults.choice_column, + self.segmentation_column]) - alts = get_data(tables = self.defaults.alternatives, - filters = self.defaults.alt_filters) + alts = get_data(tables = self.defaults.alternatives, + filters = self.defaults.alt_filters) - df = pd.merge(obs, alts, how='inner', - left_on=self.defaults.choice_column, right_index=True) + df = pd.merge(obs, alts, how='inner', + left_on=self.defaults.choice_column, right_index=True) return df[self.segmentation_column] - def build_submodels(self): + def build_submodels(self, mct=None): """ Create a submodel for each category of choosers identified in the segmentation column. Only categories with at least one observation remaining after applying @@ -152,11 +162,18 @@ def build_submodels(self): Running this method will overwrite any previous submodels. + Parameters + ---------- + mct : choicemodels.tools.MergedChoiceTable + This parameter is a temporary backdoor allowing us to pass in a more + complicated choice table than can be generated within the template, for + example including sampling weights or interaction terms. + """ self.submodels = {} submodel = LargeMultinomialLogitStep.from_dict(self.defaults.to_dict()) - col = self.get_segmentation_column() + col = self.get_segmentation_column(mct=mct) if (len(col) == 0): print("Warning: No valid observations after applying the chooser and "+ @@ -214,7 +231,7 @@ def update_submodels(self, param, value): def fit_all(self, mct=None): """ - Fit all the submodels. Build the subomdels first, if they don't exist yet. This + Fit all the submodels. Build the submodels first, if they don't exist yet. This method can be run as many times as desired. Parameters @@ -227,7 +244,7 @@ def fit_all(self, mct=None): """ if (len(self.submodels) == 0): - self.build_submodels() + self.build_submodels(mct=mct) for k, m in self.submodels.items(): print(' SEGMENT: {0} = {1} '.format( From 0f345b2cabc648002626eb74bfd1cac4664425fe Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 17 Apr 2020 10:32:44 -0700 Subject: [PATCH 105/121] Updating requirements to fix travis failure --- requirements.txt | 9 --------- setup.py | 15 ++++++++++----- 2 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c3584b4..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# minimal requirements for model management and core templates - -choicemodels >= 0.2.dev4 -numpy >= 1.14 -orca >= 1.4 -pandas >= 0.23 -patsy >= 0.4 -statsmodels >= 0.8 -urbansim >= 3.1 diff --git a/setup.py b/setup.py index 511b093..a12d56f 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,5 @@ from setuptools import setup, find_packages -with open('requirements.txt') as f: - requirements = f.readlines() -requirements = [item.strip() for item in requirements] - setup( name='urbansim_templates', version='0.2.dev7', @@ -20,5 +16,14 @@ 'License :: OSI Approved :: BSD License' ], packages=find_packages(exclude=['*.tests']), - install_requires=requirements + install_requires=[ + 'choicemodels >= 0.2.dev4', + 'numpy >= 1.14', + 'orca >= 1.4', + 'pandas >= 0.23', + 'patsy >= 0.4', + 'statsmodels >= 0.8, <0.11; python_version <"3.6"', + 'statsmodels >= 0.8; python_version >="3.6"' + 'urbansim >= 3.1' + ] ) \ No newline at end of file From 5ae4fe5cc2625cf82f8e2b90397db9c448fabfc5 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 17 Apr 2020 10:37:08 -0700 Subject: [PATCH 106/121] Fixing syntax --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a12d56f..97ca0ad 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ 'pandas >= 0.23', 'patsy >= 0.4', 'statsmodels >= 0.8, <0.11; python_version <"3.6"', - 'statsmodels >= 0.8; python_version >="3.6"' + 'statsmodels >= 0.8; python_version >="3.6"', 'urbansim >= 3.1' ] ) \ No newline at end of file From 6ab79ab4c87941ff1679fdacc0e5c70f4102a167 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 17 Apr 2020 10:41:09 -0700 Subject: [PATCH 107/121] Adding newer python versions --- .travis.yml | 8 ++------ setup.py | 2 ++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e294b14..423a934 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,8 @@ python: - '2.7' - '3.5' - '3.6' - -matrix: - include: - - python: '3.7' # temp solution until python 3.7 is more cleanly supported - dist: xenial - sudo: true + - '3.7' + - '3.8' install: - pip install . diff --git a/setup.py b/setup.py index 97ca0ad..b06e97b 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: BSD License' ], packages=find_packages(exclude=['*.tests']), From 246a8dc616cbceda246ed9343889389046c3a6b3 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Fri, 17 Apr 2020 18:51:29 +0000 Subject: [PATCH 108/121] version bump and requirements bump --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b06e97b..ec63593 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='urbansim_templates', - version='0.2.dev7', + version='0.2.dev8', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', @@ -19,7 +19,7 @@ ], packages=find_packages(exclude=['*.tests']), install_requires=[ - 'choicemodels >= 0.2.dev4', + 'choicemodels >= 0.2.2.dev1', 'numpy >= 1.14', 'orca >= 1.4', 'pandas >= 0.23', From 58e8556271f09ab50bc20267234a8e6705967609 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Fri, 17 Apr 2020 20:07:21 +0000 Subject: [PATCH 109/121] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec63593..3fb451e 100644 --- a/setup.py +++ b/setup.py @@ -28,4 +28,4 @@ 'statsmodels >= 0.8; python_version >="3.6"', 'urbansim >= 3.1' ] -) \ No newline at end of file +) From 27672150c5d511b2b6b48fe162f3835527c6f764 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Fri, 17 Apr 2020 20:44:20 +0000 Subject: [PATCH 110/121] rebuild trigger --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3fb451e..2d7a28f 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='urbansim_templates', - version='0.2.dev8', + version='0.2.dev8', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', From b86b73542f5da3a135cd6a38cf9bb7912c706f7a Mon Sep 17 00:00:00 2001 From: cvanoli Date: Wed, 13 May 2020 12:14:59 -0300 Subject: [PATCH 111/121] include the out_column to create df in binary logit --- urbansim_templates/models/binary_logit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/urbansim_templates/models/binary_logit.py b/urbansim_templates/models/binary_logit.py index 9253c24..79b3fbe 100644 --- a/urbansim_templates/models/binary_logit.py +++ b/urbansim_templates/models/binary_logit.py @@ -227,7 +227,8 @@ def run(self): df = get_data(tables = self.out_tables, fallback_tables = self.tables, filters = self.out_filters, - model_expression = self.model_expression) + model_expression = self.model_expression, + extra_columns = self.out_column) dm = patsy.dmatrices(data=df, formula_like=self.model_expression, return_type='dataframe')[1] # right-hand-side design matrix From a2a34486c5f5c83d28df5a3d9a13ce7730af8161 Mon Sep 17 00:00:00 2001 From: cvanoli Date: Wed, 13 May 2020 12:15:57 -0300 Subject: [PATCH 112/121] Save regression residuals as object element --- urbansim_templates/models/regression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/urbansim_templates/models/regression.py b/urbansim_templates/models/regression.py index 9d6b843..d8cbb05 100644 --- a/urbansim_templates/models/regression.py +++ b/urbansim_templates/models/regression.py @@ -101,6 +101,7 @@ def __init__(self, tables=None, model_expression=None, filters=None, out_tables= # Placeholders for model fit data, filled in by fit() or from_dict() self.summary_table = None self.fitted_parameters = None + self.residuals = None self.model = None @@ -189,7 +190,7 @@ def fit(self): # code later on to not rely on RegressionModel any more. self.fitted_parameters = results.params.tolist() - + self.residuals = results.resid def run(self): """ From 061fabff82cf72ce1df8ee50f6791262201cfa31 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 15 May 2020 17:43:28 -0700 Subject: [PATCH 113/121] Updating version and changelog --- CHANGELOG.md | 9 +++++++++ docs/source/index.rst | 2 +- setup.py | 2 +- urbansim_templates/__init__.py | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 026f9d5..eb537a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## 0.2 (not yet released) +#### 0.2.dev9 (2020-05-15) + +- fixes a bug in `BinaryLogitStep` simulation where the output is not updated correctly +- adds a `resid` attribute to fitted `OLSRegressionStep` models, for diagnostics + +#### 0.2.dev8 (2020-04-17) + +- allows segmented large MNL models to be estimated with a `MergedChoiceTable` that's passed in by the user (rather than generated automatically), thus achieving parity with the non-segmented model class + #### 0.2.dev7 (2019-07-15) - fixes a bug with the `out_transform` parameter for `OLSRegressionStep` diff --git a/docs/source/index.rst b/docs/source/index.rst index 4f888e6..a676c6b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ UrbanSim Templates provides building blocks for Orca-based simulation models. It The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the `Orca `__ task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. -v0.2.dev7, released July 22, 2019 +v0.2.dev9, released July 22, 2019 Contents diff --git a/setup.py b/setup.py index 2d7a28f..1143390 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='urbansim_templates', - version='0.2.dev8', + version='0.2.dev9', description='UrbanSim extension for managing model steps', author='UrbanSim Inc.', author_email='info@urbansim.com', diff --git a/urbansim_templates/__init__.py b/urbansim_templates/__init__.py index baa94ae..c57ac9f 100644 --- a/urbansim_templates/__init__.py +++ b/urbansim_templates/__init__.py @@ -1 +1 @@ -version = __version__ = '0.2.dev7' +version = __version__ = '0.2.dev9' From 11f2de38a5254744bb2acb94310d61623a8ac7a4 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 5 Feb 2021 15:00:18 -0800 Subject: [PATCH 114/121] Updating docs --- docs/README.md | 32 +++++++++++++++++++------- docs/source/data-templates.rst | 18 --------------- docs/source/utilities.rst | 41 ---------------------------------- 3 files changed, 24 insertions(+), 67 deletions(-) diff --git a/docs/README.md b/docs/README.md index eaa350e..8e7831a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,19 +1,35 @@ -## Documentation +This folder generates the UrbanSim Templates online documentation, hosted at https://udst.github.io/urbansim_templates/. -Documentation is generated using [Sphinx](http://sphinx-doc.org) and hosted with Github Pages at http://udst.github.io/urbansim_templates. +### How it works -We maintain multiple versions of the documentation, for developer releases and production releases. For example, `/docs/_source_latest/` has documentation source files for the most recent developer release, and `/docs/_source_stable/` has source files for the most recent production release. +HTML files are generated using [Sphinx](http://sphinx-doc.org) and hosted with GitHub Pages from the `gh-pages` branch of the repository. The online documentation is rendered and updated **manually**. -Sphinx reads from the source files, plus the docstrings in the code, and renders html files to `/docs/latest/` and `/docs/stable/`. The Github Pages settings are [online](https://github.com/UDST/urbansim_templates/settings). +### Editing the documentation -For now we're building the docs manually, which gives us maximum control: +The files in `docs/source`, along with docstrings in the source code, determine what appears in the rendered documentation. Here's a [good tutorial](https://pythonhosted.org/an_example_pypi_project/sphinx.html) for Sphinx. + +### Previewing changes locally + +Install the copy of UrbanSim Templates that the documentation is meant to reflect. Install the documentation tools. + +``` +pip install . +pip install sphinx sphinx_rtd_theme numpydoc +``` + +Build the documentation. There should be status messages and warnings, but no errors. ``` +cd docs sphinx-build -b html source build ``` -Building the docs requires the python libraries `sphinx`, `numpydoc`, and `sphinx_rtd_theme`, plus the baseline requirements for `urbansim_templates`. You should rebuild the docs for each release. Rendered files won't appear online until they're merged into the master branch, but you can preview them locally. +The HTML files will show up in `docs/build/`. + +### Uploading changes + +Clone a second copy of the repository and check out the `gh-pages` branch. Copy over the updated HTML files, commit them, and push the changes to GitHub. -When you build the docs, docstrings are drawn from whatever version of the library is loaded by `import urbansim_templates`, not necessarily the copy that you're building the docs within -- so watch out for this when updating older versions. +### Discussion -See more discussion of docs in [PR #81](https://github.com/UDST/urbansim_templates/pull/81) and [Issue #83](https://github.com/UDST/urbansim_templates/issues/83) \ No newline at end of file +There are various discussions about documentation in the issue threads. [Issue #120](https://github.com/UDST/urbansim_templates/issues/120) is a good starting point. diff --git a/docs/source/data-templates.rst b/docs/source/data-templates.rst index ab8b676..8aeacf9 100644 --- a/docs/source/data-templates.rst +++ b/docs/source/data-templates.rst @@ -75,11 +75,6 @@ Unlike the templates, Orca relies on user-specified "`broadcast Date: Fri, 5 Feb 2021 15:46:17 -0800 Subject: [PATCH 115/121] Cleanup --- README.md | 7 +++---- docs/README.md | 2 +- docs/source/conf.py | 10 ++++------ 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8ec043e..83907fd 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ UrbanSim Templates is a Python library that provides building blocks for Orca-ba The library contains templates for common types of model steps, plus a tool called ModelManager that runs as an extension to the [Orca](https://udst.github.io/orca) task orchestrator. ModelManager can register template-based model steps with the orchestrator, save them to disk, and automatically reload them for future sessions. The package was developed to make it easier to set up new simulation models — model step templates reduce the need for custom code and make settings more portable between models. ### Installation -UrbanSim Templates can be installed using the Pip or Conda package managers. With Conda, you (currently) need to install UrbanSim separately; Pip will handle this automatically. +UrbanSim Templates can be installed using the Pip or Conda package managers. ``` pip install urbansim_templates @@ -17,13 +17,12 @@ pip install urbansim_templates ``` conda install urbansim_templates --channel conda-forge -conda install urbansim --channel udst ``` ### Documentation -See the online documentation for much more: https://urbansim-templates.readthedocs.io +See the online documentation for much more: https://udst.github.io/urbansim_templates Some additional documentation is available within the repo in `CHANGELOG.md`, `CONTRIBUTING.md`, `/docs/README.md`, and `/tests/README.md`. -There's discussion of current and planned features in the [Pull requests](https://github.com/udst/urbansim_templates/pulls?utf8=✓&q=is%3Apr) and [Issues](https://github.com/udst/urbansim_templates/issues?utf8=✓&q=is%3Aissue), both open and closed. +There's discussion of current and planned features in the [pull requests](https://github.com/udst/urbansim_templates/pulls?utf8=✓&q=is%3Apr) and [issues](https://github.com/udst/urbansim_templates/issues?utf8=✓&q=is%3Aissue), both open and closed. diff --git a/docs/README.md b/docs/README.md index 8e7831a..dd6840f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,7 @@ Install the copy of UrbanSim Templates that the documentation is meant to reflec ``` pip install . -pip install sphinx sphinx_rtd_theme numpydoc +pip install sphinx sphinx_rtd_theme ``` Build the documentation. There should be status messages and warnings, but no errors. diff --git a/docs/source/conf.py b/docs/source/conf.py index 1faf741..739a254 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,11 +17,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -import os -import sys - -# go up two levels from /docs/source to the package root -sys.path.insert(0, os.path.abspath('../..')) +# import os +# import sys +# sys.path.insert(0, os.path.abspath('../..')) import sphinx_rtd_theme @@ -55,7 +53,7 @@ # General information about the project. project = 'UrbanSim Templates' -copyright = '2019, UDST' +copyright = '2021, UDST' author = 'UDST' # The version info for the project you're documenting, acts as replacement for From 234405fc824340d31a2e649e125ea85297754df9 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 5 Feb 2021 22:11:55 -0800 Subject: [PATCH 116/121] Adjusting travis python versions --- .travis.yml | 4 +--- README.md | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 423a934..c0f0d5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,9 @@ language: python python: - - '2.7' - - '3.5' - '3.6' - - '3.7' - '3.8' + - '3.9' install: - pip install . diff --git a/README.md b/README.md index 83907fd..b4a8407 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ [![Build Status](https://travis-ci.org/UDST/urbansim_templates.svg?branch=master)](https://travis-ci.org/UDST/urbansim_templates) [![Coverage Status](https://coveralls.io/repos/github/UDST/urbansim_templates/badge.svg?branch=master)](https://coveralls.io/github/UDST/urbansim_templates?branch=master) -[![Docs Status](https://readthedocs.org/projects/urbansim_templates/badge/?version=latest)](https://docs.udst.org/projects/urbansim-templates/en/latest) # UrbanSim Templates From f68b04a425c82fdc0969823a51cf4e100a265f3e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 9 Feb 2021 12:03:04 -0800 Subject: [PATCH 117/121] Demo notebook --- examples/UrbanSim-Templates-demo.ipynb | 623 +++++++++++++++ examples/configs/README.md | 1 + examples/data/buildings-demo.csv | 1001 ++++++++++++++++++++++++ 3 files changed, 1625 insertions(+) create mode 100644 examples/UrbanSim-Templates-demo.ipynb create mode 100644 examples/configs/README.md create mode 100644 examples/data/buildings-demo.csv diff --git a/examples/UrbanSim-Templates-demo.ipynb b/examples/UrbanSim-Templates-demo.ipynb new file mode 100644 index 0000000..310d52b --- /dev/null +++ b/examples/UrbanSim-Templates-demo.ipynb @@ -0,0 +1,623 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "mature-episode", + "metadata": {}, + "source": [ + "# UrbanSim Templates demo\n", + "\n", + "Sam Maurer, Feb 2021\n", + "\n", + "### Background\n", + "\n", + "[UrbanSim](https://github.com/udst/urbansim) is a platform for modeling land use in cities. It runs in Python and uses the [Orca](https://github.com/udst/orca) task orchestration system. \n", + "\n", + "Orca breaks a model into \"steps\", Python functions that can be assembled on the fly into linear or cyclical pipelines. Orca is designed for workflows like city simulation where the data representing a model's state is so large that it needs to be managed outside the task graph. Steps refer to tables and columns of data by name rather than passing the data directly.\n", + "\n", + "UrbanSim [Templates](https://github.com/udst/urbansim_templates) is a library that provides automated building blocks for Orca-based models. The templates were developed to reduce the need for custom code and improve the portability of model steps.\n", + "\n", + "Currently we have templates for (a) regression, (b) binary Logit, (c) multinomial Logit estimated with [PyLogit](https://github.com/timothyb0912/pylogit) (best choice for flexible utility expressions), and (d) multinomial Logit estimated with [ChoiceModels](https://github.com/udst/choicemodels) (best choice for sampling of interchangeable alternatives).\n", + "\n", + "### Documentation\n", + "\n", + "Full UrbanSim Templates documentation: https://udst.github.io/urbansim_templates/\n", + "\n", + "### Installation\n", + "\n", + "You can install `orca` and `urbansim_templates` from Pip or Conda Forge." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "natural-frequency", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.2.1\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "print(pd.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "lyric-gardening", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.5.4\n" + ] + } + ], + "source": [ + "import orca\n", + "\n", + "print(orca.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "featured-return", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.2.dev9\n" + ] + } + ], + "source": [ + "import urbansim_templates\n", + "\n", + "print(urbansim_templates.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "floppy-brunei", + "metadata": {}, + "outputs": [], + "source": [ + "# Making the notebook output clearer\n", + "import warnings\n", + "warnings.simplefilter(action='ignore', category=FutureWarning)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "worst-thong", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "sporting-orange", + "metadata": {}, + "source": [ + "### Setting up ModelManager\n", + "\n", + "[ModelManager](https://udst.github.io/urbansim_templates/modelmanager.html) is part of the Templates library. It's an extension to Orca for saving and loading template-based model steps. \n", + "\n", + "By default it will look for a folder named `configs` in your current working directory, where it will read and save yaml representations of model steps. If there are already model steps there, the corresponding template classes need to be loaded before initializing ModelManager." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "coupled-turning", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No yaml files found in path 'configs'\n" + ] + } + ], + "source": [ + "from urbansim_templates.models import OLSRegressionStep\n", + "from urbansim_templates import modelmanager\n", + "\n", + "modelmanager.initialize()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "structured-potential", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "abandoned-somalia", + "metadata": {}, + "source": [ + "### Setting up data\n", + "\n", + "We'll load a DataFrame and register it with Orca, so that our statistical models can refer to it." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cheap-darkness", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "482" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_csv('data/buildings-demo.csv').dropna()\n", + "len(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "impossible-dressing", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "orca.add_table('buildings', df)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "balanced-chair", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
building_idparcel_iddevelopment_type_idimprovement_valueresidential_unitsresidential_sqftsqft_per_unitnon_residential_sqftbuilding_sqftres_price_per_sqftstoriesyear_builtsale_pricesale_yearbuilding_type_id
3732871210.0113931393.000.00.00000012008670250.02008.01
497426611116580.0110181018.001018.0474.35053411946703000.02007.01
6117166261457526.0136933693.003693.0124.8244321199895000.01996.01
1015742822195050.0111061106.001106.0448.07426111957675000.02005.01
13187434441166000.0113541354.001354.0411.5064011195118500.02006.01
\n", + "
" + ], + "text/plain": [ + " building_id parcel_id development_type_id improvement_value \\\n", + "3 7 328712 1 0.0 \n", + "4 9 742661 1 116580.0 \n", + "6 11 716626 1 457526.0 \n", + "10 15 742822 1 95050.0 \n", + "13 18 743444 1 166000.0 \n", + "\n", + " residential_units residential_sqft sqft_per_unit non_residential_sqft \\\n", + "3 1 1393 1393.0 0 \n", + "4 1 1018 1018.0 0 \n", + "6 1 3693 3693.0 0 \n", + "10 1 1106 1106.0 0 \n", + "13 1 1354 1354.0 0 \n", + "\n", + " building_sqft res_price_per_sqft stories year_built sale_price \\\n", + "3 0.0 0.000000 1 2008 670250.0 \n", + "4 1018.0 474.350534 1 1946 703000.0 \n", + "6 3693.0 124.824432 1 1998 95000.0 \n", + "10 1106.0 448.074261 1 1957 675000.0 \n", + "13 1354.0 411.506401 1 1951 18500.0 \n", + "\n", + " sale_year building_type_id \n", + "3 2008.0 1 \n", + "4 2007.0 1 \n", + "6 1996.0 1 \n", + "10 2005.0 1 \n", + "13 2006.0 1 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "orca.get_table('buildings').to_frame().head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "wooden-appendix", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "horizontal-defeat", + "metadata": {}, + "source": [ + "### Creating a model step\n", + "\n", + "Now we can choose a [template](https://udst.github.io/urbansim_templates/model-steps.html) and use it to fit a model." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "collaborative-channels", + "metadata": {}, + "outputs": [], + "source": [ + "from urbansim_templates.models import OLSRegressionStep\n", + "\n", + "m = OLSRegressionStep()\n", + "m.name = 'price-prediction'\n", + "m.tables = 'buildings'\n", + "m.model_expression = 'np.log1p(res_price_per_sqft) ~ non_residential_sqft>0 + year_built<1960'" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "domestic-messaging", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " OLS Regression Results \n", + "========================================================================================\n", + "Dep. Variable: np.log1p(res_price_per_sqft) R-squared: 0.398\n", + "Model: OLS Adj. R-squared: 0.395\n", + "Method: Least Squares F-statistic: 158.1\n", + "Date: Tue, 09 Feb 2021 Prob (F-statistic): 1.93e-53\n", + "Time: 12:02:09 Log-Likelihood: -598.98\n", + "No. Observations: 482 AIC: 1204.\n", + "Df Residuals: 479 BIC: 1216.\n", + "Df Model: 2 \n", + "Covariance Type: nonrobust \n", + "====================================================================================================\n", + " coef std err t P>|t| [0.025 0.975]\n", + "----------------------------------------------------------------------------------------------------\n", + "Intercept 5.5567 0.047 118.870 0.000 5.465 5.649\n", + "non_residential_sqft > 0[T.True] -5.6513 0.320 -17.642 0.000 -6.281 -5.022\n", + "year_built < 1960[T.True] 0.2206 0.082 2.693 0.007 0.060 0.382\n", + "==============================================================================\n", + "Omnibus: 511.938 Durbin-Watson: 1.611\n", + "Prob(Omnibus): 0.000 Jarque-Bera (JB): 21647.939\n", + "Skew: -4.895 Prob(JB): 0.00\n", + "Kurtosis: 34.338 Cond. No. 8.89\n", + "==============================================================================\n", + "\n", + "Notes:\n", + "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n" + ] + } + ], + "source": [ + "m.fit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sought-evanescence", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "expected-overhead", + "metadata": {}, + "source": [ + "### Registering the step\n", + "\n", + "Now we can \"register\" the step with ModelManager. This saves a copy to disk (in the `configs` folder), and passes a copy to Orca so it can be run as part of a sequence of other steps for validation or simulation." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "global-mistress", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving 'price-prediction.yaml': /Users/maurer/Dropbox/Git-imac/udst/urbansim_templates/examples/configs\n", + "Registering model step 'price-prediction'\n" + ] + } + ], + "source": [ + "modelmanager.register(m)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "hairy-binary", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "polished-casting", + "metadata": {}, + "source": [ + "### Making changes\n", + "\n", + "Previously registered steps can be retrieved, modified, and re-registered as needed." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "atmospheric-colorado", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'price-prediction', 'template': 'OLSRegressionStep', 'tags': []}]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "modelmanager.list_steps()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "limited-wilson", + "metadata": {}, + "outputs": [], + "source": [ + "m2 = modelmanager.get_step('price-prediction')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "solved-glass", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving 'better-price-prediction.yaml': /Users/maurer/Dropbox/Git-imac/udst/urbansim_templates/examples/configs\n", + "Registering model step 'better-price-prediction'\n" + ] + } + ], + "source": [ + "m2.name = 'better-price-prediction'\n", + "# here you can edit the specification and re-fit, etc.\n", + "\n", + "modelmanager.register(m2)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "secondary-baking", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Removing 'better-price-prediction' and 'better-price-prediction.yaml'\n" + ] + } + ], + "source": [ + "modelmanager.remove_step('better-price-prediction')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "classical-attachment", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acoustic-insight", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:template-demo] *", + "language": "python", + "name": "conda-env-template-demo-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/configs/README.md b/examples/configs/README.md new file mode 100644 index 0000000..6e48d7b --- /dev/null +++ b/examples/configs/README.md @@ -0,0 +1 @@ +This folder stores configs that are generated by the demo notebook. \ No newline at end of file diff --git a/examples/data/buildings-demo.csv b/examples/data/buildings-demo.csv new file mode 100644 index 0000000..2108ec3 --- /dev/null +++ b/examples/data/buildings-demo.csv @@ -0,0 +1,1001 @@ +building_id,parcel_id,development_type_id,improvement_value,residential_units,residential_sqft,sqft_per_unit,non_residential_sqft,building_sqft,res_price_per_sqft,stories,year_built,sale_price,sale_year,building_type_id +1,742974,1,0,1,2029,2029.42425,0,2029.42425,302.7697507,1,1945,,,1 +2,744961,1,0,1,2029,2029.42425,0,2029.42425,254.4292789,1,1965,,,1 +3,1442641,1,53262.87,1,1568,1568,0,1568,183.4741663,1,1964,,,1 +7,328712,1,0,1,1393,1393,0,0,0,1,2008,670250,2008,1 +9,742661,1,116580,1,1018,1018,0,1018,474.3505343,1,1946,703000,2007,1 +10,155,19,0,0,0,0,145231,145231,0,3,1997,,,0 +11,716626,1,457526,1,3693,3693,0,3693,124.8244317,1,1998,95000,1996,1 +12,742716,1,345454,1,2029,2029.42425,0,2029.42425,324.7160246,1,1992,,,1 +13,742810,1,53636,1,2029,2029.42425,0,2029.42425,275.357742,1,1931,,,1 +14,742814,1,87577,1,2029,2029.42425,0,2029.42425,305.4611798,1,1964,,,1 +15,742822,1,95050,1,1106,1106,0,1106,448.0742613,1,1957,675000,2005,1 +16,743438,1,9923,1,2029,2029.42425,0,2029.42425,325.1975099,1,1968,,,1 +17,743440,1,194004,1,2029,2029.42425,0,2029.42425,325.9342553,1,1972,,,1 +18,743444,1,166000,1,1354,1354,0,1354,411.5064009,1,1951,18500,2006,1 +19,743453,1,0,1,2029,2029.42425,0,2029.42425,325.1975099,1,1968,,,1 +20,743482,2,0,1,2029,2029.42425,0,2029.42425,342.4565206,1,1968,,,3 +21,743655,2,182066,1,2029,2029.42425,0,2029.42425,341.2121108,1,1960,,,3 +22,743967,1,101570,1,851,851,0,851,546.8725439,1,1953,315000,2012,1 +23,743993,1,45934,1,2029,2029.42425,0,2029.42425,303.0678318,1,1947,,,1 +24,744008,1,194339,1,2261,2261,0,2261,259.028036,1,1914,384000,1998,1 +25,744029,1,109288,1,2029,2029.42425,0,2029.42425,275.493275,1,1932,,,1 +26,744314,1,84213,1,2029,2029.42425,0,2029.42425,305.3109946,1,1963,,,1 +27,744321,1,42116,1,2029,2029.42425,0,2029.42425,274.2771409,1,1923,,,1 +28,744558,1,32500,1,993,993,0,993,437.9080323,1,1924,575000,2010,1 +29,744559,1,0,1,2029,2029.42425,0,2029.42425,302.7697507,1,1945,,,1 +30,744566,1,54776,1,2029,2029.42425,0,2029.42425,274.4117582,1,1924,,,1 +31,744570,1,45722,1,2029,2029.42425,0,2029.42425,272.1278439,1,1907,,,1 +32,744575,1,446877,1,3409,3409,0,3409,268.9639176,1,1994,280000,1990,1 +33,744679,2,252324,2,2762,1381,0,2762,403.7313456,1,1958,330000,1991,3 +34,744680,1,69644,1,2052,2052,0,2052,304.0314766,1,1966,800000,2008,1 +35,744692,1,224372,1,1326,1326,0,1326,356.2906781,1,1931,286000,1990,1 +36,744843,1,116457,1,2029,2029.42425,0,2029.42425,303.0678318,1,1947,,,1 +37,744864,1,258794,1,2029,2029.42425,0,2029.42425,256.200407,1,1979,,,1 +38,744865,1,157148,1,1406,1406,0,1406,317.8849821,1,1974,715000,2005,1 +39,744883,1,50180,1,2029,2029.42425,0,2029.42425,304.2619874,1,1956,,,1 +40,744895,1,217095,1,2029,2029.42425,0,2029.42425,309.2382467,1,1989,,,1 +41,745007,1,180327,1,2029,2029.42425,0,2029.42425,305.7615503,1,1966,,,1 +42,745014,1,62848,1,2029,2029.42425,0,2029.42425,305.7615503,1,1966,,,1 +43,745019,1,0,1,2029,2029.42425,0,2029.42425,302.7697507,1,1945,,,1 +44,746120,2,49676,2,4058,2029.42425,0,4058.848501,384.5782234,1,1913,,,3 +45,746707,1,202478,1,1504,1504,0,1504,437.9092342,1,1928,1200000,2005,1 +46,746849,1,541040,1,2029,2029.42425,0,2029.42425,367.7181301,1,1927,,,1 +47,747968,1,0,1,2029,2029.42425,0,2029.42425,462.9766927,1,1948,,,1 +48,748912,1,386168,1,3123,3123,0,3123,324.5618445,1,1918,551000,1998,1 +49,749372,1,110443,1,2029,2029.42425,0,2029.42425,418.8695045,1,1923,,,1 +50,749556,1,257281,1,2031,2031,0,2031,463.5661611,1,1951,460000,1995,1 +51,749561,1,72983,1,2367,2367,0,2367,386.5971077,1,1909,1272000,2010,1 +52,749575,1,286377,1,2247,2247,0,2247,396.3277142,1,1912,1250000,2009,1 +53,749989,1,250772,1,2029,2029.42425,0,2029.42425,496.149161,1,1992,,,1 +54,750307,2,29012,1,2029,2029.42425,0,2029.42425,151.26952,1,1963,,,3 +55,750310,1,29012,1,2029,2029.42425,0,2029.42425,143.6457015,1,1963,,,1 +56,750586,1,308493,2,3978,1989,0,3978,389.5527907,1,1951,844045,1990,1 +57,751607,1,311235,1,1790,1790,0,1790,155.8674881,1,1990,590409,1989,1 +58,751723,2,3149076,4,8117,2029.42425,0,8117.697002,152.3516213,1,1977,,,3 +59,751727,1,288759,1,2029,2029.42425,0,2029.42425,152.9251294,1,1994,,,1 +60,751774,-1,228588,0,0,0,1334,1334,0,1,1980,,,0 +61,751779,-1,157794,0,0,0,990,990,0,1,1980,,,0 +62,751864,1,327780,1,2029,2029.42425,0,2029.42425,153.2265598,1,1998,,,1 +63,752248,1,126166,1,2029,2029.42425,0,2029.42425,346.7484086,1,1944,,,1 +64,752254,1,156885,1,1420,1420,0,1420,428.4710144,1,1944,258000,1994,1 +65,752676,1,149651,1,2029,2029.42425,0,2029.42425,346.2376055,1,1941,,,1 +66,753021,1,104749,2,4058,2029.42425,0,4058.848501,348.5250501,1,1954,,,1 +67,754120,1,39568,1,2400,2400,0,2400,291.0397053,1,1928,708000,2005,1 +68,754135,1,144610,2,4058,2029.42425,0,4058.848501,349.4855982,1,1960,,,1 +69,754337,2,97170,2,4058,2029.42425,0,4058.848501,367.0226736,1,1954,,,3 +70,755501,1,805118,2,4058,2029.42425,0,4058.848501,395.7567753,1,1990,,,1 +71,755509,1,38246,2,4058,2029.42425,0,4058.848501,350.3302997,1,1921,,,1 +72,755557,1,62868,1,2029,2029.42425,0,2029.42425,388.6270084,1,1953,,,1 +73,755639,1,64991,1,1795,1795,0,1795,370.8651111,1,1901,755000,2006,1 +74,756130,1,270457,1,3159,3159,0,3159,292.2625924,1,1904,926150,2011,1 +75,756134,1,56458,1,2029,2029.42425,0,2029.42425,388.2448964,1,1951,,,1 +76,756142,2,283582,2,4058,2029.42425,0,4058.848501,407.2865106,1,1943,,,3 +77,756504,7,133131,0,0,0,2021,2021,0,1,1980,,,10 +78,756702,2,1178087,48,97412,2029.42425,0,97412.36402,308.5934072,1,1958,,,3 +79,756918,1,274354,1,2029,2029.42425,0,2029.42425,313.3164923,1,1998,,,1 +80,757065,1,57594,1,2029,2029.42425,0,2029.42425,293.141626,1,1959,,,1 +81,757075,7,795855,0,0,0,8094,8094,0,1,1953,,,10 +82,757600,2,0,1,2029,2029.42425,0,2029.42425,367.0226736,1,1954,,,3 +83,757798,1,17215,1,2029,2029.42425,0,2029.42425,386.0259805,1,1954,,,1 +84,758577,1,97994,1,2029,2029.42425,0,2029.42425,385.8741815,1,1953,,,1 +85,758582,2,360175,2,4632,2316,0,4632,381.4706754,1,1954,555000,1993,3 +86,759256,1,0,1,2029,2029.42425,0,2029.42425,419.9437122,1,1972,,,1 +87,2370,1,160390,3,4497,1499,0,4497,140.6151479,2,2015,217000,2006,1 +88,759818,1,80900,1,2029,2029.42425,0,2029.42425,417.9867661,1,1963,,,1 +89,760210,1,33408,1,905,905,0,905,246.1718094,1,1962,120000,2008,1 +90,760670,-1,25350,0,0,0,0,0,0,1,1973,,,0 +91,760700,-1,0,0,0,0,44750,44750,0,1,1968,,,0 +92,760687,13,485003,0,0,0,37722,37722,0,1,1963,,,8 +93,761678,2,561000,3,6088,2029.42425,0,6088.272751,481.5554516,1,1912,925000,2004,3 +94,762354,2,157472,1,1761,1761,0,1761,581.1373111,1,1957,1364500,2012,3 +95,762755,1,209370,1,2029,2029.42425,0,2029.42425,507.7873777,1,1946,,,1 +96,762784,2,0,2,4058,2029.42425,0,4058.848501,539.2835951,1,1963,,,3 +97,763011,1,340000,1,2301,2301,0,2301,483.0526727,1,1968,1191000,2004,1 +98,763272,1,432404,1,2333,2333,0,2333,480.5530142,1,1970,1050000,2001,1 +99,763789,2,180528,1,1101,1101,0,1101,631.1682979,1,1974,530500,2011.5,3 +100,763805,1,150726,1,1300,1300,0,1300,531.52048,1,1971,800000,2007,1 +101,763808,1,278900,1,2029,2029.42425,0,2029.42425,406.0332629,1,1975,,,1 +102,763818,1,71653,1,2029,2029.42425,0,2029.42425,406.0332629,1,1975,,,1 +103,763903,1,119763,1,2029,2029.42425,0,2029.42425,406.0332629,1,1975,,,1 +104,765019,2,0,1,2029,2029.42425,0,2029.42425,478.53912,1,1955,,,3 +105,766126,2,0,1,2029,2029.42425,0,2029.42425,478.53912,1,1955,,,3 +106,766231,2,131147,1,2029,2029.42425,0,2029.42425,479.5050402,1,1959,,,3 +107,766549,1,166468,1,2029,2029.42425,0,2029.42425,453.9973372,1,1953,,,1 +108,766886,2,333926,1,2557,2557,0,2557,341.9675034,1,1964,320000,1997,3 +109,766909,1,132296,1,2029,2029.42425,0,2029.42425,360.9148034,1,1965,,,1 +110,767328,1,278740,1,1989,1989,0,1989,363.6400802,1,1959,375000,1995,1 +111,767772,1,247542,1,2029,2029.42425,0,2029.42425,664.4179707,1,1973,,,1 +112,768012,1,248331,1,2029,2029.42425,0,2029.42425,742.8100418,1,1963,,,1 +113,768029,1,852,1,2618,2618,0,2618,656.5041889,1,1944,1600000,2008,1 +114,768608,1,378502,1,2962,2962,0,2962,640.5433106,1,1978,1855000,2009,1 +115,768937,1,31446,1,1270,1270,0,1270,884.7280061,1,1913,925000,2012,1 +116,768945,1,191178,2,3266,1633,0,3266,747.026286,1,1904,850000,2011,1 +117,769918,1,0,1,2029,2029.42425,0,2029.42425,657.070894,1,1951,,,1 +118,770033,1,413777,1,2376,2376,0,2376,619.4227922,1,1925,575000,1989,1 +119,770899,1,69791,1,2029,2029.42425,0,2029.42425,596.2850876,1,1932,,,1 +120,770929,1,238500,1,1293,1293,0,1293,864.8838442,1,1950,285000,1991,1 +121,771352,1,114980,1,1555,1555,0,1555,693.1795723,1,1927,114000,1999,1 +122,4203,13,145738,0,0,0,3092,3092,0,1,1947,635000,2013,8 +123,773234,2,85023,1,1014,1014,0,1014,619.3346575,1,1947,550000,2010,3 +124,773576,1,331612,1,2029,2029.42425,0,2029.42425,378.3421297,1,1968,,,1 +125,773774,2,488348,1,3194,3194,0,3194,658.4465093,1,1949,890000,2003,3 +126,774513,-1,51689,0,0,0,0,0,0,1,1963,,,0 +127,774519,1,110000,1,2029,2029.42425,0,2029.42425,751.8689907,1,1967,,,1 +128,774534,1,351900,1,1235,1235,0,0,0,1,1963,575000,2004,1 +129,774995,1,87555,1,2164,2164,0,2164,372.6469281,1,1999,13500,2000,1 +130,775350,1,141501,1,2762,2762,0,2762,661.2612638,1,1977,1550000,2013,1 +131,775634,1,193614,1,2029,2029.42425,0,2029.42425,360.5601602,1,1963,,,1 +132,776023,1,524408,1,2029,2029.42425,0,2029.42425,756.6945383,1,1980,,,1 +133,776026,1,345024,1,3043,3043,0,3043,642.7860581,1,1980,1320000,2013,1 +134,776123,1,951450,1,4722,4722,0,4722,653.2447881,1,1995,2580000,2011,1 +135,777066,2,534062,1,2029,2029.42425,0,2029.42425,509.9357584,1,1987,,,3 +136,777545,1,403351,1,2534,2534,0,2534,436.162937,1,1981,1450000,2011,1 +137,777986,1,60249,1,984,984,0,984,845.3917053,1,1927,165000,1995,1 +138,778194,1,89165,1,2029,2029.42425,0,2029.42425,585.9136341,1,1967,,,1 +139,778206,1,65300,1,2029,2029.42425,0,2029.42425,585.7406868,1,1966,,,1 +140,778223,1,0,1,2029,2029.42425,0,2029.42425,585.7406868,1,1966,,,1 +141,778723,1,54519,1,848,848,0,848,1060.761705,1,1973,550000,2011,1 +142,778727,1,139755,1,1609,1609,0,1609,672.4057308,1,1979,734500,2010,1 +143,778748,1,143998,1,2029,2029.42425,0,2029.42425,586.202071,1,1968,,,1 +144,779185,1,1200000,1,3980,3980,0,3980,508.4262769,1,2005,3725000,2006,1 +145,779580,1,53760,1,2029,2029.42425,0,2029.42425,582.1765948,1,1954,,,1 +146,779908,1,277388,1,2283,2283,0,2283,553.3349005,1,1967,422500,1996,1 +147,780419,1,38005,1,980,980,0,980,617.3105071,1,1922,576000,2008,1 +148,780804,1,230885,1,1844,1844,0,1844,451.275788,1,1972,349000,1992,1 +149,780953,1,57294,1,2029,2029.42425,0,2029.42425,584.4737478,1,1962,,,1 +150,780964,1,363867,1,2029,2029.42425,0,2029.42425,625.315726,1,2003,,,1 +151,780993,1,325849,1,1749,1749,0,1749,675.5966622,1,1994,444000,1991,1 +152,781022,1,373546,1,1801,1801,0,1801,664.2850373,1,1994,510000,1995,1 +153,781024,1,424512,1,1480,1480,0,1480,752.0614065,1,2003,665000,1999,1 +154,781253,1,87043,1,1616,1616,0,1616,487.1900597,1,1970,825000,2013,1 +155,781409,1,0,1,2029,2029.42425,0,2029.42425,427.9057247,1,1969,,,1 +157,782374,1,133246,1,1361,1361,0,1361,539.2479973,1,1951,227500,1996,1 +158,782551,1,211964,1,1270,1270,0,1270,566.3048508,1,1954,500000,2010,1 +159,782709,1,47481,1,2029,2029.42425,0,2029.42425,210.5916397,1,1970,,,1 +160,782738,2,0,11,22323,2029.42425,0,22323.66675,234.9419681,1,1991,,,3 +161,782742,1,35167,1,2029,2029.42425,0,2029.42425,210.3840134,1,1968,,,1 +162,782756,1,154880,2,2184,1092,0,2184,312.9033792,1,1970,220000,1997,1 +163,782798,1,164953,2,2496,1248,0,2496,283.7586117,1,1967,224000,1998,1 +164,782980,2,44084,1,2029,2029.42425,0,2029.42425,565.5056265,1,1971,,,3 +165,783173,1,170386,1,825,825,0,825,795.1720315,1,1983,214000,1992,1 +166,783423,1,233588,1,1521,1521,0,1521,264.8276241,1,2000,701000,2005,1 +167,5852,7,2085407,0,0,0,46071,46071,0,1,1981,,,10 +168,5861,7,0,0,0,0,29935,29935,0,1,1988,,,10 +169,783812,-1,182030,0,0,0,2854,1736,0,1,1962,,,0 +170,783815,2,192856,1,2029,2029.42425,0,2029.42425,926.2263402,1,1958,,,3 +171,783979,1,151276,1,1896,1896,0,1896,500.0918734,1,1979,500000,1996,1 +172,783999,1,539456,1,5604,5604,0,5604,751.343304,1,1968,5975000,2008,1 +173,784007,1,428956,1,2813,2813,0,2813,816.1044174,1,1993,780000,2012,1 +174,784008,1,454297,1,2889,2889,0,2889,808.3860575,1,1991,975000,2001,1 +175,784083,1,394495,1,3454,3454,0,3454,777.9792481,1,2002,2700000,2013,1 +176,784090,1,180018,1,2029,2029.42425,0,2029.42425,882.1477974,1,1964,,,1 +177,784173,2,1683268,17,34500,2029.42425,0,34500.21226,934.3322762,1,1976,,,3 +178,784401,1,537504,1,5023,5023,0,5023,732.5764882,1,1976,3370000,2010,1 +179,784879,1,203086,1,1496,1496,0,1496,939.5341316,1,1910,749000,1999,1 +180,784886,1,352460,1,2350,2350,0,2350,732.6235883,1,1908,1800000,2013,1 +181,785086,1,648606,1,1543,1543,0,1543,1029.497205,1,1957,1400000,2001,1 +182,785236,2,0,1,2029,2029.42425,0,2029.42425,934.3322762,1,1976,,,3 +183,785669,1,106400,1,2611,2611,0,2611,1046.550926,1,1950,1780000,2009,1 +184,785671,1,244288,1,2029,2029.42425,0,2029.42425,1174.241422,1,1954,,,1 +185,786055,1,516100,1,3139,3139,0,3139,990.3583479,1,1956,1070000,1995,1 +186,786354,1,621367,1,3136,3136,0,3136,882.5162963,1,1900,1100000,1991,1 +187,786469,1,77184,1,2029,2029.42425,0,2029.42425,1173.779905,1,1953,,,1 +188,786499,1,2703000,1,4530,4530,0,4530,956.9745675,1,1957,5200000,2009,1 +189,788239,2,49823,3,6088,2029.42425,0,6088.272751,564.6693366,1,1968,52500,2011,3 +190,788518,2,0,2,4058,2029.42425,0,4058.848501,564.6693366,1,1968,,,3 +191,6487,2,72689,2,1460,730,0,1460,215.3614936,1,1900,,,3 +192,788880,1,359222,1,2029,2029.42425,0,2029.42425,536.7396664,1,1970,,,1 +193,788882,1,364791,1,2029,2029.42425,0,2029.42425,539.393453,1,1980,,,1 +194,789085,2,0,1,2029,2029.42425,0,2029.42425,564.6693366,1,1968,,,3 +195,789198,2,0,1,2029,2029.42425,0,2029.42425,564.6693366,1,1968,,,3 +196,789205,1,21420,1,2029,2029.42425,0,2029.42425,536.2108212,1,1968,,,1 +197,789702,1,233931,1,2029,2029.42425,0,2029.42425,537.269518,1,1972,,,1 +198,789915,1,185219,1,2029,2029.42425,0,2029.42425,401.8632775,1,1954,,,1 +199,790442,1,235590,1,2029,2029.42425,0,2029.42425,402.2587734,1,1956,,,1 +200,790895,1,420240,1,2943,2943,0,2943,346.0006772,1,1962,1055000,2002,1 +201,791295,1,459000,1,2610,2610,0,2610,779.0295221,1,1968,1550000,2009,1 +202,791563,1,140835,1,2029,2029.42425,0,2029.42425,877.836177,1,1981,,,1 +203,791744,1,810757,1,3761,3761,0,3761,712.9635652,1,1972,1500000,1998,1 +204,791745,1,516980,1,2029,2029.42425,0,2029.42425,872.6667514,1,1969,,,1 +205,792509,1,1212422,1,2029,2029.42425,0,2029.42425,876.5023524,1,2003,,,1 +206,792616,1,408993,1,2029,2029.42425,0,2029.42425,811.2600054,1,1942,,,1 +207,792619,1,98927,1,2029,2029.42425,0,2029.42425,815.2575917,1,1952,,,1 +208,792631,1,144520,1,2866,2866,0,2866,703.8447645,1,1949,1775000,2010,1 +209,793395,1,184666,1,2029,2029.42425,0,2029.42425,868.809737,1,1960,,,1 +210,793415,1,1645060,1,5609,5609,0,5609,735.8470063,1,1952,3650000,2011,1 +211,793746,1,388006,1,2034,2034,0,2034,865.6693663,1,1955,805000,1994,1 +212,793747,1,120414,1,2029,2029.42425,0,2029.42425,876.9726399,1,1979,,,1 +213,793759,1,83097,1,2029,2029.42425,0,2029.42425,781.6331906,1,1924,,,1 +214,793764,1,249702,1,2267,2267,0,2267,482.5189719,1,1952,1279000,2012,1 +215,793876,1,578162,1,2029,2029.42425,0,2029.42425,872.2373423,1,1968,,,1 +216,793895,1,398214,1,2029,2029.42425,0,2029.42425,869.6650163,1,1962,,,1 +217,793896,1,446580,1,2029,2029.42425,0,2029.42425,870.9502945,1,1965,,,1 +218,7281,1,19374,3,5931,1977,0,5931,121.7117696,2,1939,,,1 +219,2020036,1,452000,1,3068,3068,0,3068,206.9395558,1,1972,,,1 +220,7320,7,750000,0,0,0,11240,11240,0,1,1917,,,10 +221,794024,-1,7003,0,0,0,0,0,0,1,1954,,,0 +222,794303,1,0,1,480,480,0,480,1021.06947,1,1933,53500,2002,1 +223,794552,1,207035,1,2151,2151,0,2151,379.0855371,1,1978,1175000,2007,1 +224,794920,2,0,1,2029,2029.42425,0,2029.42425,410.5925753,1,1976,,,3 +225,794924,1,113537,1,2029,2029.42425,0,2029.42425,412.6751259,1,1996,,,1 +226,794934,1,0,1,3752,3752,0,0,0,1,1901,2000000,2008,1 +227,794942,-1,135677,0,0,0,0,0,0,1,1954,,,0 +228,795084,2,0,1,2029,2029.42425,0,2029.42425,410.5925753,1,1976,,,3 +229,795558,1,56673,2,4058,2029.42425,0,4058.848501,349.0981859,1,1953,,,1 +230,796090,1,0,1,2029,2029.42425,0,2029.42425,352.8894752,1,1975,,,1 +231,7779,2,109200,3,3547,1182.5,0,3547.5,148.8451071,1,1910,175000,2010,3 +232,796388,1,0,1,2029,2029.42425,0,2029.42425,352.8894752,1,1975,,,1 +233,796499,1,0,1,2029,2029.42425,0,2029.42425,352.8894752,1,1975,,,1 +234,796606,1,0,1,2029,2029.42425,0,2029.42425,352.8894752,1,1975,,,1 +235,796611,1,0,1,2029,2029.42425,0,2029.42425,352.8894752,1,1975,,,1 +236,796885,1,215664,2,3194,1597,0,3194,473.2185014,1,1997,91000,1989,1 +237,797185,1,262393,2,2998,1499,0,2998,492.2567362,1,1996,400000,1995,1 +238,797345,1,477984,2,5474,2737,0,5474,363.9018032,1,2005,308000,1997,1 +239,8015,2,213500,2,2218,1109,0,2218,166.6318096,2,1912,,,3 +240,797575,2,0,1,2029,2029.42425,0,2029.42425,410.5925753,1,1976,,,3 +241,797859,1,233000,1,1808,1808,0,1808,278.7780013,1,1967,735000,2004,1 +242,797860,1,173405,1,2029,2029.42425,0,2029.42425,261.9703724,1,1968,,,1 +243,797890,1,72326,1,2029,2029.42425,0,2029.42425,262.0993318,1,1969,,,1 +244,797960,1,213196,1,2029,2029.42425,0,2029.42425,262.0993318,1,1969,,,1 +245,797986,1,77933,1,2029,2029.42425,0,2029.42425,262.4862102,1,1972,,,1 +246,8116,10,3414464,0,0,0,39318,39318,0,5,1960,,,4 +247,798777,1,110151,1,2029,2029.42425,0,2029.42425,263.1331567,1,1977,,,1 +248,798034,1,330000,1,1702,1702,0,1702,289.309693,1,1971,805550,2005,1 +249,798180,2,88552,1,2029,2029.42425,0,2029.42425,276.6468819,1,1980,,,3 +250,798362,1,377400,1,2059,2059,0,2059,260.508908,1,1978,670000,2003,1 +251,798444,1,64084,1,2029,2029.42425,0,2029.42425,261.8414129,1,1967,,,1 +252,8173,24,0,4,6386,1596.5,0,6386,249.3464765,1,1991,447500,2009,16 +253,798617,1,155686,1,1521,1521,0,1521,309.2862858,1,1966,462500,2010,1 +254,798618,1,167169,1,2029,2029.42425,0,2029.42425,261.7124535,1,1966,,,1 +255,799249,1,514560,1,3431,3431,0,3431,342.9346458,1,2007,799000,2001,1 +256,799468,1,80680,1,1376,1376,0,0,0,1,1963,440000,2010,1 +257,799469,1,131426,1,1592,1592,0,1592,330.4042091,1,1964,400000,2010,1 +258,799627,-1,865057,0,0,0,11268,11268,0,1,1969,,,0 +259,799673,1,49898,1,2029,2029.42425,0,2029.42425,287.997917,1,1966,,,1 +260,799686,1,170611,1,1268,1268,0,1268,384.3334855,1,1966,440000,2011,1 +261,799792,1,674220,1,4013,4013,0,4013,236.5019355,1,1989,1031000,2003,1 +262,799804,1,306000,1,1715,1715,0,1715,316.0013789,1,1965,575000,2003,1 +263,799995,1,153841,1,1221,1221,0,1221,396.2297123,1,1974,250000,2012,1 +264,800028,1,250015,1,2029,2029.42425,0,2029.42425,290.9860458,1,1987,,,1 +265,800189,1,132357,1,2029,2029.42425,0,2029.42425,263.781178,1,1982,,,1 +266,800210,1,131511,1,2029,2029.42425,0,2029.42425,263.781178,1,1982,,,1 +267,800225,1,145193,1,1548,1548,0,1548,308.2747813,1,1982,517000,2005,1 +268,800227,1,319056,1,1548,1548,0,1548,308.2747813,1,1982,463000,2004,1 +269,800232,1,97092,1,2029,2029.42425,0,2029.42425,263.781178,1,1982,,,1 +270,800378,1,303012,2,6064,3032,0,6064,224.7384249,1,1987,359000,1997,1 +271,800798,1,67709,1,2029,2029.42425,0,2029.42425,285.8823537,1,1951,,,1 +272,800799,1,46598,1,2029,2029.42425,0,2029.42425,286.1636596,1,1953,,,1 +273,800914,1,290866,1,2323,2323,0,2323,218.6831843,1,1912,524000,2010,1 +274,800955,1,112859,1,1590,1590,0,1590,299.5960392,1,1958,404000,2011,1 +275,800972,1,145571,3,3516,1172,0,3516,367.8521473,1,1958,222500,1991,1 +276,801093,1,301872,1,2256,2256,0,2256,272.2642054,1,1957,731000,2010,1 +277,801414,1,40005,1,2029,2029.42425,0,2029.42425,286.4449654,1,1955,,,1 +278,801566,1,73862,1,2029,2029.42425,0,2029.42425,288.8485987,1,1972,,,1 +279,801705,1,204600,1,2029,2029.42425,0,2029.42425,289.4167808,1,1976,,,1 +280,801695,1,88402,1,2029,2029.42425,0,2029.42425,287.573372,1,1963,,,1 +281,801699,1,140560,1,1416,1416,0,1416,356.3636382,1,1965,680000,2006,1 +282,801751,1,247427,1,2886,2886,0,2886,248.3425996,1,1965,370000,1998,1 +283,801781,1,199132,1,2029,2029.42425,0,2029.42425,289.9865545,1,1980,,,1 +284,801873,1,290662,1,2252,2252,0,2252,276.1276028,1,1984,515000,2001,1 +285,801879,1,92456,1,2029,2029.42425,0,2029.42425,286.7266692,1,1957,,,1 +286,8930,2,1415730,22,20635,937.9545455,0,20635,176.5381861,4,1998,1600000,1999,3 +287,802367,2,58033,1,2029,2029.42425,0,2029.42425,214.9554871,1,1967,,,3 +288,802623,1,191760,1,1576,1576,0,1576,235.7695433,1,1966,530000,2003,1 +289,8961,2,434000,2,2578,1289,0,2578,166.3943277,2,1934,620000,2004,3 +290,802813,2,0,1,2029,2029.42425,0,2029.42425,215.6758545,1,1973,,,3 +291,8959,1,301110,2,2234,1117,0,2234,180.1963508,1,1947,94000,2001,1 +292,802875,7,1282123,0,0,0,38241,22735,0,1,1988,,,10 +293,803039,1,152380,1,1612,1612,0,1612,231.2388316,1,1955,233000,1996,1 +294,803279,1,230000,1,2412,2412,0,2412,187.6136036,1,1961,519000,2012,1 +295,803297,-1,374921,0,0,0,24904,20010,0,1,1965,,,0 +296,803390,2,198303,2,4058,2029.42425,0,4058.848501,214.6385413,1,1964,,,3 +297,803969,2,135656,1,624,624,0,624,379.1306878,1,1990,163000,2012,3 +298,803982,1,86743,1,624,624,0,624,360.0229766,1,1990,117000,1998,1 +299,804204,1,59080,1,2029,2029.42425,0,2029.42425,260.812424,1,1959,,,1 +300,804360,1,59352,1,2029,2029.42425,0,2029.42425,261.5840314,1,1965,,,1 +301,804364,1,198409,1,2029,2029.42425,0,2029.42425,261.7124535,1,1966,,,1 +302,804378,1,231039,1,1795,1795,0,1795,279.6512506,1,1965,454000,2011,1 +303,804398,1,181606,1,2029,2029.42425,0,2029.42425,263.781178,1,1982,,,1 +304,804425,1,95568,1,2029,2029.42425,0,2029.42425,259.9156185,1,1952,,,1 +305,804636,-1,344634,0,0,0,0,0,0,1,1961,,,0 +306,804570,1,76945,1,2029,2029.42425,0,2029.42425,263.1331567,1,1977,,,1 +307,804572,1,333540,1,2782,2782,0,2782,242.652503,1,1993,767000,2004,1 +308,804862,-1,163077,0,0,0,1613,1500,0,1,1962,,,0 +309,805109,1,190822,1,2029,2029.42425,0,2029.42425,235.2214962,1,1928,,,1 +310,9368,10,0,0,0,0,6325,5000,0,1,1924,,,4 +311,9370,10,0,0,0,0,2435,1925,0,1,1906,,,4 +312,805395,1,255000,1,1360,1360,0,1360,334.9955977,1,1979,236000,2009,1 +313,805434,1,163000,2,1900,950,0,1900,436.5770311,1,1988,169000,1990,1 +314,9474,7,24692,0,0,0,3807,3010,0,1,1933,,,10 +315,806032,1,192198,1,2029,2029.42425,0,2029.42425,352.1559328,1,1979,,,1 +316,806046,1,511237,1,5158,5158,0,5158,308.8408792,1,1994,1250000,1999,1 +318,806710,1,311600,1,2029,2029.42425,0,2029.42425,259.2157785,1,1986,,,1 +501,834516,1,275136,1,2029,2029.42425,0,2029.42425,417.7810134,1,1962,,,1 +319,806830,1,558960,1,2872,2872,0,2872,305.3285823,1,1986,995000,2003,1 +320,806848,1,324142,1,2936,2936,0,2936,234.636971,1,1998,399000,1994,1 +321,807104,1,305724,1,2029,2029.42425,0,2029.42425,288.42286,1,1969,,,1 +322,807326,1,391636,1,3250,3250,0,3250,172.0065172,1,1984,192608,1998,1 +323,807224,1,400731,1,2970,2970,0,2970,249.1279509,1,1989,570000,1997,1 +324,807935,1,423300,1,2584,2584,0,2584,183.9068651,1,1977,78500,2009,1 +325,808004,1,78450,1,2029,2029.42425,0,2029.42425,183.7329555,1,1932,,,1 +326,808074,1,93212,1,2029,2029.42425,0,2029.42425,287.7150196,1,1964,,,1 +327,808176,1,252204,1,2029,2029.42425,0,2029.42425,291.1288871,1,1988,,,1 +328,808271,1,78261,1,2212,2212,0,2212,276.5399118,1,1970,515000,2011,1 +329,808273,1,70345,1,2029,2029.42425,0,2029.42425,288.5645076,1,1970,,,1 +330,808325,1,268751,1,2596,2596,0,2596,258.5077048,1,1972,349000,1992,1 +331,9956,1,485428,1,2422,2422,0,2422,252.269068,2,1920,1160000,2013,1 +332,808658,1,140760,1,1524,1524,0,1524,311.6693161,1,1960,270000,2012,1 +333,876804,1,81500,1,1274,1274,0,1274,232.7601266,1,1949,433000,2004,1 +334,808738,1,146288,1,2029,2029.42425,0,2029.42425,264.3200339,1,1962,,,1 +335,808752,1,147064,1,1608,1608,0,1608,301.6675172,1,1962,133500,2012,1 +336,808840,1,74738,1,2029,2029.42425,0,2029.42425,263.8000601,1,1958,,,1 +337,808975,1,208397,1,1495,1495,0,1495,316.6125644,1,1967,490000,2010,1 +338,809209,1,118839,1,2029,2029.42425,0,2029.42425,265.1017429,1,1968,,,1 +339,809501,1,289687,1,2589,2589,0,2589,239.7315416,1,1987,455000,1999,1 +340,809683,1,65063,1,2029,2029.42425,0,2029.42425,264.4499025,1,1963,,,1 +341,809703,1,57336,1,2029,2029.42425,0,2029.42425,264.4499025,1,1963,,,1 +342,809796,1,66136,1,2029,2029.42425,0,2029.42425,264.0597973,1,1960,,,1 +343,809921,1,202909,1,1506,1506,0,1506,314.5066304,1,1963,248000,1993,1 +344,810197,1,203511,1,1788,1788,0,1788,285.0272914,1,1975,285000,1997,1 +345,810305,1,61257,1,2029,2029.42425,0,2029.42425,144.9294051,1,1952,,,1 +346,810318,1,51579,1,2029,2029.42425,0,2029.42425,144.9294051,1,1952,,,1 +347,810319,1,124861,1,2029,2029.42425,0,2029.42425,144.9294051,1,1952,,,1 +348,810523,1,138677,1,1376,1376,0,1376,185.3261695,1,1979,211000,1990,1 +349,810550,1,158318,1,1627,1627,0,1627,166.3312522,1,1978,526000,2009,1 +350,811166,1,150706,1,872,872,0,872,258.5285316,1,1973,145000,2012,1 +352,811670,1,146997,1,2029,2029.42425,0,2029.42425,146.9381909,1,1980,,,1 +353,811835,1,267750,1,1395,1395,0,1395,193.5833827,1,1990,350000,2010,1 +354,811841,1,268009,1,1395,1395,0,1395,193.5833827,1,1990,430000,2003,1 +355,812075,7,215012,0,0,0,3690,3432,0,1,1953,,,10 +356,881091,24,109656,0,0,0,0,0,0,1,1940,,,16 +357,812249,1,205391,1,1242,1242,0,1242,352.6212706,1,1956,410000,2001,1 +358,812260,1,99668,1,2029,2029.42425,0,2029.42425,236.0334034,1,1935,,,1 +359,812487,-1,0,0,0,0,538,500,0,1,1961,,,0 +360,812490,2,0,1,2029,2029.42425,0,2029.42425,276.6760481,1,1974,,,3 +361,812494,1,144656,1,2029,2029.42425,0,2029.42425,260.6840019,1,1958,,,1 +362,812506,7,1080000,0,0,0,41777,24837,0,1,1978,,,10 +363,812732,1,266754,1,2414,2414,0,2414,238.0495816,1,1979,245000,1997,1 +364,10622,1,507906,1,1840,1840,0,1840,290.85575,2,1933,320000,1996,1 +365,812628,1,167000,1,1635,1635,0,1635,290.5395889,1,1970,667000,2004,1 +366,812826,1,299915,1,2194,2194,0,2194,253.6457233,1,1981,450000,2000,1 +367,812933,2,10991000,207,420090,2029.42425,0,420090.8198,263.31571,1,1999,,,3 +368,813144,13,180678,0,0,0,5671,5492,0,1,1965,,,8 +369,813149,13,263968,0,0,0,36348,35200,0,1,1970,,,8 +370,813150,13,122890,0,0,0,8002,7749,0,1,1970,,,8 +371,813182,2,68239,1,1128,1128,0,1128,312.1351608,1,1975,1323136,2006,3 +372,814028,2,180000,1,1396,1396,0,1396,330.5225789,1,2007,350000,2005,3 +373,10933,2,65718,2,2802,1401,0,2802,559.1732694,2,1921,,,3 +374,815307,13,477921,0,0,0,13428,13428,0,1,1975,,,8 +375,815310,10,315863,0,0,0,6468,6468,0,1,1976,,,4 +376,815491,1,253604,1,1596,1596,0,1596,405.5092297,1,1987,470000,1999,1 +377,815707,1,81645,1,2029,2029.42425,0,2029.42425,232.3273313,1,1945,,,1 +378,11019,1,408800,1,1337,1337,0,1337,513.6035728,1,1922,681500,2005,1 +379,815909,1,482218,1,2029,2029.42425,0,2029.42425,250.4019765,1,2001,,,1 +380,816085,1,263528,1,1428,1428,0,1428,308.2719904,1,2001,329000,1999,1 +381,881053,1,54644,1,935,935,0,935,440.3358483,1,1941,115000,1989,1 +382,11070,1,24700,1,1111,1111,0,1111,586.1962954,1,1924,,,1 +383,11091,1,63718,1,2532,2532,0,2532,386.1350641,2,1928,,,1 +384,816569,1,180275,1,2029,2029.42425,0,2029.42425,250.6486475,1,2003,,,1 +385,816866,1,415260,1,1992,1992,0,1992,286.2452874,1,2008,880000,2005,1 +386,816955,2,166110,1,2029,2029.42425,0,2029.42425,271.378395,1,1964,,,3 +387,817009,1,259655,1,2090,2090,0,2090,253.8769345,1,1964,566500,2001,1 +388,817089,1,93969,1,2029,2029.42425,0,2029.42425,259.3537401,1,1977,,,1 +389,817223,1,94972,1,2029,2029.42425,0,2029.42425,257.8279064,1,1965,,,1 +390,817219,1,123912,1,2029,2029.42425,0,2029.42425,258.3355435,1,1969,,,1 +391,817243,1,245901,1,2029,2029.42425,0,2029.42425,257.7014145,1,1964,,,1 +392,817285,1,80927,1,2029,2029.42425,0,2029.42425,258.3355435,1,1969,,,1 +393,817532,1,248946,1,1356,1356,0,1356,327.4434935,1,1957,500000,2001,1 +394,817550,1,149998,1,1287,1287,0,1287,339.3631591,1,1957,265000,1997,1 +395,817567,1,174364,1,1434,1434,0,1434,315.2967987,1,1956,264500,1994,1 +396,817591,1,271189,1,1434,1434,0,1434,315.2967987,1,1956,576000,2003,1 +397,817613,1,291000,1,1341,1341,0,1341,329.7604872,1,1956,542000,2011,1 +398,817703,1,264715,1,2029,2029.42425,0,2029.42425,259.2259958,1,1976,,,1 +399,817772,1,102007,1,2029,2029.42425,0,2029.42425,257.8279064,1,1965,,,1 +400,817873,1,258233,1,2300,2300,0,2300,243.0333945,1,1968,315000,1997,1 +401,817877,1,220402,1,2029,2029.42425,0,2029.42425,258.2086342,1,1968,,,1 +402,817881,1,286484,1,2640,2640,0,2640,229.747187,1,1969,537000,2000,1 +403,817886,1,256230,1,2300,2300,0,2300,243.1527411,1,1969,355000,1995,1 +404,818118,1,147878,1,2029,2029.42425,0,2029.42425,258.9713423,1,1974,,,1 +405,818362,1,263506,1,1790,1790,0,1790,277.5735198,1,1976,118627,1992,1 +406,818606,1,555128,1,2363,2363,0,2363,241.6776682,1,1981,1005128,2004,1 +407,818625,1,592341,1,4384,4384,0,4384,211.8602866,1,1989,1160000,2010,1 +408,11415,1,259745,1,5356,5356,0,5356,357.544572,2,1924,3450000,2013,1 +409,818848,1,484508,1,3651,3651,0,3651,213.4550066,1,1988,319355,2001,1 +410,818970,1,217260,1,2029,2029.42425,0,2029.42425,260.6315997,1,1987,,,1 +411,819289,1,263895,1,1731,1731,0,1731,298.9443584,1,1991,415000,2002,1 +412,819586,1,481561,1,2752,2752,0,2752,248.2861227,1,2007,790000,2010,1 +413,819447,1,314470,1,2534,2534,0,2534,255.2185465,1,2001,176216,1997,1 +414,819487,2,0,1,2029,2029.42425,0,2029.42425,272.8770935,1,1975,,,3 +415,819758,1,125551,1,878,878,0,878,497.7915909,1,2008,230551,2004,1 +416,819762,1,151517,1,1113,1113,0,1113,415.3445577,1,2008,254000,2012,1 +418,11598,1,312892,1,4377,4377,0,4377,346.681225,2,1926,,,1 +419,820041,2,53194,1,1385,1385,0,1385,375.20101,1,1960,240000,1998,3 +420,820425,1,310000,1,1608,1608,0,1608,323.4996519,1,1959,770000,2012,1 +421,820468,1,60234,1,2029,2029.42425,0,2029.42425,283.448898,1,1959,,,1 +422,11680,1,26311,1,1417,1417,0,0,0,1,1924,479999,2011,1 +423,820780,1,203319,1,2029,2029.42425,0,2029.42425,283.1704493,1,1957,,,1 +424,820783,1,198131,1,2029,2029.42425,0,2029.42425,283.1704493,1,1957,,,1 +425,821520,1,81340,1,2029,2029.42425,0,2029.42425,284.2859923,1,1965,,,1 +426,821524,1,254235,1,1989,1989,0,1989,287.2784676,1,1965,174117,1996,1 +427,821595,1,97350,1,2029,2029.42425,0,2029.42425,284.4258723,1,1966,,,1 +428,11788,1,0,1,2318,2318,0,0,0,1,1962,790000,2003,1 +429,821662,2,0,1,2029,2029.42425,0,2029.42425,299.8751812,1,1968,,,3 +430,821665,1,115699,1,2029,2029.42425,0,2029.42425,284.5657523,1,1967,,,1 +431,11801,1,563199,1,1909,1909,0,1909,413.146752,1,1926,630000,1998,1 +432,821826,1,160659,1,2029,2029.42425,0,2029.42425,284.7056323,1,1968,,,1 +433,822021,1,1,1,2029,2029.42425,0,2029.42425,284.7615843,1,1968,,,1 +434,822051,1,199920,1,2029,2029.42425,0,2029.42425,285.4063437,1,1973,,,1 +435,11839,1,407051,1,2988,2988,0,2988,364.7079991,2,1927,442000,2000,1 +436,822541,2,167147,1,2029,2029.42425,0,2029.42425,326.5213289,1,1973,,,3 +437,822579,2,0,1,2029,2029.42425,0,2029.42425,324.7141223,1,1962,,,3 +438,822723,1,161148,1,2029,2029.42425,0,2029.42425,308.5457904,1,1963,,,1 +439,11894,1,643300,1,2514,2514,0,2514,386.9091245,2,1926,1201000,2004,1 +440,11897,1,518016,1,2759,2759,0,2759,374.8876145,2,1932,520000,1997,1 +441,822809,1,183709,1,1512,1512,0,1512,366.3758352,1,1965,725000,2006,1 +442,822811,1,136954,1,1728,1728,0,1728,337.54183,1,1965,699000,2007,1 +443,822813,1,76503,1,2029,2029.42425,0,2029.42425,308.8490228,1,1965,,,1 +444,822830,1,369800,1,2652,2652,0,2652,274.212124,1,1965,315000,1991,1 +445,822991,1,287170,1,1966,1966,0,1966,314.3308143,1,1967,399000,1991,1 +446,823080,1,141014,1,2029,2029.42425,0,2029.42425,308.8490228,1,1965,,,1 +447,823082,1,239276,1,2029,2029.42425,0,2029.42425,308.8490228,1,1965,,,1 +448,823165,1,83097,1,2029,2029.42425,0,2029.42425,309.000639,1,1966,,,1 +449,823180,1,321300,1,1906,1906,0,1906,319.458563,1,1966,740000,2004,1 +450,823192,1,341412,1,2029,2029.42425,0,2029.42425,309.1527427,1,1967,,,1 +451,1495515,1,395945,1,1472,1472,0,1472,186.5400059,1,1957,739000,2004,1 +452,823357,1,404273,1,2029,2029.42425,0,2029.42425,328.636643,1,1995,,,1 +453,823366,1,423838,1,2948,2948,0,2948,282.1160002,1,1998,1450000,2005,1 +454,11979,1,385543,1,2483,2483,0,2483,389.2338409,2,1928,350000,1988,1 +455,823821,2,209194,2,4188,2094,0,4188,263.897772,1,1949,869000,2006,3 +456,823504,1,217261,1,2854,2854,0,2854,269.6299428,1,1900,350000,2000,1 +457,12022,1,126641,1,3273,3273,0,3273,356.2146917,2,1926,,,1 +458,824095,2,68496,1,2029,2029.42425,0,2029.42425,269.5847073,1,1960,,,3 +460,824641,1,0,1,2029,2029.42425,0,2029.42425,255.3565512,1,1955,,,1 +461,824648,1,150888,1,1128,1128,0,1128,331.6880468,1,1909,183000,1995,1 +462,824812,-1,86242,0,0,0,2675,2000,0,1,1952,,,0 +463,824837,1,154754,1,2906,2906,0,2906,220.8210763,1,1964,1000000,2008,1 +464,825030,1,113741,1,2029,2029.42425,0,2029.42425,258.1478052,1,1977,,,1 +465,825379,1,0,1,2029,2029.42425,0,2029.42425,255.3565512,1,1955,,,1 +466,825399,1,0,1,2029,2029.42425,0,2029.42425,255.3565512,1,1955,,,1 +467,825422,1,0,1,2029,2029.42425,0,2029.42425,255.3565512,1,1955,,,1 +468,825447,1,244170,1,2029,2029.42425,0,2029.42425,258.5286925,1,1980,,,1 +469,825539,1,0,1,2029,2029.42425,0,2029.42425,255.3565512,1,1955,,,1 +470,825789,1,355381,1,2029,2029.42425,0,2029.42425,254.0666456,1,1962,,,1 +471,825716,1,56285,1,1891,1891,0,1891,335.935139,1,1957,58500,2002,1 +472,825719,1,254356,1,2029,2029.42425,0,2029.42425,323.6966348,1,1958,,,1 +473,825722,2,0,1,2029,2029.42425,0,2029.42425,342.4565206,1,1968,,,3 +474,825822,2,0,1,2029,2029.42425,0,2029.42425,342.4565206,1,1968,,,3 +475,12246,1,85230,1,1946,1946,0,1946,437.1077474,2,1929,,,1 +476,826367,2,84851,1,2029,2029.42425,0,2029.42425,300.4321269,1,1966,,,3 +477,826571,1,72326,1,2029,2029.42425,0,2029.42425,285.7121036,1,1969,,,1 +478,826584,1,87937,1,1868,1868,0,1868,323.0139724,1,1966,835000,2014,1 +479,826798,1,61328,1,2029,2029.42425,0,2029.42425,308.242558,1,1961,,,1 +480,826811,1,201619,1,1526,1526,0,1526,363.5151083,1,1961,127686,1994,1 +481,826812,1,312751,1,1526,1526,0,1526,363.5151083,1,1961,601000,2003,1 +482,827191,1,235321,1,2237,2237,0,2237,271.9651928,1,1966,448800,1989,1 +483,827200,1,267232,1,1839,1839,0,1839,300.6934006,1,1965,345000,1991,1 +484,827203,1,219510,1,2029,2029.42425,0,2029.42425,285.1504038,1,1965,,,1 +485,827332,1,77050,1,2029,2029.42425,0,2029.42425,284.869554,1,1963,,,1 +486,827336,1,179241,1,2029,2029.42425,0,2029.42425,284.869554,1,1963,,,1 +487,828416,1,357399,1,2457,2457,0,2457,372.0363729,1,1953,1175000,2012,1 +488,828822,1,327587,1,2029,2029.42425,0,2029.42425,412.150954,1,1980,,,1 +489,829214,2,112924,1,2029,2029.42425,0,2029.42425,428.892928,1,1956,,,3 +490,829228,1,174432,1,1211,1211,0,1211,561.1825639,1,1955,297500,1991,1 +491,830435,1,53413,1,2029,2029.42425,0,2029.42425,307.3357858,1,1955,,,1 +492,830539,1,138228,1,2029,2029.42425,0,2029.42425,283.6102253,1,1954,,,1 +493,830877,2,214127,1,2029,2029.42425,0,2029.42425,263.8349384,1,1975,,,3 +494,831127,1,54079,1,2029,2029.42425,0,2029.42425,247.7183094,1,1952,,,1 +495,831612,2,192956,2,4058,2029.42425,0,4058.848501,260.9939776,1,1953,,,3 +496,832556,2,0,2,4058,2029.42425,0,4058.848501,263.3806346,1,1972,,,3 +497,834008,1,64887,1,2029,2029.42425,0,2029.42425,416.755309,1,1957,,,1 +498,834084,1,185793,1,2029,2029.42425,0,2029.42425,416.755309,1,1957,,,1 +499,834274,1,210092,1,2029,2029.42425,0,2029.42425,418.8093948,1,1967,,,1 +500,834278,1,253504,1,1963,1963,0,1963,426.1702859,1,1967,455000,1999,1 +502,835076,1,240729,1,2029,2029.42425,0,2029.42425,423.3646389,1,1989,,,1 +503,835652,1,0,1,2029,2029.42425,0,2029.42425,921.2054748,1,1966,,,1 +504,835792,1,125970,1,2029,2029.42425,0,2029.42425,920.9783187,1,1966,,,1 +505,836645,1,129397,1,600,600,0,600,1996.273569,1,1934,235000,1991,1 +506,836820,1,72105,1,2029,2029.42425,0,2029.42425,818.0335617,1,1904,,,1 +507,13827,1,265390,1,1675,1675,0,1675,283.6986241,2,1939,,,1 +508,837220,1,397262,1,2464,2464,0,2464,897.6015791,1,2001,300000,1995,1 +509,837235,1,87430,1,2029,2029.42425,0,2029.42425,911.0606549,1,1944,,,1 +510,837542,1,41765,1,2029,2029.42425,0,2029.42425,910.6122815,1,1943,,,1 +511,837869,1,1462070,1,3651,3651,0,3651,804.2035282,1,2001,6300000,2008,1 +512,837884,1,49241,1,2029,2029.42425,0,2029.42425,255.9486019,1,1977,,,1 +513,837976,-1,0,0,0,0,1903,1500,0,1,1944,22500,2010,0 +514,2020868,1,272903,1,2835,2835,0,2835,213.5053558,1,1988,,,1 +515,838115,1,86324,1,2029,2029.42425,0,2029.42425,563.6236494,1,1973,,,1 +516,838372,1,283179,1,1718,1718,0,1718,469.5595124,1,1970,732500,2012,1 +517,838760,2,0,1,2029,2029.42425,0,2029.42425,564.6693366,1,1968,,,3 +518,892340,10,83758,0,0,0,7752,7752,0,2,1906,,,4 +519,890220,2,123597,8,3600,450,0,3600,1900.991744,3,1931,,,3 +520,225306,7,1306401,0,0,0,67037,67037,0,1,1961,,,10 +521,225354,1,199500,1,1038,1038,0,1038,281.0013874,1,1956,,,1 +522,225415,1,183400,1,992,992,0,992,290.9642181,1,1957,262000,2010,1 +523,225517,1,36513,1,1068,1068,0,1068,275.4769027,1,1959,,,1 +524,225533,1,35139,1,1146,1146,0,1146,261.9389151,1,1962,,,1 +525,225563,1,153019,3,3744,1248,0,3744,245.6108837,1,1955,355000,2012,1 +526,225570,1,164327,1,1256,1256,0,1256,244.5131283,1,1955,,,1 +527,225573,1,245000,1,1284,1284,0,1284,240.7849612,1,1955,390000,2004,1 +528,225964,1,182000,1,951,951,0,951,300.5518398,1,1957,377000,2003,1 +529,226039,1,162526,1,1384,1384,0,1384,229.799265,1,1964,178500,1994,1 +530,226040,1,36089,1,1204,1204,0,1204,253.0449433,1,1964,350000,2012,1 +531,225989,1,181839,1,2016,2016,0,2016,182.6369177,1,1955,210000,1998,1 +532,225992,1,28301,1,1056,1056,0,1056,277.8105191,1,1959,,,1 +533,226082,1,245000,1,1329,1329,0,1329,236.0718734,1,1963,520000,2006,1 +534,226256,2,117112,1,1242,1242,0,1242,247.1609566,1,1984,129000,1994,3 +535,226259,2,157500,2,2484,1242,0,2484,247.1609566,1,1984,275000,2003,3 +536,226294,2,115213,1,1242,1242,0,1242,263.8306391,2,1984,125000,1993,3 +537,226391,7,1775156,0,0,0,9429,9429,0,1,1990,,,10 +538,226504,2,192500,2,2952,1476,0,2952,216.5500787,2,1985,335000,2012,3 +539,226505,2,150510,1,1622,1622,0,1622,204.0466481,2,1985,315000,2011,3 +540,226521,2,210000,2,3244,1622,0,3244,204.0466481,2,1985,380000,2004,3 +541,226628,1,199083,1,1196,1196,1196,1196,0,1,1971,239000,2000,1 +542,227038,1,223300,1,1392,1392,1392,1392,0,1,1963,389900,2008,1 +543,227437,1,81060,1,2170,2170,0,2170,171.3784284,2,1976,,,1 +544,227528,1,178372,1,1553,1553,0,1553,193.0435092,1,1970,198000,1996,1 +545,227679,2,191757,1,1840,1840,0,1840,191.7338719,2,1999,213000,1996,3 +546,228051,1,375063,1,1838,1838,0,1838,255.4019691,1,1998,459000,2000,1 +547,227927,1,408800,1,2240,2240,0,2240,245.8670478,2,1995,530000,2013,1 +548,227947,1,399238,1,2240,2240,0,2240,245.8670478,2,1995,479000,2000,1 +549,228023,1,408800,1,2239,2239,0,2239,246.0391906,2,1996,,,1 +550,228056,1,399700,1,2239,2239,0,2239,246.1601633,2,1997,584000,2002,1 +551,228202,1,51854,1,1750,1750,0,1750,247.3355992,1,1972,,,1 +552,228335,2,149323,1,1337,1337,0,1337,305.469906,2,1988,155000,1991,3 +553,228251,1,204282,1,1266,1266,0,1266,304.4404132,1,1971,205000,1990,1 +554,228266,1,177403,1,1400,1400,0,1400,284.2492636,1,1971,430000,2010,1 +555,228345,2,226800,1,1519,1519,0,1519,280.8077478,2,1988,455000,2006,3 +556,228348,2,145663,1,1519,1519,0,1519,280.8077478,2,1988,146000,1989,3 +557,228364,2,156318,1,1436,1436,0,1436,291.2115294,2,1988,165000,1998,3 +558,228365,2,190800,1,1337,1337,0,1337,305.469906,2,1988,275000,2011,3 +559,228429,1,193368,1,1048,1048,0,1048,349.6358385,1,1974,425000,2013,1 +560,228646,1,137330,1,1427,1427,0,1427,281.0882664,1,1974,,,1 +561,228653,1,71346,1,1783,1783,0,1783,253.0425324,2,1974,,,1 +562,229017,2,514500,1,2834,2834,0,2834,267.8465104,2,2002,650000,2001,3 +563,228679,1,291200,1,1452,1452,0,1452,278.0387559,1,1975,364500,2002,1 +564,229041,1,275503,1,2424,2424,0,2424,260.8583768,1,2001,745000,2009,1 +565,228743,1,296100,1,1427,1427,0,1427,281.226602,1,1975,639000,2007,1 +566,229030,1,665000,1,3837,3837,0,3837,238.678716,2,2002,,,1 +567,15377,1,37880,2,2518,1259,0,2518,330.1035674,1,1925,,,1 +568,228755,1,275800,1,1343,1343,0,1343,292.8848736,1,1975,381000,2002,1 +569,15395,2,296395,7,6611,944.5,0,6611.5,491.5781645,2,1963,,,3 +570,15424,2,19582,5,5052,1010.5,0,5052.5,421.7384408,2,1935,,,3 +571,980887,10,225262,0,0,0,28000,28000,0,4,1906,,,4 +572,980888,10,106373,0,0,0,13840,13840,0,2,1911,,,4 +573,980933,-1,0,0,0,0,9037,9037,0,1,1900,,,0 +574,981712,10,338112,0,0,0,5400,5400,0,1,1906,,,4 +575,981860,13,395962,0,0,0,5780,5780,0,2,1924,,,8 +576,981884,10,527952,0,0,0,7513,7513,0,2,1957,,,4 +577,983798,10,974087,0,0,0,12400,12400,0,2,1911,,,4 +578,990156,-1,502662,0,0,0,6912,6912,0,1,1988,,,0 +579,990158,-1,0,0,0,0,18000,18000,0,1,1978,,,0 +580,990232,13,86641,0,0,0,11250,11250,0,1,1950,,,8 +581,992002,-1,8257,0,0,0,6177,6177,0,2,1950,,,0 +582,992080,13,269155,0,0,0,58613,58613,0,1,1900,,,8 +583,998625,-1,281220,0,0,0,2919,2919,0,1,1971,,,0 +584,998789,13,1170128,0,0,0,8200,8200,0,1,1951,,,8 +585,1041825,5,0,16,24269,1516.833919,16112,40381.34271,4.643665739,1,2007,,,12 +586,1041768,-1,0,0,0,0,1976,1750,0,1,1900,,,0 +587,1041994,-1,0,0,0,0,7763,7763,0,1,1936,,,0 +588,15689,7,45586,0,0,0,3835,3835,0,1,1948,,,10 +589,15765,1,262547,1,1264,1264,0,1264,350.046869,1,1947,315000,2000,1 +590,1199752,1,640265,1,1945,1945.370097,0,1945.370097,311.3661065,1,1980,,,1 +591,15791,1,520450,1,1516,1516,0,1516,312.4304644,1,1964,749000,2009,1 +592,15793,1,61124,1,1771,1771,0,1771,290.8966975,2,1944,,,1 +593,15803,1,198167,1,1495,1495,0,1495,323.1112351,2,1947,215500,1994,1 +594,1866143,1,214500,1,1062,1062,0,1062,205.3341201,1,1983,336000,2004,1 +595,15804,1,44583,1,1518,1518,0,1518,291.6994077,2,1938,,,1 +596,2020008,1,203584,1,1522,1522,0,1522,287.8463509,1,1968,490000,2010,1 +597,2020041,1,211500,1,1306,1306,0,1306,318.9088955,1,1970,385000,2013,1 +598,2020059,1,173344,1,1704,1704,0,1704,269.5759984,1,1976,,,1 +599,2020065,1,138437,1,1496,1496,0,1496,292.1842095,1,1976,452500,2013,1 +600,1214304,-1,404075,0,0,0,0,0,0,1,1942,,,0 +601,1214305,-1,272280,0,0,0,0,0,0,1,1942,,,0 +602,1214299,1,772855,1,3010,3010,0,3010,257.135811,1,1985,1400000,2004,1 +603,15986,1,266887,1,1691,1691,0,1691,507.9829934,2,1943,287500,1997,1 +604,2020073,1,389250,1,1968,1968,0,1968,248.9628368,1,1979,535000,2003,1 +605,16026,1,22033,5,8250,1650,0,8250,282.0082755,1,1936,,,1 +606,2020094,1,160118,1,2104,2104,0,2104,239.727414,1,1972,,,1 +607,16089,1,420822,1,1598,1598,0,1598,299.0913446,2,1912,691000,2011,1 +608,16090,2,301386,2,2710,1355,0,2710,350.5169917,2,1912,46000,1996,3 +609,1217805,1,263033,1,1945,1945.370097,0,1945.370097,304.4922953,1,1957,,,1 +610,1219427,-1,1217736,0,0,0,0,0,0,1,1942,,,0 +611,2020095,1,127365,1,1901,1901,0,1901,252.9009659,1,1973,,,1 +612,2020096,1,96714,1,1635,1635,0,1635,275.6846714,1,1971,,,1 +613,2020108,1,146206,1,2010,2010,0,2010,245.4912457,1,1973,,,1 +614,389704,1,417200,2,3686,1843,0,3686,270.7079017,2,2001,585000,2012,1 +615,1233800,1,317421,1,1945,1945.370097,0,1945.370097,340.3167749,1,1986,,,1 +616,2020119,1,150565,2,6954,3477,0,6954,201.4224674,1,1973,,,1 +617,2020138,1,170316,1,1450,1450,0,1450,297.1366644,1,1969,215000,1998,1 +618,2020164,1,121290,1,1675,1675,0,1675,271.0152685,1,1966,,,1 +619,2020166,1,67584,1,1851,1851,0,1851,255.8950046,1,1967,,,1 +620,2020169,1,94637,1,1203,1203,0,1203,337.1414944,1,1966,,,1 +621,1866519,1,148525,1,2016,2016,0,2016,135.7627834,1,1982,,,1 +622,1867136,1,92897,1,1496,1496,0,1496,160.5684117,1,1969,,,1 +623,1867138,1,99740,1,1457,1457,0,1457,163.0178785,1,1965,139000,1996,1 +624,1867302,1,286000,1,2138,2138,0,2138,130.4464487,1,1961,436000,2008,1 +625,1867381,1,207750,1,1144,1144,0,1144,194.0976025,1,1980,485000,2005,1 +626,1867448,1,306226,1,2544,2544,0,2544,122.4569454,1,1989,298000,1991,1 +627,1867597,1,208500,1,1660,1660,0,1660,160.2801136,1,2001,335000,2008,1 +628,1868454,10,321792,0,0,0,2959,2959,0,1,1945,,,4 +629,1868919,1,1000000,1,2163,2163,0,2163,200.2540138,1,1921,1800000,2007,1 +630,1868920,1,40543,1,1610,1610,0,1610,261.0268803,1,1949,820000,2013,1 +631,1868922,1,195741,1,1105,1105,0,1105,305.7047934,1,1929,239000,2000,1 +632,1868928,1,159228,1,1962,1962,0,1962,233.5450946,1,1954,105000,1997,1 +633,1869193,1,69305,1,1910,1910,0,1910,329.1357849,1,1958,,,1 +634,1869227,1,49607,1,1636,1636,0,1636,359.0478323,1,1952,,,1 +635,1869356,1,308000,1,1494,1494,0,1494,277.81268,1,1980,585000,2006,1 +636,1869358,1,254735,1,2601,2601,0,2601,207.7344738,1,1979,,,1 +637,1869583,1,22067,1,1025,1025,0,1025,372.9899696,1,1956,,,1 +638,1869587,7,1569526,0,0,0,3167,3167,0,1,2004,,,10 +639,1869599,1,94383,1,1245,1245,0,1245,323.6368005,1,1956,,,1 +640,2020179,1,259750,1,1447,1447,0,1447,297.1012632,1,1966,580000,2005,1 +641,1869638,1,280831,1,1940,1940,0,1940,330.0864593,1,1981,285000,1990,1 +642,1869639,1,168818,1,1873,1873,0,1873,335.8085904,1,1977,610000,2011,1 +643,1869715,1,182869,1,2006,2006,0,2006,241.9417806,1,1965,225000,1994,1 +644,1869726,1,130071,1,1977,1977,0,1977,326.3002549,1,1978,,,1 +645,1869839,1,137554,1,1792,1792,0,1792,257.5996933,1,1967,71000,2007,1 +646,1869921,1,264307,1,1266,1266,0,1266,323.0310112,1,1976,336000,2013,1 +647,1869925,1,77320,1,1200,1200,0,1200,335.3055153,1,1975,,,1 +648,1869926,1,42832,1,1230,1230,0,1230,329.4777249,1,1975,,,1 +698,18288,1,224921,2,2714,1357,0,2714,498.2554622,2,1920,,,1 +649,1870000,1,83072,1,1080,1080,0,1080,360.7965657,1,1968,225000,2013,1 +650,1870043,1,104577,1,1362,1362,0,1362,307.6753513,1,1979,,,1 +651,1870331,1,234030,2,2592,1296,0,2592,318.446693,1,1980,160000,2004,1 +652,1870336,1,148792,1,1296,1296,0,1296,318.446693,1,1980,175000,1997,1 +653,1870347,1,74747,1,1175,1175,0,1175,341.239786,1,1980,,,1 +654,1870602,1,174352,1,1655,1655.200203,0,1655.200203,362.700794,1,1988,,,1 +655,1870969,1,110806,1,1992,1992,0,1992,243.7864886,1,1973,,,1 +656,1871321,1,447043,1,2091,2091,0,2091,253.0385809,1,2003,590000,2004,1 +657,1871412,-1,0,0,0,0,0,0,0,1,1967,260000,2010,0 +658,1871491,13,788679,0,0,0,19200,19200,0,1,1988,,,8 +659,1871503,1,261275,1,1320,1320,0,1320,310.2189421,1,1936,588000,2006,1 +660,1871509,1,65560,1,1233,1233,0,1233,320.516124,1,1906,,,1 +661,1871514,1,25355,1,1123,1123,0,1123,342.9605645,1,1907,,,1 +662,1871624,7,251096,0,0,0,1810,1810,0,1,1903,,,10 +663,17391,1,224000,1,1660,1660,0,1660,248.1576144,2,1915,,,1 +664,17390,2,315000,3,2217,739.3333333,0,2218,485.6573888,2,1911,515000,2004,3 +665,2020188,1,302550,1,1777,1777,0,1777,261.7556095,1,1966,379000,2002,1 +666,1872304,7,302519,0,0,0,8400,8400,0,1,1911,,,10 +667,1872666,1,250711,1,1703,1703,0,1703,289.7011352,1,1950,316500,2000,1 +668,1872667,1,119777,1,1364,1364,0,1364,334.2424267,1,1954,165000,1992,1 +669,1872672,1,300825,1,1606,1606,0,1606,300.166844,1,1950,590000,2013,1 +670,1873069,7,605853,0,0,0,8287,7936,0,1,1965,345000,2010,10 +671,1872741,1,188700,1,1384,1384,0,1384,332.6212208,1,1964,285000,1999,1 +672,1872789,1,232167,1,1325,1325,0,1325,341.2273025,1,1956,229000,1999,1 +673,1872793,1,80896,1,834,834,0,834,482.1617824,1,1956,160000,1993,1 +674,1872943,1,98141,2,3366,1683,0,3366,296.2310895,1,1981,,,1 +675,1873174,1,228202,1,2628,2628,0,2628,237.9481551,1,1983,,,1 +676,1874141,7,541758,0,0,0,1152,1152,0,1,1998,,,10 +677,1874170,-1,6084,0,0,0,0,0,0,1,1953,,,0 +678,1874536,2,236250,1,1785,1785,0,1785,206.1142999,1,1968,,,3 +679,1874801,1,190225,1,1152,1152,0,1152,260.9996479,1,1970,438182,2003,1 +680,1874735,1,117650,1,960,960,0,960,299.1689527,1,1970,164000,1991,1 +681,1875030,1,171110,1,1972,1972,0,1972,185.5264803,1,1971,228000,1998,1 +682,1875036,1,131925,1,1785,1785,0,1785,196.0148503,1,1971,186000,1994,1 +683,1875106,1,133913,1,1600,1600,0,1600,209.4378734,1,1973,220000,1999,1 +684,1875107,1,166745,1,1628,1628,0,1628,207.2160706,1,1973,322500,2000,1 +685,1875661,1,7028,1,645,645,0,645,466.6805372,1,1915,,,1 +686,1876047,1,162818,1,2264,2264,0,2264,197.1260806,1,1930,,,1 +687,1876400,1,176525,1,2248,2248,0,2248,217.9958618,1,1949,325000,1999,1 +688,1876571,1,96529,2,3564,1782,0,3564,213.3039808,1,1957,,,1 +689,1876769,10,128003,0,0,0,2356,2356,0,1,1910,,,4 +690,1877118,1,35212,1,1144,1144,0,1144,286.1129526,1,1963,220600,2012,1 +691,1877125,1,209750,1,1186,1186,0,1186,278.7617423,1,1963,275000,2011,1 +692,1877817,1,155030,1,1448,1448,0,1448,285.2963118,1,1990,230000,1995,1 +693,1877904,1,357500,2,5474,2737,0,5474,273.0806899,1,2007,145500,2000,1 +694,18281,7,125842,0,0,0,25901,23800,0,1,1956,,,10 +695,18282,2,28638,4,3492,873,0,3492,706.3981397,1,1929,,,3 +696,18285,2,37229,4,3668,917,0,3668,724.3711646,2,1926,,,3 +697,18286,1,221037,1,1235,1235,0,1235,529.935524,2,1913,,,1 +699,1878230,1,217281,1,1007,1007,0,1007,362.2145622,1,1889,397000,2003,1 +700,1878234,1,164405,1,1112,1112,0,1112,335.5184431,1,1884,322500,2001,1 +701,1878509,-1,0,0,0,0,9440,9440,0,1,1953,,,0 +702,1878685,1,100631,1,1216,1216,0,1216,238.2263365,1,1952,285000,2012,1 +703,1878898,1,82682,1,1025,1025,0,1025,270.4200665,1,1955,65000,1998,1 +704,1878992,1,35590,2,3072,1536,0,3072,233.4217508,1,1958,,,1 +705,1879467,1,38604,2,1916,958,0,1916,285.4424898,1,1960,,,1 +866,1934956,-1,321157,0,0,0,0,0,0,1,1985,,,0 +706,1879910,1,125000,1,960,960,0,960,298.1428795,1,1963,281000,2012,1 +707,1880038,1,62438,1,960,960,0,960,298.5818471,1,1966,,,1 +708,1880211,1,38449,1,1232,1232,0,1232,249.3321956,1,1964,,,1 +709,1880357,13,82804,0,0,0,14896,14896,0,1,1985,,,8 +710,1880667,1,96265,1,916,916,0,916,489.2388941,1,1947,180000,1991,1 +711,1880719,1,76873,1,1335,1335,0,1335,335.9222696,1,1924,162000,1998,1 +712,1880720,1,45772,1,943,943,0,943,431.8080323,1,1918,515000,2013,1 +713,1880725,1,84143,1,972,972,0,972,422.2495228,1,1920,385000,2013,1 +714,1881236,1,18274,1,1036,1036,0,1036,359.6393169,1,1918,,,1 +715,1881642,1,311500,1,2047,2047,0,2047,262.0017067,1,1989,845000,2005,1 +716,1882861,1,152250,1,1482,1482,0,1482,340.1525704,1,1972,420500,2011,1 +717,1883058,1,202000,3,3000,1000,0,3000,409.6487198,1,1949,460000,2006,1 +718,1883092,1,328053,2,5760,2880,0,5760,226.3027615,1,1981,369000,1996,1 +719,1883610,1,347784,1,2510,2510,0,2510,272.6092711,1,1998,735000,2003,1 +720,1883654,1,128250,2,2464,1232,0,2464,357.1407758,1,1982,445000,2004,1 +721,1883742,1,178265,2,2440,1220,0,2440,359.9729218,1,1984,329000,2010,1 +722,1883769,1,162800,1,1232,1232,0,1232,357.6641223,1,1985,276500,2000,1 +723,19097,1,558672,1,1906,1906,0,1906,401.7811105,1,1916,800000,2009,1 +724,1884505,1,154500,1,2074,2074,0,2074,190.9074939,1,1948,350000,2001,1 +725,1884512,1,231309,1,1680,1680,0,1680,214.7356701,1,1950,275000,1998,1 +726,1884532,1,190061,1,2262,2262,0,2262,182.9851789,1,1947,,,1 +727,1884546,1,266500,1,1261,1261,0,1261,258.4773777,1,1947,400000,2013,1 +728,1884790,1,159951,1,1689,1689,0,1689,194.7681066,1,1937,189000,1998,1 +729,1884958,1,112835,1,1863,1863,0,1863,155.7002931,1,1916,156000,1997,1 +730,1885628,1,39028,1,1041,1041,0,1041,298.2532977,1,1955,,,1 +731,1885632,1,52239,1,1551,1551,0,1551,226.2081313,1,1956,,,1 +732,1889117,1,70689,1,1555,1555,0,1555,225.2878759,1,1951,,,1 +734,1889727,1,185328,1,1762,1762,0,1762,208.7891736,1,1950,158000,1996,1 +735,1889894,1,275556,1,1508,1508,0,1508,241.7992693,1,1954,428500,2013,1 +736,1890369,1,74632,1,2277,2277,0,2277,194.3036272,1,1974,,,1 +737,1890375,1,271250,1,1453,1453,0,1453,247.9225859,1,1956,340000,2012,1 +738,1891325,1,88543,1,1603,1603,0,1603,221.0606911,1,1951,,,1 +739,1892512,1,233101,1,1970,1970,0,1970,190.4793849,1,1960,310000,1999,1 +740,1892868,1,165936,1,2168,2168,0,2168,145.6726505,1,1988,,,1 +741,1893199,1,288250,1,1406,1406,0,1406,185.4333738,1,1968,494000,2004,1 +742,1893209,1,130192,1,1360,1360,0,1360,189.2396782,1,1964,175000,1997,1 +743,1893240,1,65695,1,1702,1702,0,1702,164.6419102,1,1971,,,1 +744,1893251,1,143401,1,2665,2665,0,2665,132.1927216,1,1969,650000,2013,1 +745,1893259,1,320750,1,1668,1668,0,1668,166.8037127,1,1973,550000,2004,1 +746,2020189,1,124042,2,2938,1469,0,2938,294.3374139,1,1967,310000,2010,1 +747,1894778,-1,585443,0,0,0,8281,8281,0,1,1985,,,0 +748,1895183,1,121140,1,1301,1301,0,1301,292.8387642,1,1969,421000,2012,1 +749,1895518,1,128952,1,1352,1352,0,1352,285.2433016,1,1969,325000,2010,1 +750,2020192,1,133030,1,1408,1408,0,1408,302.3527423,1,1965,,,1 +751,1895558,1,361250,1,1805,1805,0,1805,238.2885783,1,1971,480000,2002,1 +752,1895808,1,254823,1,1750,1750,0,1750,242.6912806,1,1972,475000,2007,1 +753,1896715,1,578875,1,2478,2478,0,2478,204.6229978,1,1979,638000,2002,1 +754,1896727,1,186482,1,1686,1686,0,1686,248.8168941,1,1978,,,1 +755,1896732,1,519550,1,2468,2468,0,2468,205.1634711,1,1981,619000,2001,1 +756,1896869,1,189587,1,2133,2133,0,2133,219.1423304,1,1981,,,1 +757,1897112,1,203485,1,2080,2080,0,2080,222.3375746,1,1985,,,1 +758,1897770,1,489360,1,2350,2350,0,2350,221.0488959,1,1994,539000,2012,1 +759,1899116,1,175861,2,3828,1914,0,3828,227.2710843,1,1987,206000,1989,1 +760,1899208,1,287875,1,1548,1548,0,1548,227.2458665,1,1990,577500,2007,1 +761,1899229,1,234690,1,1979,1979,0,1979,187.8100746,1,1989,350000,2000,1 +762,1900825,1,138354,1,2311,2311,0,2311,236.1153577,1,1941,,,1 +763,1901384,1,63784,3,5316,1772,0,5316,272.0922264,1,1951,,,1 +764,1901673,1,205495,1,1600,1600,0,1600,292.794325,1,1976,285000,1993,1 +765,1901681,1,165312,1,1425,1425,0,1425,313.4487079,1,1964,185000,1989,1 +766,21157,1,12346,1,1239,1239,0,1239,299.1925632,1,1910,,,1 +767,892994,19,1304318,0,0,0,14709,14709,0,1,1983,,,0 +768,1901864,1,106126,3,4422,1474,0,4422,308.2547449,1,1975,372000,2001,1 +769,1902102,1,388500,1,2202,2202,0,2202,245.74617,1,1977,925500,2005,1 +770,1902112,1,300774,1,2604,2604,0,2604,229.5250738,1,1984,457000,1999,1 +771,1903186,1,106777,1,1180,1180,0,1180,356.2116474,1,1928,207500,1990,1 +772,1903368,1,116021,1,1960,1960,0,1960,288.9924307,1,1977,,,1 +773,1903448,1,84801,1,2313,2313,0,2313,266.5479637,1,1978,,,1 +774,1903693,1,265023,1,2083,2083,0,2083,281.4103703,1,1987,343000,1991,1 +775,1903762,1,333001,1,2505,2505,0,2505,272.5594777,1,1996,449000,1999,1 +776,1903763,1,298051,1,2737,2737,0,2737,263.5309803,1,1995,665000,2003,1 +777,1903766,1,204837,1,2351,2351,0,2351,279.7937816,1,1995,290500,1994,1 +778,229120,1,591972,1,3491,3491,0,3491,291.8554228,2,2003,875000,2010,1 +779,1903807,1,416750,1,2274,2274,0,2274,283.9998117,1,1995,730000,2008,1 +780,1905337,-1,743182,0,0,0,1369,1359,0,1,1920,,,0 +781,1905349,1,18498,1,1284,1284,0,1284,406.2113007,1,1968,,,1 +782,1905886,1,121069,1,1527,1527,0,1527,364.2168118,1,1981,,,1 +783,1905485,1,480500,1,1878,1878,0,1878,263.1847008,1,1950,1150000,2013,1 +784,21753,1,40256,2,2005,1002.5,0,2005,518.9135927,1,1911,,,1 +785,1905579,1,266589,1,2825,2825,0,2825,222.4828832,1,1979,325000,1996,1 +786,1905571,1,196387,1,2202,2202,0,2202,245.9873361,1,1979,262000,1990,1 +787,1905596,1,178366,1,2202,2202,0,2202,245.9873361,1,1979,,,1 +788,1906186,1,312018,2,4636,2318,0,4636,186.6278765,1,1967,388000,1997,1 +789,2020200,1,87052,1,1384,1384,0,1384,306.2895951,1,1968,,,1 +790,2020201,1,196935,1,1792,1792,0,1792,261.0081344,1,1970,265000,1999,1 +791,2020218,7,1113500,0,0,0,100305,81385,0,1,1965,,,10 +792,2020220,1,101614,1,1772,1772,0,1772,262.3100571,1,1967,120000,2011,1 +793,2020222,1,182613,1,1523,1523,0,1523,287.7255499,1,1968,250000,1999,1 +794,2020228,1,57487,1,1466,1466,0,1466,294.5834856,1,1966,427000,2013,1 +795,1907235,1,642963,2,6572,3286,0,6572,217.0011019,1,2006,1525000,2010,1 +796,2020231,1,203500,1,1378,1378,0,1378,307.1830622,1,1968,365000,2008,1 +797,1907291,1,55405,2,1856,928,0,1856,370.034229,1,1940,,,1 +798,1907688,1,178820,1,2195,2195,0,2195,233.5681386,1,1961,,,1 +799,1907852,1,193229,1,1761,1761,0,1761,245.9318014,1,1980,,,1 +800,1908009,1,173764,1,1944,1944,0,1944,251.3373365,1,1985,,,1 +801,1908016,1,285302,1,2490,2490,0,2490,197.0996124,1,1908,550000,2005,1 +802,22232,10,0,0,0,0,11848,8963,0,1,2011,995000,2011,4 +803,1908095,-1,422127,0,0,0,2551,2070,0,1,1994,,,0 +804,1908145,1,916022,1,4423,4423,0,4423,193.7768046,1,2006,,,1 +805,1908155,1,483009,1,3803,3803,0,3803,183.9227564,1,1990,,,1 +806,1908494,1,205372,1,2509,2509,0,2509,215.0495325,1,1994,695000,2009,1 +807,1908503,1,142592,1,1452,1452,0,1452,269.8853983,1,1953,220000,1997,1 +808,1908712,1,323000,1,1737,1737,0,1737,263.6777313,1,1954,690000,2007,1 +809,1909010,7,364732,0,0,0,5612,4622,0,1,1957,,,10 +810,1909307,1,343744,1,3282,3282,0,3282,187.1824954,1,1979,674500,2010,1 +811,1909351,1,734670,1,3905,3905,0,3905,183.1729923,1,1986,650000,2012,1 +812,1909601,1,483250,2,6116,3058,0,6116,85.69835167,1,2004,841000,2013,1 +813,1909618,1,274750,1,1290,1290,0,1290,133.1445216,1,2003,,,1 +814,1909720,1,473914,1,2736,2736,0,2736,211.114286,1,1996,635000,2012,1 +815,1909956,1,241150,1,1618,1618,0,1618,191.1941632,1,1988,291000,2009,1 +816,1910264,1,246447,1,1680,1680,0,1680,210.1683466,1,1960,,,1 +817,1910512,1,69270,1,1655,1655,7151,4473,178.3255362,1,1953,,,1 +818,1910750,1,57579,1,1816,1816,0,1816,170.7170491,1,1975,,,1 +819,1910952,1,90952,1,1816,1816,0,1816,170.8006391,1,1976,,,1 +820,22880,2,314253,4,3600,900,0,3600,417.2142007,2,1924,450000,2008,3 +821,1911584,1,100058,1,1234,1234,0,1234,228.6208596,1,1987,406000,2004,1 +822,1913010,1,173468,1,2048,2048,0,2048,200.4890973,1,1992,220000,1994,1 +823,1913028,1,269594,1,1882,1882,0,1882,209.5941689,1,1991,340000,2002,1 +824,1913657,1,267050,1,1870,1870,0,1870,211.1636571,1,1999,545000,2006,1 +825,1914674,13,0,0,0,0,17694,17694,0,1,1987,,,8 +1033,1974845,1,64860,1,804,804,0,804,281.3503324,1,1978,,,1 +826,1915703,1,115772,2,2532,1266,0,2532,252.0167483,1,1981,172000,1999,1 +827,1915996,1,175092,1,1800,1800,0,1800,164.9668711,1,1983,227000,1999,1 +828,1916001,1,168589,1,1452,1452,0,1452,188.2028491,1,1983,193000,1990,1 +829,23609,13,0,0,0,0,22000,22000,0,1,1972,,,8 +830,1917975,1,350000,1,2262,2262,0,2262,96.29465131,1,2002,367000,2013,1 +831,23880,14,232939,0,0,0,2500,2500,0,1,1953,,,7 +832,1918963,1,68544,1,1782,1782,0,1782,164.0543663,1,1960,,,1 +833,1919870,1,127921,1,1699,1699,0,1699,153.7505157,1,1984,,,1 +834,23975,10,959047,0,0,2976,3576,3576,0,1,1908,325000,2013,4 +836,1920529,1,149900,1,1060,1060,0,1060,235.0043073,1,1985,250000,2008,1 +837,1920733,1,146853,1,842,842,0,842,280.9847708,1,1986,202000,2000,1 +838,1920736,1,184800,2,2120,1060,0,2120,235.1202401,1,1986,238000,2009,1 +839,1920760,1,99085,1,1088,1088,0,1088,230.6051231,1,1986,,,1 +840,1922049,1,148048,1,1490,1490,0,1490,145.9640575,1,2001,,,1 +841,1922331,1,161750,1,1062,1062,0,1062,178.0008723,1,1971,195000,2012,1 +842,1922935,7,192191,0,0,0,2185,2185,0,1,1976,,,10 +843,1923980,1,458193,1,2505,2505,0,2505,265.3370197,1,1960,569000,2009,1 +844,229780,1,44546,1,1120,1120,0,1120,315.01823,1,1961,,,1 +845,1926219,1,128000,1,1302,1302,0,1302,152.9963271,1,1961,260000,2009,1 +846,1926894,7,553930,0,0,0,2530,2530,0,1,1998,,,10 +847,230054,1,117152,1,1448,1448,0,1448,323.5061672,1,1961,68500,1991,1 +848,2020240,1,63587,1,1631,1631,0,1631,275.9612654,1,1970,,,1 +849,25145,7,34787,0,0,0,990,990,0,1,1934,,,10 +850,230178,1,277430,1,1328,1328,0,1328,279.4764051,1,1963,385000,2010,1 +851,1930565,1,43207,1,1344,1344,0,1344,216.7315938,1,1971,,,1 +852,1930595,1,550000,1,2675,2675,0,2675,223.6273025,1,2005,1287000,2006,1 +853,1930570,1,61582,1,1344,1344,0,1344,216.7315938,1,1971,,,1 +854,230435,1,107172,1,1200,1200,0,1200,299.8323042,1,1962,310000,2010,1 +855,230444,1,270200,1,1484,1484,0,1484,259.7135639,1,1964,,,1 +856,230456,1,266000,1,1484,1484,0,1484,259.7135639,1,1964,520000,2013,1 +857,1930834,1,119057,2,3774,1887,0,3774,245.1488956,1,1961,,,1 +948,1961046,1,286624,1,1707,1707,0,1707,310.4823511,1,1997,,,1 +858,1931330,1,230125,1,1627,1627,0,1627,193.1509312,1,1988,639000,2006,1 +859,1932352,1,38656,1,1120,1120,0,1120,342.0768616,1,1950,,,1 +860,1932421,1,114811,1,1918,1918,0,1918,222.6004316,1,1947,,,1 +861,1932797,1,701760,1,3097,3097,0,3097,156.9227925,1,2007,30000,2000,1 +862,893502,5,351037,0,0,0,3040,3040,0,3,1912,,,12 +863,1933304,1,167107,1,1821,1821,0,1821,130.4366993,1,1989,190000,1989,1 +864,1933313,1,268429,1,1973,1973,0,1973,124.8363264,1,1989,315000,1999,1 +865,1934955,-1,78867,0,0,0,0,0,0,1,1985,,,0 +867,1935007,1,711750,1,3921,3921,0,3921,241.686988,1,2006,1700000,2012,1 +868,1935451,1,117192,1,2784,2784,0,2784,216.4245387,1,1974,,,1 +869,1935452,1,314779,1,3320,3320,0,3320,205.6475734,1,1968,,,1 +870,1935645,1,255791,1,2218,2218,0,2218,151.5340047,1,1994,,,1 +871,230704,1,364507,1,3020,3020,0,3020,184.142649,1,1964,455000,2001,1 +872,230707,1,155700,1,1760,1760,0,1760,233.898934,1,1964,,,1 +873,230723,1,219737,1,1794,1794,0,1794,303.3163759,2,1965,225000,1997,1 +874,230764,1,76428,1,1536,1536,0,1536,312.3656041,1,1967,,,1 +875,1935793,1,185000,1,1891,1891,0,1891,154.4165592,1,1963,275000,1999,1 +876,1936185,1,133774,1,1456,1456,0,1456,307.301418,1,1964,490500,2009,1 +877,1936207,1,133540,1,1796,1796,0,1796,265.3913155,1,1977,,,1 +878,1936602,1,65140,1,1660,1660,0,1660,211.4805538,1,1975,340000,2012,1 +879,1937027,1,249250,1,1629,1629,0,1629,282.2711234,1,1985,600000,2005,1 +880,1937041,1,185574,1,2121,2121,0,2121,243.0921211,1,1979,,,1 +881,1937187,1,276000,2,4224,2112,0,4224,147.0429071,1,1981,670000,2006,1 +882,1937200,1,136757,1,1949,1949,0,1949,153.2598154,1,1981,510000,2013,1 +883,1937497,1,254392,1,2334,2334,0,2334,233.2303245,1,1986,,,1 +884,1938333,1,70978,1,1250,1250,0,1250,340.1001254,1,1960,,,1 +885,1939340,1,205375,1,1683,1683,0,1683,220.7517583,1,1968,160000,1992,1 +886,26810,2,102017,2,1596,798,0,1596,223.5936067,1,1928,120000,1999,3 +887,1940970,1,529957,1,2488,2488,0,2488,283.4892948,1,1984,755000,2010,1 +888,26993,2,52325,2,1598,799,0,1598,272.7795912,1,1913,,,3 +889,1941380,1,354521,1,5003,5003,0,5003,209.5740165,1,1974,249000,1993,1 +890,893879,2,191778,20,18000,900,0,18000,553.527275,3,1958,,,3 +891,1942997,-1,599605,0,0,0,1607,1607,0,1,1978,,,0 +892,1943066,1,542976,1,4403,4403,0,4403,201.2048084,1,1972,815000,2000,1 +893,1943162,1,94070,1,1884,1884,0,1884,258.111679,1,1975,,,1 +894,1944181,1,340206,1,904,904,0,904,337.2021486,1,1960,449000,2004,1 +895,1945040,1,256262,1,1049,1049,0,1049,299.644973,1,1952,,,1 +896,1945154,1,18306,1,888,888,0,888,341.2978828,1,1956,,,1 +897,1945793,1,271484,1,1957,1957,0,1957,213.425311,1,1993,373000,1998,1 +898,1946033,-1,29472,0,0,0,4737,3852,0,1,1928,,,0 +899,1946311,1,37814,2,1972,986,0,1972,307.476778,1,1987,,,1 +900,1946921,1,176338,1,1712,1712,0,1712,249.5010581,1,1966,225000,1994,1 +901,1946888,1,121568,2,3240,1620,0,3240,258.4506193,1,1970,,,1 +902,1948028,13,393651,0,0,0,7264,7264,0,1,1988,,,8 +903,28086,1,142723,1,1341,1341,0,1341,327.9681322,1,1919,158500,1996,1 +904,1948372,-1,224120,0,0,0,1844,1844,0,1,1999,,,0 +905,1950865,1,65307,2,2080,1040,0,2080,275.8446645,1,1931,140000,1994,1 +906,1952152,1,263656,2,5576,2788,0,5576,173.929673,1,1971,332000,1991,1 +907,1952381,1,323582,1,2210,2210,0,2210,248.0932171,1,1975,334000,1990,1 +908,1952402,1,202242,2,2768,1384,0,2768,325.5883408,1,1977,260000,1993,1 +909,28699,24,0,1,922,922,0,922,261.3485413,1,1952,191750,2010.5,16 +910,231836,1,120070,1,1864,1864,0,1864,299.187787,2,1981,,,1 +911,231846,1,252000,1,1376,1376,0,1376,335.9504845,1,1969,405000,2002,1 +912,231847,1,48625,1,1704,1704,0,1704,293.3780335,1,1969,,,1 +913,28715,5,470177,0,0,0,16266,11022,0,2,1927,,,12 +914,1952631,1,356740,1,3334,3334,0,3334,258.2599851,1,1979,,,1 +915,1952780,1,38656,1,876,876,0,876,326.9083072,1,1921,123500,2011,1 +916,1953498,1,31975,1,1212,1212,0,1212,286.227994,1,1969,225000,2009,1 +917,1953505,1,107671,1,1560,1560,0,1560,241.5208336,1,1968,240000,2003,1 +918,1954091,1,242876,1,2063,2063,0,2063,207.7588031,1,1986,280000,1990,1 +919,1954093,1,401524,1,2063,2063,0,2063,207.7588031,1,1986,687500,2002,1 +920,1954611,1,127278,1,2756,2756,0,2756,214.2546288,1,1976,365000,1998,1 +921,1956046,1,68103,1,1256,1256,0,1256,182.3534968,1,1952,,,1 +922,1956192,1,154871,1,1487,1487,0,1487,162.6052518,1,1950,285000,2002,1 +923,1956334,1,155000,1,660,660,0,660,300.7239994,1,1950,,,1 +924,1956349,1,56234,1,637,637,0,637,309.8761274,1,1950,140000,2002,1 +925,1956336,1,85216,1,1487,1487,0,1487,148.1750055,1,1940,,,1 +926,1956634,1,37195,1,630,630,0,630,312.7965937,1,1950,47500,1989,1 +927,1956656,7,37195,0,0,0,1660,1660,0,1,1954,,,10 +928,1956880,1,17894,1,754,754,0,754,270.2666229,1,1958,,,1 +929,1957791,1,175099,1,912,912,0,912,230.7820234,1,1947,165000,2009,1 +930,1958582,1,100764,1,1314,1314,0,1314,174.5009157,1,1937,,,1 +931,1958407,1,15413,1,1423,1423,0,1423,165.3816755,1,1937,,,1 +932,1958645,1,117014,1,768,768,0,768,289.7300375,1,1962,160000,2000,1 +933,1958647,1,28364,1,1056,1056,0,1056,226.8208948,1,1972,,,1 +934,1958854,1,270500,1,1640,1640,0,1640,155.6475297,1,1986,584000,2005,1 +935,1958861,1,273391,1,2345,2345,0,2345,127.9213031,1,1972,190000,1994,1 +936,1959152,1,97526,1,1320,1320,0,1320,178.4562561,1,1978,122000,1996,1 +937,1959294,1,89514,1,728,728,0,728,301.4087114,1,1953,120000,1999,1 +938,1959447,1,55679,1,1384,1384,0,1384,154.3471174,1,1927,,,1 +939,1959547,-1,97510,0,0,0,1526,1526,0,1,1930,,,0 +940,1959768,1,25121,1,944,944,0,944,245.826353,1,1962,,,1 +941,1960257,1,140302,1,1240,1240,0,1240,186.6511384,1,1981,,,1 +942,1960481,1,168562,1,2174,2174,0,2174,327.9367163,1,1978,,,1 +943,1960613,1,231046,1,1088,1088,0,1088,504.8918964,1,1975,268000,1992,1 +944,30018,1,34759,1,2496,2496,0,2496,170.247181,2,1918,,,1 +945,232334,1,392443,1,2105,2105,0,2105,297.7238895,2,2005,,,1 +946,1960963,1,127616,1,4182,4182,0,4182,228.9168227,1,1998,,,1 +947,1961045,1,389578,1,3411,3411,0,3411,233.7111689,1,2000,,,1 +949,1961150,1,276020,1,1152,1152,0,1152,380.9501808,1,1979,650000,2004,1 +950,1961166,1,460033,1,1152,1152,0,1152,379.8647703,1,1974,875000,2004,1 +951,30043,1,389200,1,1136,1136,0,1136,370.8495099,1,1931,350000,2011,1 +952,232389,1,275800,1,1271,1271,0,1271,355.2251767,1,1972,570000,2007,1 +953,232409,1,170516,1,1785,1785,0,1785,285.9542128,1,1972,650000,2013,1 +954,30063,2,395791,11,7786,707.9090909,0,7787,669.0394538,2,1960,712500,2012,3 +955,1961456,1,27597,1,1534,1534,0,1534,307.283426,1,1948,,,1 +956,30089,2,189065,3,2628,876.3333333,0,2629,525.0056215,1,1952,,,3 +957,1961719,1,179679,1,2131,2131,0,2131,260.2801434,1,1980,215000,2005,1 +958,1961876,1,34574,1,712,712,0,712,553.4212814,1,1973,36745,2001,1 +959,1962333,1,3874,1,470,470,0,470,711.0431454,1,1935,,,1 +960,1962984,1,41301,1,1220,1220,0,1220,353.4345646,1,1967,,,1 +961,1962988,1,955087,2,5880,2940,0,5880,235.9842058,1,2003,1300000,2009,1 +962,1963000,1,270500,1,1074,1074,0,1074,386.7783924,1,1962,570000,2006,1 +963,1963003,1,32928,1,1090,1090,0,1090,380.8707284,1,1953,,,1 +964,1963004,1,230250,1,1947,1947,0,1947,261.6266919,1,1953,272000,1997,1 +965,1963181,1,300837,1,3163,3163,0,3163,230.0918212,1,1992,,,1 +966,1964210,1,300068,1,4220,4220,0,4220,216.5039479,1,1990,420000,1995,1 +967,1964226,1,463941,1,3003,3003,0,3003,201.0269935,1,1917,525000,1992,1 +968,1964526,1,347187,1,4596,4596,0,4596,132.2431877,1,1987,,,1 +969,1964687,1,144033,1,1585,1585,0,1585,261.6873099,1,1951,,,1 +970,1964668,1,278196,1,1380,1380,0,1380,288.0407043,1,1964,490000,2007,1 +971,1964694,1,69896,3,5904,1968,0,5904,232.4153081,1,1964,,,1 +972,1964876,1,241338,1,2226,2226,0,2226,174.4307498,1,1983,,,1 +973,1964850,1,173365,1,3123,3123,0,3123,153.1106904,1,1971,,,1 +974,232563,1,319418,1,1728,1728,0,1728,352.2658908,1,1974,441000,2008,1 +975,1992688,1,16936,1,864,864,0,864,755.87608,1,1962,,,1 +976,1965515,1,91137,1,1104,1104,0,1104,391.8260799,1,1973,,,1 +977,1965555,1,159772,1,1440,1440,0,1440,195.1289963,1,1950,160000,1992,1 +978,1965639,1,15596,1,696,696,0,696,339.8550888,1,1961,,,1 +979,1965679,1,172250,1,1364,1364,0,1364,201.4053258,1,1941,250000,2003,1 +980,1965668,1,0,1,1142,1142,0,0,0,1,1950,265000,2013,1 +981,1965800,1,188392,1,2430,2430,0,2430,133.3280207,1,1938,231000,2000,1 +982,30657,2,41218,1,2123,2123,0,2123,340.5751814,2,1926,278182,2003,3 +983,1966714,1,170500,1,1077,1077,0,1077,240.3018421,1,1959,290000,2013,1 +984,1966733,1,0,1,1662,1662,0,1662,191.6544664,1,2001,,,1 +985,1966746,1,111593,1,1206,1206,0,1206,221.4174415,1,1960,150000,1997,1 +986,1966753,1,52782,1,884,884,0,884,279.0357977,1,1955,,,1 +987,1967100,1,204605,1,1174,1174,0,1174,224.8366441,1,1952,,,1 +988,1967223,1,178513,1,1480,1480,0,1480,192.0534283,1,1954,,,1 +989,30706,1,242378,1,1563,1563,0,1563,464.3754516,1,1925,274500,1996,1 +990,1967652,1,200000,2,2160,1080,0,2160,310.0723299,1,1940,485000,2005,1 +991,1967675,1,117014,1,702,702,0,702,433.7415002,1,1930,177000,2000,1 +992,1967703,1,35170,1,1116,1116,0,1116,301.9174831,1,1935,,,1 +993,1967876,1,117250,1,708,708,0,708,431.1048827,1,1932,315000,2006,1 +994,1967940,1,95598,1,1116,1116,0,1116,301.4729949,1,1932,,,1 +996,1968296,7,256179,0,0,0,3588,3588,0,1,1944,,,10 +997,1968312,-1,92994,0,0,0,4620,4620,0,1,1922,,,0 +998,1968346,-1,159034,0,0,0,8394,5700,0,1,1985,,,0 +999,1968535,1,438250,1,3463,3463,0,3463,140.9687923,1,1995,,,1 +1000,1968538,1,183305,1,2033,2033,0,2033,162.1497341,1,1987,863000,2013,1 +1001,1968865,1,44546,1,1152,1152,0,1152,230.4075629,1,1974,,,1 +1002,232607,1,267945,1,2039,2039,0,2039,321.3836332,1,1973,673000,2007,1 +1003,232615,1,257157,1,2300,2300,0,2300,303.5274463,1,1975,279000,1994,1 +1004,232618,1,350896,1,2368,2368,0,2368,309.5751843,2,1975,600000,2012,1 +1005,30833,1,170121,1,1499,1499,0,1499,369.3534643,1,1924,185000,1994,1 +1006,30851,1,136423,1,1841,1841,0,1841,337.6424416,2,1927,225000,1999,1 +1007,2020242,1,238750,2,2344,1172,0,2344,343.6771042,1,1967,273000,2011,1 +1008,1968833,1,40221,1,1655,1655.200203,0,1655.200203,256.4554755,1,1964,,,1 +1009,1968912,1,274033,2,5256,2628,0,5256,206.0583091,1,1987,290000,1989,1 +1010,1968892,1,198873,1,1158,1158,0,1158,326.826077,1,1972,237000,1990,1 +1011,1968915,1,0,1,1607,1607,0,1607,262.411022,1,1974,,,1 \ No newline at end of file From 22791a7f331b3199dd8b92a469e37c4ecc781eaa Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 9 Feb 2021 13:28:20 -0800 Subject: [PATCH 118/121] Cleanup --- docs/.nojekyll | 0 docs/source/getting-started.rst | 16 ++---- examples/UrbanSim-Templates-demo.ipynb | 68 +++++++++++++------------- 3 files changed, 39 insertions(+), 45 deletions(-) delete mode 100644 docs/.nojekyll diff --git a/docs/.nojekyll b/docs/.nojekyll deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index 1dce43d..1056d93 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -18,17 +18,12 @@ UrbanSim Templates was created in 2018 by Sam Maurer (maurer@urbansim.com), who Installation ------------ -UrbanSim Templates is tested with Python versions 2.7, 3.5, 3.6, and 3.7. - -As of Feb. 2019, there is an installation problem in Python 3.7 when using Pip (because of an issue with Orca's PyTables dependency). Conda should work. - -.. note:: - It can be helpful to set up a dedicated Python environment for each project you work on. This lets you use a stable and replicable set of libraries that won't be affected by other projects. Here are some good `environment settings `__ for UrbanSim Templates projects. +UrbanSim Templates is currently tested with Python versions 3.6, 3.7, 3.8, and 3.9. Production releases ~~~~~~~~~~~~~~~~~~~ -UrbanSim Templates can be installed using the Pip or Conda package managers. With Conda, you (currently) need to install UrbanSim separately; Pip will handle this automatically. +UrbanSim Templates can be installed using the Pip or Conda package managers. .. code-block:: python @@ -37,11 +32,10 @@ UrbanSim Templates can be installed using the Pip or Conda package managers. Wit .. code-block:: python conda install urbansim_templates --channel conda-forge - conda install urbansim --channel udst Dependencies include `NumPy `__, `Pandas `__, and `Statsmodels `__, plus two other UDST libraries: `Orca `__ and `ChoiceModels `__. These will be included automatically when you install UrbanSim Templates. -Certain less-commonly-used templates require additional packages: currently, `PyLogit `__ and `Scikit-learn `__. You'll need to install these manually to use the associated templates. +Certain less-commonly-used templates require additional packages: currently, `PyLogit `__ and `Scikit-learn `__. You'll need to install these separately to use the associated templates. When new production releases of UrbanSim Templates come out, you can upgrade like this: @@ -105,7 +99,7 @@ The default file location is a ``configs`` folder located in the current working In [2]: import urbansim_templates print(urbansim_templates.__version__) - Out[2]: '0.2.dev0' + Out[2]: '0.2' Creating a model step @@ -131,7 +125,7 @@ This sets up ``m`` as an instance of the OLS regression template. The ``tables`` import orca import pandas as pd - url = "https://www.dropbox.com/s/vxg5pdfzxrh6osz/buildings-demo.csv?dl=1" + url = "https://raw.githubusercontent.com/UDST/urbansim_templates/dev/examples/data/buildings-demo.csv" df = pd.read_csv(url).dropna() orca.add_table('buildings', df) diff --git a/examples/UrbanSim-Templates-demo.ipynb b/examples/UrbanSim-Templates-demo.ipynb index 310d52b..302704d 100644 --- a/examples/UrbanSim-Templates-demo.ipynb +++ b/examples/UrbanSim-Templates-demo.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "mature-episode", + "id": "enclosed-combat", "metadata": {}, "source": [ "# UrbanSim Templates demo\n", @@ -13,11 +13,11 @@ "\n", "[UrbanSim](https://github.com/udst/urbansim) is a platform for modeling land use in cities. It runs in Python and uses the [Orca](https://github.com/udst/orca) task orchestration system. \n", "\n", - "Orca breaks a model into \"steps\", Python functions that can be assembled on the fly into linear or cyclical pipelines. Orca is designed for workflows like city simulation where the data representing a model's state is so large that it needs to be managed outside the task graph. Steps refer to tables and columns of data by name rather than passing the data directly.\n", + "Orca breaks a model into \"steps\", Python functions that can be assembled on the fly into linear or cyclical pipelines. (Typically each step is a statistical model capturing one aspect of the dynamics being studied.) Orca is designed for workflows like city simulation where the data representing a model's state is so large that it needs to be managed outside the task graph. Steps refer to tables and columns of data by name rather than passing the data directly.\n", "\n", "UrbanSim [Templates](https://github.com/udst/urbansim_templates) is a library that provides automated building blocks for Orca-based models. The templates were developed to reduce the need for custom code and improve the portability of model steps.\n", "\n", - "Currently we have templates for (a) regression, (b) binary Logit, (c) multinomial Logit estimated with [PyLogit](https://github.com/timothyb0912/pylogit) (best choice for flexible utility expressions), and (d) multinomial Logit estimated with [ChoiceModels](https://github.com/udst/choicemodels) (best choice for sampling of interchangeable alternatives).\n", + "Currently we have templates for (a) regression, (b) binary logit, (c) multinomial logit estimated with [PyLogit](https://github.com/timothyb0912/pylogit) (best choice for flexible utility expressions), and (d) multinomial logit estimated with [ChoiceModels](https://github.com/udst/choicemodels) (best choice for sampling of interchangeable alternatives).\n", "\n", "### Documentation\n", "\n", @@ -25,13 +25,13 @@ "\n", "### Installation\n", "\n", - "You can install `orca` and `urbansim_templates` from Pip or Conda Forge." + "You can install `orca` and `urbansim_templates` with Pip or from Conda Forge." ] }, { "cell_type": "code", "execution_count": 1, - "id": "natural-frequency", + "id": "romantic-auction", "metadata": {}, "outputs": [ { @@ -51,7 +51,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "lyric-gardening", + "id": "representative-alaska", "metadata": {}, "outputs": [ { @@ -71,7 +71,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "featured-return", + "id": "medieval-tourism", "metadata": {}, "outputs": [ { @@ -91,11 +91,11 @@ { "cell_type": "code", "execution_count": 4, - "id": "floppy-brunei", + "id": "approved-burke", "metadata": {}, "outputs": [], "source": [ - "# Making the notebook output clearer\n", + "# This makes the notebook output clearer\n", "import warnings\n", "warnings.simplefilter(action='ignore', category=FutureWarning)" ] @@ -103,14 +103,14 @@ { "cell_type": "code", "execution_count": null, - "id": "worst-thong", + "id": "derived-navigation", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "sporting-orange", + "id": "potential-atmosphere", "metadata": {}, "source": [ "### Setting up ModelManager\n", @@ -123,7 +123,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "coupled-turning", + "id": "abstract-object", "metadata": {}, "outputs": [ { @@ -144,14 +144,14 @@ { "cell_type": "code", "execution_count": null, - "id": "structured-potential", + "id": "scenic-jacksonville", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "abandoned-somalia", + "id": "christian-deputy", "metadata": {}, "source": [ "### Setting up data\n", @@ -162,7 +162,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "cheap-darkness", + "id": "invisible-greensboro", "metadata": {}, "outputs": [ { @@ -184,7 +184,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "impossible-dressing", + "id": "suspected-walker", "metadata": {}, "outputs": [ { @@ -205,7 +205,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "balanced-chair", + "id": "adverse-queue", "metadata": {}, "outputs": [ { @@ -383,17 +383,17 @@ { "cell_type": "code", "execution_count": null, - "id": "wooden-appendix", + "id": "material-client", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "horizontal-defeat", + "id": "legislative-halloween", "metadata": {}, "source": [ - "### Creating a model step\n", + "### Fitting a model\n", "\n", "Now we can choose a [template](https://udst.github.io/urbansim_templates/model-steps.html) and use it to fit a model." ] @@ -401,7 +401,7 @@ { "cell_type": "code", "execution_count": 9, - "id": "collaborative-channels", + "id": "minor-northwest", "metadata": {}, "outputs": [], "source": [ @@ -416,7 +416,7 @@ { "cell_type": "code", "execution_count": 10, - "id": "domestic-messaging", + "id": "through-leather", "metadata": {}, "outputs": [ { @@ -459,25 +459,25 @@ { "cell_type": "code", "execution_count": null, - "id": "sought-evanescence", + "id": "raised-thumb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "expected-overhead", + "id": "applied-counter", "metadata": {}, "source": [ "### Registering the step\n", "\n", - "Now we can \"register\" the step with ModelManager. This saves a copy to disk (in the `configs` folder), and passes a copy to Orca so it can be run as part of a sequence of other steps for validation or simulation." + "When we're happy with the specification, we can \"register\" the step with ModelManager. This saves a copy to disk and also passes it to Orca so it can be run as part of a sequence of other steps for validation or simulation." ] }, { "cell_type": "code", "execution_count": 11, - "id": "global-mistress", + "id": "least-unemployment", "metadata": {}, "outputs": [ { @@ -496,14 +496,14 @@ { "cell_type": "code", "execution_count": null, - "id": "hairy-binary", + "id": "square-dryer", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "polished-casting", + "id": "christian-continent", "metadata": {}, "source": [ "### Making changes\n", @@ -514,7 +514,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "atmospheric-colorado", + "id": "constitutional-harvey", "metadata": {}, "outputs": [ { @@ -535,7 +535,7 @@ { "cell_type": "code", "execution_count": 13, - "id": "limited-wilson", + "id": "filled-display", "metadata": {}, "outputs": [], "source": [ @@ -545,7 +545,7 @@ { "cell_type": "code", "execution_count": 14, - "id": "solved-glass", + "id": "front-elder", "metadata": {}, "outputs": [ { @@ -567,7 +567,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "secondary-baking", + "id": "above-contamination", "metadata": {}, "outputs": [ { @@ -585,7 +585,7 @@ { "cell_type": "code", "execution_count": null, - "id": "classical-attachment", + "id": "quality-essay", "metadata": {}, "outputs": [], "source": [] @@ -593,7 +593,7 @@ { "cell_type": "code", "execution_count": null, - "id": "acoustic-insight", + "id": "noticed-convention", "metadata": {}, "outputs": [], "source": [] From b39ec8e5686cd29b9e86871b5d8cabc0bed6dcc3 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 9 Feb 2021 13:29:00 -0800 Subject: [PATCH 119/121] Cleanup --- examples/UrbanSim-Templates-demo.ipynb | 58 +++++++++++++------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/examples/UrbanSim-Templates-demo.ipynb b/examples/UrbanSim-Templates-demo.ipynb index 302704d..acbc003 100644 --- a/examples/UrbanSim-Templates-demo.ipynb +++ b/examples/UrbanSim-Templates-demo.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "enclosed-combat", + "id": "fixed-tenant", "metadata": {}, "source": [ "# UrbanSim Templates demo\n", @@ -15,7 +15,7 @@ "\n", "Orca breaks a model into \"steps\", Python functions that can be assembled on the fly into linear or cyclical pipelines. (Typically each step is a statistical model capturing one aspect of the dynamics being studied.) Orca is designed for workflows like city simulation where the data representing a model's state is so large that it needs to be managed outside the task graph. Steps refer to tables and columns of data by name rather than passing the data directly.\n", "\n", - "UrbanSim [Templates](https://github.com/udst/urbansim_templates) is a library that provides automated building blocks for Orca-based models. The templates were developed to reduce the need for custom code and improve the portability of model steps.\n", + "UrbanSim [Templates](https://github.com/udst/urbansim_templates) is a library that provides automated building blocks for Orca-based models. The templates were developed to reduce the need for custom code and improve the portability of model components.\n", "\n", "Currently we have templates for (a) regression, (b) binary logit, (c) multinomial logit estimated with [PyLogit](https://github.com/timothyb0912/pylogit) (best choice for flexible utility expressions), and (d) multinomial logit estimated with [ChoiceModels](https://github.com/udst/choicemodels) (best choice for sampling of interchangeable alternatives).\n", "\n", @@ -31,7 +31,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "romantic-auction", + "id": "hearing-rescue", "metadata": {}, "outputs": [ { @@ -51,7 +51,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "representative-alaska", + "id": "ultimate-durham", "metadata": {}, "outputs": [ { @@ -71,7 +71,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "medieval-tourism", + "id": "taken-membership", "metadata": {}, "outputs": [ { @@ -91,7 +91,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "approved-burke", + "id": "independent-macedonia", "metadata": {}, "outputs": [], "source": [ @@ -103,14 +103,14 @@ { "cell_type": "code", "execution_count": null, - "id": "derived-navigation", + "id": "ultimate-partner", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "potential-atmosphere", + "id": "stylish-problem", "metadata": {}, "source": [ "### Setting up ModelManager\n", @@ -123,7 +123,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "abstract-object", + "id": "fatal-welsh", "metadata": {}, "outputs": [ { @@ -144,14 +144,14 @@ { "cell_type": "code", "execution_count": null, - "id": "scenic-jacksonville", + "id": "acute-savings", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "christian-deputy", + "id": "legitimate-square", "metadata": {}, "source": [ "### Setting up data\n", @@ -162,7 +162,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "invisible-greensboro", + "id": "polyphonic-pointer", "metadata": {}, "outputs": [ { @@ -184,7 +184,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "suspected-walker", + "id": "quarterly-rugby", "metadata": {}, "outputs": [ { @@ -205,7 +205,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "adverse-queue", + "id": "broken-manchester", "metadata": {}, "outputs": [ { @@ -383,14 +383,14 @@ { "cell_type": "code", "execution_count": null, - "id": "material-client", + "id": "cheap-sugar", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "legislative-halloween", + "id": "international-ordering", "metadata": {}, "source": [ "### Fitting a model\n", @@ -401,7 +401,7 @@ { "cell_type": "code", "execution_count": 9, - "id": "minor-northwest", + "id": "studied-federation", "metadata": {}, "outputs": [], "source": [ @@ -416,7 +416,7 @@ { "cell_type": "code", "execution_count": 10, - "id": "through-leather", + "id": "checked-addition", "metadata": {}, "outputs": [ { @@ -459,14 +459,14 @@ { "cell_type": "code", "execution_count": null, - "id": "raised-thumb", + "id": "concerned-argument", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "applied-counter", + "id": "widespread-cache", "metadata": {}, "source": [ "### Registering the step\n", @@ -477,7 +477,7 @@ { "cell_type": "code", "execution_count": 11, - "id": "least-unemployment", + "id": "thick-steam", "metadata": {}, "outputs": [ { @@ -496,14 +496,14 @@ { "cell_type": "code", "execution_count": null, - "id": "square-dryer", + "id": "egyptian-newport", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "christian-continent", + "id": "bound-rapid", "metadata": {}, "source": [ "### Making changes\n", @@ -514,7 +514,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "constitutional-harvey", + "id": "portable-supplier", "metadata": {}, "outputs": [ { @@ -535,7 +535,7 @@ { "cell_type": "code", "execution_count": 13, - "id": "filled-display", + "id": "married-delay", "metadata": {}, "outputs": [], "source": [ @@ -545,7 +545,7 @@ { "cell_type": "code", "execution_count": 14, - "id": "front-elder", + "id": "responsible-logic", "metadata": {}, "outputs": [ { @@ -567,7 +567,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "above-contamination", + "id": "productive-wyoming", "metadata": {}, "outputs": [ { @@ -585,7 +585,7 @@ { "cell_type": "code", "execution_count": null, - "id": "quality-essay", + "id": "inclusive-annual", "metadata": {}, "outputs": [], "source": [] @@ -593,7 +593,7 @@ { "cell_type": "code", "execution_count": null, - "id": "noticed-convention", + "id": "martial-fortune", "metadata": {}, "outputs": [], "source": [] From 723b83b4187da53a50ee03fdba4842a464f68240 Mon Sep 17 00:00:00 2001 From: Max Gardner Date: Mon, 27 Sep 2021 14:00:00 -0700 Subject: [PATCH 120/121] Post-MCT interaction term operations (#126) --- .../models/large_multinomial_logit.py | 458 ++++++++++++------ 1 file changed, 303 insertions(+), 155 deletions(-) diff --git a/urbansim_templates/models/large_multinomial_logit.py b/urbansim_templates/models/large_multinomial_logit.py index fab92ce..3eba2c7 100644 --- a/urbansim_templates/models/large_multinomial_logit.py +++ b/urbansim_templates/models/large_multinomial_logit.py @@ -3,21 +3,21 @@ import orca from urbansim.models.util import columns_in_formula, apply_filter_query from choicemodels.tools import MergedChoiceTable +import pandas as pd from .. import modelmanager from ..utils import get_data, update_column, to_list, version_greater_or_equal from .shared import TemplateStep - def check_choicemodels_version(): try: import choicemodels assert version_greater_or_equal(choicemodels.__version__, '0.2.dev4') except: raise ImportError("LargeMultinomialLogitStep requires choicemodels 0.2.dev4 or " - "later. For installation instructions, see " - "https://github.com/udst/choicemodels.") + "later. For installation instructions, see " + "https://github.com/udst/choicemodels.") @modelmanager.template @@ -26,9 +26,9 @@ class LargeMultinomialLogitStep(TemplateStep): Class for building standard multinomial logit model steps where alternatives are interchangeable and all have the same model expression. Supports random sampling of alternatives. - + Estimation and simulation are performed using ChoiceModels. - + Parameters ---------- choosers : str or list of str, optional @@ -39,7 +39,7 @@ class LargeMultinomialLogitStep(TemplateStep): 'alternatives' parameters replace the 'tables' parameter. Both are required for fitting a model, but do not have to be provided when the object is created. Reserved column names: 'chosen'. - + alternatives : str or list of str, optional Name(s) of Orca tables containing data about alternatives. The first table is the primary one. Any additional tables need to have merge relationships ("broadcasts") @@ -48,19 +48,19 @@ class LargeMultinomialLogitStep(TemplateStep): 'alternatives' parameters replace the 'tables' parameter. Both are required for fitting a model, but do not have to be provided when the object is created. Reserved column names: 'chosen'. - + model_expression : str, optional Patsy-style right-hand-side model expression representing the utility of a single alternative. Passed to `choicemodels.MultinomialLogit()`. This parameter is required for fitting a model, but does not have to be provided when the object is created. - + choice_column : str, optional Name of the column indicating observed choices, for model estimation. The column should contain integers matching the id of the primary `alternatives` table. This parameter is required for fitting a model, but it does not have to be provided when the object is created. Not required for simulation. - + chooser_filters : str or list of str, optional Filters to apply to the chooser data before fitting the model. These are passed to `pd.DataFrame.query()`. Filters are applied after any additional tables are merged @@ -101,7 +101,7 @@ class LargeMultinomialLogitStep(TemplateStep): to match its data type. If the column is generated on the fly, it will be given the same data type as the index of the alternatives table. Replaces the `out_fname` argument in UrbanSim. - + out_chooser_filters : str or list of str, optional Filters to apply to the chooser data before simulation. If not provided, no filters will be applied. Replaces the `predict_filters` argument in UrbanSim. @@ -113,69 +113,70 @@ class LargeMultinomialLogitStep(TemplateStep): constrained_choices : bool, optional "True" means alternatives have limited capacity. "False" (default) means that alternatives can accommodate an unlimited number of choosers. - + alt_capacity : str, optional Name of a column in the out_alternatives table that expresses the capacity of alternatives. If not provided and constrained_choices is True, each alternative is interpreted as accommodating a single chooser. - + chooser_size : str, optional Name of a column in the out_choosers table that expresses the size of choosers. Choosers might have varying sizes if the alternative capacities are amounts rather than counts -- e.g. square footage. Chooser sizes must be in the same units as alternative capacities. If not provided and constrained_choices is True, each chooser has a size of 1. - + max_iter : int or None, optional Maximum number of choice simulation iterations. If None (default), the algorithm will iterate until all choosers are matched or no alternatives remain. - + name : str, optional Name of the model step, passed to ModelManager. If none is provided, a name is generated each time the `fit()` method runs. - + tags : list of str, optional Tags, passed to ModelManager. - + Attributes ---------- All parameters can also be get and set as properties. The following attributes should be treated as read-only. - + choices : pd.Series Available after the model step is run. List of chosen alternative id's, indexed with the chooser id. Does not persist when the model step is reloaded from storage. - + mergedchoicetable : choicemodels.tools.MergedChoiceTable Table built for estimation or simulation. Does not persist when the model step is reloaded from storage. Not available if choices have capacity constraints, because multiple choice tables are generated iteratively. - + model : choicemodels.MultinomialLogitResults Available after a model has been fit. Persists when reloaded from storage. - + probabilities : pd.Series Available after the model step is run -- but not if choices have capacity constraints, which requires probabilities to be calculated multiple times. Provides list of probabilities corresponding to the sampled alternatives, indexed with the chooser and alternative id's. Does not persist when the model step is reloaded from storage. - + """ - def __init__(self, choosers=None, alternatives=None, model_expression=None, - choice_column=None, chooser_filters=None, chooser_sample_size=None, - alt_filters=None, alt_sample_size=None, out_choosers=None, - out_alternatives=None, out_column=None, out_chooser_filters=None, - out_alt_filters=None, constrained_choices=False, alt_capacity=None, - chooser_size=None, max_iter=None, name=None, tags=[]): - + + def __init__(self, choosers=None, alternatives=None, model_expression=None, + choice_column=None, chooser_filters=None, chooser_sample_size=None, + alt_filters=None, alt_sample_size=None, out_choosers=None, + out_alternatives=None, out_column=None, out_chooser_filters=None, + out_alt_filters=None, constrained_choices=False, alt_capacity=None, + chooser_size=None, max_iter=None, mct_intx_ops=None, name=None, tags=[]): + self._listeners = [] - + # Parent class can initialize the standard parameters - TemplateStep.__init__(self, tables=None, model_expression=model_expression, - filters=None, out_tables=None, out_column=out_column, out_transform=None, - out_filters=None, name=name, tags=tags) + TemplateStep.__init__(self, tables=None, model_expression=model_expression, + filters=None, out_tables=None, out_column=out_column, out_transform=None, + out_filters=None, name=name, tags=tags) # Custom parameters not in parent class self.choosers = choosers @@ -193,76 +194,76 @@ def __init__(self, choosers=None, alternatives=None, model_expression=None, self.alt_capacity = alt_capacity self.chooser_size = chooser_size self.max_iter = max_iter - + self.mct_intx_ops = mct_intx_ops + # Placeholders for model fit data, filled in by fit() or from_dict() - self.summary_table = None + self.summary_table = None self.fitted_parameters = None self.model = None - + # Placeholders for diagnostic data, filled in by fit() or run() self.mergedchoicetable = None self.probabilities = None self.choices = None + def bind_to(self, callback): self._listeners.append(callback) - - + def send_to_listeners(self, param, value): for callback in self._listeners: callback(param, value) - - + @classmethod def from_dict(cls, d): """ Create an object instance from a saved dictionary representation. - + Parameters ---------- d : dict - + Returns ------- LargeMultinomialLogitStep - + """ check_choicemodels_version() from choicemodels import MultinomialLogitResults - + # Pass values from the dictionary to the __init__() method - obj = cls(choosers=d['choosers'], alternatives=d['alternatives'], - model_expression=d['model_expression'], choice_column=d['choice_column'], - chooser_filters=d['chooser_filters'], - chooser_sample_size=d['chooser_sample_size'], - alt_filters=d['alt_filters'], alt_sample_size=d['alt_sample_size'], - out_choosers=d['out_choosers'], out_alternatives=d['out_alternatives'], - out_column=d['out_column'], out_chooser_filters=d['out_chooser_filters'], - out_alt_filters=d['out_alt_filters'], - constrained_choices=d['constrained_choices'], alt_capacity=d['alt_capacity'], - chooser_size=d['chooser_size'], max_iter=d['max_iter'], name=d['name'], - tags=d['tags']) + obj = cls(choosers=d['choosers'], alternatives=d['alternatives'], + model_expression=d['model_expression'], choice_column=d['choice_column'], + chooser_filters=d['chooser_filters'], + chooser_sample_size=d['chooser_sample_size'], + alt_filters=d['alt_filters'], alt_sample_size=d['alt_sample_size'], + out_choosers=d['out_choosers'], out_alternatives=d['out_alternatives'], + out_column=d['out_column'], out_chooser_filters=d['out_chooser_filters'], + out_alt_filters=d['out_alt_filters'], + constrained_choices=d['constrained_choices'], alt_capacity=d['alt_capacity'], + chooser_size=d['chooser_size'], max_iter=d['max_iter'], + mct_intx_ops=d.get('mct_intx_ops', None), name=d['name'], + tags=d['tags']) # Load model fit data obj.summary_table = d['summary_table'] obj.fitted_parameters = d['fitted_parameters'] - + if obj.fitted_parameters is not None: - obj.model = MultinomialLogitResults(model_expression = obj.model_expression, - fitted_parameters = obj.fitted_parameters) - - return obj + obj.model = MultinomialLogitResults(model_expression=obj.model_expression, + fitted_parameters=obj.fitted_parameters) + return obj def to_dict(self): """ Create a dictionary representation of the object. - + Returns ------- dict - + """ d = { 'template': self.template, @@ -286,65 +287,72 @@ def to_dict(self): 'alt_capacity': self.alt_capacity, 'chooser_size': self.chooser_size, 'max_iter': self.max_iter, + 'mct_intx_ops': self.mct_intx_ops, 'summary_table': self.summary_table, 'fitted_parameters': self.fitted_parameters, } return d - # TO DO - there has got to be a less verbose way to handle getting and setting - + @property def choosers(self): return self.__choosers + @choosers.setter def choosers(self, value): self.__choosers = self._normalize_table_param(value) self.send_to_listeners('choosers', value) - + @property def alternatives(self): return self.__alternatives + @alternatives.setter def alternatives(self, value): self.__alternatives = self._normalize_table_param(value) self.send_to_listeners('alternatives', value) - + @property def model_expression(self): return self.__model_expression + @model_expression.setter def model_expression(self, value): self.__model_expression = value self.send_to_listeners('model_expression', value) - + @property def choice_column(self): return self.__choice_column + @choice_column.setter def choice_column(self, value): self.__choice_column = value self.send_to_listeners('choice_column', value) - + @property def chooser_filters(self): return self.__chooser_filters + @chooser_filters.setter def chooser_filters(self, value): self.__chooser_filters = value self.send_to_listeners('chooser_filters', value) - + @property def chooser_sample_size(self): return self.__chooser_sample_size + @chooser_sample_size.setter def chooser_sample_size(self, value): self.__chooser_sample_size = value self.send_to_listeners('chooser_sample_size', value) - + @property def alt_filters(self): return self.__alt_filters + @alt_filters.setter def alt_filters(self, value): self.__alt_filters = value @@ -353,6 +361,7 @@ def alt_filters(self, value): @property def alt_sample_size(self): return self.__alt_sample_size + @alt_sample_size.setter def alt_sample_size(self, value): self.__alt_sample_size = value @@ -361,104 +370,219 @@ def alt_sample_size(self, value): @property def out_choosers(self): return self.__out_choosers + @out_choosers.setter def out_choosers(self, value): self.__out_choosers = self._normalize_table_param(value) self.send_to_listeners('out_choosers', value) - + @property def out_alternatives(self): return self.__out_alternatives + @out_alternatives.setter def out_alternatives(self, value): - self.__out_alternatives = self._normalize_table_param(value) + self.__out_alternatives = self._normalize_table_param(value) self.send_to_listeners('out_alternatives', value) @property def out_column(self): return self.__out_column + @out_column.setter def out_column(self, value): self.__out_column = value self.send_to_listeners('out_column', value) - + @property def out_chooser_filters(self): return self.__out_chooser_filters + @out_chooser_filters.setter def out_chooser_filters(self, value): self.__out_chooser_filters = value self.send_to_listeners('out_chooser_filters', value) - + @property def out_alt_filters(self): return self.__out_alt_filters + @out_alt_filters.setter def out_alt_filters(self, value): self.__out_alt_filters = value self.send_to_listeners('out_alt_filters', value) - + @property def constrained_choices(self): return self.__constrained_choices + @constrained_choices.setter def constrained_choices(self, value): self.__constrained_choices = value self.send_to_listeners('constrained_choices', value) - + @property def alt_capacity(self): return self.__alt_capacity + @alt_capacity.setter def alt_capacity(self, value): self.__alt_capacity = value self.send_to_listeners('alt_capacity', value) - + @property def chooser_size(self): return self.__chooser_size + @chooser_size.setter def chooser_size(self, value): self.__chooser_size = value self.send_to_listeners('chooser_size', value) - + @property def max_iter(self): return self.__max_iter + @max_iter.setter def max_iter(self, value): self.__max_iter = value self.send_to_listeners('max_iter', value) - - + + @property + def mct_intx_ops(self): + return self.__mct_intx_ops + + @mct_intx_ops.setter + def mct_intx_ops(self, value): + self.__mct_intx_ops = value + self.send_to_listeners('mct_intx_ops', value) + + def perform_mct_intx_ops(self, mct, nan_handling='zero'): + """ + Method to dynamically update a MergedChoiceTable object according to + a pre-defined set of operations specified in the model .yaml config. + Operations are performed sequentially as follows: 1) Pandas merges + with other Orca tables; 2) Pandas group-by aggregations; 3) rename + existing columns; 4) create new columns via Pandas `eval()`. + + Parameters + ---------- + mct : choicemodels.tools.MergedChoiceTable + nan_handling : str + Either 'zero' or 'drop', where the former will replace all NaN's + and None's with 0 integers and the latter will drop all rows with + any NaN or Null values. + + Returns + ------- + MergedChoiceTable + """ + + intx_ops = self.mct_intx_ops + mct_df = mct.to_frame() + og_mct_index = mct_df.index.names + mct_df.reset_index(inplace=True) + mct_df.index.name = 'mct_index' + + # merges + intx_df = mct_df.copy() + for merge_args in intx_ops.get('successive_merges', []): + + # make sure mct index is preserved during merge + left_cols = merge_args.get('mct_cols', intx_df.columns) + left_idx = merge_args.get('left_index', False) + + if intx_df.index.name == mct_df.index.name: + if not left_idx: + intx_df.reset_index(inplace=True) + if mct_df.index.name not in left_cols: + left_cols += [mct_df.index.name] + elif mct_df.index.name in intx_df.columns: + if mct_df.index.name not in left_cols: + left_cols += [mct_df.index.name] + else: + raise KeyError( + 'Column {0} must be preserved in intx ops!'.format( + mct_df.index.name)) + + left = intx_df[left_cols] + + right = get_data( + merge_args['right_table'], + extra_columns=merge_args.get('right_cols', None)) + + intx_df = pd.merge( + left, right, + how=merge_args.get('how', 'inner'), + on=merge_args.get('on_cols', None), + left_on=merge_args.get('left_on', None), + right_on=merge_args.get('right_on', None), + left_index=left_idx, + right_index=merge_args.get('right_index', False), + suffixes=merge_args.get('suffixes', ('_x', '_y'))) + + # aggs + aggs = intx_ops.get('aggregations', False) + if aggs: + intx_df = intx_df.groupby('mct_index').agg(aggs) + + # rename cols + if intx_ops.get('rename_cols', False): + intx_df = intx_df.rename( + columns=intx_ops['rename_cols']) + + # update mct + mct_df = pd.merge(mct_df, intx_df, on='mct_index') + + # create new cols from expressions + for eval_op in intx_ops.get('sequential_eval_ops', []): + new_col = eval_op['name'] + expr = eval_op['expr'] + engine = eval_op.get('engine', 'numexpr') + mct_df[new_col] = mct_df.eval(expr, engine=engine) + + # restore original mct index + mct_df.set_index(og_mct_index, inplace=True) + + # handle NaNs and Nones + if mct_df.isna().values.any(): + if nan_handling == 'zero': + print("Replacing MCT None's and NaN's with 0") + mct_df = mct_df.fillna(0) + elif nan_handling == 'drop': + print("Dropping rows with None's/NaN's from MCT") + mct_df = mct_df.dropna(axis=0) + + return MergedChoiceTable.from_df(mct_df) + def fit(self, mct=None): """ Fit the model; save and report results. This uses the ChoiceModels estimation engine (originally from UrbanSim MNL). - + The `fit()` method can be run as many times as desired. Results will not be saved with Orca or ModelManager until the `register()` method is run. - + After sampling alternatives for each chooser, the merged choice table is saved to the class object for diagnostic use (`mergedchoicetable` with type choicemodels.tools.MergedChoiceTable). - + Parameters ---------- mct : choicemodels.tools.MergedChoiceTable This parameter is a temporary backdoor allowing us to pass in a more complicated choice table than can be generated within the template, for example including sampling weights or interaction terms. - + Returns ------- None - + """ check_choicemodels_version() from choicemodels import MultinomialLogit from choicemodels.tools import MergedChoiceTable - + if (mct is not None): df_from_mct = mct.to_frame() idx_names = df_from_mct.index.names @@ -467,44 +591,43 @@ def fit(self, mct=None): df_from_mct, self.chooser_filters).set_index(idx_names) mct = MergedChoiceTable.from_df(df_from_mct) - else: - observations = get_data(tables = self.choosers, - filters = self.chooser_filters, - model_expression = self.model_expression, - extra_columns = self.choice_column) - + else: + observations = get_data(tables=self.choosers, + filters=self.chooser_filters, + model_expression=self.model_expression, + extra_columns=self.choice_column) + if (self.chooser_sample_size is not None): observations = observations.sample(self.chooser_sample_size) - - alternatives = get_data(tables = self.alternatives, - filters = self.alt_filters, - model_expression = self.model_expression) - - mct = MergedChoiceTable(observations = observations, - alternatives = alternatives, - chosen_alternatives = self.choice_column, - sample_size = self.alt_sample_size) - - model = MultinomialLogit(data = mct, - model_expression = self.model_expression) + + alternatives = get_data(tables=self.alternatives, + filters=self.alt_filters, + model_expression=self.model_expression) + + mct = MergedChoiceTable(observations=observations, + alternatives=alternatives, + chosen_alternatives=self.choice_column, + sample_size=self.alt_sample_size) + + model = MultinomialLogit(data=mct, + model_expression=self.model_expression) results = model.fit() - + self.name = self._generate_name() self.summary_table = str(results) print(self.summary_table) - + coefs = results.get_raw_results()['fit_parameters']['Coefficient'] self.fitted_parameters = coefs.tolist() self.model = results - + # Save merged choice table to the class object for diagnostics self.mergedchoicetable = mct - - + def run(self, chooser_batch_size=None, interaction_terms=None): """ Run the model step: simulate choices and use them to update an Orca column. - + The simulated choices are saved to the class object for diagnostics. If choices are unconstrained, the choice table and the probabilities of sampled alternatives are saved as well. @@ -526,90 +649,115 @@ def run(self, chooser_batch_size=None, interaction_terms=None): MultiIndex. One level's name and values should match an index or column from the observations table, and the other should match an index or column from the alternatives table. - + Returns ------- None - + """ check_choicemodels_version() from choicemodels import MultinomialLogit - from choicemodels.tools import (MergedChoiceTable, monte_carlo_choices, - iterative_lottery_choices) + from choicemodels.tools import (MergedChoiceTable, monte_carlo_choices, + iterative_lottery_choices) # Clear simulation attributes from the class object self.mergedchoicetable = None self.probabilities = None self.choices = None - + if interaction_terms is not None: - uniq_intx_idx_names = set([idx for intx in interaction_terms for idx in intx.index.names]) - obs_extra_cols = to_list(self.chooser_size) + list(uniq_intx_idx_names) - alts_extra_cols = to_list(self.alt_capacity) + list(uniq_intx_idx_names) + uniq_intx_idx_names = set([ + idx for intx in interaction_terms for idx in intx.index.names]) + obs_extra_cols = to_list(self.chooser_size) + \ + list(uniq_intx_idx_names) + alts_extra_cols = to_list( + self.alt_capacity) + list(uniq_intx_idx_names) else: - obs_extra_cols = self.chooser_size - alts_extra_cols = self.alt_capacity - - observations = get_data(tables = self.out_choosers, - fallback_tables = self.choosers, - filters = self.out_chooser_filters, - model_expression = self.model_expression, - extra_columns = obs_extra_cols) - + obs_extra_cols = to_list(self.chooser_size) + alts_extra_cols = to_list(self.alt_capacity) + + # get any necessary extra columns from the mct intx operations spec + if self.mct_intx_ops: + intx_extra_obs_cols = self.mct_intx_ops.get('extra_obs_cols', []) + intx_extra_obs_cols = to_list(intx_extra_obs_cols) + obs_extra_cols += intx_extra_obs_cols + intx_extra_alts_cols = self.mct_intx_ops.get('extra_alts_cols', []) + intx_extra_alts_cols = to_list(intx_extra_alts_cols) + alts_extra_cols += intx_extra_alts_cols + + observations = get_data(tables=self.out_choosers, + fallback_tables=self.choosers, + filters=self.out_chooser_filters, + model_expression=self.model_expression, + extra_columns=obs_extra_cols) + if len(observations) == 0: print("No valid choosers") return - - alternatives = get_data(tables = self.out_alternatives, - fallback_tables = self.alternatives, - filters = self.out_alt_filters, - model_expression = self.model_expression, - extra_columns = alts_extra_cols) - + + alternatives = get_data(tables=self.out_alternatives, + fallback_tables=self.alternatives, + filters=self.out_alt_filters, + model_expression=self.model_expression, + extra_columns=alts_extra_cols) + if len(alternatives) == 0: print("No valid alternatives") return - + # Remove filter columns before merging, in case column names overlap expr_cols = columns_in_formula(self.model_expression) - - obs_cols = set(observations.columns) & set(expr_cols + to_list(obs_extra_cols)) + + obs_cols = set(observations.columns) & set( + expr_cols + to_list(obs_extra_cols)) observations = observations[list(obs_cols)] - - alt_cols = set(alternatives.columns) & set(expr_cols + to_list(alts_extra_cols)) + + alt_cols = set(alternatives.columns) & set( + expr_cols + to_list(alts_extra_cols)) alternatives = alternatives[list(alt_cols)] - + # Callables for iterative choices - def mct(obs, alts): - return MergedChoiceTable( + def mct(obs, alts, intx_ops=None): + + this_mct = MergedChoiceTable( obs, alts, sample_size=self.alt_sample_size, interaction_terms=interaction_terms) + if intx_ops: + this_mct = self.perform_mct_intx_ops(this_mct) + this_mct.sample_size = self.alt_sample_size + + return this_mct + def probs(mct): return self.model.probabilities(mct) - if (self.constrained_choices == True): - choices = iterative_lottery_choices(observations, alternatives, - mct_callable=mct, probs_callable=probs, - alt_capacity=self.alt_capacity, chooser_size=self.chooser_size, - max_iter=self.max_iter, chooser_batch_size=chooser_batch_size) - + if self.constrained_choices is True: + choices = iterative_lottery_choices( + observations, alternatives, + mct_callable=mct, + probs_callable=probs, + alt_capacity=self.alt_capacity, chooser_size=self.chooser_size, + max_iter=self.max_iter, chooser_batch_size=chooser_batch_size, + mct_intx_ops=self.mct_intx_ops) + else: - choicetable = mct(observations, alternatives) + choicetable = mct( + observations, alternatives, intx_ops=self.mct_intx_ops) probabilities = probs(choicetable) choices = monte_carlo_choices(probabilities) - + # Save data to class object if available self.mergedchoicetable = choicetable self.probabilities = probabilities - + # Save choices to class object for diagnostics self.choices = choices # Update Orca - update_column(table = self.out_choosers, - fallback_table = self.choosers, - column = self.out_column, - fallback_column = self.choice_column, - data = choices) + update_column(table=self.out_choosers, + fallback_table=self.choosers, + column=self.out_column, + fallback_column=self.choice_column, + data=choices) From 2acd3f3f9471f59999beaa5651f5435b543b822a Mon Sep 17 00:00:00 2001 From: sol Date: Thu, 21 Apr 2022 14:52:20 -0300 Subject: [PATCH 121/121] remove filter columns --- urbansim_templates/models/large_multinomial_logit.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/urbansim_templates/models/large_multinomial_logit.py b/urbansim_templates/models/large_multinomial_logit.py index 3eba2c7..474e08c 100644 --- a/urbansim_templates/models/large_multinomial_logit.py +++ b/urbansim_templates/models/large_multinomial_logit.py @@ -1,7 +1,7 @@ from __future__ import print_function import orca -from urbansim.models.util import columns_in_formula, apply_filter_query +from urbansim.models.util import columns_in_formula, apply_filter_query, columns_in_filters from choicemodels.tools import MergedChoiceTable import pandas as pd @@ -604,6 +604,10 @@ def fit(self, mct=None): filters=self.alt_filters, model_expression=self.model_expression) + # Remove filter columns before merging, in case column names overlap + observations.drop(columns_in_filters(self.chooser_filters), axis = 1, inplace = True) + alternatives.drop(columns_in_filters(self.alt_filters), axis = 1, inplace = True) + mct = MergedChoiceTable(observations=observations, alternatives=alternatives, chosen_alternatives=self.choice_column,