diff --git a/README.md b/README.md index 723500a..ed85e29 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,20 @@ -# AlphaClean v0.1 +# AlphaClean v0.1 for Python3 + In many data science applications, data cleaning is effectively manual with ad-hoc, single-use scripts and programs. This practice is highly problematic for reproducibility (sharing analysis between researchers), interpretability (explaining the findings of a particular analysis), and scalability (replicating the analyses on larger corpora). Most existing relational data cleaning solutions are designed as stand-alone systems -- often coupled with a DBMS -- with poor support for languages widely in data science, such as Python and R. -In order to address this problem, we designed a Python 2 library that declaratively synthesizes data cleaning programs. +In order to address this problem, we designed a Python 2 library that declaratively synthesizes data cleaning programs. AlphaClean is given a specification of quality (e.g. integrity constraints or a statistical model the data must conform to) and a language of allowed data transformations, then it searches to find a sequence of transformations that best satisfies the quality specification. The discovered sequence of transformations defines an intermediate representation, which can be easily transferred between languages or optimized with a compiler. ## Requirements + As an initial research prototype, we have not yet designed AlphaClean for deployment. It is a package that researchers can locally develop and test to evaluate different data cleaning approaches and techniques. The dependencies are: -``` -dateparser==0.6.0 -Distance==0.1.3 -numpy==1.12.1 -pandas==0.20.1 -pyparsing==2.2.0 -python-dateutil==2.6.0 -pytz==2017.2 -regex==2017.7.11 -ruamel.ordereddict==0.4.9 -ruamel.yaml==0.15.18 -scikit-learn==0.18.1 -scipy==0.19.0 -six==1.10.0 -gensim==2.2.0 -``` -These can be installed by running the following code from the command line: -``` -pip install -r requirements.txt -``` + pip install -r requirements.txt ## Using the Package + The docs folder contains a number of tutorials on how to use AlphaClean. 0. [Getting Started](https://github.com/sjyk/alphaclean/blob/master/docs/00-GettingStarted.md) @@ -38,4 +22,4 @@ The docs folder contains a number of tutorials on how to use AlphaClean. 2. [Scaling Up](https://github.com/sjyk/alphaclean/blob/master/docs/02-Scaling.md) 3. [External Knowledge](https://github.com/sjyk/alphaclean/blob/master/docs/03-ExternalInfo.md) 4. [Basic Numerical Cleaning](https://github.com/sjyk/alphaclean/blob/master/docs/04-Numerical.md) -5. [Advanced Numerical Cleaning](https://github.com/sjyk/alphaclean/blob/master/docs/05-AdvancedNumerical.md) \ No newline at end of file +5. [Advanced Numerical Cleaning](https://github.com/sjyk/alphaclean/blob/master/docs/05-AdvancedNumerical.md) diff --git a/alphaclean/core.py b/alphaclean/core.py index 7d3483f..bf990f3 100644 --- a/alphaclean/core.py +++ b/alphaclean/core.py @@ -9,8 +9,8 @@ from sklearn.neighbors import BallTree import distance from sklearn import tree -from constraints import * -from type_inference import * +from .constraints import * +from .type_inference import * class Dataset: diff --git a/alphaclean/generators.py b/alphaclean/generators.py index 8f4e3b5..e1fbee3 100644 --- a/alphaclean/generators.py +++ b/alphaclean/generators.py @@ -4,9 +4,9 @@ """ from itertools import combinations, product -from ops import * -from constraints import * -from core import * +from .ops import * +from .constraints import * +from .core import * import copy import string import logging @@ -38,7 +38,9 @@ def getParameterGrid(self): if p[0] == ParametrizedOperation.COLUMN: #remove one of the cols - origParam = copy.copy(orig) + # origParam = copy.deepcopy(list(orig.items())) + origParam = list(orig) + orig = list(orig) orig.remove(p[0]) colParams = [] diff --git a/alphaclean/ops.py b/alphaclean/ops.py index e6befa9..42e9dd0 100644 --- a/alphaclean/ops.py +++ b/alphaclean/ops.py @@ -3,7 +3,7 @@ Operations define a monoid """ -from sets import Set + from dateparser import DateDataParser import time import re diff --git a/alphaclean/search.py b/alphaclean/search.py index a9a3c50..12089d3 100644 --- a/alphaclean/search.py +++ b/alphaclean/search.py @@ -10,7 +10,7 @@ import numpy as np import datetime -from generators import * +from .generators import * from heapq import * from alphaclean.learning import * @@ -125,7 +125,7 @@ def solve(df, patterns=[], dependencies=[], partitionOn=None, config=DEFAULT_SOL op1, df = patternConstraints(df, patterns, config['pattern']) - op2, df = dependencyConstraints(df, dependencies, config['dependency']) + op2, df, _ = dependencyConstraints(df, dependencies, config['dependency']) op = op * (op1*op2) diff --git a/requirements.txt b/requirements.txt index be110f6..2ea51d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,20 @@ -appdirs==1.4.3 -dateparser==0.6.0 -Distance==0.1.3 -numpy==1.12.1 -packaging==16.8 -pandas==0.20.1 -pip==9.0.1 -pyparsing==2.2.0 -python-dateutil==2.6.0 -pytz==2017.2 -regex==2017.7.11 -ruamel.ordereddict==0.4.9 -ruamel.yaml==0.15.18 -scikit-learn==0.18.1 -scipy==0.19.0 -setuptools==35.0.2 -six==1.10.0 -sklearn==0.0 -tzlocal==1.4 -wheel==0.30.0a0 -gensim==2.2.0 +appdirs +dateparser +Distance +numpy +packaging +pandas +pip +pyparsing +python-dateutil +pytz +regex +ruamel.yaml +scikit-learn +scipy +setuptools +six +sklearn +tzlocal +wheel +gensim