From fe4e184601c2ff6508ef4bc1f5124dba5e6027fe Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Mon, 26 Sep 2022 18:37:11 +0900 Subject: [PATCH 01/10] Add package management --- docs/01_create_environemnt.md | 13 +++++++++++++ environment.yml | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docs/01_create_environemnt.md create mode 100644 environment.yml diff --git a/docs/01_create_environemnt.md b/docs/01_create_environemnt.md new file mode 100644 index 0000000..68fb53b --- /dev/null +++ b/docs/01_create_environemnt.md @@ -0,0 +1,13 @@ +# create environment + +``` +conda create --prefix ./.env --file environment.yml +``` + +If you place this repository on WSL2 /mnt/* or AzureML Compute Instance working directory, we recommend to change ```--prefix ./.env``` to ```--prefix ~/azureml-mlops-env``` because of storage latency. + + +# update environment + +1. Add new package name and version to environment.yml. +1. ```conda env update --prefix ./.env --file environment.yml``` \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..141b149 --- /dev/null +++ b/environment.yml @@ -0,0 +1,19 @@ +name: azureml-mlops-env +channels: + - defaults + - conda-forge +dependencies: + - python=3.10.6 + - pip + - flake8=5.0.4 + - isort=5.10.1 + - black=22.8.0 + - notebook + - ipykernel + - numpy=1.23.1 + - matplotlib=3.5.2 + - tqdm=4.64.0 + - mlflow=1.29.0 + - pip: + - azureml-core==1.43.0 + - azureml-mlflow==1.43.0.post1 From 8fc72aa39f14ddaf5178cd3f91229c70853e9287 Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Mon, 26 Sep 2022 18:37:39 +0900 Subject: [PATCH 02/10] auto-activate environment --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index d9d7eb9..49aad3e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "python.linting.flake8Enabled": true, "python.linting.pylintEnabled": false, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.defaultInterpreterPath": "./.env" } \ No newline at end of file From aaf85df073458c4e8dd7c7f98906fba22b6fb4e5 Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Mon, 26 Sep 2022 18:42:33 +0900 Subject: [PATCH 03/10] fix command --- docs/01_create_environemnt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/01_create_environemnt.md b/docs/01_create_environemnt.md index 68fb53b..ccb34e2 100644 --- a/docs/01_create_environemnt.md +++ b/docs/01_create_environemnt.md @@ -1,7 +1,7 @@ # create environment ``` -conda create --prefix ./.env --file environment.yml +conda env create --prefix ./.env --file environment.yml ``` If you place this repository on WSL2 /mnt/* or AzureML Compute Instance working directory, we recommend to change ```--prefix ./.env``` to ```--prefix ~/azureml-mlops-env``` because of storage latency. From 69407fc838f47c74bf11999db407b64209b59518 Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Mon, 26 Sep 2022 18:51:42 +0900 Subject: [PATCH 04/10] Add command alias --- Makefile | 4 ++++ docs/01_create_environemnt.md | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e440251 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +env-create: + conda env create --prefix=.env --file=environment.yml +env-update: + conda env update --prefix=.env --file=environment.yml \ No newline at end of file diff --git a/docs/01_create_environemnt.md b/docs/01_create_environemnt.md index ccb34e2..a22b5d3 100644 --- a/docs/01_create_environemnt.md +++ b/docs/01_create_environemnt.md @@ -1,13 +1,26 @@ # create environment +``` +make env-create +``` + +```make env-create``` command is alias of command below defined in Makefile. + ``` conda env create --prefix ./.env --file environment.yml ``` If you place this repository on WSL2 /mnt/* or AzureML Compute Instance working directory, we recommend to change ```--prefix ./.env``` to ```--prefix ~/azureml-mlops-env``` because of storage latency. +Affected: +- .vscode/setting.json ```python.defaultInterpreterPath``` +- Makefile # update environment 1. Add new package name and version to environment.yml. -1. ```conda env update --prefix ./.env --file environment.yml``` \ No newline at end of file +1. ```make env-update``` + +```make env-update``` command is alias of command below defined in Makefile. + +```conda env update --prefix ./.env --file environment.yml``` \ No newline at end of file From ee46f75e6e972b48c21a5468c4f9a4d2e6af383d Mon Sep 17 00:00:00 2001 From: shunta_ito Date: Mon, 7 Nov 2022 10:02:50 +0900 Subject: [PATCH 05/10] Update docs/01_create_environemnt.md Co-authored-by: Keita Onabuta --- docs/01_create_environemnt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/01_create_environemnt.md b/docs/01_create_environemnt.md index a22b5d3..4b02ff9 100644 --- a/docs/01_create_environemnt.md +++ b/docs/01_create_environemnt.md @@ -4,7 +4,7 @@ make env-create ``` -```make env-create``` command is alias of command below defined in Makefile. +```make env-create``` command is an alias of command below defined in Makefile. ``` conda env create --prefix ./.env --file environment.yml From b6a87a73931cbe0aee6d676a17b7dec8c88d99db Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Mon, 7 Nov 2022 10:09:46 +0900 Subject: [PATCH 06/10] format --- src/data/prepare_data.py | 56 +++++++++------------------------- src/deploy/score-nyx-taxi.py | 18 ++++++----- src/model/register_nyx_taxi.py | 37 ++++++++++++---------- src/model/train.py | 1 - src/model/train_nyx_taxi.py | 44 ++++++++++++++------------ 5 files changed, 69 insertions(+), 87 deletions(-) diff --git a/src/data/prepare_data.py b/src/data/prepare_data.py index 5f41aec..9a5bb1f 100644 --- a/src/data/prepare_data.py +++ b/src/data/prepare_data.py @@ -13,33 +13,19 @@ def register_dataset(ws: Workspace) -> None: dataset_name = "nyc_taxi_dataset" try: - dataset = Dataset.get_by_name( - ws, dataset_name - ) + dataset = Dataset.get_by_name(ws, dataset_name) df = dataset.to_pandas_dataframe() except Exception: raw_df = pd.DataFrame([]) - start = datetime.strptime( - "1/1/2015", "%m/%d/%Y" - ) - end = datetime.strptime( - "1/31/2015", "%m/%d/%Y" - ) + start = datetime.strptime("1/1/2015", "%m/%d/%Y") + end = datetime.strptime("1/31/2015", "%m/%d/%Y") for sample_month in range(3): temp_df_green = NycTlcGreen( - start - + relativedelta( - months=sample_monthっs - ), - end - + relativedelta( - months=sample_month - ), + start + relativedelta(months=sample_monthっs), + end + relativedelta(months=sample_month), ).to_pandas_dataframe() - raw_df = raw_df.append( - temp_df_green.sample(2000) - ) + raw_df = raw_df.append(temp_df_green.sample(2000)) print(raw_df.head(10)) @@ -64,27 +50,15 @@ def register_dataset(ws: Workspace) -> None: for col in columns_to_remove: df.pop(col) - df = df.query( - "pickupLatitude>=40.53 and pickupLatitude<=40.88" - ) - df = df.query( - "pickupLongitude>=-74.09 and pickupLongitude<=-73.72" - ) - df = df.query( - "tripDistance>=0.25 and tripDistance<31" - ) - df = df.query( - "passengerCount>0 and totalAmount>0" - ) - - df["lpepPickupDatetime"] = df[ - "lpepPickupDatetime" - ].map(lambda x: x.timestamp()) + df = df.query("pickupLatitude>=40.53 and pickupLatitude<=40.88") + df = df.query("pickupLongitude>=-74.09 and pickupLongitude<=-73.72") + df = df.query("tripDistance>=0.25 and tripDistance<31") + df = df.query("passengerCount>0 and totalAmount>0") + + df["lpepPickupDatetime"] = df["lpepPickupDatetime"].map(lambda x: x.timestamp()) datastore = Datastore.get_default(ws) - dataset = Dataset.Tabular.register_pandas_dataframe( - df, datastore, dataset_name - ) + dataset = Dataset.Tabular.register_pandas_dataframe(df, datastore, dataset_name) print(df.head(5)) df.to_csv( @@ -95,9 +69,7 @@ def register_dataset(ws: Workspace) -> None: def main() -> None: - subscription_id = ( - "9c0f91b8-eb2f-484c-979c-15848c098a6b" - ) + subscription_id = "9c0f91b8-eb2f-484c-979c-15848c098a6b" resource_group = "azureml-mlopshack" workspace_name = "azureml-mlopshack" diff --git a/src/deploy/score-nyx-taxi.py b/src/deploy/score-nyx-taxi.py index 09dedc9..596f342 100644 --- a/src/deploy/score-nyx-taxi.py +++ b/src/deploy/score-nyx-taxi.py @@ -5,19 +5,21 @@ # Called when the service is loaded def init(): global model - + model_path = os.path.join(os.environ["AZUREML_MODEL_DIR"], "nyx_taxi") -# model_path = Model.get_model_path(args.model_name) - model = load_model(model_path) + # model_path = Model.get_model_path(args.model_name) + model = load_model(model_path) def run(mini_batch): print(f"run method start: {__file__}, run({mini_batch})") - + for input in mini_batch: - input_np = np.loadtxt(input, delimiter=',',skiprows=1) + input_np = np.loadtxt(input, delimiter=",", skiprows=1) predictions = model.predict(input_np) - log_txt = 'Predictions:' + str(predictions) - print (log_txt) + log_txt = "Predictions:" + str(predictions) + print(log_txt) - return [[row, pred] for row, pred in enumerate(predictions)] # return a dataframe or a list \ No newline at end of file + return [ + [row, pred] for row, pred in enumerate(predictions) + ] # return a dataframe or a list diff --git a/src/model/register_nyx_taxi.py b/src/model/register_nyx_taxi.py index 9214a75..364a784 100644 --- a/src/model/register_nyx_taxi.py +++ b/src/model/register_nyx_taxi.py @@ -4,15 +4,20 @@ import mlflow from mlflow.pyfunc import load_model + def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--model_name', type=str, help='Name under which model will be registered') - parser.add_argument('--model_path', type=str, help='Model directory') - parser.add_argument('--deploy_flag', type=str, help='A deploy flag whether to deploy or no') - + parser.add_argument( + "--model_name", type=str, help="Name under which model will be registered" + ) + parser.add_argument("--model_path", type=str, help="Model directory") + parser.add_argument( + "--deploy_flag", type=str, help="A deploy flag whether to deploy or no" + ) + args, _ = parser.parse_known_args() - print(f'Arguments: {args}') + print(f"Arguments: {args}") return args @@ -29,25 +34,25 @@ def main(): model_name = args.model_name model_path = args.model_path - if len(args.deploy_flag) == 1: # this is the case where deploy_flag is a digit + if len(args.deploy_flag) == 1: # this is the case where deploy_flag is a digit deploy_flag = int(args.deploy_flag) - else: # this is the case where deploy_flag is a path name - with open((Path(args.deploy_flag) / "deploy_flag"), 'rb') as f: + else: # this is the case where deploy_flag is a path name + with open((Path(args.deploy_flag) / "deploy_flag"), "rb") as f: deploy_flag = int(f.read()) - if deploy_flag==1: - + if deploy_flag == 1: + print("Registering ", model_name) - model = load_model(os.path.join(model_path, 'models')) - # log model using mlflow + model = load_model(os.path.join(model_path, "models")) + # log model using mlflow mlflow.sklearn.log_model(model, model_name) - #mlflow.sklearn.log_model(sk_model=model, artifact_path="models", registered_model_name=model_name) + # mlflow.sklearn.log_model(sk_model=model, artifact_path="models", registered_model_name=model_name) # register model using mlflow model - model_uri = f'runs:/{run_id}/{args.model_name}' + model_uri = f"runs:/{run_id}/{args.model_name}" mlflow.register_model(model_uri, model_name) - + else: print("Model will not be registered!") @@ -55,4 +60,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/model/train.py b/src/model/train.py index 9c46dc8..3ccbaf7 100644 --- a/src/model/train.py +++ b/src/model/train.py @@ -9,4 +9,3 @@ def sample(): if __name__ == "__main__": print("main 関数です") - diff --git a/src/model/train_nyx_taxi.py b/src/model/train_nyx_taxi.py index d797c10..c270f20 100644 --- a/src/model/train_nyx_taxi.py +++ b/src/model/train_nyx_taxi.py @@ -16,7 +16,9 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--input_data", type=str, help="input data") - parser.add_argument("--output_dir", type=str, help="output dir", default="./outputs") + parser.add_argument( + "--output_dir", type=str, help="output dir", default="./outputs" + ) args = parser.parse_args() return args @@ -24,22 +26,24 @@ def parse_args(): args = parse_args() lines = [ f"Training data path: {args.input_data}", - f"output dir path: {args.output_dir}" + f"output dir path: {args.output_dir}", ] for line in lines: print(line) df = pd.read_csv(args.input_data) -X = df.drop(columns='totalAmount') -y = df['totalAmount'] -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=0) +X = df.drop(columns="totalAmount") +y = df["totalAmount"] +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.30, random_state=0 +) with mlflow.start_run(): run_id = mlflow.active_run().info.run_id mlflow.autolog(log_models=False, exclusive=True) - print('run_id = ', run_id) + print("run_id = ", run_id) mlflow.log_metric("Training samples", len(X_train)) mlflow.log_metric("Test samples", len(X_test)) @@ -61,28 +65,28 @@ def parse_args(): mlflow.log_metric("r2", r2) # Finally save the model to the outputs directory for capture - os.makedirs(os.path.join(args.output_dir, 'models'), exist_ok=True) - mlflow.sklearn.save_model(model, os.path.join(args.output_dir, 'models')) + os.makedirs(os.path.join(args.output_dir, "models"), exist_ok=True) + mlflow.sklearn.save_model(model, os.path.join(args.output_dir, "models")) # Plot actuals vs predictions and save the plot within the run plt.figure(figsize=(10, 7)) - #scatterplot of y_test and pred - plt.scatter(y_test, y_pred) - plt.plot(y_test, y_test, color='r') + # scatterplot of y_test and pred + plt.scatter(y_test, y_pred) + plt.plot(y_test, y_test, color="r") - plt.title('Actual VS Predicted Values (Test set)') - plt.xlabel('Actual Values') - plt.ylabel('Predicted Values') - plt.savefig('actuals_vs_predictions.png') + plt.title("Actual VS Predicted Values (Test set)") + plt.xlabel("Actual Values") + plt.ylabel("Predicted Values") + plt.savefig("actuals_vs_predictions.png") mlflow.log_artifact("actuals_vs_predictions.png") metric = {} -metric['run_id'] = run_id -metric['RMSE'] = rmse -metric['R2'] = r2 +metric["run_id"] = run_id +metric["RMSE"] = rmse +metric["R2"] = r2 print(metric) -with open(os.path.join(args.output_dir, 'metric.json'), "w") as outfile: - json.dump(metric, outfile) \ No newline at end of file +with open(os.path.join(args.output_dir, "metric.json"), "w") as outfile: + json.dump(metric, outfile) From aa127f85a71a8273c3535dac21ed25446de9efef Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Mon, 7 Nov 2022 10:12:50 +0900 Subject: [PATCH 07/10] Add black formatter for jupyter --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d9ab010..cf8d0e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,10 @@ repos: hooks: - id: black language_version: python3.8 +- repo: https://github.com/psf/black + rev: 22.1.0 + hooks: + - id: black-jupyter # import 並び替え - repo: https://github.com/pycqa/isort From 7fb2e1b417a39927715c4f53d2bf67dd2e515e38 Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Mon, 7 Nov 2022 10:14:42 +0900 Subject: [PATCH 08/10] Move environment.yml on root to on environments dir --- Makefile | 4 ++-- environment.yml => environments/environment.yml | 0 notebooks/train-regression.ipynb | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) rename environment.yml => environments/environment.yml (100%) diff --git a/Makefile b/Makefile index e440251..139d8ba 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ env-create: - conda env create --prefix=.env --file=environment.yml + conda env create --prefix=.env --file=environments/environment.yml env-update: - conda env update --prefix=.env --file=environment.yml \ No newline at end of file + conda env update --prefix=.env --file=environments/environment.yml \ No newline at end of file diff --git a/environment.yml b/environments/environment.yml similarity index 100% rename from environment.yml rename to environments/environment.yml diff --git a/notebooks/train-regression.ipynb b/notebooks/train-regression.ipynb index 681168a..29a13c5 100644 --- a/notebooks/train-regression.ipynb +++ b/notebooks/train-regression.ipynb @@ -300,7 +300,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.8.13 ('mlops-train')", + "display_name": "Python 3.10.6 (conda)", "language": "python", "name": "python3" }, @@ -314,12 +314,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.10.6" }, "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "74419d3d9274bcbfe6ecb9acd0596b867bc1ac63effdfbb8a6e0b958ebbd5c34" + "hash": "38e473d28400b2903a173eeae82f0fe034393fd50eb0c5d12497f83fc604afcc" } } }, From 78438d7e5b480f1b317abf59ae3f37b8f1bac31f Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Wed, 23 Nov 2022 11:30:01 +0900 Subject: [PATCH 09/10] Use black in notebook --- environments/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/environment.yml b/environments/environment.yml index 141b149..e2e15e6 100644 --- a/environments/environment.yml +++ b/environments/environment.yml @@ -17,3 +17,4 @@ dependencies: - pip: - azureml-core==1.43.0 - azureml-mlflow==1.43.0.post1 + - black_nbconvert==0.4.0 From fbb2962d4f6b72c91f9d241916e39adfac87a51f Mon Sep 17 00:00:00 2001 From: ShuntaIto Date: Tue, 29 Nov 2022 18:29:35 +0900 Subject: [PATCH 10/10] change virtual env dir name --- .vscode/settings.json | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 12892bd..0baab2f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,6 @@ "python.linting.flake8Enabled": true, "python.linting.pylintEnabled": false, "python.linting.enabled": true, - "python.defaultInterpreterPath": "./.env" + "python.defaultInterpreterPath": "./.venv" } diff --git a/Makefile b/Makefile index 139d8ba..ca56664 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ env-create: - conda env create --prefix=.env --file=environments/environment.yml + conda env create --prefix=.venv --file=environments/environment.yml env-update: - conda env update --prefix=.env --file=environments/environment.yml \ No newline at end of file + conda env update --prefix=.venv --file=environments/environment.yml \ No newline at end of file