From 2eaeb54a3e934c661d2dac455c5cf93f1c06ba88 Mon Sep 17 00:00:00 2001 From: "richard.m.marcus@gmail.com" Date: Mon, 20 Apr 2026 21:11:27 +0000 Subject: [PATCH] Update Databricks cluster configs and library versions of dependencies --- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../raw_orders/default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../default/configs/dataset.json | 2 +- .../AirflowEndToEndJob/code/prophecy-job.json | 29 +- .../tasks/__init__.py | 6 +- .../tasks/gold_sales.py | 20 +- .../tasks/gold_top_customers.py | 20 +- .../tasks/raw_bronze.py | 20 +- .../tasks/silver_customers_orders.py | 20 +- .../tasks/silver_zip_codes.py | 20 +- .../utils/prophecy.py | 14 +- .../DatabricksJob2/code/databricks-job.json | 26 +- .../DatabricksJob2/code/prophecy-job.json | 348 +----------------- prophecy_scala/pbt_project.yml | 24 +- .../code/.prophecy/workflow.latest.json | 36 +- .../pipelines/gold_sales/code/pom.xml | 75 +++- .../prophecy/pipelines/gold_sales/main.scala | 1 - .../pipelines/gold_sales/udfs/UDFs.scala | 5 +- .../code/.prophecy/workflow.latest.json | 12 +- .../pipelines/gold_top_customers/code/pom.xml | 75 +++- .../pipelines/gold_top_customers/main.scala | 1 - .../gold_top_customers/udfs/UDFs.scala | 5 +- .../code/.prophecy/workflow.latest.json | 14 +- .../pipelines/raw_bronze/code/pom.xml | 75 +++- .../prophecy/pipelines/raw_bronze/main.scala | 1 - .../pipelines/raw_bronze/udfs/UDFs.scala | 5 +- .../code/.prophecy/workflow.latest.json | 14 +- .../silver_customers_orders/code/pom.xml | 75 +++- .../silver_customers_orders/main.scala | 1 - .../silver_customers_orders/udfs/UDFs.scala | 5 +- .../code/.prophecy/workflow.latest.json | 16 +- .../pipelines/silver_zip_codes/code/pom.xml | 75 +++- .../pipelines/silver_zip_codes/main.scala | 1 - .../silver_zip_codes/udfs/UDFs.scala | 5 +- 44 files changed, 533 insertions(+), 537 deletions(-) diff --git a/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json b/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json index 1da93fb..9ba8127 100644 --- a/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json +++ b/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}},{"name":"full_name","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}},{"name":"full_name","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json b/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json index 6b5fa9a..a1a53f0 100644 --- a/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json +++ b/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json b/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json index fd7a5b5..3294905 100644 --- a/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json +++ b/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json b/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json index be5d943..9825728 100644 --- a/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json +++ b/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"avg_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_count","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_sales_by_zip_date","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"avg_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_count","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_sales_by_zip_date","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json b/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json index a231992..c053f9b 100644 --- a/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json +++ b/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_top50_customers_by_spend","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_top50_customers_by_spend","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json b/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json index 95cf266..aee4960 100644 --- a/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json +++ b/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_total_sales_by_customer","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_total_sales_by_customer","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/raw_customers/default/configs/dataset.json b/prophecy_scala/datasets/raw_customers/default/configs/dataset.json index eb45f79..9371859 100644 --- a/prophecy_scala/datasets/raw_customers/default/configs/dataset.json +++ b/prophecy_scala/datasets/raw_customers/default/configs/dataset.json @@ -1 +1 @@ -{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/CustomersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"string","nullable":true,"metadata":{"description":"The date the the Customer opened their account","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null} \ No newline at end of file +{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/CustomersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"string","nullable":true,"metadata":{"description":"The date the the Customer opened their account","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null} \ No newline at end of file diff --git a/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json b/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json index 3805ea5..71c80c4 100644 --- a/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json +++ b/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json @@ -1 +1 @@ -{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/databricks-datasets/data.gov/irs_zip_code_data/data-001/2013_soi_zipcode_agi.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null} \ No newline at end of file +{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/databricks-datasets/data.gov/irs_zip_code_data/data-001/2013_soi_zipcode_agi.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null} \ No newline at end of file diff --git a/prophecy_scala/datasets/raw_orders/default/configs/dataset.json b/prophecy_scala/datasets/raw_orders/default/configs/dataset.json index 2ed1214..5cac832 100644 --- a/prophecy_scala/datasets/raw_orders/default/configs/dataset.json +++ b/prophecy_scala/datasets/raw_orders/default/configs/dataset.json @@ -1 +1 @@ -{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/OrdersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"string","nullable":true,"metadata":{"description":"Order ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"string","nullable":true,"metadata":{"description":"Date the order was made","mappings":[],"tags":[]}},{"name":"amount","type":"string","nullable":true,"metadata":{"description":"Total amount for the order","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null} \ No newline at end of file +{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/OrdersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"string","nullable":true,"metadata":{"description":"Order ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"string","nullable":true,"metadata":{"description":"Date the order was made","mappings":[],"tags":[]}},{"name":"amount","type":"string","nullable":true,"metadata":{"description":"Total amount for the order","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null} \ No newline at end of file diff --git a/prophecy_scala/datasets/silver_customers/default/configs/dataset.json b/prophecy_scala/datasets/silver_customers/default/configs/dataset.json index 5957d99..663898c 100644 --- a/prophecy_scala/datasets/silver_customers/default/configs/dataset.json +++ b/prophecy_scala/datasets/silver_customers/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json b/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json index a6fed3f..45cd566 100644 --- a/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json +++ b/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"state","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_percent","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"low_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"all_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"is_high_income","type":"boolean","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"state","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_percent","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"low_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"all_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"is_high_income","type":"boolean","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/silver_orders/default/configs/dataset.json b/prophecy_scala/datasets/silver_orders/default/configs/dataset.json index 67f7bc3..a8556e0 100644 --- a/prophecy_scala/datasets/silver_orders/default/configs/dataset.json +++ b/prophecy_scala/datasets/silver_orders/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json b/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json index bbed526..450e127 100644 --- a/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json +++ b/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json @@ -1 +1 @@ -{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_order_customer_details","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file +{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_order_customer_details","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"} \ No newline at end of file diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json b/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json index a4508bc..db2f714 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json @@ -11,31 +11,14 @@ "emails" : "", "onStart" : false, "onSuccess" : false, - "onFailure" : false + "onFailure" : false, + "onDurationWarningThresholdExceeded" : false }, "maxConcurrentRuns" : 1, "processDefaultSettings" : { "ignore_first_depends_on_past" : true, "do_xcom_push" : true }, - "availableFabrics" : [ { - "id" : "6817", - "name" : "workdayAirflowFabric" - }, { - "id" : "8615", - "name" : "AirflowFab3", - "trialStartDate" : 1705001938144 - }, { - "id" : "10547", - "name" : "rjAirflowFabricMWAA1" - }, { - "id" : "12737", - "name" : "Airflow-prophecy-development", - "trialStartDate" : 1716477924239 - }, { - "id" : "16432", - "name" : "prophecyTeamAirflowTesting" - } ], "maxActiveRuns" : 1, "catchup" : false, "scheduleTimeZone" : "UTC", @@ -51,7 +34,8 @@ "databricks" : false, "dataproc" : false, "emr" : false, - "onprem" : false + "onprem" : false, + "emr_serverless" : false } }, "processes" : { @@ -77,6 +61,7 @@ "settings" : { }, "properties" : { "runJson" : null, + "projectId" : "10729", "databricksConnId" : "", "clusterSize" : "16433/small", "configurations" : { @@ -121,6 +106,7 @@ "settings" : { }, "properties" : { "runJson" : null, + "projectId" : "10729", "databricksConnId" : "", "clusterSize" : "16433/small", "configurations" : { @@ -165,6 +151,7 @@ "settings" : { }, "properties" : { "runJson" : null, + "projectId" : "10729", "databricksConnId" : "", "clusterSize" : "16433/small", "configurations" : { @@ -209,6 +196,7 @@ "settings" : { }, "properties" : { "runJson" : null, + "projectId" : "10729", "databricksConnId" : "", "clusterSize" : "16433/small", "configurations" : { @@ -253,6 +241,7 @@ "settings" : { }, "properties" : { "runJson" : null, + "projectId" : "10729", "databricksConnId" : "", "clusterSize" : "16433/small", "configurations" : { diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py index bb22f83..19d6b5e 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py @@ -1,5 +1,5 @@ from .gold_sales import * -from .silver_customers_orders import * from .gold_top_customers import * -from .silver_zip_codes import * -from .raw_bronze import * \ No newline at end of file +from .raw_bronze import * +from .silver_customers_orders import * +from .silver_zip_codes import * \ No newline at end of file diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py index 5290452..de2da86 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py @@ -11,11 +11,11 @@ def gold_sales(ti=None, params=None, **context): "task_key": "gold_sales", "new_cluster": { "node_type_id": "i3.xlarge", - "spark_version": "12.2.x-scala2.12", - "runtime_engine": "STANDARD", + "spark_version": "16.4.x-scala2.12", + "runtime_engine": "PHOTON", "num_workers": 0.0, - "data_security_mode": "SINGLE_USER", - "custom_tags": {"ResourceClass" : "SingleNode"}, + "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", + "custom_tags": {}, "spark_conf": { "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", "spark.prophecy.metadata.is.interactive.run": "false", @@ -23,14 +23,13 @@ def gold_sales(ti=None, params=None, **context): "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", "spark.prophecy.metadata.user.id": "2169", - "spark.master": "local[*, 4]", "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", "spark.prophecy.execution.metrics.disabled": "true", "spark.databricks.isv.product": "prophecy", "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", - "spark.databricks.cluster.profile": "singleNode", - "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws" + "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws" }, + "is_single_node": True, "aws_attributes": { "first_on_demand": 1.0, "availability": "SPOT_WITH_FALLBACK", @@ -38,15 +37,16 @@ def gold_sales(ti=None, params=None, **context): "spot_bid_price_percent": 100.0 }, "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, + "kind": "CLASSIC_PREVIEW", "enable_elastic_disk": False }, "spark_jar_task": { "main_class_name": "io.prophecy.pipelines.gold_sales.Main", "parameters": ["-i", "default", "-O", "{}"] }, - "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}}, { - "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales.jar" + "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}}, { + "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales_2.12.jar" }] }, - databricks_conn_id = "pbt-only-cicd-databricks", + databricks_conn_id = "pbt-only-cicd-databricks" ) diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py index a0db7ff..bd479f8 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py @@ -11,11 +11,11 @@ def gold_top_customers(ti=None, params=None, **context): "task_key": "gold_top_customers", "new_cluster": { "node_type_id": "i3.xlarge", - "spark_version": "12.2.x-scala2.12", - "runtime_engine": "STANDARD", + "spark_version": "16.4.x-scala2.12", + "runtime_engine": "PHOTON", "num_workers": 0.0, - "data_security_mode": "SINGLE_USER", - "custom_tags": {"ResourceClass" : "SingleNode"}, + "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", + "custom_tags": {}, "spark_conf": { "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", "spark.prophecy.metadata.is.interactive.run": "false", @@ -23,14 +23,13 @@ def gold_top_customers(ti=None, params=None, **context): "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", "spark.prophecy.metadata.user.id": "2169", - "spark.master": "local[*, 4]", "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", "spark.prophecy.execution.metrics.disabled": "true", "spark.databricks.isv.product": "prophecy", "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", - "spark.databricks.cluster.profile": "singleNode", - "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws" + "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws" }, + "is_single_node": True, "aws_attributes": { "first_on_demand": 1.0, "availability": "SPOT_WITH_FALLBACK", @@ -38,15 +37,16 @@ def gold_top_customers(ti=None, params=None, **context): "spot_bid_price_percent": 100.0 }, "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, + "kind": "CLASSIC_PREVIEW", "enable_elastic_disk": False }, "spark_jar_task": { "main_class_name": "io.prophecy.pipelines.gold_top_customers.Main", "parameters": ["-i", "default", "-O", "{}"] }, - "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}}, { - "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers.jar" + "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}}, { + "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers_2.12.jar" }] }, - databricks_conn_id = "pbt-only-cicd-databricks", + databricks_conn_id = "pbt-only-cicd-databricks" ) diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py index fdb2fe0..5422984 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py @@ -11,11 +11,11 @@ def raw_bronze(ti=None, params=None, **context): "task_key": "raw_bronze", "new_cluster": { "node_type_id": "i3.xlarge", - "spark_version": "12.2.x-scala2.12", - "runtime_engine": "STANDARD", + "spark_version": "16.4.x-scala2.12", + "runtime_engine": "PHOTON", "num_workers": 0.0, - "data_security_mode": "SINGLE_USER", - "custom_tags": {"ResourceClass" : "SingleNode"}, + "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", + "custom_tags": {}, "spark_conf": { "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", "spark.prophecy.metadata.is.interactive.run": "false", @@ -23,14 +23,13 @@ def raw_bronze(ti=None, params=None, **context): "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", "spark.prophecy.metadata.user.id": "2169", - "spark.master": "local[*, 4]", "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", "spark.prophecy.execution.metrics.disabled": "true", "spark.databricks.isv.product": "prophecy", "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", - "spark.databricks.cluster.profile": "singleNode", - "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws" + "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws" }, + "is_single_node": True, "aws_attributes": { "first_on_demand": 1.0, "availability": "SPOT_WITH_FALLBACK", @@ -38,15 +37,16 @@ def raw_bronze(ti=None, params=None, **context): "spot_bid_price_percent": 100.0 }, "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, + "kind": "CLASSIC_PREVIEW", "enable_elastic_disk": False }, "spark_jar_task": { "main_class_name": "io.prophecy.pipelines.raw_bronze.Main", "parameters": ["-i", "default", "-O", "{}"] }, - "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}}, { - "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze.jar" + "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}}, { + "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze_2.12.jar" }] }, - databricks_conn_id = "pbt-only-cicd-databricks", + databricks_conn_id = "pbt-only-cicd-databricks" ) diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py index b208a3b..172bc9c 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py @@ -11,11 +11,11 @@ def silver_customers_orders(ti=None, params=None, **context): "task_key": "silver_customers_orders", "new_cluster": { "node_type_id": "i3.xlarge", - "spark_version": "12.2.x-scala2.12", - "runtime_engine": "STANDARD", + "spark_version": "16.4.x-scala2.12", + "runtime_engine": "PHOTON", "num_workers": 0.0, - "data_security_mode": "SINGLE_USER", - "custom_tags": {"ResourceClass" : "SingleNode"}, + "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", + "custom_tags": {}, "spark_conf": { "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", "spark.prophecy.metadata.is.interactive.run": "false", @@ -23,14 +23,13 @@ def silver_customers_orders(ti=None, params=None, **context): "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", "spark.prophecy.metadata.user.id": "2169", - "spark.master": "local[*, 4]", "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", "spark.prophecy.execution.metrics.disabled": "true", "spark.databricks.isv.product": "prophecy", "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", - "spark.databricks.cluster.profile": "singleNode", - "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws" + "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws" }, + "is_single_node": True, "aws_attributes": { "first_on_demand": 1.0, "availability": "SPOT_WITH_FALLBACK", @@ -38,15 +37,16 @@ def silver_customers_orders(ti=None, params=None, **context): "spot_bid_price_percent": 100.0 }, "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, + "kind": "CLASSIC_PREVIEW", "enable_elastic_disk": False }, "spark_jar_task": { "main_class_name": "io.prophecy.pipelines.silver_customers_orders.Main", "parameters": ["-i", "default", "-O", "{}"] }, - "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}}, { - "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders.jar" + "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}}, { + "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders_2.12.jar" }] }, - databricks_conn_id = "pbt-only-cicd-databricks", + databricks_conn_id = "pbt-only-cicd-databricks" ) diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py index f24bd86..be81740 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py @@ -11,11 +11,11 @@ def silver_zip_codes(ti=None, params=None, **context): "task_key": "silver_zip_codes", "new_cluster": { "node_type_id": "i3.xlarge", - "spark_version": "12.2.x-scala2.12", - "runtime_engine": "STANDARD", + "spark_version": "16.4.x-scala2.12", + "runtime_engine": "PHOTON", "num_workers": 0.0, - "data_security_mode": "SINGLE_USER", - "custom_tags": {"ResourceClass" : "SingleNode"}, + "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", + "custom_tags": {}, "spark_conf": { "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", "spark.prophecy.metadata.is.interactive.run": "false", @@ -23,14 +23,13 @@ def silver_zip_codes(ti=None, params=None, **context): "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", "spark.prophecy.metadata.user.id": "2169", - "spark.master": "local[*, 4]", "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", "spark.prophecy.execution.metrics.disabled": "true", "spark.databricks.isv.product": "prophecy", "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", - "spark.databricks.cluster.profile": "singleNode", - "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws" + "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws" }, + "is_single_node": True, "aws_attributes": { "first_on_demand": 1.0, "availability": "SPOT_WITH_FALLBACK", @@ -38,15 +37,16 @@ def silver_zip_codes(ti=None, params=None, **context): "spot_bid_price_percent": 100.0 }, "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, + "kind": "CLASSIC_PREVIEW", "enable_elastic_disk": False }, "spark_jar_task": { "main_class_name": "io.prophecy.pipelines.silver_zip_codes.Main", "parameters": ["-i", "default", "-O", "{}"] }, - "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}}, { - "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes.jar" + "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}}, { + "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes_2.12.jar" }] }, - databricks_conn_id = "pbt-only-cicd-databricks", + databricks_conn_id = "pbt-only-cicd-databricks" ) diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py index 3573317..eb03b10 100644 --- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py +++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py @@ -1,19 +1,19 @@ from airflow.decorators import task db_pipeline_id_to_path_dict = { - "pipelines/gold_sales": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales.jar", - "pipelines/gold_top_customers": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers.jar", - "pipelines/raw_bronze": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze.jar", - "pipelines/silver_customers_orders": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders.jar", - "pipelines/silver_zip_codes": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes.jar" + "pipelines/gold_sales": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales_2.12.jar", + "pipelines/gold_top_customers": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers_2.12.jar", + "pipelines/raw_bronze": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze_2.12.jar", + "pipelines/silver_customers_orders": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders_2.12.jar", + "pipelines/silver_zip_codes": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes_2.12.jar" } -def task_wrapper(task_id): +def task_wrapper(task_id, **task_kwargs): def decorator(func): - @task(task_id = task_id) + @task(task_id = task_id, **task_kwargs) def wrapper(*args, **context): ## running the actual method. return func(*args, **context).execute(context) diff --git a/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json b/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json index 892c97c..8a46e73 100644 --- a/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json +++ b/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json @@ -2,7 +2,7 @@ "fabric_id" : "16433", "components" : [ { "ScriptComponent" : { - "path" : "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py", + "path" : "dbfs:/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py", "nodeName" : "Script_0", "id" : "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l", "language" : "python", @@ -15,16 +15,12 @@ "job_clusters" : [ { "job_cluster_key" : "DatabricksJob2_default_small", "new_cluster" : { - "spark_version" : "12.2.x-scala2.12", + "spark_version" : "16.4.x-scala2.12", "node_type_id" : "i3.xlarge", "num_workers" : 0, - "custom_tags" : { - "ResourceClass" : "SingleNode" - }, + "custom_tags" : { }, "enable_elastic_disk" : false, "spark_conf" : { - "spark.databricks.cluster.profile" : "singleNode", - "spark.master" : "local[*, 4]", "spark.prophecy.metadata.fabric.id" : "16433", "spark.prophecy.metadata.job.uri" : "__PROJECT_ID_PLACEHOLDER__/jobs/DatabricksJob2", "spark.prophecy.metadata.is.interactive.run" : "false", @@ -34,20 +30,22 @@ "spark.prophecy.metadata.job.branch" : "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", "spark.prophecy.metadata.url" : "__PROPHECY_URL_PLACEHOLDER__", "spark.prophecy.execution.metrics.disabled" : "true", - "spark.prophecy.execution.service.url" : "wss://execution.dp.app.prophecy.io/eventws", + "spark.prophecy.execution.service.url" : "wss://app.prophecy.io/execution/eventws", "spark.databricks.isv.product" : "prophecy" }, "spark_env_vars" : { "PYSPARK_PYTHON" : "/databricks/python3/bin/python3" }, - "runtime_engine" : "STANDARD", + "runtime_engine" : "PHOTON", "aws_attributes" : { "first_on_demand" : 1, "availability" : "SPOT_WITH_FALLBACK", "zone_id" : "auto", "spot_bid_price_percent" : 100 }, - "data_security_mode" : "SINGLE_USER" + "data_security_mode" : "DATA_SECURITY_MODE_DEDICATED", + "is_single_node" : true, + "kind" : "CLASSIC_PREVIEW" } } ], "email_notifications" : { }, @@ -55,10 +53,11 @@ "task_key" : "Script_0", "job_cluster_key" : "DatabricksJob2_default_small", "spark_python_task" : { - "python_file" : "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py" + "python_file" : "dbfs:/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py" }, "email_notifications" : { }, - "max_retries" : 0 + "max_retries" : 0, + "run_if" : "ALL_SUCCESS" } ], "max_concurrent_runs" : 1, "schedule" : { @@ -71,5 +70,6 @@ "clusterMode" : "Single" }, "secret_scope" : "prophecy_jobs_2905", - "sorted_processes" : [ "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l" ] + "sorted_processes" : [ "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l" ], + "is_prophecy_managed_fabric" : false } \ No newline at end of file diff --git a/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json b/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json index 95f7281..a3f6fbb 100644 --- a/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json +++ b/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json @@ -15,7 +15,8 @@ "emails" : "", "onStart" : false, "onSuccess" : false, - "onFailure" : false + "onFailure" : false, + "onDurationWarningThresholdExceeded" : false }, "maxConcurrentRuns" : 1, "processDefaultSettings" : { @@ -25,335 +26,10 @@ "emails" : "", "onStart" : false, "onSuccess" : false, - "onFailure" : false + "onFailure" : false, + "onDurationWarningThresholdExceeded" : false } - }, - "availableFabrics" : [ { - "id" : "2807", - "name" : "demos-unity-catalog-guides-dev", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "small-with-new-java", "large" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "12042", - "name" : "databricks-cluster", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "1684", - "name" : "workday-default", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "medium", "large" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "1571", - "name" : "HelloDev", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "Unity", "Photon" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "9635", - "name" : "meisaas", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "2055", - "name" : "bt-dataplatform-bi-dev01", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "prophecy_policy_dev" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "3911", - "name" : "fedex-demo-fabric", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "12128", - "name" : "databricks_execution_env", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "6671", - "name" : "workshop-fabric", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "7603", - "name" : "rjTestTCS1", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "4615", - "name" : "product-workspace", - "clusterSizes" : { - "default" : "medium-64-cores", - "availableClusterSizes" : [ "medium-64-cores", "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "7175", - "name" : "test1delete", - "clusterSizes" : { - "default" : "test1", - "availableClusterSizes" : [ "test1" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "4005", - "name" : "demos-qa", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "6863", - "name" : "workshop-example-fabric", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "9636", - "name" : "test1", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "4004", - "name" : "demos-prod", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "test1" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "1174", - "name" : "demos", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "large_autoscale" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "2588", - "name" : "HealthVerityDemoFabric", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "largeGroup" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "8387", - "name" : "dev", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "6834", - "name" : "lab-samples-dbx", - "clusterSizes" : { - "default" : "sumit-lab-samples", - "availableClusterSizes" : [ "sumit-lab-samples" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "9907", - "name" : "dev1", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small", "large_autoscale" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "7697", - "name" : "testSharedUC", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "16433", - "name" : "prophecyTeamDatabricksTestingGuidesProd3", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "6250", - "name" : "databricks-E2-prod", - "clusterSizes" : { - "default" : "adn_prod", - "availableClusterSizes" : [ "adn_prod", "astellas_single", "astellas_full", "ucb_single", "ucb_full", "healthverity_single", "healthverity_full", "bayer_single", "bayer_full", "moderna_single", "moderna_full", "demo-single_node" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - }, { - "id" : "1572", - "name" : "HelloProd", - "clusterSizes" : { - "default" : "small", - "availableClusterSizes" : [ "small" ] - }, - "pylibsResolutionMode" : { - "PublicCentral" : { } - }, - "scalalibsResolutionMode" : { - "PublicCentral" : { } - } - } ] + } }, "processes" : { "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l" : { @@ -381,12 +57,22 @@ "emails" : "", "onStart" : false, "onSuccess" : false, - "onFailure" : false + "onFailure" : false, + "onDurationWarningThresholdExceeded" : false } }, "properties" : { "code" : "print(\"helloWorld\")" - } + }, + "conditional" : { + "runIf" : "ALL_SUCCESS", + "forEach" : false, + "forEachConfig" : "", + "forEachConcurrency" : 1, + "forEachInputVariable" : "", + "forEachIterationCount" : null + }, + "parent" : null } }, "connections" : [ ] diff --git a/prophecy_scala/pbt_project.yml b/prophecy_scala/pbt_project.yml index f0652d6..a2edb3f 100644 --- a/prophecy_scala/pbt_project.yml +++ b/prophecy_scala/pbt_project.yml @@ -297,6 +297,7 @@ jobs: createdAt: '2023-08-15 00:41:43' author: scott+demo2@prophecy.io models: [] + sqlProjects: [] jobs/AirflowEndToEndJob: name: AirflowEndToEndJob description: test job for airflow scala @@ -326,6 +327,7 @@ jobs: createdAt: '2024-08-13 22:14:02' author: rj@prophecy.io models: [] + sqlProjects: [] jobs/DatabricksJob2: name: DatabricksJob2 description: '' @@ -350,6 +352,7 @@ jobs: createdAt: '2024-08-16 18:46:21' author: rj@prophecy.io models: [] + sqlProjects: [] gemContainer: languageVersion: 2.12.15 gemLibsVersion: 1.1.26 @@ -382,28 +385,35 @@ sqlUnreferencedSources: {} sqlPipeline: null dependencies: - type: plibMaven - modifiedAt: '2024-08-13T18:42:31.331081Z' - name: plibMaven - version: 8.0.31 + modifiedAt: '2026-04-20T21:09:09.461Z' + name: ProphecyLibsScala + version: 9.2.0 id: '106310315' createdAt: '2024-08-13T18:42:31.33104Z' + exclusions: [] + enabled: true + description: Default dependency for Prophecy Libs + editable: true + scope: Project + entity: Project projectDependencies: '12651': projectUID: 12651 projectName: ProphecySparkBasicsScala repositoryPath: gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath= - version: 0.2.5.5 + version: 0.2.6.7 isDefault: true author: null - createdAt: '2024-08-13 18:42:25' + createdAt: '2026-04-20 21:09:09' '12652': projectUID: 12652 projectName: ProphecyWarehouseScala repositoryPath: gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12652.git&subPath= - version: 0.0.5 + version: 0.0.6 isDefault: true author: null - createdAt: '2024-08-13 18:42:25' + createdAt: '2026-04-20 21:09:09' flags: shouldAddDefaultDependenciesIfAbsent: true pipelineConfigurations: {} +projectConfigurations: {} diff --git a/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json index d664bdc..17a6d31 100644 --- a/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json +++ b/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json @@ -1,17 +1,21 @@ { "metainfo" : { "migrationVersions" : { - "gemBuilderVersion" : "v1" + "gemBuilderVersion" : "v1", + "unitTestNullDataMigrated" : true }, "functions" : { }, "codeGenConfiguration" : { "editableConfig" : false, "plibVersion" : { - "mavenVersion" : "8.0.31" + "mavenVersion" : "9.2.0" } }, "id" : "1", "uri" : "pipelines/gold_sales", + "pipelineSettingsInfo" : { + "isNullValuesInUnitTestsEnabled" : true + }, "language" : "scala", "fabricId" : "1571", "frontEndLanguage" : "sql", @@ -283,7 +287,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/OrderBy", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/OrderBy", "projectName" : "ProphecySparkBasicsScala" } }, @@ -365,7 +369,7 @@ "type" : "string", "nullable" : true, "metadata" : { - "description" : "", + "description" : "Customer account flags", "mappings" : [ ], "tags" : [ ] } @@ -417,7 +421,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Aggregate", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Aggregate", "projectName" : "ProphecySparkBasicsScala" } }, @@ -567,18 +571,24 @@ "type" : "string", "nullable" : true, "metadata" : { - "description" : "", + "description" : "Customer first name", "mappings" : [ ], - "tags" : [ ] + "tags" : [ { + "type" : "user", + "name" : "pii" + } ] } }, { "name" : "last_name", "type" : "string", "nullable" : true, "metadata" : { - "description" : "", + "description" : "Customer last name", "mappings" : [ ], - "tags" : [ ] + "tags" : [ { + "type" : "user", + "name" : "pii" + } ] } }, { "name" : "zipcode", @@ -594,7 +604,7 @@ "type" : "string", "nullable" : true, "metadata" : { - "description" : "", + "description" : "Customer account flags", "mappings" : [ ], "tags" : [ ] } @@ -664,7 +674,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat", "projectName" : "ProphecySparkBasicsScala" } }, @@ -746,7 +756,7 @@ "type" : "string", "nullable" : true, "metadata" : { - "description" : "", + "description" : "Customer account flags", "mappings" : [ ], "tags" : [ ] } @@ -814,7 +824,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Aggregate", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Aggregate", "projectName" : "ProphecySparkBasicsScala" } } diff --git a/prophecy_scala/pipelines/gold_sales/code/pom.xml b/prophecy_scala/pipelines/gold_sales/code/pom.xml index f55e9df..408e0a5 100644 --- a/prophecy_scala/pipelines/gold_sales/code/pom.xml +++ b/prophecy_scala/pipelines/gold_sales/code/pom.xml @@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.12 2.12.15 3.3.0 - 8.0.31 + 3.5.6 + 9.2.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + 3.2.1 @@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs snapshots Central Snapshots - https://s01.oss.sonatype.org/content/repositories/snapshots/ + https://central.sonatype.com/repository/maven-snapshots/ default true @@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.json4s json4s-native_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided org.json4s json4s-jackson_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided io.delta - delta-core_${scalaBaseVersion} - 1.1.0 + delta-spark_${scalaBaseVersion} + ${delta.version} provided com.databricks - dbutils-api_${scalaBaseVersion} - 0.0.5 + databricks-dbutils-scala_${scalaBaseVersion} + 0.1.5 io.prophecy @@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.scalatest scalatest_${scalaBaseVersion} - 3.0.1 + ${scalatestVersion} test com.holdenkarau spark-testing-base_${scalaBaseVersion} - 3.0.1_1.1.0 + ${sparkTestingBaseVersion} test @@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + + spark3-scala-2.12 + + true + + + 2.12 + 2.12.15 + 3.5.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + + + ${project.artifactId}_2.12 + + + + spark4-scala-2.13 + + 2.13 + 2.13.14 + 4.0.0 + 4.0.0 + 3.6.7 + 3.0.8 + 4.0.0_2.1.3 + 4.0.0 + + + ${project.artifactId}_2.13 + + + + spark3-scala-2.13 + + 2.13 + 2.13.14 + 3.5.0 + 3.6.7 + 3.0.8 + 3.5.0_2.1.3 + + + ${project.artifactId}_2.13 + + + + ${project.artifactId}_${scalaBaseVersion} net.alchim31.maven diff --git a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala index cffa796..f6f7d48 100644 --- a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala +++ b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala @@ -30,7 +30,6 @@ object Main { val config = ConfigurationFactoryImpl.getConfig(args) val spark: SparkSession = SparkSession .builder() - .appName("Prophecy Pipeline") .config("spark.default.parallelism", "4") .config("spark.sql.legacy.allowUntypedScalaUDF", "true") .enableHiveSupport() diff --git a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala index 519e42a..eea2ffc 100644 --- a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala +++ b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala @@ -10,7 +10,10 @@ object UDFs extends Serializable { def registerUDFs(spark: SparkSession) = { spark.udf.register("rand_zip_index", rand_zip_index) - registerAllUDFs(spark) + try registerAllUDFs(spark) + catch { + case _ => () + } } def rand_zip_index = { diff --git a/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json index 5fbaa34..6fa958a 100644 --- a/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json +++ b/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json @@ -1,17 +1,21 @@ { "metainfo" : { "migrationVersions" : { - "gemBuilderVersion" : "v1" + "gemBuilderVersion" : "v1", + "unitTestNullDataMigrated" : true }, "functions" : { }, "codeGenConfiguration" : { "editableConfig" : false, "plibVersion" : { - "mavenVersion" : "8.0.31" + "mavenVersion" : "9.2.0" } }, "id" : "1", "uri" : "pipelines/gold_top_customers", + "pipelineSettingsInfo" : { + "isNullValuesInUnitTestsEnabled" : true + }, "language" : "scala", "fabricId" : "1571", "frontEndLanguage" : "sql", @@ -182,7 +186,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/OrderBy", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/OrderBy", "projectName" : "ProphecySparkBasicsScala" } }, @@ -248,7 +252,7 @@ "limit" : "50" }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Limit", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Limit", "projectName" : "ProphecySparkBasicsScala" } }, diff --git a/prophecy_scala/pipelines/gold_top_customers/code/pom.xml b/prophecy_scala/pipelines/gold_top_customers/code/pom.xml index 7be847c..da31cbf 100644 --- a/prophecy_scala/pipelines/gold_top_customers/code/pom.xml +++ b/prophecy_scala/pipelines/gold_top_customers/code/pom.xml @@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.12 2.12.15 3.3.0 - 8.0.31 + 3.5.6 + 9.2.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + 3.2.1 @@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs snapshots Central Snapshots - https://s01.oss.sonatype.org/content/repositories/snapshots/ + https://central.sonatype.com/repository/maven-snapshots/ default true @@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.json4s json4s-native_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided org.json4s json4s-jackson_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided io.delta - delta-core_${scalaBaseVersion} - 1.1.0 + delta-spark_${scalaBaseVersion} + ${delta.version} provided com.databricks - dbutils-api_${scalaBaseVersion} - 0.0.5 + databricks-dbutils-scala_${scalaBaseVersion} + 0.1.5 io.prophecy @@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.scalatest scalatest_${scalaBaseVersion} - 3.0.1 + ${scalatestVersion} test com.holdenkarau spark-testing-base_${scalaBaseVersion} - 3.0.1_1.1.0 + ${sparkTestingBaseVersion} test @@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + + spark3-scala-2.12 + + true + + + 2.12 + 2.12.15 + 3.5.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + + + ${project.artifactId}_2.12 + + + + spark4-scala-2.13 + + 2.13 + 2.13.14 + 4.0.0 + 4.0.0 + 3.6.7 + 3.0.8 + 4.0.0_2.1.3 + 4.0.0 + + + ${project.artifactId}_2.13 + + + + spark3-scala-2.13 + + 2.13 + 2.13.14 + 3.5.0 + 3.6.7 + 3.0.8 + 3.5.0_2.1.3 + + + ${project.artifactId}_2.13 + + + + ${project.artifactId}_${scalaBaseVersion} net.alchim31.maven diff --git a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala index dba0e09..5047fec 100644 --- a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala +++ b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala @@ -25,7 +25,6 @@ object Main { val config = ConfigurationFactoryImpl.getConfig(args) val spark: SparkSession = SparkSession .builder() - .appName("Prophecy Pipeline") .config("spark.default.parallelism", "4") .config("spark.sql.legacy.allowUntypedScalaUDF", "true") .enableHiveSupport() diff --git a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala index c0939b1..37fae26 100644 --- a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala +++ b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala @@ -10,7 +10,10 @@ object UDFs extends Serializable { def registerUDFs(spark: SparkSession) = { spark.udf.register("rand_zip_index", rand_zip_index) - registerAllUDFs(spark) + try registerAllUDFs(spark) + catch { + case _ => () + } } def rand_zip_index = { diff --git a/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json index 8eb268f..5f42bb8 100644 --- a/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json +++ b/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json @@ -1,17 +1,21 @@ { "metainfo" : { "migrationVersions" : { - "gemBuilderVersion" : "v1" + "gemBuilderVersion" : "v1", + "unitTestNullDataMigrated" : true }, "functions" : { }, "codeGenConfiguration" : { "editableConfig" : false, "plibVersion" : { - "mavenVersion" : "8.0.31" + "mavenVersion" : "9.2.0" } }, "id" : "1", "uri" : "pipelines/raw_bronze", + "pipelineSettingsInfo" : { + "isNullValuesInUnitTestsEnabled" : true + }, "language" : "scala", "fabricId" : "16433", "frontEndLanguage" : "sql", @@ -290,7 +294,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat", "projectName" : "ProphecySparkBasicsScala" } }, @@ -3241,7 +3245,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat", "projectName" : "ProphecySparkBasicsScala" } }, @@ -3412,7 +3416,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat", "projectName" : "ProphecySparkBasicsScala" } }, diff --git a/prophecy_scala/pipelines/raw_bronze/code/pom.xml b/prophecy_scala/pipelines/raw_bronze/code/pom.xml index fde7048..23d0f2f 100644 --- a/prophecy_scala/pipelines/raw_bronze/code/pom.xml +++ b/prophecy_scala/pipelines/raw_bronze/code/pom.xml @@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.12 2.12.15 3.3.0 - 8.0.31 + 3.5.6 + 9.2.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + 3.2.1 @@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs snapshots Central Snapshots - https://s01.oss.sonatype.org/content/repositories/snapshots/ + https://central.sonatype.com/repository/maven-snapshots/ default true @@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.json4s json4s-native_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided org.json4s json4s-jackson_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided io.delta - delta-core_${scalaBaseVersion} - 1.1.0 + delta-spark_${scalaBaseVersion} + ${delta.version} provided com.databricks - dbutils-api_${scalaBaseVersion} - 0.0.5 + databricks-dbutils-scala_${scalaBaseVersion} + 0.1.5 io.prophecy @@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.scalatest scalatest_${scalaBaseVersion} - 3.0.1 + ${scalatestVersion} test com.holdenkarau spark-testing-base_${scalaBaseVersion} - 3.0.1_1.1.0 + ${sparkTestingBaseVersion} test @@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + + spark3-scala-2.12 + + true + + + 2.12 + 2.12.15 + 3.5.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + + + ${project.artifactId}_2.12 + + + + spark4-scala-2.13 + + 2.13 + 2.13.14 + 4.0.0 + 4.0.0 + 3.6.7 + 3.0.8 + 4.0.0_2.1.3 + 4.0.0 + + + ${project.artifactId}_2.13 + + + + spark3-scala-2.13 + + 2.13 + 2.13.14 + 3.5.0 + 3.6.7 + 3.0.8 + 3.5.0_2.1.3 + + + ${project.artifactId}_2.13 + + + + ${project.artifactId}_${scalaBaseVersion} net.alchim31.maven diff --git a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala index 56f54c2..c7c0c5b 100644 --- a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala +++ b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala @@ -30,7 +30,6 @@ object Main { val config = ConfigurationFactoryImpl.getConfig(args) val spark: SparkSession = SparkSession .builder() - .appName("Prophecy Pipeline") .config("spark.default.parallelism", "4") .config("spark.sql.legacy.allowUntypedScalaUDF", "true") .enableHiveSupport() diff --git a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala index 00ea31e..db435f7 100644 --- a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala +++ b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala @@ -10,7 +10,10 @@ object UDFs extends Serializable { def registerUDFs(spark: SparkSession) = { spark.udf.register("rand_zip_index", rand_zip_index) - registerAllUDFs(spark) + try registerAllUDFs(spark) + catch { + case _ => () + } } def rand_zip_index = { diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json index ffd6b1f..bf2401a 100644 --- a/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json +++ b/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json @@ -1,17 +1,21 @@ { "metainfo" : { "migrationVersions" : { - "gemBuilderVersion" : "v1" + "gemBuilderVersion" : "v1", + "unitTestNullDataMigrated" : true }, "functions" : { }, "codeGenConfiguration" : { "editableConfig" : false, "plibVersion" : { - "mavenVersion" : "8.0.31" + "mavenVersion" : "9.2.0" } }, "id" : "1", "uri" : "pipelines/silver_customers_orders", + "pipelineSettingsInfo" : { + "isNullValuesInUnitTestsEnabled" : true + }, "language" : "scala", "fabricId" : "1571", "frontEndLanguage" : "sql", @@ -476,7 +480,7 @@ "headAlias" : "in0" }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Join", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Join", "projectName" : "ProphecySparkBasicsScala" } }, @@ -611,7 +615,7 @@ "inputPortNames" : [ "in0" ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/SQLStatement", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/SQLStatement", "projectName" : "ProphecySparkBasicsScala" } }, @@ -1004,7 +1008,7 @@ "headAlias" : "in0" }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Join", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Join", "projectName" : "ProphecySparkBasicsScala" } }, diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml b/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml index 95ddd62..b5a8b5f 100644 --- a/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml +++ b/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml @@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.12 2.12.15 3.3.0 - 8.0.31 + 3.5.6 + 9.2.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + 3.2.1 @@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs snapshots Central Snapshots - https://s01.oss.sonatype.org/content/repositories/snapshots/ + https://central.sonatype.com/repository/maven-snapshots/ default true @@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.json4s json4s-native_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided org.json4s json4s-jackson_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided io.delta - delta-core_${scalaBaseVersion} - 1.1.0 + delta-spark_${scalaBaseVersion} + ${delta.version} provided com.databricks - dbutils-api_${scalaBaseVersion} - 0.0.5 + databricks-dbutils-scala_${scalaBaseVersion} + 0.1.5 io.prophecy @@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.scalatest scalatest_${scalaBaseVersion} - 3.0.1 + ${scalatestVersion} test com.holdenkarau spark-testing-base_${scalaBaseVersion} - 3.0.1_1.1.0 + ${sparkTestingBaseVersion} test @@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + + spark3-scala-2.12 + + true + + + 2.12 + 2.12.15 + 3.5.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + + + ${project.artifactId}_2.12 + + + + spark4-scala-2.13 + + 2.13 + 2.13.14 + 4.0.0 + 4.0.0 + 3.6.7 + 3.0.8 + 4.0.0_2.1.3 + 4.0.0 + + + ${project.artifactId}_2.13 + + + + spark3-scala-2.13 + + 2.13 + 2.13.14 + 3.5.0 + 3.6.7 + 3.0.8 + 3.5.0_2.1.3 + + + ${project.artifactId}_2.13 + + + + ${project.artifactId}_${scalaBaseVersion} net.alchim31.maven diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala index 6eed538..3db373f 100644 --- a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala +++ b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala @@ -32,7 +32,6 @@ object Main { val config = ConfigurationFactoryImpl.getConfig(args) val spark: SparkSession = SparkSession .builder() - .appName("Prophecy Pipeline") .config("spark.default.parallelism", "4") .config("spark.sql.legacy.allowUntypedScalaUDF", "true") .enableHiveSupport() diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala index 5fa7841..eaa8627 100644 --- a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala +++ b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala @@ -10,7 +10,10 @@ object UDFs extends Serializable { def registerUDFs(spark: SparkSession) = { spark.udf.register("rand_zip_index", rand_zip_index) - registerAllUDFs(spark) + try registerAllUDFs(spark) + catch { + case _ => () + } } def rand_zip_index = { diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json index de2a060..1d37e35 100644 --- a/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json +++ b/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json @@ -1,17 +1,21 @@ { "metainfo" : { "migrationVersions" : { - "gemBuilderVersion" : "v1" + "gemBuilderVersion" : "v1", + "unitTestNullDataMigrated" : true }, "functions" : { }, "codeGenConfiguration" : { "editableConfig" : false, "plibVersion" : { - "mavenVersion" : "8.0.31" + "mavenVersion" : "9.2.0" } }, "id" : "1", "uri" : "pipelines/silver_zip_codes", + "pipelineSettingsInfo" : { + "isNullValuesInUnitTestsEnabled" : true + }, "language" : "scala", "fabricId" : "1571", "frontEndLanguage" : "sql", @@ -1163,7 +1167,7 @@ } }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Filter", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Filter", "projectName" : "ProphecySparkBasicsScala" } }, @@ -1324,7 +1328,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat", "projectName" : "ProphecySparkBasicsScala" } }, @@ -2418,7 +2422,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat", "projectName" : "ProphecySparkBasicsScala" } }, @@ -2636,7 +2640,7 @@ } ] }, "componentInfo" : { - "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Aggregate", + "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Aggregate", "projectName" : "ProphecySparkBasicsScala" } } diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml b/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml index f8cd7c9..137cf80 100644 --- a/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml +++ b/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml @@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.12 2.12.15 3.3.0 - 8.0.31 + 3.5.6 + 9.2.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + 3.2.1 @@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs snapshots Central Snapshots - https://s01.oss.sonatype.org/content/repositories/snapshots/ + https://central.sonatype.com/repository/maven-snapshots/ default true @@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.json4s json4s-native_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided org.json4s json4s-jackson_${scalaBaseVersion} - 3.5.3 + ${json4sVersion} provided io.delta - delta-core_${scalaBaseVersion} - 1.1.0 + delta-spark_${scalaBaseVersion} + ${delta.version} provided com.databricks - dbutils-api_${scalaBaseVersion} - 0.0.5 + databricks-dbutils-scala_${scalaBaseVersion} + 0.1.5 io.prophecy @@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.scalatest scalatest_${scalaBaseVersion} - 3.0.1 + ${scalatestVersion} test com.holdenkarau spark-testing-base_${scalaBaseVersion} - 3.0.1_1.1.0 + ${sparkTestingBaseVersion} test @@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + + spark3-scala-2.12 + + true + + + 2.12 + 2.12.15 + 3.5.0 + 3.5.3 + 3.0.1 + 3.3.0_1.1.2 + + + ${project.artifactId}_2.12 + + + + spark4-scala-2.13 + + 2.13 + 2.13.14 + 4.0.0 + 4.0.0 + 3.6.7 + 3.0.8 + 4.0.0_2.1.3 + 4.0.0 + + + ${project.artifactId}_2.13 + + + + spark3-scala-2.13 + + 2.13 + 2.13.14 + 3.5.0 + 3.6.7 + 3.0.8 + 3.5.0_2.1.3 + + + ${project.artifactId}_2.13 + + + + ${project.artifactId}_${scalaBaseVersion} net.alchim31.maven diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala index 2960871..fdaa508 100644 --- a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala +++ b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala @@ -29,7 +29,6 @@ object Main { val config = ConfigurationFactoryImpl.getConfig(args) val spark: SparkSession = SparkSession .builder() - .appName("Prophecy Pipeline") .config("spark.default.parallelism", "4") .config("spark.sql.legacy.allowUntypedScalaUDF", "true") .enableHiveSupport() diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala index 5e922a2..99bb170 100644 --- a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala +++ b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala @@ -10,7 +10,10 @@ object UDFs extends Serializable { def registerUDFs(spark: SparkSession) = { spark.udf.register("rand_zip_index", rand_zip_index) - registerAllUDFs(spark) + try registerAllUDFs(spark) + catch { + case _ => () + } } def rand_zip_index = {