diff --git a/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json b/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json
index 1da93fb..9ba8127 100644
--- a/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json
+++ b/prophecy_scala/datasets/bronze_customers/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}},{"name":"full_name","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}},{"name":"full_name","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json b/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json
index 6b5fa9a..a1a53f0 100644
--- a/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json
+++ b/prophecy_scala/datasets/bronze_irs_zipcode/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json b/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json
index fd7a5b5..3294905 100644
--- a/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json
+++ b/prophecy_scala/datasets/bronze_orders/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json b/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json
index be5d943..9825728 100644
--- a/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json
+++ b/prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"avg_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_count","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_sales_by_zip_date","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"avg_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_count","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_sales_by_zip_date","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json b/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json
index a231992..c053f9b 100644
--- a/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json
+++ b/prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_top50_customers_by_spend","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_top50_customers_by_spend","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json b/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json
index 95cf266..aee4960 100644
--- a/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json
+++ b/prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_total_sales_by_customer","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_total_sales_by_customer","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/raw_customers/default/configs/dataset.json b/prophecy_scala/datasets/raw_customers/default/configs/dataset.json
index eb45f79..9371859 100644
--- a/prophecy_scala/datasets/raw_customers/default/configs/dataset.json
+++ b/prophecy_scala/datasets/raw_customers/default/configs/dataset.json
@@ -1 +1 @@
-{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/CustomersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"string","nullable":true,"metadata":{"description":"The date the the Customer opened their account","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null}
\ No newline at end of file
+{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/CustomersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"string","nullable":true,"metadata":{"description":"The date the the Customer opened their account","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json b/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json
index 3805ea5..71c80c4 100644
--- a/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json
+++ b/prophecy_scala/datasets/raw_irs_zipcode/default/configs/dataset.json
@@ -1 +1 @@
-{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/databricks-datasets/data.gov/irs_zip_code_data/data-001/2013_soi_zipcode_agi.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null}
\ No newline at end of file
+{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/databricks-datasets/data.gov/irs_zip_code_data/data-001/2013_soi_zipcode_agi.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"STATEFIPS","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"STATE","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"agi_stub","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS1","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"MARS4","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"PREP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N2","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"NUMDEP","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00200","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00650","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01000","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A01700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"SCHF","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A26270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A02900","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03270","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03150","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03210","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A03240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04470","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A00101","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18425","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18450","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A18300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A19700","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A04800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A05800","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07100","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07180","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07230","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07240","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07220","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A07260","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A09400","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10600","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59660","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A59720","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11070","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10960","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A06500","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A10300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85330","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A85300","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11901","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"N11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"A11902","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/raw_orders/default/configs/dataset.json b/prophecy_scala/datasets/raw_orders/default/configs/dataset.json
index 2ed1214..5cac832 100644
--- a/prophecy_scala/datasets/raw_orders/default/configs/dataset.json
+++ b/prophecy_scala/datasets/raw_orders/default/configs/dataset.json
@@ -1 +1 @@
-{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/OrdersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"string","nullable":true,"metadata":{"description":"Order ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"string","nullable":true,"metadata":{"description":"Date the order was made","mappings":[],"tags":[]}},{"name":"amount","type":"string","nullable":true,"metadata":{"description":"Total amount for the order","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null}
\ No newline at end of file
+{"format":"csv","ignoreLeadingWhiteSpaceReading":null,"multiLine":null,"path":"dbfs:/prophecy_training/HelloWorldDefault/OrdersDatasetInput.csv","ignoreTrailingWhiteSpaceReading":null,"maxColumns":null,"writeMode":"error","description":"","separator":",","compression":null,"nullValue":null,"nanValue":null,"escapeQuotes":null,"charToEscapeQuoteEscaping":null,"skipFooters":null,"encoding":null,"inferSchema":null,"enforceSchema":null,"quoteAll":null,"ignoreLeadingWhiteSpaceWriting":null,"locale":null,"quote":null,"timestampFormat":null,"emptyValue":null,"recursiveFileLookup":null,"partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"string","nullable":true,"metadata":{"description":"Order ID","mappings":[],"tags":[{"type":"user","name":"primary"}]}},{"name":"customer_id","type":"string","nullable":true,"metadata":{"description":"Customer ID","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"string","nullable":true,"metadata":{"description":"Date the order was made","mappings":[],"tags":[]}},{"name":"amount","type":"string","nullable":true,"metadata":{"description":"Total amount for the order","mappings":[],"tags":[]}}]},"escape":null,"unescapedQuoteHandling":null,"positiveInf":null,"skipHeaders":null,"comment":null,"lineSep":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/csv","projectName":"ProphecySparkBasicsScala"},"samplingRatio":null,"dateFormat":null,"mode":null,"header":true,"columnNameOfCorruptRecord":null,"modifiedBefore":null,"ignoreTrailingWhiteSpaceWriting":null,"maxCharsPerColumn":null,"negativeInf":null,"createSingleOutputFile":null,"pathGlobFilter":null,"datasetType":"File","useSchema":true,"modifiedAfter":null}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/silver_customers/default/configs/dataset.json b/prophecy_scala/datasets/silver_customers/default/configs/dataset.json
index 5957d99..663898c 100644
--- a/prophecy_scala/datasets/silver_customers/default/configs/dataset.json
+++ b/prophecy_scala/datasets/silver_customers/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json b/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json
index a6fed3f..45cd566 100644
--- a/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json
+++ b/prophecy_scala/datasets/silver_irs_zipcode/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"state","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_percent","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"low_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"all_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"is_high_income","type":"boolean","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"state","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_percent","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"high_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"low_income_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"all_returns","type":"long","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"is_high_income","type":"boolean","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_irs_zipcode","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/silver_orders/default/configs/dataset.json b/prophecy_scala/datasets/silver_orders/default/configs/dataset.json
index 67f7bc3..a8556e0 100644
--- a/prophecy_scala/datasets/silver_orders/default/configs/dataset.json
+++ b/prophecy_scala/datasets/silver_orders/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json b/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json
index bbed526..450e127 100644
--- a/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json
+++ b/prophecy_scala/datasets/silver_orders_customer_details/default/configs/dataset.json
@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_order_customer_details","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"silver_order_customer_details","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
\ No newline at end of file
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json b/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json
index a4508bc..db2f714 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/prophecy-job.json
@@ -11,31 +11,14 @@
       "emails" : "",
       "onStart" : false,
       "onSuccess" : false,
-      "onFailure" : false
+      "onFailure" : false,
+      "onDurationWarningThresholdExceeded" : false
     },
     "maxConcurrentRuns" : 1,
     "processDefaultSettings" : {
       "ignore_first_depends_on_past" : true,
       "do_xcom_push" : true
     },
-    "availableFabrics" : [ {
-      "id" : "6817",
-      "name" : "workdayAirflowFabric"
-    }, {
-      "id" : "8615",
-      "name" : "AirflowFab3",
-      "trialStartDate" : 1705001938144
-    }, {
-      "id" : "10547",
-      "name" : "rjAirflowFabricMWAA1"
-    }, {
-      "id" : "12737",
-      "name" : "Airflow-prophecy-development",
-      "trialStartDate" : 1716477924239
-    }, {
-      "id" : "16432",
-      "name" : "prophecyTeamAirflowTesting"
-    } ],
     "maxActiveRuns" : 1,
     "catchup" : false,
     "scheduleTimeZone" : "UTC",
@@ -51,7 +34,8 @@
       "databricks" : false,
       "dataproc" : false,
       "emr" : false,
-      "onprem" : false
+      "onprem" : false,
+      "emr_serverless" : false
     }
   },
   "processes" : {
@@ -77,6 +61,7 @@
       "settings" : { },
       "properties" : {
         "runJson" : null,
+        "projectId" : "10729",
         "databricksConnId" : "",
         "clusterSize" : "16433/small",
         "configurations" : {
@@ -121,6 +106,7 @@
       "settings" : { },
       "properties" : {
         "runJson" : null,
+        "projectId" : "10729",
         "databricksConnId" : "",
         "clusterSize" : "16433/small",
         "configurations" : {
@@ -165,6 +151,7 @@
       "settings" : { },
       "properties" : {
         "runJson" : null,
+        "projectId" : "10729",
         "databricksConnId" : "",
         "clusterSize" : "16433/small",
         "configurations" : {
@@ -209,6 +196,7 @@
       "settings" : { },
       "properties" : {
         "runJson" : null,
+        "projectId" : "10729",
         "databricksConnId" : "",
         "clusterSize" : "16433/small",
         "configurations" : {
@@ -253,6 +241,7 @@
       "settings" : { },
       "properties" : {
         "runJson" : null,
+        "projectId" : "10729",
         "databricksConnId" : "",
         "clusterSize" : "16433/small",
         "configurations" : {
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py
index bb22f83..19d6b5e 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/__init__.py
@@ -1,5 +1,5 @@
 from .gold_sales import *
-from .silver_customers_orders import *
 from .gold_top_customers import *
-from .silver_zip_codes import *
-from .raw_bronze import *
\ No newline at end of file
+from .raw_bronze import *
+from .silver_customers_orders import *
+from .silver_zip_codes import *
\ No newline at end of file
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py
index 5290452..de2da86 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_sales.py
@@ -11,11 +11,11 @@ def gold_sales(ti=None, params=None, **context):
           "task_key": "gold_sales", 
           "new_cluster": {
             "node_type_id": "i3.xlarge", 
-            "spark_version": "12.2.x-scala2.12", 
-            "runtime_engine": "STANDARD", 
+            "spark_version": "16.4.x-scala2.12", 
+            "runtime_engine": "PHOTON", 
             "num_workers": 0.0, 
-            "data_security_mode": "SINGLE_USER", 
-            "custom_tags": {"ResourceClass" : "SingleNode"}, 
+            "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", 
+            "custom_tags": {}, 
             "spark_conf": {
               "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", 
               "spark.prophecy.metadata.is.interactive.run": "false", 
@@ -23,14 +23,13 @@ def gold_sales(ti=None, params=None, **context):
               "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", 
               "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", 
               "spark.prophecy.metadata.user.id": "2169", 
-              "spark.master": "local[*, 4]", 
               "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", 
               "spark.prophecy.execution.metrics.disabled": "true", 
               "spark.databricks.isv.product": "prophecy", 
               "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", 
-              "spark.databricks.cluster.profile": "singleNode", 
-              "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws"
+              "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws"
             }, 
+            "is_single_node": True, 
             "aws_attributes": {
               "first_on_demand": 1.0, 
               "availability": "SPOT_WITH_FALLBACK", 
@@ -38,15 +37,16 @@ def gold_sales(ti=None, params=None, **context):
               "spot_bid_price_percent": 100.0
             }, 
             "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, 
+            "kind": "CLASSIC_PREVIEW", 
             "enable_elastic_disk": False
           }, 
           "spark_jar_task": {
             "main_class_name": "io.prophecy.pipelines.gold_sales.Main", 
             "parameters": ["-i", "default", "-O", "{}"]
           }, 
-          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}},                          {
-                           "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales.jar"
+          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}},                          {
+                           "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales_2.12.jar"
                          }]
         },
-        databricks_conn_id = "pbt-only-cicd-databricks",
+        databricks_conn_id = "pbt-only-cicd-databricks"
     )
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py
index a0db7ff..bd479f8 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/gold_top_customers.py
@@ -11,11 +11,11 @@ def gold_top_customers(ti=None, params=None, **context):
           "task_key": "gold_top_customers", 
           "new_cluster": {
             "node_type_id": "i3.xlarge", 
-            "spark_version": "12.2.x-scala2.12", 
-            "runtime_engine": "STANDARD", 
+            "spark_version": "16.4.x-scala2.12", 
+            "runtime_engine": "PHOTON", 
             "num_workers": 0.0, 
-            "data_security_mode": "SINGLE_USER", 
-            "custom_tags": {"ResourceClass" : "SingleNode"}, 
+            "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", 
+            "custom_tags": {}, 
             "spark_conf": {
               "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", 
               "spark.prophecy.metadata.is.interactive.run": "false", 
@@ -23,14 +23,13 @@ def gold_top_customers(ti=None, params=None, **context):
               "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", 
               "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", 
               "spark.prophecy.metadata.user.id": "2169", 
-              "spark.master": "local[*, 4]", 
               "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", 
               "spark.prophecy.execution.metrics.disabled": "true", 
               "spark.databricks.isv.product": "prophecy", 
               "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", 
-              "spark.databricks.cluster.profile": "singleNode", 
-              "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws"
+              "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws"
             }, 
+            "is_single_node": True, 
             "aws_attributes": {
               "first_on_demand": 1.0, 
               "availability": "SPOT_WITH_FALLBACK", 
@@ -38,15 +37,16 @@ def gold_top_customers(ti=None, params=None, **context):
               "spot_bid_price_percent": 100.0
             }, 
             "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, 
+            "kind": "CLASSIC_PREVIEW", 
             "enable_elastic_disk": False
           }, 
           "spark_jar_task": {
             "main_class_name": "io.prophecy.pipelines.gold_top_customers.Main", 
             "parameters": ["-i", "default", "-O", "{}"]
           }, 
-          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}},                          {
-                           "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers.jar"
+          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}},                          {
+                           "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers_2.12.jar"
                          }]
         },
-        databricks_conn_id = "pbt-only-cicd-databricks",
+        databricks_conn_id = "pbt-only-cicd-databricks"
     )
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py
index fdb2fe0..5422984 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/raw_bronze.py
@@ -11,11 +11,11 @@ def raw_bronze(ti=None, params=None, **context):
           "task_key": "raw_bronze", 
           "new_cluster": {
             "node_type_id": "i3.xlarge", 
-            "spark_version": "12.2.x-scala2.12", 
-            "runtime_engine": "STANDARD", 
+            "spark_version": "16.4.x-scala2.12", 
+            "runtime_engine": "PHOTON", 
             "num_workers": 0.0, 
-            "data_security_mode": "SINGLE_USER", 
-            "custom_tags": {"ResourceClass" : "SingleNode"}, 
+            "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", 
+            "custom_tags": {}, 
             "spark_conf": {
               "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", 
               "spark.prophecy.metadata.is.interactive.run": "false", 
@@ -23,14 +23,13 @@ def raw_bronze(ti=None, params=None, **context):
               "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", 
               "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", 
               "spark.prophecy.metadata.user.id": "2169", 
-              "spark.master": "local[*, 4]", 
               "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", 
               "spark.prophecy.execution.metrics.disabled": "true", 
               "spark.databricks.isv.product": "prophecy", 
               "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", 
-              "spark.databricks.cluster.profile": "singleNode", 
-              "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws"
+              "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws"
             }, 
+            "is_single_node": True, 
             "aws_attributes": {
               "first_on_demand": 1.0, 
               "availability": "SPOT_WITH_FALLBACK", 
@@ -38,15 +37,16 @@ def raw_bronze(ti=None, params=None, **context):
               "spot_bid_price_percent": 100.0
             }, 
             "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, 
+            "kind": "CLASSIC_PREVIEW", 
             "enable_elastic_disk": False
           }, 
           "spark_jar_task": {
             "main_class_name": "io.prophecy.pipelines.raw_bronze.Main", 
             "parameters": ["-i", "default", "-O", "{}"]
           }, 
-          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}},                          {
-                           "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze.jar"
+          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}},                          {
+                           "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze_2.12.jar"
                          }]
         },
-        databricks_conn_id = "pbt-only-cicd-databricks",
+        databricks_conn_id = "pbt-only-cicd-databricks"
     )
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py
index b208a3b..172bc9c 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_customers_orders.py
@@ -11,11 +11,11 @@ def silver_customers_orders(ti=None, params=None, **context):
           "task_key": "silver_customers_orders", 
           "new_cluster": {
             "node_type_id": "i3.xlarge", 
-            "spark_version": "12.2.x-scala2.12", 
-            "runtime_engine": "STANDARD", 
+            "spark_version": "16.4.x-scala2.12", 
+            "runtime_engine": "PHOTON", 
             "num_workers": 0.0, 
-            "data_security_mode": "SINGLE_USER", 
-            "custom_tags": {"ResourceClass" : "SingleNode"}, 
+            "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", 
+            "custom_tags": {}, 
             "spark_conf": {
               "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", 
               "spark.prophecy.metadata.is.interactive.run": "false", 
@@ -23,14 +23,13 @@ def silver_customers_orders(ti=None, params=None, **context):
               "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", 
               "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", 
               "spark.prophecy.metadata.user.id": "2169", 
-              "spark.master": "local[*, 4]", 
               "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", 
               "spark.prophecy.execution.metrics.disabled": "true", 
               "spark.databricks.isv.product": "prophecy", 
               "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", 
-              "spark.databricks.cluster.profile": "singleNode", 
-              "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws"
+              "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws"
             }, 
+            "is_single_node": True, 
             "aws_attributes": {
               "first_on_demand": 1.0, 
               "availability": "SPOT_WITH_FALLBACK", 
@@ -38,15 +37,16 @@ def silver_customers_orders(ti=None, params=None, **context):
               "spot_bid_price_percent": 100.0
             }, 
             "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, 
+            "kind": "CLASSIC_PREVIEW", 
             "enable_elastic_disk": False
           }, 
           "spark_jar_task": {
             "main_class_name": "io.prophecy.pipelines.silver_customers_orders.Main", 
             "parameters": ["-i", "default", "-O", "{}"]
           }, 
-          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}},                          {
-                           "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders.jar"
+          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}},                          {
+                           "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders_2.12.jar"
                          }]
         },
-        databricks_conn_id = "pbt-only-cicd-databricks",
+        databricks_conn_id = "pbt-only-cicd-databricks"
     )
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py
index f24bd86..be81740 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/tasks/silver_zip_codes.py
@@ -11,11 +11,11 @@ def silver_zip_codes(ti=None, params=None, **context):
           "task_key": "silver_zip_codes", 
           "new_cluster": {
             "node_type_id": "i3.xlarge", 
-            "spark_version": "12.2.x-scala2.12", 
-            "runtime_engine": "STANDARD", 
+            "spark_version": "16.4.x-scala2.12", 
+            "runtime_engine": "PHOTON", 
             "num_workers": 0.0, 
-            "data_security_mode": "SINGLE_USER", 
-            "custom_tags": {"ResourceClass" : "SingleNode"}, 
+            "data_security_mode": "DATA_SECURITY_MODE_DEDICATED", 
+            "custom_tags": {}, 
             "spark_conf": {
               "spark.prophecy.metadata.job.uri": "__PROJECT_ID_PLACEHOLDER__/jobs/AirflowEndToEndJob", 
               "spark.prophecy.metadata.is.interactive.run": "false", 
@@ -23,14 +23,13 @@ def silver_zip_codes(ti=None, params=None, **context):
               "spark.prophecy.tasks": "H4sIAAAAAAAAAKuuBQBDv6ajAgAAAA==", 
               "spark.prophecy.metadata.url": "__PROPHECY_URL_PLACEHOLDER__", 
               "spark.prophecy.metadata.user.id": "2169", 
-              "spark.master": "local[*, 4]", 
               "spark.prophecy.project.id": "__PROJECT_ID_PLACEHOLDER__", 
               "spark.prophecy.execution.metrics.disabled": "true", 
               "spark.databricks.isv.product": "prophecy", 
               "spark.prophecy.metadata.job.branch": "__PROJECT_RELEASE_VERSION_PLACEHOLDER__", 
-              "spark.databricks.cluster.profile": "singleNode", 
-              "spark.prophecy.execution.service.url": "wss://execution.dp.app.prophecy.io/eventws"
+              "spark.prophecy.execution.service.url": "wss://app.prophecy.io/execution/eventws"
             }, 
+            "is_single_node": True, 
             "aws_attributes": {
               "first_on_demand": 1.0, 
               "availability": "SPOT_WITH_FALLBACK", 
@@ -38,15 +37,16 @@ def silver_zip_codes(ti=None, params=None, **context):
               "spot_bid_price_percent": 100.0
             }, 
             "spark_env_vars": {"PYSPARK_PYTHON" : "/databricks/python3/bin/python3"}, 
+            "kind": "CLASSIC_PREVIEW", 
             "enable_elastic_disk": False
           }, 
           "spark_jar_task": {
             "main_class_name": "io.prophecy.pipelines.silver_zip_codes.Main", 
             "parameters": ["-i", "default", "-O", "{}"]
           }, 
-          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.3.0-8.0.31"}},                          {
-                           "jar": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes.jar"
+          "libraries": [{"maven" : {"coordinates" : "io.prophecy:prophecy-libs_2.12:3.5.0-9.2.0"}},                          {
+                           "jar": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes_2.12.jar"
                          }]
         },
-        databricks_conn_id = "pbt-only-cicd-databricks",
+        databricks_conn_id = "pbt-only-cicd-databricks"
     )
diff --git a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py
index 3573317..eb03b10 100644
--- a/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py
+++ b/prophecy_scala/jobs/AirflowEndToEndJob/code/scott_demo2_team_helloprophecyscala_airflowendtoendjob/utils/prophecy.py
@@ -1,19 +1,19 @@
 from airflow.decorators import task
 
 db_pipeline_id_to_path_dict = {
-    "pipelines/gold_sales": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales.jar", 
-    "pipelines/gold_top_customers": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers.jar", 
-    "pipelines/raw_bronze": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze.jar", 
-    "pipelines/silver_customers_orders": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders.jar", 
-    "pipelines/silver_zip_codes": "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes.jar"
+    "pipelines/gold_sales": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_sales_2.12.jar", 
+    "pipelines/gold_top_customers": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/gold_top_customers_2.12.jar", 
+    "pipelines/raw_bronze": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/raw_bronze_2.12.jar", 
+    "pipelines/silver_customers_orders": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_customers_orders_2.12.jar", 
+    "pipelines/silver_zip_codes": "/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/pipeline/silver_zip_codes_2.12.jar"
 }
 
 
-def task_wrapper(task_id):
+def task_wrapper(task_id, **task_kwargs):
 
     def decorator(func):
 
-        @task(task_id = task_id)
+        @task(task_id = task_id, **task_kwargs)
         def wrapper(*args, **context):
             ## running the actual method.
             return func(*args, **context).execute(context)
diff --git a/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json b/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json
index 892c97c..8a46e73 100644
--- a/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json
+++ b/prophecy_scala/jobs/DatabricksJob2/code/databricks-job.json
@@ -2,7 +2,7 @@
   "fabric_id" : "16433",
   "components" : [ {
     "ScriptComponent" : {
-      "path" : "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py",
+      "path" : "dbfs:/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py",
       "nodeName" : "Script_0",
       "id" : "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l",
       "language" : "python",
@@ -15,16 +15,12 @@
     "job_clusters" : [ {
       "job_cluster_key" : "DatabricksJob2_default_small",
       "new_cluster" : {
-        "spark_version" : "12.2.x-scala2.12",
+        "spark_version" : "16.4.x-scala2.12",
         "node_type_id" : "i3.xlarge",
         "num_workers" : 0,
-        "custom_tags" : {
-          "ResourceClass" : "SingleNode"
-        },
+        "custom_tags" : { },
         "enable_elastic_disk" : false,
         "spark_conf" : {
-          "spark.databricks.cluster.profile" : "singleNode",
-          "spark.master" : "local[*, 4]",
           "spark.prophecy.metadata.fabric.id" : "16433",
           "spark.prophecy.metadata.job.uri" : "__PROJECT_ID_PLACEHOLDER__/jobs/DatabricksJob2",
           "spark.prophecy.metadata.is.interactive.run" : "false",
@@ -34,20 +30,22 @@
           "spark.prophecy.metadata.job.branch" : "__PROJECT_RELEASE_VERSION_PLACEHOLDER__",
           "spark.prophecy.metadata.url" : "__PROPHECY_URL_PLACEHOLDER__",
           "spark.prophecy.execution.metrics.disabled" : "true",
-          "spark.prophecy.execution.service.url" : "wss://execution.dp.app.prophecy.io/eventws",
+          "spark.prophecy.execution.service.url" : "wss://app.prophecy.io/execution/eventws",
           "spark.databricks.isv.product" : "prophecy"
         },
         "spark_env_vars" : {
           "PYSPARK_PYTHON" : "/databricks/python3/bin/python3"
         },
-        "runtime_engine" : "STANDARD",
+        "runtime_engine" : "PHOTON",
         "aws_attributes" : {
           "first_on_demand" : 1,
           "availability" : "SPOT_WITH_FALLBACK",
           "zone_id" : "auto",
           "spot_bid_price_percent" : 100
         },
-        "data_security_mode" : "SINGLE_USER"
+        "data_security_mode" : "DATA_SECURITY_MODE_DEDICATED",
+        "is_single_node" : true,
+        "kind" : "CLASSIC_PREVIEW"
       }
     } ],
     "email_notifications" : { },
@@ -55,10 +53,11 @@
       "task_key" : "Script_0",
       "job_cluster_key" : "DatabricksJob2_default_small",
       "spark_python_task" : {
-        "python_file" : "dbfs:/FileStore/prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py"
+        "python_file" : "dbfs:/Volumes/pbt_testing/default/prophecy//prophecy/artifacts/saas/app/__PROJECT_ID_PLACEHOLDER__/__PROJECT_RELEASE_VERSION_PLACEHOLDER__/DatabricksJob2/Script_0--342935076.py"
       },
       "email_notifications" : { },
-      "max_retries" : 0
+      "max_retries" : 0,
+      "run_if" : "ALL_SUCCESS"
     } ],
     "max_concurrent_runs" : 1,
     "schedule" : {
@@ -71,5 +70,6 @@
     "clusterMode" : "Single"
   },
   "secret_scope" : "prophecy_jobs_2905",
-  "sorted_processes" : [ "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l" ]
+  "sorted_processes" : [ "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l" ],
+  "is_prophecy_managed_fabric" : false
 }
\ No newline at end of file
diff --git a/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json b/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json
index 95f7281..a3f6fbb 100644
--- a/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json
+++ b/prophecy_scala/jobs/DatabricksJob2/code/prophecy-job.json
@@ -15,7 +15,8 @@
       "emails" : "",
       "onStart" : false,
       "onSuccess" : false,
-      "onFailure" : false
+      "onFailure" : false,
+      "onDurationWarningThresholdExceeded" : false
     },
     "maxConcurrentRuns" : 1,
     "processDefaultSettings" : {
@@ -25,335 +26,10 @@
         "emails" : "",
         "onStart" : false,
         "onSuccess" : false,
-        "onFailure" : false
+        "onFailure" : false,
+        "onDurationWarningThresholdExceeded" : false
       }
-    },
-    "availableFabrics" : [ {
-      "id" : "2807",
-      "name" : "demos-unity-catalog-guides-dev",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "small-with-new-java", "large" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "12042",
-      "name" : "databricks-cluster",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "1684",
-      "name" : "workday-default",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "medium", "large" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "1571",
-      "name" : "HelloDev",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "Unity", "Photon" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "9635",
-      "name" : "meisaas",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "2055",
-      "name" : "bt-dataplatform-bi-dev01",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "prophecy_policy_dev" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "3911",
-      "name" : "fedex-demo-fabric",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "12128",
-      "name" : "databricks_execution_env",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "6671",
-      "name" : "workshop-fabric",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "7603",
-      "name" : "rjTestTCS1",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "4615",
-      "name" : "product-workspace",
-      "clusterSizes" : {
-        "default" : "medium-64-cores",
-        "availableClusterSizes" : [ "medium-64-cores", "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "7175",
-      "name" : "test1delete",
-      "clusterSizes" : {
-        "default" : "test1",
-        "availableClusterSizes" : [ "test1" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "4005",
-      "name" : "demos-qa",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "6863",
-      "name" : "workshop-example-fabric",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "9636",
-      "name" : "test1",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "4004",
-      "name" : "demos-prod",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "test1" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "1174",
-      "name" : "demos",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "large_autoscale" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "2588",
-      "name" : "HealthVerityDemoFabric",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "largeGroup" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "8387",
-      "name" : "dev",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "6834",
-      "name" : "lab-samples-dbx",
-      "clusterSizes" : {
-        "default" : "sumit-lab-samples",
-        "availableClusterSizes" : [ "sumit-lab-samples" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "9907",
-      "name" : "dev1",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small", "large_autoscale" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "7697",
-      "name" : "testSharedUC",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "16433",
-      "name" : "prophecyTeamDatabricksTestingGuidesProd3",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "6250",
-      "name" : "databricks-E2-prod",
-      "clusterSizes" : {
-        "default" : "adn_prod",
-        "availableClusterSizes" : [ "adn_prod", "astellas_single", "astellas_full", "ucb_single", "ucb_full", "healthverity_single", "healthverity_full", "bayer_single", "bayer_full", "moderna_single", "moderna_full", "demo-single_node" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    }, {
-      "id" : "1572",
-      "name" : "HelloProd",
-      "clusterSizes" : {
-        "default" : "small",
-        "availableClusterSizes" : [ "small" ]
-      },
-      "pylibsResolutionMode" : {
-        "PublicCentral" : { }
-      },
-      "scalalibsResolutionMode" : {
-        "PublicCentral" : { }
-      }
-    } ]
+    }
   },
   "processes" : {
     "yfd2K0EiwSIXOL3ECJHgp$$j63W3omPlneLhSc03hF7l" : {
@@ -381,12 +57,22 @@
           "emails" : "",
           "onStart" : false,
           "onSuccess" : false,
-          "onFailure" : false
+          "onFailure" : false,
+          "onDurationWarningThresholdExceeded" : false
         }
       },
       "properties" : {
         "code" : "print(\"helloWorld\")"
-      }
+      },
+      "conditional" : {
+        "runIf" : "ALL_SUCCESS",
+        "forEach" : false,
+        "forEachConfig" : "",
+        "forEachConcurrency" : 1,
+        "forEachInputVariable" : "",
+        "forEachIterationCount" : null
+      },
+      "parent" : null
     }
   },
   "connections" : [ ]
diff --git a/prophecy_scala/pbt_project.yml b/prophecy_scala/pbt_project.yml
index f0652d6..a2edb3f 100644
--- a/prophecy_scala/pbt_project.yml
+++ b/prophecy_scala/pbt_project.yml
@@ -297,6 +297,7 @@ jobs:
     createdAt: '2023-08-15 00:41:43'
     author: scott+demo2@prophecy.io
     models: []
+    sqlProjects: []
   jobs/AirflowEndToEndJob:
     name: AirflowEndToEndJob
     description: test job for airflow scala
@@ -326,6 +327,7 @@ jobs:
     createdAt: '2024-08-13 22:14:02'
     author: rj@prophecy.io
     models: []
+    sqlProjects: []
   jobs/DatabricksJob2:
     name: DatabricksJob2
     description: ''
@@ -350,6 +352,7 @@ jobs:
     createdAt: '2024-08-16 18:46:21'
     author: rj@prophecy.io
     models: []
+    sqlProjects: []
 gemContainer:
   languageVersion: 2.12.15
   gemLibsVersion: 1.1.26
@@ -382,28 +385,35 @@ sqlUnreferencedSources: {}
 sqlPipeline: null
 dependencies:
 - type: plibMaven
-  modifiedAt: '2024-08-13T18:42:31.331081Z'
-  name: plibMaven
-  version: 8.0.31
+  modifiedAt: '2026-04-20T21:09:09.461Z'
+  name: ProphecyLibsScala
+  version: 9.2.0
   id: '106310315'
   createdAt: '2024-08-13T18:42:31.33104Z'
+  exclusions: []
+  enabled: true
+  description: Default dependency for Prophecy Libs
+  editable: true
+  scope: Project
+  entity: Project
 projectDependencies:
   '12651':
     projectUID: 12651
     projectName: ProphecySparkBasicsScala
     repositoryPath: gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=
-    version: 0.2.5.5
+    version: 0.2.6.7
     isDefault: true
     author: null
-    createdAt: '2024-08-13 18:42:25'
+    createdAt: '2026-04-20 21:09:09'
   '12652':
     projectUID: 12652
     projectName: ProphecyWarehouseScala
     repositoryPath: gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12652.git&subPath=
-    version: 0.0.5
+    version: 0.0.6
     isDefault: true
     author: null
-    createdAt: '2024-08-13 18:42:25'
+    createdAt: '2026-04-20 21:09:09'
 flags:
   shouldAddDefaultDependenciesIfAbsent: true
 pipelineConfigurations: {}
+projectConfigurations: {}
diff --git a/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json
index d664bdc..17a6d31 100644
--- a/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json
+++ b/prophecy_scala/pipelines/gold_sales/code/.prophecy/workflow.latest.json
@@ -1,17 +1,21 @@
 {
   "metainfo" : {
     "migrationVersions" : {
-      "gemBuilderVersion" : "v1"
+      "gemBuilderVersion" : "v1",
+      "unitTestNullDataMigrated" : true
     },
     "functions" : { },
     "codeGenConfiguration" : {
       "editableConfig" : false,
       "plibVersion" : {
-        "mavenVersion" : "8.0.31"
+        "mavenVersion" : "9.2.0"
       }
     },
     "id" : "1",
     "uri" : "pipelines/gold_sales",
+    "pipelineSettingsInfo" : {
+      "isNullValuesInUnitTestsEnabled" : true
+    },
     "language" : "scala",
     "fabricId" : "1571",
     "frontEndLanguage" : "sql",
@@ -283,7 +287,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/OrderBy",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/OrderBy",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -365,7 +369,7 @@
               "type" : "string",
               "nullable" : true,
               "metadata" : {
-                "description" : "",
+                "description" : "Customer account flags",
                 "mappings" : [ ],
                 "tags" : [ ]
               }
@@ -417,7 +421,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Aggregate",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Aggregate",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -567,18 +571,24 @@
               "type" : "string",
               "nullable" : true,
               "metadata" : {
-                "description" : "",
+                "description" : "Customer first name",
                 "mappings" : [ ],
-                "tags" : [ ]
+                "tags" : [ {
+                  "type" : "user",
+                  "name" : "pii"
+                } ]
               }
             }, {
               "name" : "last_name",
               "type" : "string",
               "nullable" : true,
               "metadata" : {
-                "description" : "",
+                "description" : "Customer last name",
                 "mappings" : [ ],
-                "tags" : [ ]
+                "tags" : [ {
+                  "type" : "user",
+                  "name" : "pii"
+                } ]
               }
             }, {
               "name" : "zipcode",
@@ -594,7 +604,7 @@
               "type" : "string",
               "nullable" : true,
               "metadata" : {
-                "description" : "",
+                "description" : "Customer account flags",
                 "mappings" : [ ],
                 "tags" : [ ]
               }
@@ -664,7 +674,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -746,7 +756,7 @@
               "type" : "string",
               "nullable" : true,
               "metadata" : {
-                "description" : "",
+                "description" : "Customer account flags",
                 "mappings" : [ ],
                 "tags" : [ ]
               }
@@ -814,7 +824,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Aggregate",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Aggregate",
         "projectName" : "ProphecySparkBasicsScala"
       }
     }
diff --git a/prophecy_scala/pipelines/gold_sales/code/pom.xml b/prophecy_scala/pipelines/gold_sales/code/pom.xml
index f55e9df..408e0a5 100644
--- a/prophecy_scala/pipelines/gold_sales/code/pom.xml
+++ b/prophecy_scala/pipelines/gold_sales/code/pom.xml
@@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <scalaBaseVersion>2.12</scalaBaseVersion>
         <scalaVersion>2.12.15</scalaVersion>
         <sparkVersion>3.3.0</sparkVersion>
-        <prophecyLibsVersion>8.0.31</prophecyLibsVersion>
+        <devMauchSparkVersion>3.5.6</devMauchSparkVersion>
+        <prophecyLibsVersion>9.2.0</prophecyLibsVersion>
+        <json4sVersion>3.5.3</json4sVersion>
+        <scalatestVersion>3.0.1</scalatestVersion>
+        <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+        <delta.version>3.2.1</delta.version>
     </properties>
     <repositories>
         <repository>
@@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <repository>
             <id>snapshots</id>
             <name>Central Snapshots</name>
-            <url>https://s01.oss.sonatype.org/content/repositories/snapshots/</url>
+            <url>https://central.sonatype.com/repository/maven-snapshots/</url>
             <layout>default</layout>
             <snapshots>
                 <enabled>true</enabled>
@@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-native_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-jackson_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_${scalaBaseVersion}</artifactId>
-            <version>1.1.0</version>
+            <artifactId>delta-spark_${scalaBaseVersion}</artifactId>
+            <version>${delta.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.databricks</groupId>
-            <artifactId>dbutils-api_${scalaBaseVersion}</artifactId>
-            <version>0.0.5</version>
+            <artifactId>databricks-dbutils-scala_${scalaBaseVersion}</artifactId>
+            <version>0.1.5</version>
         </dependency>
         <dependency>
             <groupId>io.prophecy</groupId>
@@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scalaBaseVersion}</artifactId>
-            <version>3.0.1</version>
+            <version>${scalatestVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.holdenkarau</groupId>
             <artifactId>spark-testing-base_${scalaBaseVersion}</artifactId>
-            <version>3.0.1_1.1.0</version>
+            <version>${sparkTestingBaseVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
             </exclusions>
         </dependency>
     </dependencies>
+    <profiles>
+        <profile>
+            <id>spark3-scala-2.12</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <scalaBaseVersion>2.12</scalaBaseVersion>
+                <scalaVersion>2.12.15</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.5.3</json4sVersion>
+                <scalatestVersion>3.0.1</scalatestVersion>
+                <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.12</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark4-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>4.0.0</sparkVersion>
+                <devMauchSparkVersion>4.0.0</devMauchSparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>4.0.0_2.1.3</sparkTestingBaseVersion>
+                <delta.version>4.0.0</delta.version>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark3-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>3.5.0_2.1.3</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+    </profiles>
     <build>
+        <finalName>${project.artifactId}_${scalaBaseVersion}</finalName>
         <plugins>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
diff --git a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala
index cffa796..f6f7d48 100644
--- a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala
+++ b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/main.scala
@@ -30,7 +30,6 @@ object Main {
     val config = ConfigurationFactoryImpl.getConfig(args)
     val spark: SparkSession = SparkSession
       .builder()
-      .appName("Prophecy Pipeline")
       .config("spark.default.parallelism",             "4")
       .config("spark.sql.legacy.allowUntypedScalaUDF", "true")
       .enableHiveSupport()
diff --git a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala
index 519e42a..eea2ffc 100644
--- a/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala
+++ b/prophecy_scala/pipelines/gold_sales/code/src/main/scala/io/prophecy/pipelines/gold_sales/udfs/UDFs.scala
@@ -10,7 +10,10 @@ object UDFs extends Serializable {
 
   def registerUDFs(spark: SparkSession) = {
     spark.udf.register("rand_zip_index", rand_zip_index)
-    registerAllUDFs(spark)
+    try registerAllUDFs(spark)
+    catch {
+      case _ => ()
+    }
   }
 
   def rand_zip_index = {
diff --git a/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json
index 5fbaa34..6fa958a 100644
--- a/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json
+++ b/prophecy_scala/pipelines/gold_top_customers/code/.prophecy/workflow.latest.json
@@ -1,17 +1,21 @@
 {
   "metainfo" : {
     "migrationVersions" : {
-      "gemBuilderVersion" : "v1"
+      "gemBuilderVersion" : "v1",
+      "unitTestNullDataMigrated" : true
     },
     "functions" : { },
     "codeGenConfiguration" : {
       "editableConfig" : false,
       "plibVersion" : {
-        "mavenVersion" : "8.0.31"
+        "mavenVersion" : "9.2.0"
       }
     },
     "id" : "1",
     "uri" : "pipelines/gold_top_customers",
+    "pipelineSettingsInfo" : {
+      "isNullValuesInUnitTestsEnabled" : true
+    },
     "language" : "scala",
     "fabricId" : "1571",
     "frontEndLanguage" : "sql",
@@ -182,7 +186,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/OrderBy",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/OrderBy",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -248,7 +252,7 @@
         "limit" : "50"
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Limit",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Limit",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
diff --git a/prophecy_scala/pipelines/gold_top_customers/code/pom.xml b/prophecy_scala/pipelines/gold_top_customers/code/pom.xml
index 7be847c..da31cbf 100644
--- a/prophecy_scala/pipelines/gold_top_customers/code/pom.xml
+++ b/prophecy_scala/pipelines/gold_top_customers/code/pom.xml
@@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <scalaBaseVersion>2.12</scalaBaseVersion>
         <scalaVersion>2.12.15</scalaVersion>
         <sparkVersion>3.3.0</sparkVersion>
-        <prophecyLibsVersion>8.0.31</prophecyLibsVersion>
+        <devMauchSparkVersion>3.5.6</devMauchSparkVersion>
+        <prophecyLibsVersion>9.2.0</prophecyLibsVersion>
+        <json4sVersion>3.5.3</json4sVersion>
+        <scalatestVersion>3.0.1</scalatestVersion>
+        <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+        <delta.version>3.2.1</delta.version>
     </properties>
     <repositories>
         <repository>
@@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <repository>
             <id>snapshots</id>
             <name>Central Snapshots</name>
-            <url>https://s01.oss.sonatype.org/content/repositories/snapshots/</url>
+            <url>https://central.sonatype.com/repository/maven-snapshots/</url>
             <layout>default</layout>
             <snapshots>
                 <enabled>true</enabled>
@@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-native_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-jackson_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_${scalaBaseVersion}</artifactId>
-            <version>1.1.0</version>
+            <artifactId>delta-spark_${scalaBaseVersion}</artifactId>
+            <version>${delta.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.databricks</groupId>
-            <artifactId>dbutils-api_${scalaBaseVersion}</artifactId>
-            <version>0.0.5</version>
+            <artifactId>databricks-dbutils-scala_${scalaBaseVersion}</artifactId>
+            <version>0.1.5</version>
         </dependency>
         <dependency>
             <groupId>io.prophecy</groupId>
@@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scalaBaseVersion}</artifactId>
-            <version>3.0.1</version>
+            <version>${scalatestVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.holdenkarau</groupId>
             <artifactId>spark-testing-base_${scalaBaseVersion}</artifactId>
-            <version>3.0.1_1.1.0</version>
+            <version>${sparkTestingBaseVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
             </exclusions>
         </dependency>
     </dependencies>
+    <profiles>
+        <profile>
+            <id>spark3-scala-2.12</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <scalaBaseVersion>2.12</scalaBaseVersion>
+                <scalaVersion>2.12.15</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.5.3</json4sVersion>
+                <scalatestVersion>3.0.1</scalatestVersion>
+                <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.12</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark4-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>4.0.0</sparkVersion>
+                <devMauchSparkVersion>4.0.0</devMauchSparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>4.0.0_2.1.3</sparkTestingBaseVersion>
+                <delta.version>4.0.0</delta.version>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark3-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>3.5.0_2.1.3</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+    </profiles>
     <build>
+        <finalName>${project.artifactId}_${scalaBaseVersion}</finalName>
         <plugins>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
diff --git a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala
index dba0e09..5047fec 100644
--- a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala
+++ b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/main.scala
@@ -25,7 +25,6 @@ object Main {
     val config = ConfigurationFactoryImpl.getConfig(args)
     val spark: SparkSession = SparkSession
       .builder()
-      .appName("Prophecy Pipeline")
       .config("spark.default.parallelism",             "4")
       .config("spark.sql.legacy.allowUntypedScalaUDF", "true")
       .enableHiveSupport()
diff --git a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala
index c0939b1..37fae26 100644
--- a/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala
+++ b/prophecy_scala/pipelines/gold_top_customers/code/src/main/scala/io/prophecy/pipelines/gold_top_customers/udfs/UDFs.scala
@@ -10,7 +10,10 @@ object UDFs extends Serializable {
 
   def registerUDFs(spark: SparkSession) = {
     spark.udf.register("rand_zip_index", rand_zip_index)
-    registerAllUDFs(spark)
+    try registerAllUDFs(spark)
+    catch {
+      case _ => ()
+    }
   }
 
   def rand_zip_index = {
diff --git a/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json
index 8eb268f..5f42bb8 100644
--- a/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json
+++ b/prophecy_scala/pipelines/raw_bronze/code/.prophecy/workflow.latest.json
@@ -1,17 +1,21 @@
 {
   "metainfo" : {
     "migrationVersions" : {
-      "gemBuilderVersion" : "v1"
+      "gemBuilderVersion" : "v1",
+      "unitTestNullDataMigrated" : true
     },
     "functions" : { },
     "codeGenConfiguration" : {
       "editableConfig" : false,
       "plibVersion" : {
-        "mavenVersion" : "8.0.31"
+        "mavenVersion" : "9.2.0"
       }
     },
     "id" : "1",
     "uri" : "pipelines/raw_bronze",
+    "pipelineSettingsInfo" : {
+      "isNullValuesInUnitTestsEnabled" : true
+    },
     "language" : "scala",
     "fabricId" : "16433",
     "frontEndLanguage" : "sql",
@@ -290,7 +294,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -3241,7 +3245,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -3412,7 +3416,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
diff --git a/prophecy_scala/pipelines/raw_bronze/code/pom.xml b/prophecy_scala/pipelines/raw_bronze/code/pom.xml
index fde7048..23d0f2f 100644
--- a/prophecy_scala/pipelines/raw_bronze/code/pom.xml
+++ b/prophecy_scala/pipelines/raw_bronze/code/pom.xml
@@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <scalaBaseVersion>2.12</scalaBaseVersion>
         <scalaVersion>2.12.15</scalaVersion>
         <sparkVersion>3.3.0</sparkVersion>
-        <prophecyLibsVersion>8.0.31</prophecyLibsVersion>
+        <devMauchSparkVersion>3.5.6</devMauchSparkVersion>
+        <prophecyLibsVersion>9.2.0</prophecyLibsVersion>
+        <json4sVersion>3.5.3</json4sVersion>
+        <scalatestVersion>3.0.1</scalatestVersion>
+        <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+        <delta.version>3.2.1</delta.version>
     </properties>
     <repositories>
         <repository>
@@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <repository>
             <id>snapshots</id>
             <name>Central Snapshots</name>
-            <url>https://s01.oss.sonatype.org/content/repositories/snapshots/</url>
+            <url>https://central.sonatype.com/repository/maven-snapshots/</url>
             <layout>default</layout>
             <snapshots>
                 <enabled>true</enabled>
@@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-native_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-jackson_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_${scalaBaseVersion}</artifactId>
-            <version>1.1.0</version>
+            <artifactId>delta-spark_${scalaBaseVersion}</artifactId>
+            <version>${delta.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.databricks</groupId>
-            <artifactId>dbutils-api_${scalaBaseVersion}</artifactId>
-            <version>0.0.5</version>
+            <artifactId>databricks-dbutils-scala_${scalaBaseVersion}</artifactId>
+            <version>0.1.5</version>
         </dependency>
         <dependency>
             <groupId>io.prophecy</groupId>
@@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scalaBaseVersion}</artifactId>
-            <version>3.0.1</version>
+            <version>${scalatestVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.holdenkarau</groupId>
             <artifactId>spark-testing-base_${scalaBaseVersion}</artifactId>
-            <version>3.0.1_1.1.0</version>
+            <version>${sparkTestingBaseVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
             </exclusions>
         </dependency>
     </dependencies>
+    <profiles>
+        <profile>
+            <id>spark3-scala-2.12</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <scalaBaseVersion>2.12</scalaBaseVersion>
+                <scalaVersion>2.12.15</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.5.3</json4sVersion>
+                <scalatestVersion>3.0.1</scalatestVersion>
+                <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.12</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark4-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>4.0.0</sparkVersion>
+                <devMauchSparkVersion>4.0.0</devMauchSparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>4.0.0_2.1.3</sparkTestingBaseVersion>
+                <delta.version>4.0.0</delta.version>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark3-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>3.5.0_2.1.3</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+    </profiles>
     <build>
+        <finalName>${project.artifactId}_${scalaBaseVersion}</finalName>
         <plugins>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
diff --git a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala
index 56f54c2..c7c0c5b 100644
--- a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala
+++ b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/main.scala
@@ -30,7 +30,6 @@ object Main {
     val config = ConfigurationFactoryImpl.getConfig(args)
     val spark: SparkSession = SparkSession
       .builder()
-      .appName("Prophecy Pipeline")
       .config("spark.default.parallelism",             "4")
       .config("spark.sql.legacy.allowUntypedScalaUDF", "true")
       .enableHiveSupport()
diff --git a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala
index 00ea31e..db435f7 100644
--- a/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala
+++ b/prophecy_scala/pipelines/raw_bronze/code/src/main/scala/io/prophecy/pipelines/raw_bronze/udfs/UDFs.scala
@@ -10,7 +10,10 @@ object UDFs extends Serializable {
 
   def registerUDFs(spark: SparkSession) = {
     spark.udf.register("rand_zip_index", rand_zip_index)
-    registerAllUDFs(spark)
+    try registerAllUDFs(spark)
+    catch {
+      case _ => ()
+    }
   }
 
   def rand_zip_index = {
diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json
index ffd6b1f..bf2401a 100644
--- a/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json
+++ b/prophecy_scala/pipelines/silver_customers_orders/code/.prophecy/workflow.latest.json
@@ -1,17 +1,21 @@
 {
   "metainfo" : {
     "migrationVersions" : {
-      "gemBuilderVersion" : "v1"
+      "gemBuilderVersion" : "v1",
+      "unitTestNullDataMigrated" : true
     },
     "functions" : { },
     "codeGenConfiguration" : {
       "editableConfig" : false,
       "plibVersion" : {
-        "mavenVersion" : "8.0.31"
+        "mavenVersion" : "9.2.0"
       }
     },
     "id" : "1",
     "uri" : "pipelines/silver_customers_orders",
+    "pipelineSettingsInfo" : {
+      "isNullValuesInUnitTestsEnabled" : true
+    },
     "language" : "scala",
     "fabricId" : "1571",
     "frontEndLanguage" : "sql",
@@ -476,7 +480,7 @@
         "headAlias" : "in0"
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Join",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Join",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -611,7 +615,7 @@
         "inputPortNames" : [ "in0" ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/SQLStatement",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/SQLStatement",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -1004,7 +1008,7 @@
         "headAlias" : "in0"
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Join",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Join",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml b/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml
index 95ddd62..b5a8b5f 100644
--- a/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml
+++ b/prophecy_scala/pipelines/silver_customers_orders/code/pom.xml
@@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <scalaBaseVersion>2.12</scalaBaseVersion>
         <scalaVersion>2.12.15</scalaVersion>
         <sparkVersion>3.3.0</sparkVersion>
-        <prophecyLibsVersion>8.0.31</prophecyLibsVersion>
+        <devMauchSparkVersion>3.5.6</devMauchSparkVersion>
+        <prophecyLibsVersion>9.2.0</prophecyLibsVersion>
+        <json4sVersion>3.5.3</json4sVersion>
+        <scalatestVersion>3.0.1</scalatestVersion>
+        <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+        <delta.version>3.2.1</delta.version>
     </properties>
     <repositories>
         <repository>
@@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <repository>
             <id>snapshots</id>
             <name>Central Snapshots</name>
-            <url>https://s01.oss.sonatype.org/content/repositories/snapshots/</url>
+            <url>https://central.sonatype.com/repository/maven-snapshots/</url>
             <layout>default</layout>
             <snapshots>
                 <enabled>true</enabled>
@@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-native_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-jackson_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_${scalaBaseVersion}</artifactId>
-            <version>1.1.0</version>
+            <artifactId>delta-spark_${scalaBaseVersion}</artifactId>
+            <version>${delta.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.databricks</groupId>
-            <artifactId>dbutils-api_${scalaBaseVersion}</artifactId>
-            <version>0.0.5</version>
+            <artifactId>databricks-dbutils-scala_${scalaBaseVersion}</artifactId>
+            <version>0.1.5</version>
         </dependency>
         <dependency>
             <groupId>io.prophecy</groupId>
@@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scalaBaseVersion}</artifactId>
-            <version>3.0.1</version>
+            <version>${scalatestVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.holdenkarau</groupId>
             <artifactId>spark-testing-base_${scalaBaseVersion}</artifactId>
-            <version>3.0.1_1.1.0</version>
+            <version>${sparkTestingBaseVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
             </exclusions>
         </dependency>
     </dependencies>
+    <profiles>
+        <profile>
+            <id>spark3-scala-2.12</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <scalaBaseVersion>2.12</scalaBaseVersion>
+                <scalaVersion>2.12.15</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.5.3</json4sVersion>
+                <scalatestVersion>3.0.1</scalatestVersion>
+                <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.12</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark4-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>4.0.0</sparkVersion>
+                <devMauchSparkVersion>4.0.0</devMauchSparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>4.0.0_2.1.3</sparkTestingBaseVersion>
+                <delta.version>4.0.0</delta.version>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark3-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>3.5.0_2.1.3</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+    </profiles>
     <build>
+        <finalName>${project.artifactId}_${scalaBaseVersion}</finalName>
         <plugins>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala
index 6eed538..3db373f 100644
--- a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala
+++ b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/main.scala
@@ -32,7 +32,6 @@ object Main {
     val config = ConfigurationFactoryImpl.getConfig(args)
     val spark: SparkSession = SparkSession
       .builder()
-      .appName("Prophecy Pipeline")
       .config("spark.default.parallelism",             "4")
       .config("spark.sql.legacy.allowUntypedScalaUDF", "true")
       .enableHiveSupport()
diff --git a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala
index 5fa7841..eaa8627 100644
--- a/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala
+++ b/prophecy_scala/pipelines/silver_customers_orders/code/src/main/scala/io/prophecy/pipelines/silver_customers_orders/udfs/UDFs.scala
@@ -10,7 +10,10 @@ object UDFs extends Serializable {
 
   def registerUDFs(spark: SparkSession) = {
     spark.udf.register("rand_zip_index", rand_zip_index)
-    registerAllUDFs(spark)
+    try registerAllUDFs(spark)
+    catch {
+      case _ => ()
+    }
   }
 
   def rand_zip_index = {
diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json b/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json
index de2a060..1d37e35 100644
--- a/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json
+++ b/prophecy_scala/pipelines/silver_zip_codes/code/.prophecy/workflow.latest.json
@@ -1,17 +1,21 @@
 {
   "metainfo" : {
     "migrationVersions" : {
-      "gemBuilderVersion" : "v1"
+      "gemBuilderVersion" : "v1",
+      "unitTestNullDataMigrated" : true
     },
     "functions" : { },
     "codeGenConfiguration" : {
       "editableConfig" : false,
       "plibVersion" : {
-        "mavenVersion" : "8.0.31"
+        "mavenVersion" : "9.2.0"
       }
     },
     "id" : "1",
     "uri" : "pipelines/silver_zip_codes",
+    "pipelineSettingsInfo" : {
+      "isNullValuesInUnitTestsEnabled" : true
+    },
     "language" : "scala",
     "fabricId" : "1571",
     "frontEndLanguage" : "sql",
@@ -1163,7 +1167,7 @@
         }
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Filter",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Filter",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -1324,7 +1328,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -2418,7 +2422,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Reformat",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Reformat",
         "projectName" : "ProphecySparkBasicsScala"
       }
     },
@@ -2636,7 +2640,7 @@
         } ]
       },
       "componentInfo" : {
-        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/Aggregate",
+        "gemId" : "gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/Aggregate",
         "projectName" : "ProphecySparkBasicsScala"
       }
     }
diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml b/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml
index f8cd7c9..137cf80 100644
--- a/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml
+++ b/prophecy_scala/pipelines/silver_zip_codes/code/pom.xml
@@ -10,7 +10,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <scalaBaseVersion>2.12</scalaBaseVersion>
         <scalaVersion>2.12.15</scalaVersion>
         <sparkVersion>3.3.0</sparkVersion>
-        <prophecyLibsVersion>8.0.31</prophecyLibsVersion>
+        <devMauchSparkVersion>3.5.6</devMauchSparkVersion>
+        <prophecyLibsVersion>9.2.0</prophecyLibsVersion>
+        <json4sVersion>3.5.3</json4sVersion>
+        <scalatestVersion>3.0.1</scalatestVersion>
+        <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+        <delta.version>3.2.1</delta.version>
     </properties>
     <repositories>
         <repository>
@@ -25,7 +30,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <repository>
             <id>snapshots</id>
             <name>Central Snapshots</name>
-            <url>https://s01.oss.sonatype.org/content/repositories/snapshots/</url>
+            <url>https://central.sonatype.com/repository/maven-snapshots/</url>
             <layout>default</layout>
             <snapshots>
                 <enabled>true</enabled>
@@ -86,25 +91,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-native_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-jackson_${scalaBaseVersion}</artifactId>
-            <version>3.5.3</version>
+            <version>${json4sVersion}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_${scalaBaseVersion}</artifactId>
-            <version>1.1.0</version>
+            <artifactId>delta-spark_${scalaBaseVersion}</artifactId>
+            <version>${delta.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.databricks</groupId>
-            <artifactId>dbutils-api_${scalaBaseVersion}</artifactId>
-            <version>0.0.5</version>
+            <artifactId>databricks-dbutils-scala_${scalaBaseVersion}</artifactId>
+            <version>0.1.5</version>
         </dependency>
         <dependency>
             <groupId>io.prophecy</groupId>
@@ -114,13 +119,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scalaBaseVersion}</artifactId>
-            <version>3.0.1</version>
+            <version>${scalatestVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.holdenkarau</groupId>
             <artifactId>spark-testing-base_${scalaBaseVersion}</artifactId>
-            <version>3.0.1_1.1.0</version>
+            <version>${sparkTestingBaseVersion}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -136,7 +141,57 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
             </exclusions>
         </dependency>
     </dependencies>
+    <profiles>
+        <profile>
+            <id>spark3-scala-2.12</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <scalaBaseVersion>2.12</scalaBaseVersion>
+                <scalaVersion>2.12.15</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.5.3</json4sVersion>
+                <scalatestVersion>3.0.1</scalatestVersion>
+                <sparkTestingBaseVersion>3.3.0_1.1.2</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.12</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark4-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>4.0.0</sparkVersion>
+                <devMauchSparkVersion>4.0.0</devMauchSparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>4.0.0_2.1.3</sparkTestingBaseVersion>
+                <delta.version>4.0.0</delta.version>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+        <profile>
+            <id>spark3-scala-2.13</id>
+            <properties>
+                <scalaBaseVersion>2.13</scalaBaseVersion>
+                <scalaVersion>2.13.14</scalaVersion>
+                <sparkVersion>3.5.0</sparkVersion>
+                <json4sVersion>3.6.7</json4sVersion>
+                <scalatestVersion>3.0.8</scalatestVersion>
+                <sparkTestingBaseVersion>3.5.0_2.1.3</sparkTestingBaseVersion>
+            </properties>
+            <build>
+                <finalName>${project.artifactId}_2.13</finalName>
+            </build>
+        </profile>
+    </profiles>
     <build>
+        <finalName>${project.artifactId}_${scalaBaseVersion}</finalName>
         <plugins>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala
index 2960871..fdaa508 100644
--- a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala
+++ b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/main.scala
@@ -29,7 +29,6 @@ object Main {
     val config = ConfigurationFactoryImpl.getConfig(args)
     val spark: SparkSession = SparkSession
       .builder()
-      .appName("Prophecy Pipeline")
       .config("spark.default.parallelism",             "4")
       .config("spark.sql.legacy.allowUntypedScalaUDF", "true")
       .enableHiveSupport()
diff --git a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala
index 5e922a2..99bb170 100644
--- a/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala
+++ b/prophecy_scala/pipelines/silver_zip_codes/code/src/main/scala/io/prophecy/pipelines/silver_zip_codes/udfs/UDFs.scala
@@ -10,7 +10,10 @@ object UDFs extends Serializable {
 
   def registerUDFs(spark: SparkSession) = {
     spark.udf.register("rand_zip_index", rand_zip_index)
-    registerAllUDFs(spark)
+    try registerAllUDFs(spark)
+    catch {
+      case _ => ()
+    }
   }
 
   def rand_zip_index = {