Update Databricks cluster configs and library versions of dependencies

prophecy_scala/datasets/bronze_customers/default/configs/dataset.json

@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}},{"name":"full_name","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":null,"path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"first_name","type":"string","nullable":true,"metadata":{"description":"Customer first name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"last_name","type":"string","nullable":true,"metadata":{"description":"Customer last name","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"phone","type":"string","nullable":true,"metadata":{"description":"Customer phone number","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"email","type":"string","nullable":true,"metadata":{"description":"Customer Email","mappings":[],"tags":[{"type":"user","name":"pii"}]}},{"name":"country_code","type":"string","nullable":true,"metadata":{"description":"Customer country code","mappings":[],"tags":[]}},{"name":"account_open_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"account_flags","type":"string","nullable":true,"metadata":{"description":"Customer account flags","mappings":[],"tags":[]}},{"name":"full_name","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_customers","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}

prophecy_scala/datasets/bronze_orders/default/configs/dataset.json

@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"order_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_status","type":"string","nullable":true,"metadata":{"description":"Order status","mappings":[],"tags":[]}},{"name":"order_category","type":"string","nullable":true,"metadata":{"description":"Order category","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"amount","type":"float","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"bronze_orders","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}

prophecy_scala/datasets/gold_sales_by_zip_date/default/configs/dataset.json

@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"avg_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_count","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_sales_by_zip_date","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"zipcode","type":"string","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_date","type":"date","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"avg_amount","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"order_count","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_sales_by_zip_date","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}

prophecy_scala/datasets/gold_top50_customers_by_spend/default/configs/dataset.json

@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_top50_customers_by_spend","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_top50_customers_by_spend","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}

prophecy_scala/datasets/gold_total_sales_by_customer/default/configs/dataset.json

@@ -1 +1 @@
-{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_total_sales_by_customer","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.5.5&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}
+{"mergeCondition":null,"minFlagCol":null,"format":"catalogTable","catalog":"spark_catalog","path":"scottdemoscala","insertInto":false,"provider":"delta","externalFilePath":"","mergeTargetAlias":"target","overwriteSchema":null,"flagValue":"integer","versionAsOf":null,"writeMode":"overwrite","description":"","replaceWhere":null,"historicColumns":[],"mergeSourceAlias":"source","filterQuery":"","notMatchedTable":[],"fromTimeCol":null,"notMatchedCondition":null,"matchedTable":[],"notMatchedAction":"insert","partitionColumns":null,"schema":{"type":"struct","fields":[{"name":"customer_id","type":"integer","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"orders","type":"long","nullable":false,"metadata":{"description":"","mappings":[],"tags":[]}},{"name":"total_spend","type":"double","nullable":true,"metadata":{"description":"","mappings":[],"tags":[]}}]},"activeTab":"whenMatched","tableName":"gold_total_sales_by_customer","maxFlagCol":null,"matchedConditionDelete":null,"isCatalogEnabled":null,"componentInfo":{"gemId":"gitUri=http://gitserver:3000/DxEheDCJ_team_5751/DxEheDCJ_project_12651.git&subPath=&tag=0.2.6.7&projectSubscriptionProjectId=12651&path=gems/catalogTable","projectName":"ProphecySparkBasicsScala"},"matchedActionDelete":"ignore","timestampAsOf":null,"mergeSchema":null,"toTimeCol":null,"optimizeWrite":null,"useExternalFilePath":false,"keyColumns":[],"matchedCondition":null,"matchedAction":"update","datasetType":"Database","fileFormat":"parquet"}