From 22674fbd94338a013d7d48f63d33e1cc317c3913 Mon Sep 17 00:00:00 2001 From: frbattid Date: Tue, 5 Nov 2024 13:35:25 +0100 Subject: [PATCH 01/35] Add Fabric datasets schemas and extensions --- .../v1.0.0/Fabric_Purposes_1.0.0.avsc | 12 +++ .../v1.0.0/Fabric_Segments_1.0.0.avsc | 12 +++ .../v1.0.0/Fabric_User_Consents_1.0.0.avsc | 23 ++++++ .../v1.0.0/Fabric_User_Data_1.0.0.avsc | 44 +++++++++++ .../Fabric_Purposes/v1.0.0/extensions.json | 16 ++++ .../Fabric_Segments/v1.0.0/extensions.json | 16 ++++ .../v1.0.0/extensions.json | 46 ++++++++++++ .../Fabric_User_Data/v1.0.0/extensions.json | 75 +++++++++++++++++++ 8 files changed, 244 insertions(+) create mode 100644 samples/datasets/Fabric_Purposes/v1.0.0/Fabric_Purposes_1.0.0.avsc create mode 100644 samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc create mode 100644 samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc create mode 100644 samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc create mode 100644 samples/extensions/Fabric_Purposes/v1.0.0/extensions.json create mode 100644 samples/extensions/Fabric_Segments/v1.0.0/extensions.json create mode 100644 samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json create mode 100644 samples/extensions/Fabric_User_Data/v1.0.0/extensions.json diff --git a/samples/datasets/Fabric_Purposes/v1.0.0/Fabric_Purposes_1.0.0.avsc b/samples/datasets/Fabric_Purposes/v1.0.0/Fabric_Purposes_1.0.0.avsc new file mode 100644 index 0000000..03f8108 --- /dev/null +++ b/samples/datasets/Fabric_Purposes/v1.0.0/Fabric_Purposes_1.0.0.avsc @@ -0,0 +1,12 @@ +{ + "name": "Fabric_Purposes", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "Purposes", + "fields": [ + { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier" }, + { "name": "LAWFUL_BASIS", "type": { "type": "enum", "name": "STATUS", "symbols": [ "LegitimateInterest", "Consent" ] }, "doc": "Lawful basis" } + ] +} diff --git a/samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc b/samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc new file mode 100644 index 0000000..dc996f8 --- /dev/null +++ b/samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc @@ -0,0 +1,12 @@ +{ + "name": "Fabric_Segments", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "Segments", + "fields": [ + { "name": "SEGMENT_ID", "type": "string", "doc": "Segment identifier" }, + { "name": "SEGMENT_NAME", "type": "string", "doc": "Segment name" } + ] +} diff --git a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc new file mode 100644 index 0000000..8ad45c7 --- /dev/null +++ b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc @@ -0,0 +1,23 @@ +{ + "name": "Fabric_User_Consents", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User consents", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier" }, + { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier" }, + { "name": "APP_ID", "type": "string", "doc": "Application identifier" }, + { "name": "APP_LEGAL_ENTITY_ID", "type": "string", "doc": "Application legal entity identifier" }, + { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"] }, "doc": "Consent status" }, + { "name": "EXPIRATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Consent expiration date" }, + { "name": "IDENTIFIER", "type": [ + "null", + { "name": "IDENTIFIER", "type": "record", "fields": [ + { "name": "TYPE", "type": "string" }, + { "name": "ID", "type": "string", "x-fp-data-protection": "pseudonymize" } + ]} + ], "doc": "User identifier" } + ] +} diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc new file mode 100644 index 0000000..47bffd1 --- /dev/null +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -0,0 +1,44 @@ +{ + "name": "Fabric_User_Data", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User data", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier" }, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier" }, + { "name": "NAME", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "User name" }, + { "name": "AGE", "type": "int", "doc": "User age" }, + { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Activation date" }, + { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to" }, + { "name": "LINE", "type": { + "type": "record", "name": "LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" }, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } + ] + }, "doc": "User IP addresses" } + ] + }, "doc": "User line at home" }, + { "name": "MOBILE_LINES", "type": { + "type": "array", "items": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" }, + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + ] + } + }, "doc": "User mobile lines" }, + { "name": "BILLING_MEANS", "type": { + "type": "map", "values": { + "type": "record", "name": "BILLING_MEAN", "fields": [ + { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type" }, + { "name": "BILLING_VALUE", "type": "string", "doc": "Billing value" } + ] + } + }, "doc": "Billing means" } + ] +} diff --git a/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json new file mode 100644 index 0000000..5c138af --- /dev/null +++ b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json @@ -0,0 +1,16 @@ +{ + "name": "Fabric_Purposes", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "Purposes", + "fields": [ + { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier", "arg.properties": { + "options": [ "LI", "C" ] + }}, + { "name": "LAWFUL_BASIS", "type": { "type": "enum", "name": "STATUS", "symbols": [ "LegitimateInterest", "Consent" ] }, "doc": "Lawful basis", "arg.properties": { + "options": [ "LegitimateInterest", "Consent" ] + }} + ] +} diff --git a/samples/extensions/Fabric_Segments/v1.0.0/extensions.json b/samples/extensions/Fabric_Segments/v1.0.0/extensions.json new file mode 100644 index 0000000..19cfc3c --- /dev/null +++ b/samples/extensions/Fabric_Segments/v1.0.0/extensions.json @@ -0,0 +1,16 @@ +{ + "name": "Fabric_Segments", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "Segments", + "fields": [ + { "name": "SEGMENT_ID", "type": "string", "doc": "Segment identifier", "arg.properties": { + "options": [ "R", "E" ] + }}, + { "name": "SEGMENT_NAME", "type": "string", "doc": "Segment name", "arg.properties": { + "options": [ "Residential", "Enterprise" ] + }} + ] +} diff --git a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json new file mode 100644 index 0000000..a787c7e --- /dev/null +++ b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json @@ -0,0 +1,46 @@ +{ + "name": "Fabric_User_Consents", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User consents", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier", "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true, + "num_repetitions": 50 + }}, + { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier", "arg.properties": { + "options": [ "LI", "C" ] + }}, + { "name": "APP_ID", "type": "string", "doc": "Application identifier", "arg.properties": { + "options": [ "baikal", "novum", "aura", "nba" ] + }}, + { "name": "APP_LEGAL_ENTITY_ID", "type": "string", "doc": "Application legal entity identifier", "arg.properties": { + "options": [ "telefonica", "movistar", "o2" ] + }}, + { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"] }, "doc": "Consent status", "arg.properties": { + "options": [ "Active", "Revoked" ] + }}, + { "name": "EXPIRATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Consent expiration date", "arg.properties": { + "range": { + "start": "${DATE_RANGE_START}", + "end": "${DATE_RANGE_END}" + } + }}, + { "name": "IDENTIFIER", "type": [ + "null", + { "name": "IDENTIFIER", "type": "record", "fields": [ + { "name": "TYPE", "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}, + { "name": "ID", "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "unique": true, + "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", + "prefix": "+34" + }} + ]} + ], "doc": "User identifier" } + ] +} diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json new file mode 100644 index 0000000..db0059d --- /dev/null +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -0,0 +1,75 @@ +{ + "name": "Fabric_User_Data", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User data", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier", "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier", "arg.properties": { + "length": 8 + }}, + { "name": "NAME", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "User name", "arg.properties": { + "length": 16 + }}, + { "name": "AGE", "type": "int", "doc": "User age", "arg.properties": { + "range": { + "start": 18, + "end": 99 + } + }}, + { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Activation date", "arg.properties": { + "range": { + "start": "${DATE_RANGE_START}", + "end": "${DATE_RANGE_END}" + } + }}, + { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to", "arg.properties": { + "options": [ "R", "E" ] + }}, + { "name": "LINE", "type": { + "type": "record", "name": "LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { + "unique": true, + "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", + "prefix": "+34" + }}, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } + ] + }, "doc": "User IP addresses" } + ] + }, "doc": "User line at home" }, + { "name": "MOBILE_LINES", "type": { + "type": "array", "items": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { + "unique": true, + "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", + "prefix": "+34" + }}, + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + ] + } + }, "doc": "User mobile lines" }, + { "name": "BILLING_MEANS", "type": { + "type": "map", "values": { + "type": "record", "name": "BILLING_MEAN", "fields": [ + { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type", "arg.properties": { + "options": [ "iban", "credit-card" ] + }}, + { "name": "BILLING_VALUE", "type": "string", "doc": "Billing value", "arg.properties": { + "length": 16 + }} + ] + } + }, "doc": "Billing means" } + ] +} From 3f57f1a9bbdca9609fd31c261b39b190d2506bb9 Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 6 Nov 2024 10:03:23 +0100 Subject: [PATCH 02/35] Refactor Fabric_User_Data --- .../v1.0.0/Fabric_User_Data_1.0.0.avsc | 58 ++++++++---- .../Fabric_User_Data/v1.0.0/extensions.json | 89 +++++++++++++------ 2 files changed, 103 insertions(+), 44 deletions(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index 47bffd1..b4be1a3 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -12,33 +12,53 @@ { "name": "AGE", "type": "int", "doc": "User age" }, { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Activation date" }, { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to" }, - { "name": "LINE", "type": { - "type": "record", "name": "LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" }, - { "name": "IP_ADDRESSES", "type": { - "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ - { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, - { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } - ] - }, "doc": "User IP addresses" } + { "name": "MOBILE_LINE", "type": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" }, + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } ] - }, "doc": "User line at home" }, - { "name": "MOBILE_LINES", "type": { - "type": "array", "items": { - "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + }, "doc": "User main mobile line" }, + { "name": "LAND_LINE", "type": [ + "null", + { + "type": "record", "name": "LAND_LINE_RECORD", "fields": [ { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" }, - { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, - { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } + ] + }, "doc": "User IP addresses" } ] } - }, "doc": "User mobile lines" }, + ], "doc": "Optional user line at home" }, + { "name": "SERVICES", "type": [ + "null", + { + "type": "array", "items": { + "type": "record", "name": "SERVICE_RECORD", "fields": [ + { "name": "SERVICE_TYPE", "type": "string", "doc": "Service type" }, + { "name": "SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": ["Activated", "Deactivated"] }, "doc": "Service name" } + ] + } + } + ], "doc": "Optional Services" }, { "name": "BILLING_MEANS", "type": { "type": "map", "values": { - "type": "record", "name": "BILLING_MEAN", "fields": [ - { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type" }, + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ + { "name": "BILLING_TYPE", "type": "string"", "doc": "Billing type" }, { "name": "BILLING_VALUE", "type": "string", "doc": "Billing value" } ] } - }, "doc": "Billing means" } + }, "doc": "Billing means" }, + { "name": "ADDRESS", "type": { + "type": "record", "name": "ADDRESS_RECORD", "fields": [ + { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Street" }, + { "name": "ZIP_CODE", "type": "string", "doc": "Zip code" }, + { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2" }, "doc": "Country" }, + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "Geographic area" } + ] + }, "doc": "User address" } ] } diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index db0059d..739272a 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -31,45 +31,84 @@ { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to", "arg.properties": { "options": [ "R", "E" ] }}, - { "name": "LINE", "type": { - "type": "record", "name": "LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { + { "name": "MOBILE_LINE", "type": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { "unique": true, "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", "prefix": "+34" }}, - { "name": "IP_ADDRESSES", "type": { - "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ - { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, - { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } - ] - }, "doc": "User IP addresses" } + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } ] - }, "doc": "User line at home" }, - { "name": "MOBILE_LINES", "type": { - "type": "array", "items": { - "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { - "unique": true, - "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", - "prefix": "+34" - }}, - { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, - { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + }, "doc": "User main mobile line" }, + { "name": "LAND_LINE", "type": [ + "null", + { + "type": "record", "name": "LAND_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { + "unique": true, + "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", + "prefix": "+34" + }}, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } + ] + }, "doc": "User IP addresses" } ] } - }, "doc": "User mobile lines" }, + ], "doc": "Optional user line at home" }, + { "name": "SERVICES", "type": [ + "null", + { + "type": "array", "items": { + "type": "record", "name": "SERVICE_RECORD", "fields": [ + { "name": "SERVICE_TYPE", "type": "string", "doc": "Service type", "arg.properties": { + "options": [ "TV", "Netflix", "Disney+", "HomeAlarm", "HomeAutomation" ] + }}, + { "name": "SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ] }, "doc": "Service name", "arg.properties": { + "options": [ "Activated", "Deactivated" ] + }} + ] + } + } + ], "doc": "Optional Services" }, { "name": "BILLING_MEANS", "type": { "type": "map", "values": { - "type": "record", "name": "BILLING_MEAN", "fields": [ + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type", "arg.properties": { - "options": [ "iban", "credit-card" ] + "options": [ "CreditCard", "IBAN" ] }}, { "name": "BILLING_VALUE", "type": "string", "doc": "Billing value", "arg.properties": { - "length": 16 + "regex": "^[0-9]{16}$" }} ] } - }, "doc": "Billing means" } + }, "doc": "Billing means" }, + { "name": "ADDRESS", "type": { + "type": "record", "name": "ADDRESS_RECORD", "fields": [ + { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Street", "arg.properties": { + "length": 20 + }}, + { "name": "ZIP_CODE", "type": "string", "doc": "Zip code", "arg.properties": { + "length": 5 + }}, + { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2" }, "doc": "Country", "arg.properties": { + "options": [ "ES" ] + }}, + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "Geographic area", "arg.properties": { + "options": [ + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [39.4699, -0.3763]}, \"properties\": {\"name\": \"Valencia\"}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.263, -2.935]}, \"properties\": {\"name\": \"Bilbao\"}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}" + ] + }} + ] + }, "doc": "User address" } ] } From 3debef52f6c736cbebaefef97d0e335b75863e2e Mon Sep 17 00:00:00 2001 From: Francisco Romero Bueno Date: Wed, 6 Nov 2024 10:12:10 +0100 Subject: [PATCH 03/35] Update samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc Co-authored-by: Eric Blanco --- .../Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index b4be1a3..8157684 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -7,7 +7,7 @@ "doc": "User data", "fields": [ { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier" }, - { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier" }, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier" }, { "name": "NAME", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "User name" }, { "name": "AGE", "type": "int", "doc": "User age" }, { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Activation date" }, From b1a731428772183a154e15e0ad87a340cff66095 Mon Sep 17 00:00:00 2001 From: Francisco Romero Bueno Date: Wed, 6 Nov 2024 10:13:10 +0100 Subject: [PATCH 04/35] Update samples/extensions/Fabric_User_Data/v1.0.0/extensions.json Co-authored-by: Eric Blanco --- samples/extensions/Fabric_User_Data/v1.0.0/extensions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 739272a..89829cd 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -13,7 +13,7 @@ { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier", "arg.properties": { "length": 8 }}, - { "name": "NAME", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "User name", "arg.properties": { + { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "User name", "arg.properties": { "length": 16 }}, { "name": "AGE", "type": "int", "doc": "User age", "arg.properties": { From ce15af860f89fd6744b8c5fd6fb0dbfc8d6f4a4b Mon Sep 17 00:00:00 2001 From: Francisco Romero Bueno Date: Wed, 6 Nov 2024 10:13:18 +0100 Subject: [PATCH 05/35] Update samples/extensions/Fabric_User_Data/v1.0.0/extensions.json Co-authored-by: Eric Blanco --- samples/extensions/Fabric_User_Data/v1.0.0/extensions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 89829cd..156d338 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -10,7 +10,7 @@ "iteration": { "start": "0" }, "hashed": true }}, - { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier", "arg.properties": { + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier", "arg.properties": { "length": 8 }}, { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "User name", "arg.properties": { From 2bf51e78eb58dbd2347fa4d0ef8b64d5e49a4bee Mon Sep 17 00:00:00 2001 From: Francisco Romero Bueno Date: Wed, 6 Nov 2024 10:13:47 +0100 Subject: [PATCH 06/35] Update samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc Co-authored-by: Eric Blanco --- .../Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index 8157684..2a5645a 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -8,7 +8,7 @@ "fields": [ { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier" }, { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier" }, - { "name": "NAME", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "User name" }, + { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "User name" }, { "name": "AGE", "type": "int", "doc": "User age" }, { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Activation date" }, { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to" }, From ae05dac2371902eba5b23e560526f280a7138577 Mon Sep 17 00:00:00 2001 From: Francisco Romero Bueno Date: Wed, 6 Nov 2024 10:14:20 +0100 Subject: [PATCH 07/35] Update samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc Co-authored-by: Eric Blanco --- .../Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc index 8ad45c7..690116b 100644 --- a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc @@ -11,7 +11,7 @@ { "name": "APP_ID", "type": "string", "doc": "Application identifier" }, { "name": "APP_LEGAL_ENTITY_ID", "type": "string", "doc": "Application legal entity identifier" }, { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"] }, "doc": "Consent status" }, - { "name": "EXPIRATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Consent expiration date" }, + { "name": "EXPIRATION_DATE", "type": [ "null", { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime" }], "doc": "Consent expiration date" }, { "name": "IDENTIFIER", "type": [ "null", { "name": "IDENTIFIER", "type": "record", "fields": [ From ce071d9d71f7158c8af60d775b163631af70ca57 Mon Sep 17 00:00:00 2001 From: Francisco Romero Bueno Date: Wed, 6 Nov 2024 10:14:30 +0100 Subject: [PATCH 08/35] Update samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json Co-authored-by: Eric Blanco --- samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json index a787c7e..647e842 100644 --- a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json @@ -23,7 +23,7 @@ { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"] }, "doc": "Consent status", "arg.properties": { "options": [ "Active", "Revoked" ] }}, - { "name": "EXPIRATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Consent expiration date", "arg.properties": { + { "name": "EXPIRATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime" }, "doc": "Consent expiration date", "arg.properties": { "range": { "start": "${DATE_RANGE_START}", "end": "${DATE_RANGE_END}" From 32fdf6480af39f04bff78935a7be3c2dbb7e2aab Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 6 Nov 2024 12:53:21 +0100 Subject: [PATCH 09/35] Add more pseudonymized fields --- .../v1.0.0/Fabric_User_Data_1.0.0.avsc | 15 ++++++++------- .../Fabric_User_Data/v1.0.0/extensions.json | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index 2a5645a..c5a9bd2 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -23,7 +23,7 @@ "null", { "type": "record", "name": "LAND_LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" }, + { "name": "PHONE_NUMBER", "type": { "type": "string", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" }, { "name": "IP_ADDRESSES", "type": { "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, @@ -33,22 +33,23 @@ ] } ], "doc": "Optional user line at home" }, - { "name": "SERVICES", "type": [ + { "name": "TV_SERVICES", "type": [ "null", { "type": "array", "items": { - "type": "record", "name": "SERVICE_RECORD", "fields": [ - { "name": "SERVICE_TYPE", "type": "string", "doc": "Service type" }, - { "name": "SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": ["Activated", "Deactivated"] }, "doc": "Service name" } + "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ + { "name": "TV_SERVICE_TYPE", "type": "string", "doc": "Service type" }, + { "name": "TV_SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": ["Activated", "Deactivated"] }, "doc": "Service name" }, + { "name": "TV_SERVICE_PARENTAL_CONTROL", "type": { "type": "boolean", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } ] } } ], "doc": "Optional Services" }, { "name": "BILLING_MEANS", "type": { - "type": "map", "values": { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ { "name": "BILLING_TYPE", "type": "string"", "doc": "Billing type" }, - { "name": "BILLING_VALUE", "type": "string", "doc": "Billing value" } + { "name": "BILLING_VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value" } ] } }, "doc": "Billing means" }, diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 156d338..d768483 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -46,7 +46,7 @@ "null", { "type": "record", "name": "LAND_LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { + { "name": "PHONE_NUMBER", "type": { "type": "string", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { "unique": true, "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", "prefix": "+34" @@ -60,28 +60,29 @@ ] } ], "doc": "Optional user line at home" }, - { "name": "SERVICES", "type": [ + { "name": "TV_SERVICES", "type": [ "null", { "type": "array", "items": { - "type": "record", "name": "SERVICE_RECORD", "fields": [ - { "name": "SERVICE_TYPE", "type": "string", "doc": "Service type", "arg.properties": { - "options": [ "TV", "Netflix", "Disney+", "HomeAlarm", "HomeAutomation" ] + "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ + { "name": "TV_SERVICE_TYPE", "type": "string", "doc": "Service type", "arg.properties": { + "options": [ "Movistar+", "Netflix", "Disney+", "DAZN" ] }}, - { "name": "SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ] }, "doc": "Service name", "arg.properties": { + { "name": "TV_SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ] }, "doc": "Service name", "arg.properties": { "options": [ "Activated", "Deactivated" ] - }} + }}, + { "name": "TV_SERVICE_PARENTAL_CONTROL", "type": { "type": "boolean", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } ] } } ], "doc": "Optional Services" }, { "name": "BILLING_MEANS", "type": { - "type": "map", "values": { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type", "arg.properties": { "options": [ "CreditCard", "IBAN" ] }}, - { "name": "BILLING_VALUE", "type": "string", "doc": "Billing value", "arg.properties": { + { "name": "BILLING_VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value", "arg.properties": { "regex": "^[0-9]{16}$" }} ] From 6ed961e2b79d7b19dca2b47f653d8540c177db65 Mon Sep 17 00:00:00 2001 From: Francisco Romero Bueno Date: Wed, 6 Nov 2024 12:54:46 +0100 Subject: [PATCH 10/35] Update samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc Co-authored-by: Eric Blanco --- .../Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc index 690116b..2d61257 100644 --- a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc @@ -10,7 +10,7 @@ { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier" }, { "name": "APP_ID", "type": "string", "doc": "Application identifier" }, { "name": "APP_LEGAL_ENTITY_ID", "type": "string", "doc": "Application legal entity identifier" }, - { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"] }, "doc": "Consent status" }, + { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"] }, "doc": "Consent status" }, { "name": "EXPIRATION_DATE", "type": [ "null", { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime" }], "doc": "Consent expiration date" }, { "name": "IDENTIFIER", "type": [ "null", From 2cdbe3bf45ee747f635c47e2d64d634f609b3128 Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 6 Nov 2024 12:56:00 +0100 Subject: [PATCH 11/35] Minor fix --- .../Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index c5a9bd2..3b58320 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -48,7 +48,7 @@ { "name": "BILLING_MEANS", "type": { "type": "map", "x-fp-data-protection": "pseudonymize", "values": { "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ - { "name": "BILLING_TYPE", "type": "string"", "doc": "Billing type" }, + { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type" }, { "name": "BILLING_VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value" } ] } From 6e30e9a37156746929ba96b26057fd6e420cd864 Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 6 Nov 2024 13:01:31 +0100 Subject: [PATCH 12/35] Boolean fields cannot be pseudonymized --- .../Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc | 4 ++-- samples/extensions/Fabric_User_Data/v1.0.0/extensions.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index 3b58320..fc53936 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -40,7 +40,7 @@ "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ { "name": "TV_SERVICE_TYPE", "type": "string", "doc": "Service type" }, { "name": "TV_SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": ["Activated", "Deactivated"] }, "doc": "Service name" }, - { "name": "TV_SERVICE_PARENTAL_CONTROL", "type": { "type": "boolean", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + { "name": "TV_SERVICE_PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } ] } } @@ -58,7 +58,7 @@ { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Street" }, { "name": "ZIP_CODE", "type": "string", "doc": "Zip code" }, { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2" }, "doc": "Country" }, - { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "Geographic area" } + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "GeoJson-like geographic area" } ] }, "doc": "User address" } ] diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index d768483..1c0a025 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -71,7 +71,7 @@ { "name": "TV_SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ] }, "doc": "Service name", "arg.properties": { "options": [ "Activated", "Deactivated" ] }}, - { "name": "TV_SERVICE_PARENTAL_CONTROL", "type": { "type": "boolean", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + { "name": "TV_SERVICE_PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } ] } } @@ -99,7 +99,7 @@ { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2" }, "doc": "Country", "arg.properties": { "options": [ "ES" ] }}, - { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "Geographic area", "arg.properties": { + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "GeoJson-like geographic area", "arg.properties": { "options": [ "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}", "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}", From a5acd658f8d9ebd53a26a534665f0c2a44f2c40b Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 10:01:17 +0100 Subject: [PATCH 13/35] Fix SEGMENT_ID and some extension values --- .../v1.0.0/Fabric_Segments_1.0.0.avsc | 2 +- .../v1.0.0/Fabric_User_Data_1.0.0.avsc | 10 +++++----- .../Fabric_Purposes/v1.0.0/extensions.json | 2 +- .../Fabric_Segments/v1.0.0/extensions.json | 4 ++-- .../Fabric_User_Data/v1.0.0/extensions.json | 12 ++++++------ 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc b/samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc index dc996f8..127320e 100644 --- a/samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc +++ b/samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc @@ -6,7 +6,7 @@ "type": "record", "doc": "Segments", "fields": [ - { "name": "SEGMENT_ID", "type": "string", "doc": "Segment identifier" }, + { "name": "SEGMENT_ID", "type": "int", "doc": "Segment identifier" }, { "name": "SEGMENT_NAME", "type": "string", "doc": "Segment name" } ] } diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index fc53936..cb1a7dd 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -38,9 +38,9 @@ { "type": "array", "items": { "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ - { "name": "TV_SERVICE_TYPE", "type": "string", "doc": "Service type" }, - { "name": "TV_SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": ["Activated", "Deactivated"] }, "doc": "Service name" }, - { "name": "TV_SERVICE_PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + { "name": "TYPE", "type": "string", "doc": "Service type" }, + { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": ["Activated", "Deactivated"] }, "doc": "Service name" }, + { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } ] } } @@ -48,8 +48,8 @@ { "name": "BILLING_MEANS", "type": { "type": "map", "x-fp-data-protection": "pseudonymize", "values": { "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ - { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type" }, - { "name": "BILLING_VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value" } + { "name": "TYPE", "type": "string", "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value" } ] } }, "doc": "Billing means" }, diff --git a/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json index 5c138af..4e9aeb7 100644 --- a/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json @@ -7,7 +7,7 @@ "doc": "Purposes", "fields": [ { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier", "arg.properties": { - "options": [ "LI", "C" ] + "options": [ "FraudPrevention", "Advertising" ] }}, { "name": "LAWFUL_BASIS", "type": { "type": "enum", "name": "STATUS", "symbols": [ "LegitimateInterest", "Consent" ] }, "doc": "Lawful basis", "arg.properties": { "options": [ "LegitimateInterest", "Consent" ] diff --git a/samples/extensions/Fabric_Segments/v1.0.0/extensions.json b/samples/extensions/Fabric_Segments/v1.0.0/extensions.json index 19cfc3c..d3d39b0 100644 --- a/samples/extensions/Fabric_Segments/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_Segments/v1.0.0/extensions.json @@ -6,8 +6,8 @@ "type": "record", "doc": "Segments", "fields": [ - { "name": "SEGMENT_ID", "type": "string", "doc": "Segment identifier", "arg.properties": { - "options": [ "R", "E" ] + { "name": "SEGMENT_ID", "type": "int", "doc": "Segment identifier", "arg.properties": { + "options": [ 0, 1 ] }}, { "name": "SEGMENT_NAME", "type": "string", "doc": "Segment name", "arg.properties": { "options": [ "Residential", "Enterprise" ] diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 1c0a025..bd810a4 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -29,7 +29,7 @@ } }}, { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to", "arg.properties": { - "options": [ "R", "E" ] + "options": [ 0, 1 ] }}, { "name": "MOBILE_LINE", "type": { "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ @@ -65,13 +65,13 @@ { "type": "array", "items": { "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ - { "name": "TV_SERVICE_TYPE", "type": "string", "doc": "Service type", "arg.properties": { + { "name": "TYPE", "type": "string", "doc": "Service type", "arg.properties": { "options": [ "Movistar+", "Netflix", "Disney+", "DAZN" ] }}, - { "name": "TV_SERVICE_STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ] }, "doc": "Service name", "arg.properties": { + { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ] }, "doc": "Service name", "arg.properties": { "options": [ "Activated", "Deactivated" ] }}, - { "name": "TV_SERVICE_PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } ] } } @@ -79,10 +79,10 @@ { "name": "BILLING_MEANS", "type": { "type": "map", "x-fp-data-protection": "pseudonymize", "values": { "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ - { "name": "BILLING_TYPE", "type": "string", "doc": "Billing type", "arg.properties": { + { "name": "TYPE", "type": "string", "doc": "Billing type", "arg.properties": { "options": [ "CreditCard", "IBAN" ] }}, - { "name": "BILLING_VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value", "arg.properties": { + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value", "arg.properties": { "regex": "^[0-9]{16}$" }} ] From fa0140ebfc4c4050666a2343fc8060b1655cc7c5 Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 11:19:15 +0100 Subject: [PATCH 14/35] Fix arg.properties --- .../v1.0.0/Fabric_User_Consents_1.0.0.avsc | 9 ++- .../Fabric_Purposes/v1.0.0/extensions.json | 8 +-- .../Fabric_Segments/v1.0.0/extensions.json | 8 +-- .../v1.0.0/extensions.json | 45 +++++++------ .../Fabric_User_Data/v1.0.0/extensions.json | 64 +++++++++---------- 5 files changed, 70 insertions(+), 64 deletions(-) diff --git a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc index 2d61257..26dfc8e 100644 --- a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc @@ -11,12 +11,15 @@ { "name": "APP_ID", "type": "string", "doc": "Application identifier" }, { "name": "APP_LEGAL_ENTITY_ID", "type": "string", "doc": "Application legal entity identifier" }, { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"] }, "doc": "Consent status" }, - { "name": "EXPIRATION_DATE", "type": [ "null", { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime" }], "doc": "Consent expiration date" }, + { "name": "EXPIRATION_DATE", "type": [ + "null", + { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime" } + ], "doc": "Consent expiration date" }, { "name": "IDENTIFIER", "type": [ "null", { "name": "IDENTIFIER", "type": "record", "fields": [ - { "name": "TYPE", "type": "string" }, - { "name": "ID", "type": "string", "x-fp-data-protection": "pseudonymize" } + { "name": "TYPE", "type": "string", "doc": "Identifier type" }, + { "name": "ID", "type": { "type": string", "x-fp-data-protection": "pseudonymize" }, "doc": "Identifier" } ]} ], "doc": "User identifier" } ] diff --git a/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json index 4e9aeb7..89a2c80 100644 --- a/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json @@ -6,11 +6,11 @@ "type": "record", "doc": "Purposes", "fields": [ - { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier", "arg.properties": { + { "name": "PURPOSE_ID", "type": { "type": "string", "arg.properties": { "options": [ "FraudPrevention", "Advertising" ] - }}, - { "name": "LAWFUL_BASIS", "type": { "type": "enum", "name": "STATUS", "symbols": [ "LegitimateInterest", "Consent" ] }, "doc": "Lawful basis", "arg.properties": { + }}, "doc": "Purpose identifier" }, + { "name": "LAWFUL_BASIS", "type": { "type": "enum", "name": "STATUS", "symbols": [ "LegitimateInterest", "Consent" ], "arg.properties": { "options": [ "LegitimateInterest", "Consent" ] - }} + }}, "doc": "Lawful basis" } ] } diff --git a/samples/extensions/Fabric_Segments/v1.0.0/extensions.json b/samples/extensions/Fabric_Segments/v1.0.0/extensions.json index d3d39b0..205123b 100644 --- a/samples/extensions/Fabric_Segments/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_Segments/v1.0.0/extensions.json @@ -6,11 +6,11 @@ "type": "record", "doc": "Segments", "fields": [ - { "name": "SEGMENT_ID", "type": "int", "doc": "Segment identifier", "arg.properties": { + { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { "options": [ 0, 1 ] - }}, - { "name": "SEGMENT_NAME", "type": "string", "doc": "Segment name", "arg.properties": { + }}, "doc": "Segment identifier" }, + { "name": "SEGMENT_NAME", "type": { "type": "string", "arg.properties": { "options": [ "Residential", "Enterprise" ] - }} + }}, "doc": "Segment name" } ] } diff --git a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json index 647e842..c5dc1df 100644 --- a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json @@ -6,40 +6,43 @@ "type": "record", "doc": "User consents", "fields": [ - { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier", "arg.properties": { + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { "iteration": { "start": "0" }, "hashed": true, "num_repetitions": 50 - }}, - { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier", "arg.properties": { - "options": [ "LI", "C" ] - }}, - { "name": "APP_ID", "type": "string", "doc": "Application identifier", "arg.properties": { + }}, "doc": "Internal user identifier" }, + { "name": "PURPOSE_ID", "type": { "type": "string", "arg.properties": { + "options": [ "FraudPrevention", "Advertising" ] + }}, "doc": "Purpose identifier" }, + { "name": "APP_ID", "type": { "type": "string", "arg.properties": { "options": [ "baikal", "novum", "aura", "nba" ] - }}, - { "name": "APP_LEGAL_ENTITY_ID", "type": "string", "doc": "Application legal entity identifier", "arg.properties": { + }}, "doc": "Application identifier" }, + { "name": "APP_LEGAL_ENTITY_ID", "type": { "type": "string", "arg.properties": { "options": [ "telefonica", "movistar", "o2" ] - }}, - { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"] }, "doc": "Consent status", "arg.properties": { + }}, "doc": "Application legal entity identifier" }, + { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"], "arg.properties": { "options": [ "Active", "Revoked" ] - }}, - { "name": "EXPIRATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime" }, "doc": "Consent expiration date", "arg.properties": { - "range": { - "start": "${DATE_RANGE_START}", - "end": "${DATE_RANGE_END}" - } - }}, + }}, "doc": "Consent status" }, + { "name": "EXPIRATION_DATE", "type": [ + "null", + { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { + "start": "2021-01-01T00:00:00.000Z", + "end": "2026-01-01T00:00:00.000Z" + } + }} + ], "doc": "Consent expiration date" }, { "name": "IDENTIFIER", "type": [ "null", { "name": "IDENTIFIER", "type": "record", "fields": [ - { "name": "TYPE", "type": "string", "arg.properties": { + { "name": "TYPE", "type": { "type": "string", "arg.properties": { "options": [ "phone-number" ] - }}, - { "name": "ID", "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + }}, "doc": "Identifier type" }, + { "name": "ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "unique": true, "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", "prefix": "+34" - }} + }}, "doc": "Identifier" } ]} ], "doc": "User identifier" } ] diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index bd810a4..e66c7e3 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -6,38 +6,38 @@ "type": "record", "doc": "User data", "fields": [ - { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier", "arg.properties": { + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { "iteration": { "start": "0" }, "hashed": true - }}, - { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier", "arg.properties": { + }}, "doc": "Internal user identifier" }, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "length": 8 - }}, - { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "User name", "arg.properties": { + }}, "doc": "Personal user identifier" }, + { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "length": 16 - }}, - { "name": "AGE", "type": "int", "doc": "User age", "arg.properties": { + }}, "doc": "User name" }, + { "name": "AGE", "type": { "type": "int", "arg.properties": { "range": { "start": 18, "end": 99 } - }}, - { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Activation date", "arg.properties": { + }}, "doc": "User age"}, + { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date", "arg.properties": { "range": { "start": "${DATE_RANGE_START}", "end": "${DATE_RANGE_END}" } - }}, - { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to", "arg.properties": { + }}, "doc": "Activation date" }, + { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { "options": [ 0, 1 ] - }}, + }}, "doc": "Segment the user belongs to" }, { "name": "MOBILE_LINE", "type": { "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { "unique": true, "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", "prefix": "+34" - }}, + }}, "doc": "Phone number" }, { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } ] @@ -46,11 +46,11 @@ "null", { "type": "record", "name": "LAND_LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number", "arg.properties": { + { "name": "PHONE_NUMBER", "type": { "type": "string", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { "unique": true, "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", "prefix": "+34" - }}, + }}, "doc": "Phone number" }, { "name": "IP_ADDRESSES", "type": { "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, @@ -65,12 +65,12 @@ { "type": "array", "items": { "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ - { "name": "TYPE", "type": "string", "doc": "Service type", "arg.properties": { + { "name": "TYPE", "type": { "type": "string", "arg.properties": { "options": [ "Movistar+", "Netflix", "Disney+", "DAZN" ] - }}, - { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ] }, "doc": "Service name", "arg.properties": { + }}, "doc": "Phone number" }, + { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ], "arg.properties": { "options": [ "Activated", "Deactivated" ] - }}, + }}, "doc": "Service name" }, { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } ] } @@ -79,27 +79,27 @@ { "name": "BILLING_MEANS", "type": { "type": "map", "x-fp-data-protection": "pseudonymize", "values": { "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ - { "name": "TYPE", "type": "string", "doc": "Billing type", "arg.properties": { + { "name": "TYPE", "type": { "type": "string", "arg.properties": { "options": [ "CreditCard", "IBAN" ] - }}, - { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value", "arg.properties": { + }}, "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "regex": "^[0-9]{16}$" - }} + }}, "doc": "Billing value" } ] } }, "doc": "Billing means" }, { "name": "ADDRESS", "type": { "type": "record", "name": "ADDRESS_RECORD", "fields": [ - { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Street", "arg.properties": { + { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "length": 20 - }}, - { "name": "ZIP_CODE", "type": "string", "doc": "Zip code", "arg.properties": { + }}, "doc": "Street" }, + { "name": "ZIP_CODE", "type": { "type": "string", "arg.properties": { "length": 5 - }}, - { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2" }, "doc": "Country", "arg.properties": { + }}, "doc": "Zip code" }, + { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2", "arg.properties": { "options": [ "ES" ] - }}, - { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "GeoJson-like geographic area", "arg.properties": { + }}, "doc": "Country" }, + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature", "arg.properties": { "options": [ "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}", "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}", @@ -108,7 +108,7 @@ "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}", "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}" ] - }} + }}, "doc": "GeoJson-like geographic area" } ] }, "doc": "User address" } ] From 27ec1244486b6dd34df8251ea260051b0406e5ba Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 11:22:51 +0100 Subject: [PATCH 15/35] Minor fix --- samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json index c5dc1df..5b18592 100644 --- a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json @@ -20,7 +20,7 @@ { "name": "APP_LEGAL_ENTITY_ID", "type": { "type": "string", "arg.properties": { "options": [ "telefonica", "movistar", "o2" ] }}, "doc": "Application legal entity identifier" }, - { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Revoked"], "arg.properties": { + { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { "options": [ "Active", "Revoked" ] }}, "doc": "Consent status" }, { "name": "EXPIRATION_DATE", "type": [ From 8d014404d61aa9e758d02216bf1348ed6c83859f Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 11:32:21 +0100 Subject: [PATCH 16/35] Minor fix --- .../Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc | 2 +- samples/datasets/Fabric_User_Consents/v1.0.0/kk.json | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 samples/datasets/Fabric_User_Consents/v1.0.0/kk.json diff --git a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc index 26dfc8e..fe06eed 100644 --- a/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc @@ -19,7 +19,7 @@ "null", { "name": "IDENTIFIER", "type": "record", "fields": [ { "name": "TYPE", "type": "string", "doc": "Identifier type" }, - { "name": "ID", "type": { "type": string", "x-fp-data-protection": "pseudonymize" }, "doc": "Identifier" } + { "name": "ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Identifier" } ]} ], "doc": "User identifier" } ] diff --git a/samples/datasets/Fabric_User_Consents/v1.0.0/kk.json b/samples/datasets/Fabric_User_Consents/v1.0.0/kk.json new file mode 100644 index 0000000..e69de29 From 8e1366a94bf7f0f91ddc8938a1d4a706d7b2e081 Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 11:45:42 +0100 Subject: [PATCH 17/35] Fix User_Data AGE and ACTIVATION_DATE in extension --- .../extensions/Fabric_User_Data/v1.0.0/extensions.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index e66c7e3..d3e6e01 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -18,14 +18,14 @@ }}, "doc": "User name" }, { "name": "AGE", "type": { "type": "int", "arg.properties": { "range": { - "start": 18, - "end": 99 + "min": 18, + "max": 99 } }}, "doc": "User age"}, { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date", "arg.properties": { "range": { - "start": "${DATE_RANGE_START}", - "end": "${DATE_RANGE_END}" + "start": "2021-01-01", + "end": "2026-01-01" } }}, "doc": "Activation date" }, { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { From b3df3a8164a66eb15feca72bdb911963caa85e36 Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 12:25:01 +0100 Subject: [PATCH 18/35] Fix ZIP code, billing mean keys and billing mean value --- samples/datasets/Fabric_User_Consents/v1.0.0/kk.json | 0 .../extensions/Fabric_User_Data/v1.0.0/extensions.json | 8 ++++++-- 2 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 samples/datasets/Fabric_User_Consents/v1.0.0/kk.json diff --git a/samples/datasets/Fabric_User_Consents/v1.0.0/kk.json b/samples/datasets/Fabric_User_Consents/v1.0.0/kk.json deleted file mode 100644 index e69de29..0000000 diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index d3e6e01..8b4acaf 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -83,9 +83,13 @@ "options": [ "CreditCard", "IBAN" ] }}, "doc": "Billing type" }, { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { - "regex": "^[0-9]{16}$" + "regex": "[0-9]{16}" }}, "doc": "Billing value" } ] + }, "arg.properties": { + "length": 2, + "unique": true, + "keys": { "options": [ "main", "other" ] } } }, "doc": "Billing means" }, { "name": "ADDRESS", "type": { @@ -94,7 +98,7 @@ "length": 20 }}, "doc": "Street" }, { "name": "ZIP_CODE", "type": { "type": "string", "arg.properties": { - "length": 5 + "regex": "[0-9]{5}" }}, "doc": "Zip code" }, { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2", "arg.properties": { "options": [ "ES" ] From 1383dff3c16120d3d330529751efd8e9656cb526 Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 13:08:56 +0100 Subject: [PATCH 19/35] Purposes sync between PURPOSES and USER_CONSENTS --- .../extensions/Fabric_Purposes/v1.0.0/extensions.json | 3 ++- .../Fabric_User_Consents/v1.0.0/extensions.json | 8 +++----- .../extensions/Fabric_User_Data/v1.0.0/extensions.json | 10 ++-------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json index 89a2c80..4cc75b0 100644 --- a/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_Purposes/v1.0.0/extensions.json @@ -7,7 +7,8 @@ "doc": "Purposes", "fields": [ { "name": "PURPOSE_ID", "type": { "type": "string", "arg.properties": { - "options": [ "FraudPrevention", "Advertising" ] + "iteration": { "start": "0" }, + "prefix": "purpose_" }}, "doc": "Purpose identifier" }, { "name": "LAWFUL_BASIS", "type": { "type": "enum", "name": "STATUS", "symbols": [ "LegitimateInterest", "Consent" ], "arg.properties": { "options": [ "LegitimateInterest", "Consent" ] diff --git a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json index 5b18592..9199eaa 100644 --- a/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Consents/v1.0.0/extensions.json @@ -12,7 +12,8 @@ "num_repetitions": 50 }}, "doc": "Internal user identifier" }, { "name": "PURPOSE_ID", "type": { "type": "string", "arg.properties": { - "options": [ "FraudPrevention", "Advertising" ] + "iteration": { "start": "0", "restart": "49" }, + "prefix": "purpose_" }}, "doc": "Purpose identifier" }, { "name": "APP_ID", "type": { "type": "string", "arg.properties": { "options": [ "baikal", "novum", "aura", "nba" ] @@ -26,10 +27,7 @@ { "name": "EXPIRATION_DATE", "type": [ "null", { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { - "start": "2021-01-01T00:00:00.000Z", - "end": "2026-01-01T00:00:00.000Z" - } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } }} ], "doc": "Consent expiration date" }, { "name": "IDENTIFIER", "type": [ diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 8b4acaf..5b82b0d 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -17,16 +17,10 @@ "length": 16 }}, "doc": "User name" }, { "name": "AGE", "type": { "type": "int", "arg.properties": { - "range": { - "min": 18, - "max": 99 - } + "range": { "min": 18, "max": 99 } }}, "doc": "User age"}, { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date", "arg.properties": { - "range": { - "start": "2021-01-01", - "end": "2026-01-01" - } + "range": { "start": "2021-01-01", "end": "2026-01-01" } }}, "doc": "Activation date" }, { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { "options": [ 0, 1 ] From ff281a20ea9381b01d710079ddac995eb617550a Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 7 Nov 2024 16:30:26 +0100 Subject: [PATCH 20/35] Fix IPv6 doc field --- .../Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc | 2 +- samples/extensions/Fabric_User_Data/v1.0.0/extensions.json | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index cb1a7dd..9c98e14 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -27,7 +27,7 @@ { "name": "IP_ADDRESSES", "type": { "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, - { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } + { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv6 address" } ] }, "doc": "User IP addresses" } ] diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 5b82b0d..36ca6db 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -11,7 +11,8 @@ "hashed": true }}, "doc": "Internal user identifier" }, { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { - "length": 8 + "regex": "[0-9]{8}[A-Z]{1}", + "unique": true }}, "doc": "Personal user identifier" }, { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "length": 16 @@ -48,7 +49,7 @@ { "name": "IP_ADDRESSES", "type": { "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, - { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc:": "IPv6 address" } + { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv6 address" } ] }, "doc": "User IP addresses" } ] From 8861456c0a8f82dc9b64ae58fd2a93208f4e34db Mon Sep 17 00:00:00 2001 From: frbattid Date: Fri, 8 Nov 2024 10:33:27 +0100 Subject: [PATCH 21/35] Do nullable BILLING_MEANS --- .../v1.0.0/Fabric_User_Data_1.0.0.avsc | 17 +++++---- .../Fabric_User_Data/v1.0.0/extensions.json | 35 ++++++++++--------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index 9c98e14..30d0070 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -45,14 +45,17 @@ } } ], "doc": "Optional Services" }, - { "name": "BILLING_MEANS", "type": { - "type": "map", "x-fp-data-protection": "pseudonymize", "values": { - "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ - { "name": "TYPE", "type": "string", "doc": "Billing type" }, - { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value" } - ] + { "name": "BILLING_MEANS", "type": [ + "null", + { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ + { "name": "TYPE", "type": "string", "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value" } + ] + } } - }, "doc": "Billing means" }, + ], "doc": "Billing means" }, { "name": "ADDRESS", "type": { "type": "record", "name": "ADDRESS_RECORD", "fields": [ { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Street" }, diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 36ca6db..604565a 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -67,26 +67,29 @@ "options": [ "Activated", "Deactivated" ] }}, "doc": "Service name" }, { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } - ] + ] } } ], "doc": "Optional Services" }, - { "name": "BILLING_MEANS", "type": { - "type": "map", "x-fp-data-protection": "pseudonymize", "values": { - "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ - { "name": "TYPE", "type": { "type": "string", "arg.properties": { - "options": [ "CreditCard", "IBAN" ] - }}, "doc": "Billing type" }, - { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { - "regex": "[0-9]{16}" - }}, "doc": "Billing value" } - ] - }, "arg.properties": { - "length": 2, - "unique": true, - "keys": { "options": [ "main", "other" ] } + { "name": "BILLING_MEANS", "type": [ + "null", + { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "CreditCard", "IBAN" ] + }}, "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{16}" + }}, "doc": "Billing value" } + ] + }, "arg.properties": { + "length": 2, + "unique": true, + "keys": { "options": [ "main", "other" ] } + } } - }, "doc": "Billing means" }, + ], "doc": "Billing means" }, { "name": "ADDRESS", "type": { "type": "record", "name": "ADDRESS_RECORD", "fields": [ { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { From aae057121eebfdea19b1809dd181ffe230291690 Mon Sep 17 00:00:00 2001 From: frbattid Date: Fri, 8 Nov 2024 13:16:17 +0100 Subject: [PATCH 22/35] Fix GEO_AREA values --- .../Fabric_User_Data/v1.0.0/extensions.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 604565a..461b8d4 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -103,12 +103,12 @@ }}, "doc": "Country" }, { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature", "arg.properties": { "options": [ - "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}", - "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}", - "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [39.4699, -0.3763]}, \"properties\": {\"name\": \"Valencia\"}", - "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.263, -2.935]}, \"properties\": {\"name\": \"Bilbao\"}", - "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}", - "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}" + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [39.4699, -0.3763]}, \"properties\": {\"name\": \"Valencia\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.263, -2.935]}, \"properties\": {\"name\": \"Bilbao\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}}" ] }}, "doc": "GeoJson-like geographic area" } ] From 847d4a1924977a7dfe0663e54a18d38d67dfe4ae Mon Sep 17 00:00:00 2001 From: frbattid Date: Tue, 12 Nov 2024 13:49:51 +0100 Subject: [PATCH 23/35] Add Consent_State v1.4.0 --- .../v1.4.0/Consent_State_1.4.0.avsc | 42 +++++++++++ .../v1.4.0/extensions_identifier_bound.json | 74 +++++++++++++++++++ .../v1.4.0/extensions_user_bound.json | 64 ++++++++++++++++ .../extensions_user_bound_by_app_0.json | 67 +++++++++++++++++ .../extensions_user_bound_by_app_1.json | 67 +++++++++++++++++ .../extensions_user_bound_by_app_2.json | 67 +++++++++++++++++ .../extensions_user_bound_by_app_3.json | 67 +++++++++++++++++ .../extensions_user_bound_by_app_4.json | 67 +++++++++++++++++ .../Fabric_User_Data/v1.0.0/extensions.json | 23 +----- 9 files changed, 519 insertions(+), 19 deletions(-) create mode 100644 samples/datasets/Consent_State/v1.4.0/Consent_State_1.4.0.avsc create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json diff --git a/samples/datasets/Consent_State/v1.4.0/Consent_State_1.4.0.avsc b/samples/datasets/Consent_State/v1.4.0/Consent_State_1.4.0.avsc new file mode 100644 index 0000000..972256a --- /dev/null +++ b/samples/datasets/Consent_State/v1.4.0/Consent_State_1.4.0.avsc @@ -0,0 +1,42 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": "string", "doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true }, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": "string", "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": "string", "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": "string"}, + { "name": "id", "type": "string"} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"] }, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime" }, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ "null", { "type": "string", "logicalType": "datetime" } ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime" } ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ "null", "string" ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json new file mode 100644 index 0000000..b2f4dfc --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json @@ -0,0 +1,74 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true, + "num_repetitions": 10 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "600000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked" ], + "distribution": { "Active": 0.9, "Revoked": 0.1 } + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json new file mode 100644 index 0000000..3effd36 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json @@ -0,0 +1,64 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "user_bound_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null" + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked" ], + "distribution": { "Active": 0.9, "Revoked": 0.1 } + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json new file mode 100644 index 0000000..af850db --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json @@ -0,0 +1,67 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "user_bound_by_app_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null" + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked" ], + "distribution": { "Active": 0.9, "Revoked": 0.1 } + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null", + { "type": "string", "arg.properties": { + "options": [ "app_0" ] + }} + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json new file mode 100644 index 0000000..3b5448f --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json @@ -0,0 +1,67 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "user_bound_by_app_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null" + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked" ], + "distribution": { "Active": 0.9, "Revoked": 0.1 } + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null", + { "type": "string", "arg.properties": { + "options": [ "app_1" ] + }} + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json new file mode 100644 index 0000000..2e504ef --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json @@ -0,0 +1,67 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "user_bound_by_app_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null" + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked" ], + "distribution": { "Active": 0.9, "Revoked": 0.1 } + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null", + { "type": "string", "arg.properties": { + "options": [ "app_2" ] + }} + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json new file mode 100644 index 0000000..1046637 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json @@ -0,0 +1,67 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "user_bound_by_app_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null" + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked" ], + "distribution": { "Active": 0.9, "Revoked": 0.1 } + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null", + { "type": "string", "arg.properties": { + "options": [ "app_3" ] + }} + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json new file mode 100644 index 0000000..0e6f786 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json @@ -0,0 +1,67 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "x-fp-join-definitions": { + "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} + }, + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "user_bound_by_app_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null" + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked" ], + "distribution": { "Active": 0.9, "Revoked": 0.1 } + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null", + { "type": "string", "arg.properties": { + "options": [ "app_4" ] + }} + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 461b8d4..29c436f 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -8,7 +8,8 @@ "fields": [ { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { "iteration": { "start": "0" }, - "hashed": true + "hashed": true, + "num_repetitions": 10 }}, "doc": "Internal user identifier" }, { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "regex": "[0-9]{8}[A-Z]{1}", @@ -29,8 +30,7 @@ { "name": "MOBILE_LINE", "type": { "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { - "unique": true, - "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", + "iteration": { "start": "600000000" }, "prefix": "+34" }}, "doc": "Phone number" }, { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, @@ -38,22 +38,7 @@ ] }, "doc": "User main mobile line" }, { "name": "LAND_LINE", "type": [ - "null", - { - "type": "record", "name": "LAND_LINE_RECORD", "fields": [ - { "name": "PHONE_NUMBER", "type": { "type": "string", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { - "unique": true, - "regex": "(6[0-9]{8})|(7[1-4][0-9]{7})", - "prefix": "+34" - }}, "doc": "Phone number" }, - { "name": "IP_ADDRESSES", "type": { - "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ - { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, - { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv6 address" } - ] - }, "doc": "User IP addresses" } - ] - } + "null" ], "doc": "Optional user line at home" }, { "name": "TV_SERVICES", "type": [ "null", From 9a9e5466e93cb91a725b0c94680447f4ee87e1ac Mon Sep 17 00:00:00 2001 From: frbattid Date: Tue, 12 Nov 2024 14:08:16 +0100 Subject: [PATCH 24/35] Minor fix --- .../Consent_State/v1.4.0/extensions_identifier_bound.json | 2 +- samples/extensions/Fabric_User_Data/v1.0.0/extensions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json index b2f4dfc..e912300 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json @@ -14,7 +14,7 @@ { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { "iteration": { "start": "0" }, "hashed": true, - "num_repetitions": 10 + "num_repetitions": 5 }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { "options": [ "telefonica" ] diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 29c436f..462e0f9 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -9,7 +9,7 @@ { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { "iteration": { "start": "0" }, "hashed": true, - "num_repetitions": 10 + "num_repetitions": 5 }}, "doc": "Internal user identifier" }, { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { "regex": "[0-9]{8}[A-Z]{1}", From 18df9a2cfdbbb1433105448e8c3e45cf47b67408 Mon Sep 17 00:00:00 2001 From: ejblanco Date: Tue, 12 Nov 2024 16:58:53 +0100 Subject: [PATCH 25/35] fix: distribution does not work with enum --- .../v1.4.0/extensions_identifier_bound.json | 6 +----- .../Consent_State/v1.4.0/extensions_user_bound.json | 6 +----- .../v1.4.0/extensions_user_bound_by_app_0.json | 9 ++------- .../v1.4.0/extensions_user_bound_by_app_1.json | 9 ++------- .../v1.4.0/extensions_user_bound_by_app_2.json | 9 ++------- .../v1.4.0/extensions_user_bound_by_app_3.json | 9 ++------- .../v1.4.0/extensions_user_bound_by_app_4.json | 9 ++------- 7 files changed, 12 insertions(+), 45 deletions(-) diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json index e912300..784b1a9 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json @@ -4,9 +4,6 @@ "x-fp-avro4p-version": "1.4", "type": "record", "doc": "The states for each consent at the most recently available point in time", - "x-fp-join-definitions": { - "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} - }, "fields": [ { "name": "id", "type": { "type": "string", "arg.properties": { "kind": "uuid" @@ -35,8 +32,7 @@ ]} ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { - "options": [ "Active", "Revoked" ], - "distribution": { "Active": 0.9, "Revoked": 0.1 } + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json index 3effd36..6f72d7d 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json @@ -4,9 +4,6 @@ "x-fp-avro4p-version": "1.4", "type": "record", "doc": "The states for each consent at the most recently available point in time", - "x-fp-join-definitions": { - "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} - }, "fields": [ { "name": "id", "type": { "type": "string", "arg.properties": { "kind": "uuid" @@ -25,8 +22,7 @@ "null" ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { - "options": [ "Active", "Revoked" ], - "distribution": { "Active": 0.9, "Revoked": 0.1 } + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json index af850db..753642d 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json @@ -4,9 +4,6 @@ "x-fp-avro4p-version": "1.4", "type": "record", "doc": "The states for each consent at the most recently available point in time", - "x-fp-join-definitions": { - "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} - }, "fields": [ { "name": "id", "type": { "type": "string", "arg.properties": { "kind": "uuid" @@ -25,8 +22,7 @@ "null" ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { - "options": [ "Active", "Revoked" ], - "distribution": { "Active": 0.9, "Revoked": 0.1 } + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } @@ -58,10 +54,9 @@ { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ - "null", { "type": "string", "arg.properties": { "options": [ "app_0" ] }} - ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ], "default": "app_0", "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } ] } diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json index 3b5448f..bb017ef 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json @@ -4,9 +4,6 @@ "x-fp-avro4p-version": "1.4", "type": "record", "doc": "The states for each consent at the most recently available point in time", - "x-fp-join-definitions": { - "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} - }, "fields": [ { "name": "id", "type": { "type": "string", "arg.properties": { "kind": "uuid" @@ -25,8 +22,7 @@ "null" ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { - "options": [ "Active", "Revoked" ], - "distribution": { "Active": 0.9, "Revoked": 0.1 } + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } @@ -58,10 +54,9 @@ { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ - "null", { "type": "string", "arg.properties": { "options": [ "app_1" ] }} - ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ], "default": "app_1", "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } ] } diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json index 2e504ef..e105dd3 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json @@ -4,9 +4,6 @@ "x-fp-avro4p-version": "1.4", "type": "record", "doc": "The states for each consent at the most recently available point in time", - "x-fp-join-definitions": { - "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} - }, "fields": [ { "name": "id", "type": { "type": "string", "arg.properties": { "kind": "uuid" @@ -25,8 +22,7 @@ "null" ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { - "options": [ "Active", "Revoked" ], - "distribution": { "Active": 0.9, "Revoked": 0.1 } + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } @@ -58,10 +54,9 @@ { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ - "null", { "type": "string", "arg.properties": { "options": [ "app_2" ] }} - ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ], "default": "app_2", "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } ] } diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json index 1046637..3db47be 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json @@ -4,9 +4,6 @@ "x-fp-avro4p-version": "1.4", "type": "record", "doc": "The states for each consent at the most recently available point in time", - "x-fp-join-definitions": { - "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} - }, "fields": [ { "name": "id", "type": { "type": "string", "arg.properties": { "kind": "uuid" @@ -25,8 +22,7 @@ "null" ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { - "options": [ "Active", "Revoked" ], - "distribution": { "Active": 0.9, "Revoked": 0.1 } + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } @@ -58,10 +54,9 @@ { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ - "null", { "type": "string", "arg.properties": { "options": [ "app_3" ] }} - ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ], "default": "app_3", "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } ] } diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json index 0e6f786..d8b868f 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json @@ -4,9 +4,6 @@ "x-fp-avro4p-version": "1.4", "type": "record", "doc": "The states for each consent at the most recently available point in time", - "x-fp-join-definitions": { - "join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]} - }, "fields": [ { "name": "id", "type": { "type": "string", "arg.properties": { "kind": "uuid" @@ -25,8 +22,7 @@ "null" ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { - "options": [ "Active", "Revoked" ], - "distribution": { "Active": 0.9, "Revoked": 0.1 } + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } @@ -58,10 +54,9 @@ { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ - "null", { "type": "string", "arg.properties": { "options": [ "app_4" ] }} - ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ], "default": "app_4", "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } ] } From fe1598601e00340c8b2ea1252126d7294ae93a16 Mon Sep 17 00:00:00 2001 From: ejblanco Date: Thu, 14 Nov 2024 12:38:39 +0100 Subject: [PATCH 26/35] chore: add legitimate interest --- .../v1.4.0/extensions_identifier_bound.json | 4 +- ..._identifier_bound_legitimate_interest.json | 76 +++++++++++++++++++ .../v1.4.0/extensions_user_bound.json | 4 +- .../extensions_user_bound_by_app_0.json | 4 +- ...nsions_user_bound_legitimate_interest.json | 66 ++++++++++++++++ 5 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_user_bound_legitimate_interest.json diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json index 784b1a9..35a1d83 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json @@ -35,7 +35,7 @@ "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }}, "doc": "The creation date of the consent" }, { "name": "expiration_date", "type": [ "null", @@ -46,7 +46,7 @@ { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }} ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, { "name": "channel", "type": [ "null", "string" ]}, diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest.json new file mode 100644 index 0000000..b830bd5 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest.json @@ -0,0 +1,76 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_leg_int_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "600000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], + "arg.properties": { + "distribution": { + "0": 0.9, + "1": 0.1 + } + }, "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json index 6f72d7d..26e9e37 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound.json @@ -25,7 +25,7 @@ "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }}, "doc": "The creation date of the consent" }, { "name": "expiration_date", "type": [ "null", @@ -36,7 +36,7 @@ { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }} ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, { "name": "channel", "type": [ "null", "string" ]}, diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json index 753642d..7ded2ea 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_0.json @@ -25,7 +25,7 @@ "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }}, "doc": "The creation date of the consent" }, { "name": "expiration_date", "type": [ "null", @@ -36,7 +36,7 @@ { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }} ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, { "name": "channel", "type": [ "null", "string" ]}, diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_legitimate_interest.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_legitimate_interest.json new file mode 100644 index 0000000..3f35588 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_legitimate_interest.json @@ -0,0 +1,66 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "0" }, + "hashed": true + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "user_bound_leg_int_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null" + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], + "arg.properties": { + "distribution": { + "0": 0.9, + "1": 0.1 + } + }, "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} From 41bf9bcac436d45ba3827ef63c7f8f6060ddd9cd Mon Sep 17 00:00:00 2001 From: ejblanco Date: Thu, 14 Nov 2024 12:39:23 +0100 Subject: [PATCH 27/35] fix: creation_date --- .../Consent_State/v1.4.0/extensions_user_bound_by_app_1.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_2.json | 4 ++-- .../Consent_State/v1.4.0/extensions_user_bound_by_app_3.json | 4 ++-- .../Consent_State/v1.4.0/extensions_user_bound_by_app_4.json | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json index bb017ef..01533ae 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json @@ -36,7 +36,7 @@ { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }} ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, { "name": "channel", "type": [ "null", "string" ]}, diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json index e105dd3..fa07d9e 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json @@ -25,7 +25,7 @@ "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }}, "doc": "The creation date of the consent" }, { "name": "expiration_date", "type": [ "null", @@ -36,7 +36,7 @@ { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }} ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, { "name": "channel", "type": [ "null", "string" ]}, diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json index 3db47be..ab212a4 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json @@ -25,7 +25,7 @@ "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }}, "doc": "The creation date of the consent" }, { "name": "expiration_date", "type": [ "null", @@ -36,7 +36,7 @@ { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }} ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, { "name": "channel", "type": [ "null", "string" ]}, diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json index d8b868f..09e873c 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json @@ -25,7 +25,7 @@ "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }}, "doc": "The creation date of the consent" }, { "name": "expiration_date", "type": [ "null", @@ -36,7 +36,7 @@ { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime", "arg.properties": { - "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } }} ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, { "name": "channel", "type": [ "null", "string" ]}, From 6073986bd67031ff1bb5bb27e5a2f4954de43a9c Mon Sep 17 00:00:00 2001 From: ejblanco Date: Thu, 14 Nov 2024 13:18:00 +0100 Subject: [PATCH 28/35] chore: add IPv4 and IPv6 with some null --- .../v1.0.0/Fabric_User_Data_1.0.0.avsc | 6 ++++- .../Fabric_User_Data/v1.0.0/extensions.json | 27 ++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc index 30d0070..e6c076a 100644 --- a/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc +++ b/samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc @@ -27,7 +27,11 @@ { "name": "IP_ADDRESSES", "type": { "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, - { "name": "IPV6", "type": { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv6 address" } + { "name": "IPV6", "type": [ + "null", + { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" } + ] + , "doc": "IPv6 address" } ] }, "doc": "User IP addresses" } ] diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 462e0f9..19e0439 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -37,9 +37,30 @@ { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } ] }, "doc": "User main mobile line" }, - { "name": "LAND_LINE", "type": [ - "null" - ], "doc": "Optional user line at home" }, + { "name": "LAND_LINE", "type": { + "type": "record", "name": "LAND_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "800000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": [ + "null", + { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" } + ], + "arg.properties": { + "distribution": { + "0": 0.2, + "1": 0.8 + } + } + , "doc": "IPv6 address" } + ] + }, "doc": "User IP addresses" } + ] + }, "doc": "Optional user line at home" }, { "name": "TV_SERVICES", "type": [ "null", { From e0f0d9de5354af5436002070a1ab3590452551a0 Mon Sep 17 00:00:00 2001 From: ejblanco Date: Tue, 19 Nov 2024 11:24:29 +0100 Subject: [PATCH 29/35] chore: add countries --- samples/extensions/Fabric_User_Data/v1.0.0/extensions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json index 19e0439..253092f 100644 --- a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json @@ -105,7 +105,7 @@ "regex": "[0-9]{5}" }}, "doc": "Zip code" }, { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2", "arg.properties": { - "options": [ "ES" ] + "options": [ "ES", "DE", "BR", "GB" ] }}, "doc": "Country" }, { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature", "arg.properties": { "options": [ From 414f13bd28d0580b634070c20df83382c8b46615 Mon Sep 17 00:00:00 2001 From: ejblanco Date: Tue, 19 Nov 2024 12:03:36 +0100 Subject: [PATCH 30/35] feat: generate hashed data and repeat X times --- Dockerfile | 2 +- README.md | 14 +++++ .../avro/random/generator/Generator.java | 61 +++++++++++++++++++ lanuza/scripts/build-data.sh | 3 +- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd91957..90c0998 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM gradle:8.1-jdk11 RUN apt-get update && \ - apt-get install -y python3.6 python3-pip jq gettext-base + apt-get install -y python3.6 python3-pip jq gettext-base uuid-runtime RUN python3 -m pip install deepmerge diff --git a/README.md b/README.md index 228a79f..237af0b 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,21 @@ if <restart> is equal to <start>. If provided with a boolean schema, only <start> may be specified; the resulting values will begin with <start> and alternate from `true` to `false` and from `false` to `true` from that point on. + + __hashed:__ If this options is set to `true`, the iteration will be hashed. + This is useful when you want to generate IDs that can join with other datasets. + + __num_repetitions:__ Number of times you want each element to be repeated. + This is useful when you want to generate a dataset with a lot of repeated IDs. +```json + "arg.properties": { + "iteration": { + "start": "0" + }, + "hashed": true, + "num_repetitions": 4 + } +``` > ITERATION_STEP environment var can be used as script argument + + __range:__ A JSON object that conforms to the following formats: - `{"min": , "max": }` (at least one of "min" or "max" must be specified). If provided, ensures that the generated number will be diff --git a/app/src/main/java/io/confluent/avro/random/generator/Generator.java b/app/src/main/java/io/confluent/avro/random/generator/Generator.java index 71df76f..9160ffb 100644 --- a/app/src/main/java/io/confluent/avro/random/generator/Generator.java +++ b/app/src/main/java/io/confluent/avro/random/generator/Generator.java @@ -36,6 +36,8 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.*; /** @@ -93,6 +95,15 @@ public class Generator { */ public static final String SUFFIX_PROP = "suffix"; + /** + * If you want to apply a hash function to generated values in iterations. As a boolean. + */ + public static final String HASHED_PROP = "hashed"; + + /** + * Number of times the same element would be repeated in iterations. As an integer. + */ + public static final String REPEAT_PROP = "num_repetitions"; /** * The name of the attribute for specifying specific values which should be randomly chosen from * when generating values for the schema. Can be given as either an array of values or an object @@ -1042,6 +1053,9 @@ private Iterator parseIterations(Schema schema, Map propertiesProp) { case DOUBLE: return getDoubleIterator(iterationProps); case STRING: + if (propertiesProp.containsKey(HASHED_PROP) && (Boolean) propertiesProp.get(HASHED_PROP)) { + return createHashedStringIterator(getIntegerIterator(iterationProps), propertiesProp); + } return createStringIterator(getIntegerIterator(iterationProps), propertiesProp); default: throw new UnsupportedOperationException(String.format( @@ -1157,6 +1171,53 @@ public Object next() { }; } + private String hashNumber(BigInteger number) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] hash = md.digest(number.toByteArray()); + StringBuilder hexString = new StringBuilder(); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) hexString.append('0'); + hexString.append(hex); + } + return hexString.toString(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + private Iterator createHashedStringIterator(Iterator inner, Map propertiesProp) { + return new Iterator() { + private Integer count = 0; + private String currentValue = ""; + private final Integer repeatValue = getIntegerNumberField( + HASHED_PROP, + REPEAT_PROP, + propertiesProp + ); + private final Integer numRepeat = repeatValue == null ? 1 : repeatValue; + + @Override + public boolean hasNext() { + return inner.hasNext(); + } + + @Override + public Object next() { + if (numRepeat == 1) { + return hashNumber(BigInteger.valueOf((Integer) inner.next())); + } + if (count % numRepeat == 0) { + currentValue = hashNumber(BigInteger.valueOf((Integer) inner.next())); + count = 0; + } + count++; + return currentValue; + } + }; + } + private Iterator getIntegerIterator(Map iterationProps) { Integer iterationStartField = getIntegerNumberField( ITERATION_PROP, diff --git a/lanuza/scripts/build-data.sh b/lanuza/scripts/build-data.sh index 2ebdc83..647a203 100755 --- a/lanuza/scripts/build-data.sh +++ b/lanuza/scripts/build-data.sh @@ -65,6 +65,7 @@ function run() { python3 lib/merge-schemas.py -s "$schema_orig" -e "$schema_extension" --out "${merged_schema_path}" export DATE_RANGE_START="${START_DATE}" + local uuid=$(uuidgen) while [ "${DATE_RANGE_START}" != "${END_DATE}" ]; do export DATE_RANGE_END=$(date -I -d "${DATE_RANGE_START} + 1 day") @@ -75,7 +76,7 @@ function run() { if [ "${SHOW}" = "true" ]; then ARGS="-j -p" else - out="${out_dir}/${DATASET_ID}.${FILE_PREFIX}.${counter}_${DATE_RANGE_START}.avro" + out="${out_dir}/${DATASET_ID}.${FILE_PREFIX}.${counter}_${uuid}_${DATE_RANGE_START}.avro" touch $out ARGS="-b -o ${out}" fi From a0a235728de4ce7835684722f7bfdb65ea4c7bbc Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 21 Nov 2024 12:51:51 +0100 Subject: [PATCH 31/35] Fix user bound by app extensions --- .../Consent_State/v1.4.0/extensions_user_bound_by_app_1.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_2.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_3.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_4.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json index 01533ae..1966aef 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "0" }, + "iteration": { "start": "999999" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json index fa07d9e..2d1f76d 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "0" }, + "iteration": { "start": "2999999" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json index ab212a4..fdc36cb 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "0" }, + "iteration": { "start": "5999999" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json index 09e873c..6b3da21 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "0" }, + "iteration": { "start": "9999999" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { From ebdf3e2059065f3f39129df6feebb1717d26f6e8 Mon Sep 17 00:00:00 2001 From: ejblanco Date: Tue, 26 Nov 2024 16:59:07 +0100 Subject: [PATCH 32/35] fix: SHA function bug --- .../io/confluent/avro/random/generator/Generator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/io/confluent/avro/random/generator/Generator.java b/app/src/main/java/io/confluent/avro/random/generator/Generator.java index 9160ffb..741f8b5 100644 --- a/app/src/main/java/io/confluent/avro/random/generator/Generator.java +++ b/app/src/main/java/io/confluent/avro/random/generator/Generator.java @@ -1171,10 +1171,10 @@ public Object next() { }; } - private String hashNumber(BigInteger number) { + private String hashNumber(String number) { try { MessageDigest md = MessageDigest.getInstance("SHA-256"); - byte[] hash = md.digest(number.toByteArray()); + byte[] hash = md.digest(number.getBytes()); StringBuilder hexString = new StringBuilder(); for (byte b : hash) { String hex = Integer.toHexString(0xff & b); @@ -1206,10 +1206,10 @@ public boolean hasNext() { @Override public Object next() { if (numRepeat == 1) { - return hashNumber(BigInteger.valueOf((Integer) inner.next())); + return hashNumber(inner.next().toString()); } if (count % numRepeat == 0) { - currentValue = hashNumber(BigInteger.valueOf((Integer) inner.next())); + currentValue = hashNumber(inner.next().toString()); count = 0; } count++; From 43ba9f1ffe5588ba5ea620ba8376751459235ff3 Mon Sep 17 00:00:00 2001 From: frbattid Date: Tue, 26 Nov 2024 21:18:36 +0100 Subject: [PATCH 33/35] Fix purpose in user_bound_by_app consents --- .../Consent_State/v1.4.0/extensions_user_bound_by_app_1.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_2.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_3.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_4.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json index 1966aef..a127677 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json @@ -16,7 +16,7 @@ "options": [ "telefonica" ] }}, "doc": "The legal entity for which the user gave consent to" }, { "name": "purpose_id", "type": { "type": "string", "arg.properties": { - "options": [ "user_bound_by_app_0" ] + "options": [ "user_bound_by_app_1" ] }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, { "name": "identifier", "type": [ "null" diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json index 2d1f76d..d737883 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json @@ -16,7 +16,7 @@ "options": [ "telefonica" ] }}, "doc": "The legal entity for which the user gave consent to" }, { "name": "purpose_id", "type": { "type": "string", "arg.properties": { - "options": [ "user_bound_by_app_0" ] + "options": [ "user_bound_by_app_2" ] }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, { "name": "identifier", "type": [ "null" diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json index fdc36cb..8dfa93b 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json @@ -16,7 +16,7 @@ "options": [ "telefonica" ] }}, "doc": "The legal entity for which the user gave consent to" }, { "name": "purpose_id", "type": { "type": "string", "arg.properties": { - "options": [ "user_bound_by_app_0" ] + "options": [ "user_bound_by_app_3" ] }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, { "name": "identifier", "type": [ "null" diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json index 6b3da21..b60a8d8 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json @@ -16,7 +16,7 @@ "options": [ "telefonica" ] }}, "doc": "The legal entity for which the user gave consent to" }, { "name": "purpose_id", "type": { "type": "string", "arg.properties": { - "options": [ "user_bound_by_app_0" ] + "options": [ "user_bound_by_app_4" ] }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, { "name": "identifier", "type": [ "null" From 298925d6bb5e09594c446e4dffbc77b37b4e702f Mon Sep 17 00:00:00 2001 From: frbattid Date: Wed, 27 Nov 2024 13:30:58 +0100 Subject: [PATCH 34/35] Fix user_id start in user_bound_by_app_* --- .../Consent_State/v1.4.0/extensions_user_bound_by_app_1.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_2.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_3.json | 2 +- .../Consent_State/v1.4.0/extensions_user_bound_by_app_4.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json index a127677..435466c 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_1.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "999999" }, + "iteration": { "start": "4000000" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json index d737883..990b210 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_2.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "2999999" }, + "iteration": { "start": "7000000" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json index 8dfa93b..77ec014 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_3.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "5999999" }, + "iteration": { "start": "10000000" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json index b60a8d8..275be00 100644 --- a/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json +++ b/samples/extensions/Consent_State/v1.4.0/extensions_user_bound_by_app_4.json @@ -9,7 +9,7 @@ "kind": "uuid" }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { - "iteration": { "start": "9999999" }, + "iteration": { "start": "13000000" }, "hashed": true }}, "doc": "The user_id to which the consent applies" }, { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { From e04f20d6306da96e4e1e019f203d6e3df06ee6e4 Mon Sep 17 00:00:00 2001 From: frbattid Date: Thu, 28 Nov 2024 09:56:27 +0100 Subject: [PATCH 35/35] Split into chunks some extensions --- ... extensions_identifier_bound_chunk_0.json} | 0 .../extensions_identifier_bound_chunk_1.json | 70 ++++++++++ .../extensions_identifier_bound_chunk_2.json | 70 ++++++++++ .../extensions_identifier_bound_chunk_3.json | 70 ++++++++++ .../extensions_identifier_bound_chunk_4.json | 70 ++++++++++ ...er_bound_legitimate_interest_chunk_0.json} | 0 ...ier_bound_legitimate_interest_chunk_1.json | 76 +++++++++++ ...ier_bound_legitimate_interest_chunk_2.json | 76 +++++++++++ ...ier_bound_legitimate_interest_chunk_3.json | 76 +++++++++++ ...ier_bound_legitimate_interest_chunk_4.json | 76 +++++++++++ ...xtensions.json => extensions_chunk_0.json} | 0 .../v1.0.0/extensions_chunk_1.json | 123 ++++++++++++++++++ .../v1.0.0/extensions_chunk_2.json | 123 ++++++++++++++++++ .../v1.0.0/extensions_chunk_3.json | 123 ++++++++++++++++++ .../v1.0.0/extensions_chunk_4.json | 123 ++++++++++++++++++ 15 files changed, 1076 insertions(+) rename samples/extensions/Consent_State/v1.4.0/{extensions_identifier_bound.json => extensions_identifier_bound_chunk_0.json} (100%) create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_1.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_2.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_3.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_4.json rename samples/extensions/Consent_State/v1.4.0/{extensions_identifier_bound_legitimate_interest.json => extensions_identifier_bound_legitimate_interest_chunk_0.json} (100%) create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_1.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_2.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_3.json create mode 100644 samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_4.json rename samples/extensions/Fabric_User_Data/v1.0.0/{extensions.json => extensions_chunk_0.json} (100%) create mode 100644 samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_1.json create mode 100644 samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_2.json create mode 100644 samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_3.json create mode 100644 samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_4.json diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_0.json similarity index 100% rename from samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound.json rename to samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_0.json diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_1.json new file mode 100644 index 0000000..2ab7002 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_1.json @@ -0,0 +1,70 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "3000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "615000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_2.json new file mode 100644 index 0000000..496ac26 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_2.json @@ -0,0 +1,70 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "6000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "630000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_3.json new file mode 100644 index 0000000..aa2d6cf --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_3.json @@ -0,0 +1,70 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "9000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "645000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_4.json new file mode 100644 index 0000000..e5b7d04 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_chunk_4.json @@ -0,0 +1,70 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "12000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "660000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_0.json similarity index 100% rename from samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest.json rename to samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_0.json diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_1.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_1.json new file mode 100644 index 0000000..4802e3b --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_1.json @@ -0,0 +1,76 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "3000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_leg_int_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "615000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], + "arg.properties": { + "distribution": { + "0": 0.9, + "1": 0.1 + } + }, "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_2.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_2.json new file mode 100644 index 0000000..c8cad12 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_2.json @@ -0,0 +1,76 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "6000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_leg_int_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "630000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], + "arg.properties": { + "distribution": { + "0": 0.9, + "1": 0.1 + } + }, "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_3.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_3.json new file mode 100644 index 0000000..3202093 --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_3.json @@ -0,0 +1,76 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "9000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_leg_int_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "645000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], + "arg.properties": { + "distribution": { + "0": 0.9, + "1": 0.1 + } + }, "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_4.json b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_4.json new file mode 100644 index 0000000..d5a890d --- /dev/null +++ b/samples/extensions/Consent_State/v1.4.0/extensions_identifier_bound_legitimate_interest_chunk_4.json @@ -0,0 +1,76 @@ +{ + "name": "Consent_State", + "x-fp-version": "1.4.0", + "x-fp-avro4p-version": "1.4", + "type": "record", + "doc": "The states for each consent at the most recently available point in time", + "fields": [ + { "name": "id", "type": { "type": "string", "arg.properties": { + "kind": "uuid" + }},"doc": "Identifier for a consent", "x-fp-unique-constraints": [1] }, + { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true, "arg.properties": { + "iteration": { "start": "12000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "The user_id to which the consent applies" }, + { "name": "legal_entity_id", "type": { "type": "string", "arg.properties": { + "options": [ "telefonica" ] + }}, "doc": "The legal entity for which the user gave consent to" }, + { "name": "purpose_id", "type": { "type": "string", "arg.properties": { + "options": [ "identifier_bound_leg_int_0" ] + }}, "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" }, + { "name": "identifier", "type": [ + "null", + { "name": "Identifier", "type": "record", "fields": [ + { "name": "type", "type": { "type": "string", "arg.properties": { + "options": [ "phone-number" ] + }}}, + { "name": "id", "type": { "type": "string", "arg.properties": { + "iteration": { "start": "660000000" }, + "prefix": "+34" + }}} + ]} + ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" }, + { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"], "arg.properties": { + "options": [ "Active", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked", "Revoked" ] + }}, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" }, + { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }}, "doc": "The creation date of the consent" }, + { "name": "expiration_date", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2026-01-01T00:00:00.000Z" } + }} + ], + "arg.properties": { + "distribution": { + "0": 0.9, + "1": 0.1 + } + }, "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" }, + { "name": "user_prompted", "type": [ + "null", + { "type": "string", "logicalType": "datetime", "arg.properties": { + "range": { "start": "2021-01-01T00:00:00.000Z", "end": "2024-11-01T00:00:00.000Z" } + }} + ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" }, + { "name": "channel", "type": [ "null", "string" ]}, + { "name": "reason", "type": [ "null", "string" ]}, + { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"}, + { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [ + "null", + { "type": "array", "items": { + "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [ + { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" }, + { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" } + ] + }} + ], "default": null, "doc": "Tags"}, + { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" }, + { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" }, + { "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ + "null" + ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" } + ] +} diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_0.json similarity index 100% rename from samples/extensions/Fabric_User_Data/v1.0.0/extensions.json rename to samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_0.json diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_1.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_1.json new file mode 100644 index 0000000..d04a68a --- /dev/null +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_1.json @@ -0,0 +1,123 @@ +{ + "name": "Fabric_User_Data", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User data", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "3000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "Internal user identifier" }, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{8}[A-Z]{1}", + "unique": true + }}, "doc": "Personal user identifier" }, + { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 16 + }}, "doc": "User name" }, + { "name": "AGE", "type": { "type": "int", "arg.properties": { + "range": { "min": 18, "max": 99 } + }}, "doc": "User age"}, + { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date", "arg.properties": { + "range": { "start": "2021-01-01", "end": "2026-01-01" } + }}, "doc": "Activation date" }, + { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { + "options": [ 0, 1 ] + }}, "doc": "Segment the user belongs to" }, + { "name": "MOBILE_LINE", "type": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "615000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + ] + }, "doc": "User main mobile line" }, + { "name": "LAND_LINE", "type": { + "type": "record", "name": "LAND_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "815000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": [ + "null", + { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" } + ], + "arg.properties": { + "distribution": { + "0": 0.2, + "1": 0.8 + } + } + , "doc": "IPv6 address" } + ] + }, "doc": "User IP addresses" } + ] + }, "doc": "Optional user line at home" }, + { "name": "TV_SERVICES", "type": [ + "null", + { + "type": "array", "items": { + "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "Movistar+", "Netflix", "Disney+", "DAZN" ] + }}, "doc": "Phone number" }, + { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ], "arg.properties": { + "options": [ "Activated", "Deactivated" ] + }}, "doc": "Service name" }, + { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + ] + } + } + ], "doc": "Optional Services" }, + { "name": "BILLING_MEANS", "type": [ + "null", + { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "CreditCard", "IBAN" ] + }}, "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{16}" + }}, "doc": "Billing value" } + ] + }, "arg.properties": { + "length": 2, + "unique": true, + "keys": { "options": [ "main", "other" ] } + } + } + ], "doc": "Billing means" }, + { "name": "ADDRESS", "type": { + "type": "record", "name": "ADDRESS_RECORD", "fields": [ + { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 20 + }}, "doc": "Street" }, + { "name": "ZIP_CODE", "type": { "type": "string", "arg.properties": { + "regex": "[0-9]{5}" + }}, "doc": "Zip code" }, + { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2", "arg.properties": { + "options": [ "ES", "DE", "BR", "GB" ] + }}, "doc": "Country" }, + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature", "arg.properties": { + "options": [ + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [39.4699, -0.3763]}, \"properties\": {\"name\": \"Valencia\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.263, -2.935]}, \"properties\": {\"name\": \"Bilbao\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}}" + ] + }}, "doc": "GeoJson-like geographic area" } + ] + }, "doc": "User address" } + ] +} diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_2.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_2.json new file mode 100644 index 0000000..d5ac282 --- /dev/null +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_2.json @@ -0,0 +1,123 @@ +{ + "name": "Fabric_User_Data", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User data", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "6000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "Internal user identifier" }, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{8}[A-Z]{1}", + "unique": true + }}, "doc": "Personal user identifier" }, + { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 16 + }}, "doc": "User name" }, + { "name": "AGE", "type": { "type": "int", "arg.properties": { + "range": { "min": 18, "max": 99 } + }}, "doc": "User age"}, + { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date", "arg.properties": { + "range": { "start": "2021-01-01", "end": "2026-01-01" } + }}, "doc": "Activation date" }, + { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { + "options": [ 0, 1 ] + }}, "doc": "Segment the user belongs to" }, + { "name": "MOBILE_LINE", "type": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "630000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + ] + }, "doc": "User main mobile line" }, + { "name": "LAND_LINE", "type": { + "type": "record", "name": "LAND_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "830000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": [ + "null", + { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" } + ], + "arg.properties": { + "distribution": { + "0": 0.2, + "1": 0.8 + } + } + , "doc": "IPv6 address" } + ] + }, "doc": "User IP addresses" } + ] + }, "doc": "Optional user line at home" }, + { "name": "TV_SERVICES", "type": [ + "null", + { + "type": "array", "items": { + "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "Movistar+", "Netflix", "Disney+", "DAZN" ] + }}, "doc": "Phone number" }, + { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ], "arg.properties": { + "options": [ "Activated", "Deactivated" ] + }}, "doc": "Service name" }, + { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + ] + } + } + ], "doc": "Optional Services" }, + { "name": "BILLING_MEANS", "type": [ + "null", + { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "CreditCard", "IBAN" ] + }}, "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{16}" + }}, "doc": "Billing value" } + ] + }, "arg.properties": { + "length": 2, + "unique": true, + "keys": { "options": [ "main", "other" ] } + } + } + ], "doc": "Billing means" }, + { "name": "ADDRESS", "type": { + "type": "record", "name": "ADDRESS_RECORD", "fields": [ + { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 20 + }}, "doc": "Street" }, + { "name": "ZIP_CODE", "type": { "type": "string", "arg.properties": { + "regex": "[0-9]{5}" + }}, "doc": "Zip code" }, + { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2", "arg.properties": { + "options": [ "ES", "DE", "BR", "GB" ] + }}, "doc": "Country" }, + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature", "arg.properties": { + "options": [ + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [39.4699, -0.3763]}, \"properties\": {\"name\": \"Valencia\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.263, -2.935]}, \"properties\": {\"name\": \"Bilbao\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}}" + ] + }}, "doc": "GeoJson-like geographic area" } + ] + }, "doc": "User address" } + ] +} diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_3.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_3.json new file mode 100644 index 0000000..4556b3e --- /dev/null +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_3.json @@ -0,0 +1,123 @@ +{ + "name": "Fabric_User_Data", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User data", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "9000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "Internal user identifier" }, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{8}[A-Z]{1}", + "unique": true + }}, "doc": "Personal user identifier" }, + { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 16 + }}, "doc": "User name" }, + { "name": "AGE", "type": { "type": "int", "arg.properties": { + "range": { "min": 18, "max": 99 } + }}, "doc": "User age"}, + { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date", "arg.properties": { + "range": { "start": "2021-01-01", "end": "2026-01-01" } + }}, "doc": "Activation date" }, + { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { + "options": [ 0, 1 ] + }}, "doc": "Segment the user belongs to" }, + { "name": "MOBILE_LINE", "type": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "645000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + ] + }, "doc": "User main mobile line" }, + { "name": "LAND_LINE", "type": { + "type": "record", "name": "LAND_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "845000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": [ + "null", + { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" } + ], + "arg.properties": { + "distribution": { + "0": 0.2, + "1": 0.8 + } + } + , "doc": "IPv6 address" } + ] + }, "doc": "User IP addresses" } + ] + }, "doc": "Optional user line at home" }, + { "name": "TV_SERVICES", "type": [ + "null", + { + "type": "array", "items": { + "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "Movistar+", "Netflix", "Disney+", "DAZN" ] + }}, "doc": "Phone number" }, + { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ], "arg.properties": { + "options": [ "Activated", "Deactivated" ] + }}, "doc": "Service name" }, + { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + ] + } + } + ], "doc": "Optional Services" }, + { "name": "BILLING_MEANS", "type": [ + "null", + { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "CreditCard", "IBAN" ] + }}, "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{16}" + }}, "doc": "Billing value" } + ] + }, "arg.properties": { + "length": 2, + "unique": true, + "keys": { "options": [ "main", "other" ] } + } + } + ], "doc": "Billing means" }, + { "name": "ADDRESS", "type": { + "type": "record", "name": "ADDRESS_RECORD", "fields": [ + { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 20 + }}, "doc": "Street" }, + { "name": "ZIP_CODE", "type": { "type": "string", "arg.properties": { + "regex": "[0-9]{5}" + }}, "doc": "Zip code" }, + { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2", "arg.properties": { + "options": [ "ES", "DE", "BR", "GB" ] + }}, "doc": "Country" }, + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature", "arg.properties": { + "options": [ + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [39.4699, -0.3763]}, \"properties\": {\"name\": \"Valencia\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.263, -2.935]}, \"properties\": {\"name\": \"Bilbao\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}}" + ] + }}, "doc": "GeoJson-like geographic area" } + ] + }, "doc": "User address" } + ] +} diff --git a/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_4.json b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_4.json new file mode 100644 index 0000000..dbeb335 --- /dev/null +++ b/samples/extensions/Fabric_User_Data/v1.0.0/extensions_chunk_4.json @@ -0,0 +1,123 @@ +{ + "name": "Fabric_User_Data", + "x-fp-version": "1.0.0", + "namespace": "com.telefonica.baikal.fabric", + "x-fp-avro4p-version": "1.5", + "type": "record", + "doc": "User data", + "fields": [ + { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "12000000" }, + "hashed": true, + "num_repetitions": 5 + }}, "doc": "Internal user identifier" }, + { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{8}[A-Z]{1}", + "unique": true + }}, "doc": "Personal user identifier" }, + { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 16 + }}, "doc": "User name" }, + { "name": "AGE", "type": { "type": "int", "arg.properties": { + "range": { "min": 18, "max": 99 } + }}, "doc": "User age"}, + { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date", "arg.properties": { + "range": { "start": "2021-01-01", "end": "2026-01-01" } + }}, "doc": "Activation date" }, + { "name": "SEGMENT_ID", "type": { "type": "int", "arg.properties": { + "options": [ 0, 1 ] + }}, "doc": "Segment the user belongs to" }, + { "name": "MOBILE_LINE", "type": { + "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "660000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" }, + { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" } + ] + }, "doc": "User main mobile line" }, + { "name": "LAND_LINE", "type": { + "type": "record", "name": "LAND_LINE_RECORD", "fields": [ + { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "iteration": { "start": "860000000" }, + "prefix": "+34" + }}, "doc": "Phone number" }, + { "name": "IP_ADDRESSES", "type": { + "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [ + { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" }, + { "name": "IPV6", "type": [ + "null", + { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" } + ], + "arg.properties": { + "distribution": { + "0": 0.2, + "1": 0.8 + } + } + , "doc": "IPv6 address" } + ] + }, "doc": "User IP addresses" } + ] + }, "doc": "Optional user line at home" }, + { "name": "TV_SERVICES", "type": [ + "null", + { + "type": "array", "items": { + "type": "record", "name": "TV_SERVICE_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "Movistar+", "Netflix", "Disney+", "DAZN" ] + }}, "doc": "Phone number" }, + { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": [ "Activated", "Deactivated" ], "arg.properties": { + "options": [ "Activated", "Deactivated" ] + }}, "doc": "Service name" }, + { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" } + ] + } + } + ], "doc": "Optional Services" }, + { "name": "BILLING_MEANS", "type": [ + "null", + { + "type": "map", "x-fp-data-protection": "pseudonymize", "values": { + "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [ + { "name": "TYPE", "type": { "type": "string", "arg.properties": { + "options": [ "CreditCard", "IBAN" ] + }}, "doc": "Billing type" }, + { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "regex": "[0-9]{16}" + }}, "doc": "Billing value" } + ] + }, "arg.properties": { + "length": 2, + "unique": true, + "keys": { "options": [ "main", "other" ] } + } + } + ], "doc": "Billing means" }, + { "name": "ADDRESS", "type": { + "type": "record", "name": "ADDRESS_RECORD", "fields": [ + { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize", "arg.properties": { + "length": 20 + }}, "doc": "Street" }, + { "name": "ZIP_CODE", "type": { "type": "string", "arg.properties": { + "regex": "[0-9]{5}" + }}, "doc": "Zip code" }, + { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2", "arg.properties": { + "options": [ "ES", "DE", "BR", "GB" ] + }}, "doc": "Country" }, + { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature", "arg.properties": { + "options": [ + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [40.4165, -3.70256]}, \"properties\": {\"name\": \"Madrid\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [41.3851, 2.1734]}, \"properties\": {\"name\": \"Barcelona\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [39.4699, -0.3763]}, \"properties\": {\"name\": \"Valencia\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.263, -2.935]}, \"properties\": {\"name\": \"Bilbao\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [37.3828, -5.9731]}, \"properties\": {\"name\": \"Sevilla\"}}", + "{\"type\": \"Feature\", \"geometry\": {\"type\": \"Point\", \"coordinates\": [43.3623, -8.4115]}, \"properties\": {\"name\": \"A Coruña\"}}" + ] + }}, "doc": "GeoJson-like geographic area" } + ] + }, "doc": "User address" } + ] +}