feat: Fabric dataset schemas FPF-17288

Dockerfile

@@ -1,7 +1,7 @@
 FROM gradle:8.1-jdk11
 
 RUN apt-get update && \
-  apt-get install -y python3.6 python3-pip jq gettext-base
+  apt-get install -y python3.6 python3-pip jq gettext-base uuid-runtime
 
 RUN python3 -m pip install deepmerge
 

README.md

@@ -86,7 +86,21 @@ if <restart> is equal to <start>. If provided with a boolean
 schema, only <start> may be specified; the resulting values will
 begin with <start> and alternate from `true` to `false` and from
 `false` to `true` from that point on.
+  + __hashed:__ If this options is set to `true`, the iteration will be hashed. 
+  This is useful when you want to generate IDs that can join with other datasets.
+  + __num_repetitions:__ Number of times you want each element to be repeated. 
+  This is useful when you want to generate a dataset with a lot of repeated IDs.
+```json
+              "arg.properties": {
+                "iteration": {
+                  "start": "0"
+                },
+                "hashed": true,
+                "num_repetitions": 4
+              }
+```
 > ITERATION_STEP environment var can be used as script argument
+
 + __range:__ A JSON object that conforms to the following formats:
     - `{"min": <min>, "max": <max>}` (at least one of "min" or "max" must be
     specified). If provided, ensures that the generated number will be

app/src/main/java/io/confluent/avro/random/generator/Generator.java

@@ -36,6 +36,8 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.*;
 
 /**
@@ -93,6 +95,15 @@ public class Generator {
    */
   public static final String SUFFIX_PROP = "suffix";
 
+    /**
+     * If you want to apply a hash function to generated values in iterations. As a boolean.
+     */
+  public static final String HASHED_PROP = "hashed";
+
+    /**
+     * Number of times the same element would be repeated in iterations. As an integer.
+     */
+  public static final String REPEAT_PROP = "num_repetitions";
   /**
    * The name of the attribute for specifying specific values which should be randomly chosen from
    * when generating values for the schema. Can be given as either an array of values or an object
@@ -1042,6 +1053,9 @@ private Iterator<Object> parseIterations(Schema schema, Map propertiesProp) {
       case DOUBLE:
         return getDoubleIterator(iterationProps);
       case STRING:
+        if (propertiesProp.containsKey(HASHED_PROP) && (Boolean) propertiesProp.get(HASHED_PROP)) {
+          return createHashedStringIterator(getIntegerIterator(iterationProps), propertiesProp);
+        }
         return createStringIterator(getIntegerIterator(iterationProps), propertiesProp);
       default:
         throw new UnsupportedOperationException(String.format(
@@ -1157,6 +1171,53 @@ public Object next() {
     };
   }
 
+  private String hashNumber(String number) {
+    try {
+      MessageDigest md = MessageDigest.getInstance("SHA-256");
+      byte[] hash = md.digest(number.getBytes());
+      StringBuilder hexString = new StringBuilder();
+      for (byte b : hash) {
+        String hex = Integer.toHexString(0xff & b);
+        if (hex.length() == 1) hexString.append('0');
+        hexString.append(hex);
+      }
+      return hexString.toString();
+    } catch (NoSuchAlgorithmException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private Iterator<Object> createHashedStringIterator(Iterator<Object> inner, Map propertiesProp) {
+    return new Iterator<Object>() {
+      private Integer count = 0;
+      private String currentValue = "";
+      private final Integer repeatValue = getIntegerNumberField(
+          HASHED_PROP,
+          REPEAT_PROP,
+          propertiesProp
+      );
+      private final Integer numRepeat = repeatValue == null ? 1 : repeatValue;
+
+      @Override
+      public boolean hasNext() {
+        return inner.hasNext();
+      }
+
+      @Override
+      public Object next() {
+        if (numRepeat == 1) {
+          return hashNumber(inner.next().toString());
+        }
+        if (count % numRepeat == 0) {
+          currentValue = hashNumber(inner.next().toString());
+          count = 0;
+        }
+        count++;
+        return currentValue;
+      }
+    };
+  }
+
   private Iterator<Object> getIntegerIterator(Map iterationProps) {
     Integer iterationStartField = getIntegerNumberField(
         ITERATION_PROP,

lanuza/scripts/build-data.sh

@@ -65,6 +65,7 @@ function run() {
   python3 lib/merge-schemas.py -s "$schema_orig" -e "$schema_extension" --out "${merged_schema_path}"
 
   export DATE_RANGE_START="${START_DATE}"
+  local uuid=$(uuidgen)
   while [ "${DATE_RANGE_START}" != "${END_DATE}" ]; do
     export DATE_RANGE_END=$(date -I -d "${DATE_RANGE_START} + 1 day")
 
@@ -75,7 +76,7 @@ function run() {
       if [ "${SHOW}" = "true" ]; then
         ARGS="-j -p"
       else
-        out="${out_dir}/${DATASET_ID}.${FILE_PREFIX}.${counter}_${DATE_RANGE_START}.avro"
+        out="${out_dir}/${DATASET_ID}.${FILE_PREFIX}.${counter}_${uuid}_${DATE_RANGE_START}.avro"
         touch $out
         ARGS="-b -o ${out}"
       fi

samples/datasets/Consent_State/v1.4.0/Consent_State_1.4.0.avsc

@@ -0,0 +1,42 @@
+{
+    "name": "Consent_State",
+    "x-fp-version": "1.4.0",
+    "x-fp-avro4p-version": "1.4",
+    "type": "record",
+    "doc": "The states for each consent at the most recently available point in time",
+    "x-fp-join-definitions": {
+		"join00001": {"dataset_id": "D_Gbl_Brand", "dataset_version": 6, "fields": ["GBL_BRAND_ID"]}
+	},
+    "fields": [
+        { "name": "id", "type": "string", "doc": "Identifier for a consent", "x-fp-unique-constraints": [1] },
+        { "name": "user_id", "type": { "type": "string", "x-fp-user-id": true }, "doc": "The user_id to which the consent applies" },
+        { "name": "legal_entity_id", "type": "string", "doc": "The legal entity for which the user gave consent to" },
+        { "name": "purpose_id", "type": "string", "doc": "A purpose_id identifies a legal basis for the consent, and restricts the usage of the requested information to the user only for those scenarios covered under the purpose definition" },
+        { "name": "identifier", "type": [
+            "null",
+            { "name": "Identifier", "type": "record", "fields": [
+                { "name": "type", "type": "string"},
+                { "name": "id", "type": "string"}
+            ]}
+        ], "doc": "The scope of a consent can be restricted to a specific identifier that the user has (e.g. just on of his cell phone numbers). If this field is null then the consent applies to all possible identifiers" },
+        { "name": "status", "type": { "name": "ConsentStatus", "type": "enum", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"] }, "doc": "The status of the consent. The only status that allows accessing the user's information is \"Active\"" },
+        { "name": "creation_date", "type": { "type": "string", "logicalType": "datetime" }, "doc": "The creation date of the consent" },
+        { "name": "expiration_date", "type": [ "null", { "type": "string", "logicalType": "datetime" } ], "doc": "The expiration date of the consent. Consents that do not expire will have a null value in this field" },
+        { "name": "user_prompted", "type": [ "null", { "type": "string", "logicalType": "datetime" } ], "doc": "The date of the consent last time that the consent request was shown to the user. If the consent request has never been shown to the user, then null" },
+        { "name": "channel", "type": [ "null", "string" ]},
+        { "name": "reason", "type": [ "null", "string" ]},
+        { "name": "BRAND_ID", "aliases": ["brand_id"], "type": [ "null", "string" ], "default": null, "x-fp-join": [{"join_id": "join00001", "field": "GBL_BRAND_ID"}], "doc": "Commercial brand identifier. In order to differentiate among different brands in the same OB (e.g. Movistar, O2, Tuenti...)"},
+        { "name": "TAGS_ARRAY", "aliases": ["tags"], "type": [
+            "null",
+            { "type": "array", "items": {
+                "name": "TAGS_ARRAY_ITEM", "type": "record", "fields": [
+                    { "name": "KEY_DES", "aliases": ["key"], "type": "string", "doc": "The key tag" },
+                    { "name": "VALUE_DES", "aliases": ["value"], "type": ["null", "string"], "doc": "The value tag" }
+                ]
+            }}
+        ], "default": null, "doc": "Tags"},
+        { "name": "TRIGGERED_BY_DES", "aliases": ["triggered_by"], "type": [ "null", "string" ], "default": null, "doc": "Name of the process that launched the creation of a consent" },
+        { "name": "DELEGATED_IND", "aliases": ["delegated"], "type": [ "null", "boolean" ], "default": null, "doc": "Flag indicator to mark if the consent has been delegated to a third party" },
+		{ "name": "KERNEL_APP_ID", "aliases": ["kernel_app_id"], "type": [ "null", "string" ], "default": null, "doc": "App ID identifier, refering to the client id used when performing an OAuth flow" }
+    ]
+}

samples/datasets/Fabric_Purposes/v1.0.0/Fabric_Purposes_1.0.0.avsc

@@ -0,0 +1,12 @@
+{
+    "name": "Fabric_Purposes",
+    "x-fp-version": "1.0.0",
+    "namespace": "com.telefonica.baikal.fabric",
+    "x-fp-avro4p-version": "1.5",
+    "type": "record",
+    "doc": "Purposes",
+    "fields": [
+        { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier" },
+        { "name": "LAWFUL_BASIS", "type": { "type": "enum", "name": "STATUS", "symbols": [ "LegitimateInterest", "Consent" ] }, "doc": "Lawful basis" }
+    ]
+}

samples/datasets/Fabric_Segments/v1.0.0/Fabric_Segments_1.0.0.avsc

@@ -0,0 +1,12 @@
+{
+    "name": "Fabric_Segments",
+    "x-fp-version": "1.0.0",
+    "namespace": "com.telefonica.baikal.fabric",
+    "x-fp-avro4p-version": "1.5",
+    "type": "record",
+    "doc": "Segments",
+    "fields": [
+        { "name": "SEGMENT_ID", "type": "int", "doc": "Segment identifier" },
+        { "name": "SEGMENT_NAME", "type": "string", "doc": "Segment name" }
+    ]
+}

samples/datasets/Fabric_User_Consents/v1.0.0/Fabric_User_Consents_1.0.0.avsc

@@ -0,0 +1,26 @@
+{
+    "name": "Fabric_User_Consents",
+    "x-fp-version": "1.0.0",
+    "namespace": "com.telefonica.baikal.fabric",
+    "x-fp-avro4p-version": "1.5",
+    "type": "record",
+    "doc": "User consents",
+    "fields": [
+        { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier" },
+        { "name": "PURPOSE_ID", "type": "string", "doc": "Purpose identifier" },
+        { "name": "APP_ID", "type": "string", "doc": "Application identifier" },
+        { "name": "APP_LEGAL_ENTITY_ID", "type": "string", "doc": "Application legal entity identifier" },
+        { "name": "STATUS", "type": { "type": "enum", "name": "STATUS", "symbols": ["Active", "Latched", "Expired", "Revoked", "Prompted", "Removed"] }, "doc": "Consent status" },
+        { "name": "EXPIRATION_DATE", "type": [
+            "null",
+            { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "datetime" }
+        ], "doc": "Consent expiration date" },
+        { "name": "IDENTIFIER", "type": [
+            "null",
+            { "name": "IDENTIFIER", "type": "record", "fields": [
+                { "name": "TYPE", "type": "string", "doc": "Identifier type" },
+                { "name": "ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Identifier" }
+            ]}
+        ], "doc": "User identifier" }
+    ]
+}

samples/datasets/Fabric_User_Data/v1.0.0/Fabric_User_Data_1.0.0.avsc

@@ -0,0 +1,72 @@
+{
+    "name": "Fabric_User_Data",
+    "x-fp-version": "1.0.0",
+    "namespace": "com.telefonica.baikal.fabric",
+    "x-fp-avro4p-version": "1.5",
+    "type": "record",
+    "doc": "User data",
+    "fields": [
+        { "name": "USER_ID", "type": { "type": "string", "x-fp-user-id": true, "x-fp-data-protection": "pseudonymize" }, "doc": "Internal user identifier" },
+        { "name": "PERSONAL_DOC_ID", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Personal user identifier" },
+        { "name": "NAME", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "User name" },
+        { "name": "AGE", "type": "int", "doc": "User age" },
+        { "name": "ACTIVATION_DATE", "type": { "x-fp-time-dimension": "logicalType", "type": "string", "logicalType": "iso-date" }, "doc": "Activation date" },
+        { "name": "SEGMENT_ID", "type": "int", "doc": "Segment the user belongs to" },
+        { "name": "MOBILE_LINE", "type": {
+            "type": "record", "name": "MOBILE_LINE_RECORD", "fields": [
+                { "name": "PHONE_NUMBER", "type": { "type": "string", "x-fp-identifier": "phone-number", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" },
+                { "name": "IMEI", "type": { "type": "string", "logicalType": "imei", "x-fp-data-protection": "pseudonymize" }, "doc": "IMEI" },
+                { "name": "IMSI", "type": { "type": "string", "logicalType": "imsi", "x-fp-data-protection": "pseudonymize" }, "doc": "IMSI" }
+            ]
+        }, "doc": "User main mobile line" },
+        { "name": "LAND_LINE", "type": [
+            "null",
+            {
+                "type": "record", "name": "LAND_LINE_RECORD", "fields": [
+                    { "name": "PHONE_NUMBER", "type": { "type": "string", "logicalType": "phone-number", "x-fp-data-protection": "pseudonymize" }, "doc": "Phone number" },
+                    { "name": "IP_ADDRESSES", "type": {
+                        "type": "record", "name": "IP_ADDRESSES_RECORD", "fields": [
+                            { "name": "IPV4", "type": { "type": "string", "logicalType": "ipv4", "x-fp-data-protection": "pseudonymize" }, "doc": "IPv4 address" },
+                            { "name": "IPV6", "type": [
+                                "null",
+                                { "type": "string", "logicalType": "ipv6", "x-fp-data-protection": "pseudonymize" }
+                            ]
+                            , "doc": "IPv6 address" }
+                        ]
+                    }, "doc": "User IP addresses" }
+                ]
+            }
+        ], "doc": "Optional user line at home" },
+        { "name": "TV_SERVICES", "type": [
+            "null",
+            {
+                "type": "array", "items": {
+                    "type": "record", "name": "TV_SERVICE_RECORD", "fields": [
+                        { "name": "TYPE", "type": "string", "doc": "Service type" },
+                        { "name": "STATUS", "type": { "type": "enum", "name": "SERVICE_STATUS", "symbols": ["Activated", "Deactivated"] }, "doc": "Service name" },
+                        { "name": "PARENTAL_PIN", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Service parental control" }
+                    ]
+                }
+            }
+        ], "doc": "Optional Services" },
+        { "name": "BILLING_MEANS", "type": [
+            "null",
+            {
+                "type": "map", "x-fp-data-protection": "pseudonymize", "values": {
+                    "type": "record", "name": "BILLING_MEAN_RECORD", "fields": [
+                        { "name": "TYPE", "type": "string", "doc": "Billing type" },
+                        { "name": "VALUE", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Billing value" }
+                    ]
+                }
+            }
+        ], "doc": "Billing means" },
+        { "name": "ADDRESS", "type": {
+            "type": "record", "name": "ADDRESS_RECORD", "fields": [
+                { "name": "STREET", "type": { "type": "string", "x-fp-data-protection": "pseudonymize" }, "doc": "Street" },
+                { "name": "ZIP_CODE", "type": "string", "doc": "Zip code" },
+                { "name": "COUNTRY", "type": { "type": "string", "logicalType": "country-code-alpha-2" }, "doc": "Country" },
+                { "name": "GEO_AREA", "type": { "type": "string", "logicalType": "feature" }, "doc": "GeoJson-like geographic area" }
+            ]
+        }, "doc": "User address" }
+    ]
+}