PR-7510 refinements to documentation, etc

Author: benbart

ctorm/cumulus-db-md-extract/README.md

@@ -128,10 +128,7 @@ echo "${BUILD_ID}"
    aws codebuild start-build \
      --project-name "${CODEBUILD_PROJECT}" \
      --region "${AWS_REGION}" \
-     --environment-variables-override '[
-        {"name": "SLICES_S3_URI", "value": "s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_RTC-S1_V1-20170910-20181203.tsv", "type": "PLAINTEXT"},
-        {"name": "COLLECTION", "value": "OPERA_L2_RTC-S1_V1", "type": "PLAINTEXT"}
-    ]' \
+     --environment-variables-override name=SLICES_S3_URI,value="s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_CSLC-S1_V1-20180707-20200522.tsv",type=PLAINTEXT  \
      --query 'build.id' \
      --output text
 # )"

ctorm/cumulus-db-md-extract/export-granules-daily.sh

@@ -1,5 +1,7 @@
 set -euo pipefail
 
+# Takes slice TSV files and creates .jsonl files out of the cumulus DB
+
 S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}"
 FAILED_SLICES_FILE="failed_slices_daily.tsv"
 

ctorm/granule-md-db-loader/README.md

@@ -26,7 +26,7 @@ export CTORM_BUCKET="ctorm-dev-scratch"
 aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/nisar/ |grep PRE
 aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/opera/ |grep PRE
 
-# If you want do them all sequentially, not recommended and not going to work with all the data:
+# If you want do them all sequentially, not recommended and not going to work with all the data because throttling::
 aws codebuild start-build --profile="${AWS_PROFILE}" \
 --project-name "${CODEBUILD_PROJECT}" \
 --region "us-west-2" \
@@ -60,6 +60,7 @@ aws codebuild start-build --profile="${AWS_PROFILE}" \
 --environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DIST-ALERT-S1_V1_/,type=PLAINTEXT \
 --query 'build.id' \
 --output text
+# nisar
 aws codebuild start-build --profile="${AWS_PROFILE}" \
 --project-name "${CODEBUILD_PROJECT}" \
 --region "us-west-2" \

ctorm/granule-md-db-loader/granule_md_db_loader.py

@@ -47,49 +47,51 @@ def limit(self, pk):
 
 
 def handle_file(
-        local_path,
-        table,
-        rate_limiter: PartitionRateLimiter,
-        records_in_file,
-        total_records,
+    local_path: str,
+    table,
+    rate_limiter: PartitionRateLimiter,
+    records_in_file: int,
+    total_records: int,
 ):
-    with gzip.open(local_path, "rt", encoding="utf-8") as f:
-        with table.batch_writer() as batch:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-
-                try:
-                    # Convert floats/doubles to Decimal for
-                    # DynamoDB compatibility
-                    item = json.loads(line, parse_float=Decimal)
-                except Exception as e:
-                    logger.error(f"Failed to parse JSON line: {e}")
-                    continue
-
-                # Add/modify fields if needed
-                newdate = datetime.datetime.utcnow().isoformat() + "Z"
-                item["imported_at"] = newdate
-
-                # Ensure partition and sort keys are present
-                pk = item.get("pk")
-                sk = item.get("sk")
-                if not pk or not sk:
-                    logger.warning(
-                        "Skipping record missing pk/sk: %s",
-                        item.get('granule_id'),
-                    )
-
-                    continue
-
-                # Apply dynamic rate limit based on the partition key
-                rate_limiter.limit(pk)
-
-                # Batch insert into DynamoDB
-                batch.put_item(Item=item)
-                records_in_file += 1
-                total_records += 1
+    with (
+        gzip.open(local_path, "rt", encoding="utf-8") as f,
+        table.batch_writer() as batch,
+    ):
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                # Convert floats/doubles to Decimal for
+                # DynamoDB compatibility
+                item = json.loads(line, parse_float=Decimal)
+            except Exception as e:
+                logger.error("Failed to parse JSON line: %s", e)
+                continue
+
+            # Add/modify fields if needed
+            newdate = datetime.datetime.utcnow().isoformat() + "Z"
+            item["imported_at"] = newdate
+
+            # Ensure partition and sort keys are present
+            pk = item.get("pk")
+            sk = item.get("sk")
+            if not pk or not sk:
+                logger.warning(
+                    "Skipping record missing pk/sk: %s",
+                    item.get("granule_id"),
+                )
+
+                continue
+
+            # Apply dynamic rate limit based on the partition key
+            rate_limiter.limit(pk)
+
+            # Batch insert into DynamoDB
+            batch.put_item(Item=item)
+            records_in_file += 1
+            total_records += 1
     return records_in_file, total_records
 
 
@@ -142,7 +144,7 @@ def main():
             if not key.endswith(".jsonl.gz"):
                 continue
 
-            logger.info(f"Processing s3://{bucket_name}/{key}")
+            logger.info("Processing s3://%s/%s", bucket_name, key)
             total_files += 1
 
             # Download and decompress the file
@@ -166,14 +168,13 @@ def main():
                     dyndb_table,
                     rate_limiter,
                     records_in_file,
-                    total_records
+                    total_records,
                 )
 
                 logger.info(
                     "Successfully imported %s records from %s",
                     records_in_file,
                     key,
-
                 )
             except Exception as e:
                 logger.error("Error processing file %s: %s", key, e)

ctorm/infra/README.md

@@ -19,16 +19,25 @@ terraform plan \
 terraform apply \
   -var-file="${VARFILE}"
 
+terraform destroy  
+```
+
+## Increasing DynamoDB table capacity
+
+For loading the DynamoDB table with the .jsonl files, we'll need to increase the capacity of the table.
+AWS will throttle drastically if we don't. There is some limitation on how often we can switch between
+PROVISIONED and PAY_PER_REQUEST, so don't do it willy-nilly.
+
+```bash
 # For temporary use while loading granule metadata:
 terraform plan \
   -var-file="${VARFILE}" \
   -var="granule_table_billing_mode=PROVISIONED" \
-  -var="granule_table_write_capacity=10000"
+  -var="granule_table_write_capacity=5000"
 
-terraform apply \
-  -var-file="${VARFILE}" \
-  -var="granule_table_billing_mode=PROVISIONED" \
-  -var="granule_table_write_capacity=10000"
+# To set it back to the default:
+terraform plan \
+  -var-file="${VARFILE}"
 
 
 terraform destroy

ctorm/infra/terraform/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.9.2",
-  "serial": 69,
+  "serial": 80,
   "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c",
   "outputs": {
     "cumulus_db_md_extract_upload_role_arn": {
@@ -45,16 +45,16 @@
           "attributes": {
             "exclude_symlink_directories": null,
             "excludes": null,
-            "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f",
-            "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=",
-            "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==",
+            "id": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6",
+            "output_base64sha256": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=",
+            "output_base64sha512": "cYqWFwuEi6LTUM2lmqkXnWuYX+bf0oG86hslOpldD33Zq+jezzEQpTt9V3MGu8xmBy+knCvhX457ql2fPXo2qw==",
             "output_file_mode": null,
-            "output_md5": "10648403b2277ad95dd6324dd58521c9",
+            "output_md5": "c7918cf0c08ea6e16f8828296cd3d06a",
             "output_path": "./.terraform/granule-md-db-loader.zip",
-            "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f",
-            "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c",
-            "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da",
-            "output_size": 3296,
+            "output_sha": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6",
+            "output_sha256": "b09c8d8bc03812252df7ed5e19fc6a7afea4871f07c97c4bde8db96c4ae22f96",
+            "output_sha512": "718a96170b848ba2d350cda59aa9179d6b985fe6dfd281bcea1b253a995d0f7dd9abe8decf3110a53b7d577306bbcc66072fa49c2be15f8e7baa5d9f3d7a36ab",
+            "output_size": 3481,
             "source": [],
             "source_content": null,
             "source_content_filename": null,
@@ -793,7 +793,7 @@
             "inline_policy": [
               {
                 "name": "ctorm-dev-cumulus-db-md-extract-upload",
-                "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}"
+                "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}"
               }
             ],
             "managed_policy_arns": [],
@@ -1195,7 +1195,7 @@
             "content_encoding": "",
             "content_language": "",
             "content_type": "application/octet-stream",
-            "etag": "10648403b2277ad95dd6324dd58521c9",
+            "etag": "c7918cf0c08ea6e16f8828296cd3d06a",
             "force_destroy": false,
             "id": "codebuild/granule-md-db-loader.zip",
             "key": "codebuild/granule-md-db-loader.zip",
@@ -1207,7 +1207,7 @@
             "override_provider": [],
             "server_side_encryption": "AES256",
             "source": "./.terraform/granule-md-db-loader.zip",
-            "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=",
+            "source_hash": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=",
             "storage_class": "STANDARD",
             "tags": {},
             "tags_all": {},

ctorm/infra/terraform/terraform.tfstate.backup

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.9.2",
-  "serial": 67,
+  "serial": 76,
   "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c",
   "outputs": {
     "cumulus_db_md_extract_upload_role_arn": {
@@ -45,16 +45,16 @@
           "attributes": {
             "exclude_symlink_directories": null,
             "excludes": null,
-            "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f",
-            "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=",
-            "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==",
+            "id": "d4be9c10fb356c2942476bb405fe6e86236ca9d9",
+            "output_base64sha256": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=",
+            "output_base64sha512": "/tl0guEuEuyrvuj+stzcDIneyXnGYZxXS+GHq3+Od3b3Rgyni0trdrvK2XAEb/SD0ccNAACcZ9AJxPkDFGv/EA==",
             "output_file_mode": null,
-            "output_md5": "10648403b2277ad95dd6324dd58521c9",
+            "output_md5": "66541e163cc3aaff54beddc162ad3aca",
             "output_path": "./.terraform/granule-md-db-loader.zip",
-            "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f",
-            "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c",
-            "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da",
-            "output_size": 3296,
+            "output_sha": "d4be9c10fb356c2942476bb405fe6e86236ca9d9",
+            "output_sha256": "935f7853bdb8561ea66efb62eea3b49ab1f8b837c552aa0e020494b58f33996d",
+            "output_sha512": "fed97482e12e12ecabbee8feb2dcdc0c89dec979c6619c574be187ab7f8e7776f7460ca78b4b6b76bbcad970046ff483d1c70d00009c67d009c4f903146bff10",
+            "output_size": 3469,
             "source": [],
             "source_content": null,
             "source_content_filename": null,
@@ -249,9 +249,9 @@
         {
           "schema_version": 0,
           "attributes": {
-            "id": "923578907",
-            "json": "{\n  \"Version\": \"2012-10-17\",\n  \"Statement\": [\n    {\n      \"Sid\": \"UploadExtractedGranules\",\n      \"Effect\": \"Allow\",\n      \"Action\": [\n        \"s3:PutObject\",\n        \"s3:AbortMultipartUpload\"\n      ],\n      \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n    },\n    {\n      \"Sid\": \"ListDestinationPrefix\",\n      \"Effect\": \"Allow\",\n      \"Action\": \"s3:ListBucket\",\n      \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n      \"Condition\": {\n        \"StringLike\": {\n          \"s3:prefix\": \"cumulus-granules/*\"\n        }\n      }\n    }\n  ]\n}",
-            "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}",
+            "id": "3026717548",
+            "json": "{\n  \"Version\": \"2012-10-17\",\n  \"Statement\": [\n    {\n      \"Sid\": \"UploadExtractedGranules\",\n      \"Effect\": \"Allow\",\n      \"Action\": [\n        \"s3:PutObject\",\n        \"s3:GetObject\",\n        \"s3:AbortMultipartUpload\"\n      ],\n      \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n    },\n    {\n      \"Sid\": \"ListDestinationPrefix\",\n      \"Effect\": \"Allow\",\n      \"Action\": \"s3:ListBucket\",\n      \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n      \"Condition\": {\n        \"StringLike\": {\n          \"s3:prefix\": \"cumulus-granules/*\"\n        }\n      }\n    }\n  ]\n}",
+            "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}",
             "override_json": null,
             "override_policy_documents": null,
             "policy_id": null,
@@ -261,6 +261,7 @@
               {
                 "actions": [
                   "s3:AbortMultipartUpload",
+                  "s3:GetObject",
                   "s3:PutObject"
                 ],
                 "condition": [],
@@ -676,7 +677,7 @@
                 "type": "S"
               }
             ],
-            "billing_mode": "PAY_PER_REQUEST",
+            "billing_mode": "PROVISIONED",
             "deletion_protection_enabled": false,
             "global_secondary_index": [
               {
@@ -686,8 +687,8 @@
                 "on_demand_throughput": [],
                 "projection_type": "ALL",
                 "range_key": "gsi1sk",
-                "read_capacity": 0,
-                "write_capacity": 0
+                "read_capacity": 5000,
+                "write_capacity": 5
               }
             ],
             "hash_key": "pk",
@@ -703,7 +704,7 @@
               }
             ],
             "range_key": "sk",
-            "read_capacity": 0,
+            "read_capacity": 5000,
             "replica": [],
             "restore_date_time": null,
             "restore_source_name": null,
@@ -715,7 +716,7 @@
             "stream_label": "",
             "stream_view_type": "",
             "table_class": "STANDARD",
-            "tags": {},
+            "tags": null,
             "tags_all": {},
             "timeouts": null,
             "ttl": [
@@ -724,7 +725,7 @@
                 "enabled": false
               }
             ],
-            "write_capacity": 0
+            "write_capacity": 5
           },
           "sensitive_attributes": [],
           "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
@@ -792,7 +793,7 @@
             "inline_policy": [
               {
                 "name": "ctorm-dev-cumulus-db-md-extract-upload",
-                "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}"
+                "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}"
               }
             ],
             "managed_policy_arns": [],
@@ -915,7 +916,7 @@
             "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload",
             "name": "ctorm-dev-cumulus-db-md-extract-upload",
             "name_prefix": "",
-            "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}",
+            "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}",
             "role": "ctorm-dev-cumulus-db-md-extract-upload"
           },
           "sensitive_attributes": [],
@@ -1194,7 +1195,7 @@
             "content_encoding": "",
             "content_language": "",
             "content_type": "application/octet-stream",
-            "etag": "10648403b2277ad95dd6324dd58521c9",
+            "etag": "66541e163cc3aaff54beddc162ad3aca",
             "force_destroy": false,
             "id": "codebuild/granule-md-db-loader.zip",
             "key": "codebuild/granule-md-db-loader.zip",
@@ -1206,7 +1207,7 @@
             "override_provider": [],
             "server_side_encryption": "AES256",
             "source": "./.terraform/granule-md-db-loader.zip",
-            "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=",
+            "source_hash": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=",
             "storage_class": "STANDARD",
             "tags": {},
             "tags_all": {},