Skip to content

Commit 03dcbf6

Browse files
benbartbenbart
committed
Fixing bugs relating to specifying a slices tsv file
1 parent 93bf745 commit 03dcbf6

12 files changed

Lines changed: 532 additions & 170 deletions

‎ctorm/cumulus-db-md-extract/README.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,17 +108,35 @@ export AWS_PROFILE="cumulus-uat-6921"
108108
export AWS_REGION="us-west-2"
109109
export CODEBUILD_PROJECT="$(terraform output -state="${STATEFILE}" -raw cumulus_db_md_extract_codebuild_project_name)"
110110

111+
# You can do one collection at a time, many in parallel if necessary:
112+
export COLLECTION="OPERA_L2_RTC-S1_V1"
113+
111114
# start build:
112115
BUILD_ID="$(
113116
aws codebuild start-build \
114117
--project-name "${CODEBUILD_PROJECT}" \
115118
--region "${AWS_REGION}" \
119+
--environment-variables-override name=COLLECTION,value="${COLLECTION}",type=PLAINTEXT \
116120
--query 'build.id' \
117121
--output text
118122
)"
119123

120124
echo "${BUILD_ID}"
121125

126+
# Or start build specifying a custom slices.tsv file from S3:
127+
# BUILD_ID="$(
128+
aws codebuild start-build \
129+
--project-name "${CODEBUILD_PROJECT}" \
130+
--region "${AWS_REGION}" \
131+
--environment-variables-override '[
132+
{"name": "SLICES_S3_URI", "value": "s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_RTC-S1_V1-20170910-20181203.tsv", "type": "PLAINTEXT"},
133+
{"name": "COLLECTION", "value": "OPERA_L2_RTC-S1_V1", "type": "PLAINTEXT"}
134+
]' \
135+
--query 'build.id' \
136+
--output text
137+
# )"
138+
139+
122140
# get status:
123141
aws codebuild batch-get-builds \
124142
--ids "${BUILD_ID}" \

‎ctorm/cumulus-db-md-extract/export-granules-daily.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do
1111
continue
1212
fi
1313

14-
YYYY="${YYYYMMDD:0:4)"
14+
YYYY="${YYYYMMDD:0:4}"
1515
YYYYMM="${YYYYMMDD:0:6}"
1616
SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_')
1717
S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYY}/${YYYYMM}/${YYYYMMDD}.jsonl.gz"

‎ctorm/cumulus-db-md-extract/generate-slices-daily.sh‎

Lines changed: 89 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,95 @@
11

22
set -euo pipefail
33

4-
psql \
5-
--host="$PGHOST" \
6-
--port="${PGPORT:-5432}" \
7-
--username="$PGUSER" \
8-
--dbname="$PGDATABASE" \
9-
--quiet \
10-
--no-psqlrc \
11-
--tuples-only \
12-
--no-align \
13-
--field-separator $'\t' \
14-
--set=ON_ERROR_STOP=1 \
15-
--command "
16-
SELECT
17-
c.name,
18-
to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') AS yyyymmdd,
19-
count(DISTINCT g.cumulus_id) AS granule_count,
20-
count(f.cumulus_id) AS file_count
21-
FROM public.granules g
22-
JOIN public.collections c
23-
ON c.cumulus_id = g.collection_cumulus_id
24-
JOIN public.files f
25-
ON f.granule_cumulus_id = g.cumulus_id
26-
AND f.checksum_value IS NOT NULL
27-
WHERE g.status = 'completed'
28-
AND c.name in (
29-
'ANTPAT',
30-
'BFPQ',
31-
'CORNER_REFL',
32-
'DCOP',
33-
'DC_RADAR',
34-
'DSG_STATIC',
35-
'EA_L0B_L_CRSD',
36-
'EA_L0B_L_RRSD',
37-
'EA_L1_L_RIFG',
38-
'EA_L1_L_ROFF',
39-
'EA_L1_L_RSLC',
40-
'EA_L1_L_RUNW',
41-
'EA_L2_L_GCOV',
42-
'EA_L2_L_GOFF',
43-
'EA_L2_L_GSLC',
44-
'EA_L2_L_GUNW',
45-
'EA_L3_L_SME2',
46-
'FOE',
47-
'FRP',
48-
'FT_PARAM',
49-
'FT_WAVEFORM',
50-
'LRCLK_UTC',
51-
'L_CHAN_DATA',
52-
'MOE',
53-
'NISAR_DEM_TIF',
54-
'NISAR_DEM_VRT',
55-
'NISAR_L0A_RRST_BETA_V1',
56-
'NISAR_L0B_CRSD_BETA_V1',
57-
'NISAR_L0B_RRSD_BETA_V1',
58-
'NISAR_L1_RIFG_BETA_V1',
59-
'NISAR_L1_ROFF_BETA_V1',
60-
'NISAR_L1_RSLC_BETA_V1',
61-
'NISAR_L1_RUNW_BETA_V1',
62-
'NISAR_L2_GCOV_BETA_V1',
63-
'NISAR_L2_GOFF_BETA_V1',
64-
'NISAR_L2_GSLC_BETA_V1',
65-
'NISAR_L2_GUNW_BETA_V1',
66-
'NISAR_L3_SME2_BETA_V1',
67-
'NISAR_VWC',
68-
'NISAR_WATERMASK_TIF',
69-
'NISAR_WATERMASK_VRT',
70-
'NOE',
71-
'NRP',
72-
'OROST',
73-
'PA_L0B_L_RRSD',
74-
'POE',
75-
'PRP',
76-
'STUF',
77-
'TEC',
78-
'TFDB',
79-
'UR_L0B_L_RRSD',
804

81-
'OPERA_L2_CSLC-S1_V1',
82-
'OPERA_L2_RTC-S1_V1',
83-
'OPERA_L3_DISP-S1_V1',
84-
'OPERA_L3_DIST-ALERT-S1_V1',
85-
'OPERA_L4_TROPO-ZENITH_V1'
86-
)
87-
GROUP BY
88-
c.name,
89-
to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD')
90-
ORDER BY
91-
c.name,
92-
to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD');
93-
" > slices_daily.tsv
5+
#ANTPAT',
6+
#BFPQ',
7+
#CORNER_REFL',
8+
#DCOP',
9+
#DC_RADAR',
10+
#DSG_STATIC',
11+
#EA_L0B_L_CRSD',
12+
#EA_L0B_L_RRSD',
13+
#EA_L1_L_RIFG',
14+
#EA_L1_L_ROFF',
15+
#EA_L1_L_RSLC',
16+
#EA_L1_L_RUNW',--
17+
#EA_L2_L_GCOV',
18+
#EA_L2_L_GOFF',
19+
#EA_L2_L_GSLC',
20+
#EA_L2_L_GUNW',
21+
#EA_L3_L_SME2',
22+
#FOE',
23+
#FRP',
24+
#FT_PARAM',
25+
#FT_WAVEFORM',
26+
#LRCLK_UTC',
27+
#L_CHAN_DATA',
28+
#MOE',
29+
#NISAR_DEM_TIF',
30+
#NISAR_DEM_VRT',
31+
#NISAR_L0A_RRST_BETA_V1',
32+
#NISAR_L0B_CRSD_BETA_V1',
33+
#NISAR_L0B_RRSD_BETA_V1',
34+
#NISAR_L1_RIFG_BETA_V1',
35+
#NISAR_L1_ROFF_BETA_V1',
36+
#NISAR_L1_RSLC_BETA_V1',
37+
#NISAR_L1_RUNW_BETA_V1',
38+
#NISAR_L2_GCOV_BETA_V1',
39+
#NISAR_L2_GOFF_BETA_V1',
40+
#NISAR_L2_GSLC_BETA_V1',
41+
#NISAR_L2_GUNW_BETA_V1',
42+
#NISAR_L3_SME2_BETA_V1',
43+
#NISAR_VWC',
44+
#NISAR_WATERMASK_TIF',
45+
#NISAR_WATERMASK_VRT',
46+
#NOE',
47+
#NRP',
48+
#OROST',
49+
#PA_L0B_L_RRSD',
50+
#POE',
51+
#PRP',
52+
#STUF',
53+
#TEC',
54+
#TFDB',
55+
#UR_L0B_L_RRSD',
56+
# Opera collections:
57+
#COLLECTION="OPERA_L2_CSLC-S1_V1"
58+
#COLLECTION="OPERA_L2_RTC-S1_V1"
59+
#COLLECTION="OPERA_L3_DISP-S1_V1"
60+
#COLLECTION="OPERA_L3_DIST-ALERT-S1_V1"
61+
#COLLECTION="OPERA_L4_TROPO-ZENITH_V1"
9462

95-
source ./refresh-ctorm-upload-role.sh
63+
echo "SLICES_S3_URI: ${SLICES_S3_URI}"
64+
echo "COLLECTION: ${COLLECTION}"
9665

97-
SLICES_S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily.tsv"
98-
aws s3 cp slices_daily.tsv "${SLICES_S3_URI}"
99-
echo "Uploaded ${SLICES_S3_URI}"
66+
if [[ -n "${SLICES_S3_URI:-}" ]]; then
67+
echo "SLICES_S3_URI is specified: ${SLICES_S3_URI}"
68+
echo "Downloading specified slices file instead of querying database..."
69+
source ./refresh-ctorm-upload-role.sh
70+
aws s3 cp "${SLICES_S3_URI}" slices_daily.tsv
71+
else
72+
# Generate slices TSV file
73+
echo "SLICES_S3_URI not set. Going to run a big query now..."
74+
psql \
75+
--host="$PGHOST" \
76+
--port="${PGPORT:-5432}" \
77+
--username="$PGUSER" \
78+
--dbname="$PGDATABASE" \
79+
--quiet \
80+
--no-psqlrc \
81+
--tuples-only \
82+
--no-align \
83+
--field-separator $'\t' \
84+
--set=ON_ERROR_STOP=1 \
85+
--set=collection="$COLLECTION" \
86+
--file=generate-slices-daily.sql \
87+
> slices_daily.tsv
88+
89+
source ./refresh-ctorm-upload-role.sh
90+
91+
# Upload slices TSV file to S3 for archive purposes.
92+
SLICES_S3_UPLOAD_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily-${COLLECTION}-${CODEBUILD_BUILD_NUMBER}.tsv"
93+
aws s3 cp slices_daily.tsv "${SLICES_S3_UPLOAD_URI}"
94+
echo "Uploaded ${SLICES_S3_UPLOAD_URI}"
95+
fi

‎ctorm/cumulus-db-md-extract/generate-slices-daily.sql‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
SELECT
2+
C.NAME,
3+
TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD') AS YYYYMMDD,
4+
COUNT(DISTINCT G.CUMULUS_ID) AS GRANULE_COUNT,
5+
COUNT(F.CUMULUS_ID) AS FILE_COUNT
6+
FROM
7+
PUBLIC.GRANULES G
8+
JOIN PUBLIC.COLLECTIONS C
9+
ON C.CUMULUS_ID = G.COLLECTION_CUMULUS_ID
10+
JOIN PUBLIC.FILES F
11+
ON F.GRANULE_CUMULUS_ID = G.CUMULUS_ID
12+
AND F.CHECKSUM_VALUE IS NOT NULL
13+
WHERE
14+
G.STATUS = 'completed'
15+
AND C.NAME = :'collection'
16+
GROUP BY
17+
C.NAME,
18+
TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD')
19+
ORDER BY
20+
C.NAME,
21+
TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD');

‎ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@ export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKey
1717
export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')"
1818
export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')"
1919

20-
aws sts get-caller-identity >&2
20+
# aws sts get-caller-identity >&2

‎ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,16 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" {
306306
name = "CTORM_S3_ROLE_ARN"
307307
value = var.db_md_extract_ctorm_s3_role_arn
308308
}
309+
310+
environment_variable {
311+
name = "COLLECTION"
312+
value = ""
313+
}
314+
315+
environment_variable {
316+
name = "SLICES_S3_URI"
317+
value = ""
318+
}
309319
}
310320

311321
vpc_config {

‎ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"version": 4,
33
"terraform_version": "1.9.2",
4-
"serial": 112,
4+
"serial": 142,
55
"lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886",
66
"outputs": {
77
"cumulus_db_md_extract_codebuild_project_name": {
@@ -34,16 +34,16 @@
3434
"terraform",
3535
"terraform/*"
3636
],
37-
"id": "15e0432811ac45b77440e902f20b433130b97e70",
38-
"output_base64sha256": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=",
39-
"output_base64sha512": "loaVRKTjqeDeRXYLTynxgJIcoeFMuaH5+Yx1GZPZymzfK4GSIarTAvusSWNs7DHEH+baqi6JJ8D5SjrMvSmT4Q==",
37+
"id": "a96216dceaf9e6e503bf02981b53784e03ef13b8",
38+
"output_base64sha256": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=",
39+
"output_base64sha512": "RLp6HJBUgfaHuTiqD2qyQKwBJyHtvZ+/VDK/urPEE58Gf5Md2MDIJeCVoQIux45wacZU5HiEe/eoFtx28IitLw==",
4040
"output_file_mode": null,
41-
"output_md5": "cdd0aab259d06df74b4ae3ddbbd7615e",
41+
"output_md5": "5a4c52cf01416b48350304df429135a5",
4242
"output_path": "./.terraform/cumulus-db-md-extract.zip",
43-
"output_sha": "15e0432811ac45b77440e902f20b433130b97e70",
44-
"output_sha256": "f37531a4a276be2b5853d2383331e343604a81c2b63926c24f83c318e3960d9f",
45-
"output_sha512": "96869544a4e3a9e0de45760b4f29f180921ca1e14cb9a1f9f98c751993d9ca6cdf2b819221aad302fbac49636cec31c41fe6daaa2e8927c0f94a3accbd2993e1",
46-
"output_size": 6341,
43+
"output_sha": "a96216dceaf9e6e503bf02981b53784e03ef13b8",
44+
"output_sha256": "8882f14372977b33e1aa89c08d14ad49217280b96c1c984d91c0f02f7042b750",
45+
"output_sha512": "44ba7a1c905481f687b938aa0f6ab240ac012721edbd9fbf5432bfbab3c4139f067f931dd8c0c825e095a1022ec78e7069c654e478847bf7a816dc76f088ad2f",
46+
"output_size": 7000,
4747
"source": [],
4848
"source_content": null,
4949
"source_content_filename": null,
@@ -371,6 +371,16 @@
371371
"name": "CTORM_S3_ROLE_ARN",
372372
"type": "PLAINTEXT",
373373
"value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload"
374+
},
375+
{
376+
"name": "COLLECTION",
377+
"type": "PLAINTEXT",
378+
"value": ""
379+
},
380+
{
381+
"name": "SLICES_S3_URI",
382+
"type": "PLAINTEXT",
383+
"value": ""
374384
}
375385
],
376386
"fleet": [],
@@ -717,7 +727,7 @@
717727
"content_encoding": "",
718728
"content_language": "",
719729
"content_type": "application/octet-stream",
720-
"etag": "cdd0aab259d06df74b4ae3ddbbd7615e",
730+
"etag": "5a4c52cf01416b48350304df429135a5",
721731
"force_destroy": false,
722732
"id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip",
723733
"key": "codebuild/cumulus-db-md-extract.zip",
@@ -730,7 +740,7 @@
730740
"region": "us-west-2",
731741
"server_side_encryption": "AES256",
732742
"source": "./.terraform/cumulus-db-md-extract.zip",
733-
"source_hash": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=",
743+
"source_hash": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=",
734744
"storage_class": "STANDARD",
735745
"tags": {},
736746
"tags_all": {},

0 commit comments

Comments
 (0)