Skip to content

Commit c23bb8a

Browse files
benbartbenbart
committed
PR-7509 progress w/ tf for codebuild stuff
1 parent a32bd6c commit c23bb8a

12 files changed

Lines changed: 753 additions & 129 deletions

‎ctorm/cumulus-db-md-extract/README.md‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ Really should try to create a EC2 next time.
7777

7878
### Terraform
7979

80-
```bash
8180
```bash
8281
export AWS_PROFILE="cumulus-uat-6921"
8382
export VARFILE=opera-uat.tfvars
@@ -94,14 +93,44 @@ terraform destroy
9493

9594
```
9695

97-
```
96+
### Codebuild
9897

9998
```bash
100-
aws codebuild create-project \
101-
--name "OneTimeDataPull" \
102-
--source '{"type": "S3", "location": "ctorm-dev-scratch/codebuild/codebuild-noop.zip"}' \
103-
--environment '{"type": "LINUX_CONTAINER", "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", "computeType": "BUILD_GENERAL1_SMALL"}' \
104-
--artifacts '{"type": "NO_ARTIFACTS"}' \
105-
--service-role "arn:aws:iam::123456789012:role/service-role/YourCodeBuildRole" \
106-
--timeout-in-minutes 240
99+
export AWS_PROFILE="cumulus-uat-6921"
100+
export AWS_REGION="us-west-2"
101+
export CODEBUILD_PROJECT="$(terraform output -raw cumulus_db_md_extract_codebuild_project_name)"
102+
103+
# start build:
104+
BUILD_ID="$(
105+
aws codebuild start-build \
106+
--project-name "${CODEBUILD_PROJECT}" \
107+
--region "${AWS_REGION}" \
108+
--query 'build.id' \
109+
--output text
110+
)"
111+
112+
echo "${BUILD_ID}"
113+
114+
# get status:
115+
aws codebuild batch-get-builds \
116+
--ids "${BUILD_ID}" \
117+
--region "${AWS_REGION}" \
118+
--query 'builds[0].{status:buildStatus,phase:currentPhase,start:startTime,end:endTime,logs:logs.deepLink}' \
119+
--output table
120+
121+
# get build logs:
122+
aws codebuild batch-get-builds \
123+
--ids "${BUILD_ID}" \
124+
--region "${AWS_REGION}" \
125+
--query 'builds[0].logs.deepLink' \
126+
--output text
127+
128+
LOG_GROUP="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.groupName' --output text)"
129+
LOG_STREAM="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.streamName' --output text)"
130+
131+
aws logs tail "${LOG_GROUP}" \
132+
--log-stream-names "${LOG_STREAM}" \
133+
--follow \
134+
--region "${AWS_REGION}"
135+
107136
```

‎ctorm/cumulus-db-md-extract/buildspec.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
version: 0.2
2+
env:
3+
variables:
4+
AWS_DEFAULT_REGION: "us-west-2"
5+
secrets-manager:
6+
PGHOST: "${DB_SECRET_ARN}:host"
7+
PGUSER: "${DB_SECRET_ARN}:username"
8+
PGPASSWORD: "${DB_SECRET_ARN}:password"
9+
PGDATABASE: "${DB_SECRET_ARN}:database"
10+
11+
phases:
12+
install:
13+
commands:
14+
- echo "Inspecting build image..."
15+
- cat /etc/os-release || true
16+
- aws --version || true
17+
- jq --version || true
18+
- unzip -v | head -n 2 || true
19+
- psql --version || true
20+
- echo "Installing required packages if missing..."
21+
- |
22+
if ! command -v jq >/dev/null 2>&1; then
23+
yum install -y jq
24+
fi
25+
- |
26+
if ! command -v unzip >/dev/null 2>&1; then
27+
yum install -y unzip
28+
fi
29+
- |
30+
if ! command -v psql >/dev/null 2>&1; then
31+
echo "psql not found; trying PostgreSQL client package installs..."
32+
33+
yum install -y postgresql15 \
34+
|| yum install -y postgresql14 \
35+
|| yum install -y postgresql13 \
36+
|| yum install -y postgresql12 \
37+
|| yum install -y postgresql \
38+
|| {
39+
echo "Unable to install psql from enabled yum repositories."
40+
echo "Available postgres-related packages:"
41+
yum search postgresql || true
42+
exit 1
43+
}
44+
fi
45+
- psql --version
46+
- aws --version
47+
48+
pre_build:
49+
commands:
50+
- ls -lah
51+
- chmod +x generate-slices-daily.sh export-granules-daily.sh
52+
- |
53+
echo 'Before assuming role: '
54+
- aws sts get-caller-identity
55+
- |
56+
echo "Assuming ctorm upload role..."
57+
- |
58+
CREDS_JSON="$(aws sts assume-role \
59+
--role-arn "${CTORM_S3_ROLE_ARN}" \
60+
--role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER}")"
61+
62+
export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')"
63+
export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')"
64+
export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')"
65+
- |
66+
echo 'After assuming role: '
67+
- aws sts get-caller-identity
68+
69+
build:
70+
commands:
71+
72+
- aws sts get-caller-identity
73+
- ./generate-slices-daily.sh
74+
- ./export-granules-daily.sh
75+
post_build:
76+
commands:
77+
- unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN
78+
- echo "Build complete."

‎ctorm/cumulus-db-md-extract/export-granules-daily.sh‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,20 @@
11
set -euo pipefail
22

33
S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}"
4+
FAILED_SLICES_FILE="failed_slices_daily.tsv"
5+
6+
rm -f "$FAILED_SLICES_FILE"
7+
8+
echo "Statement timeout: "
9+
psql \
10+
--host="$PGHOST" \
11+
--port="${PGPORT:-5432}" \
12+
--username="$PGUSER" \
13+
--dbname="$PGDATABASE" \
14+
--no-psqlrc \
15+
-c "SHOW statement_timeout;"
16+
17+
418

519
while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do
620
if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then
@@ -15,7 +29,7 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do
1529
echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}"
1630
echo "Destination: ${S3_URI}"
1731

18-
psql \
32+
if ! PGOPTIONS="-c statement_timeout=${PG_STATEMENT_TIMEOUT:-0}" psql \
1933
--host="$PGHOST" \
2034
--port="${PGPORT:-5432}" \
2135
--username="$PGUSER" \
@@ -26,22 +40,22 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do
2640
--set=collection="$COLLECTION" \
2741
--set=yyyymmdd="$YYYYMMDD" \
2842
--file=export-granules-slice-daily.sql \
29-
| awk -v label="${COLLECTION} ${YYYYMMDD}" '
30-
{
31-
print;
32-
count++;
33-
if (count % 1000 == 0) {
34-
printf("[%s] exported %d JSONL rows\n", label, count) > "/dev/stderr";
35-
fflush("/dev/stderr");
36-
}
37-
}
38-
END {
39-
printf("[%s] export complete: %d JSONL rows\n", label, count) > "/dev/stderr";
40-
}
41-
' \
4243
| gzip -c \
43-
| aws --profile=ctorm s3 cp - "$S3_URI"
44+
| aws s3 cp - "$S3_URI"; then
45+
46+
echo "FAILED collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2
47+
printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE"
48+
continue
49+
fi
4450

4551
echo "Uploaded ${S3_URI}"
4652

4753
done < slices_daily.tsv
54+
55+
if [[ -s "$FAILED_SLICES_FILE" ]]; then
56+
FAILED_SLICES_S3_URI="${S3_PREFIX}/failed_slices_daily.tsv"
57+
aws s3 cp "$FAILED_SLICES_FILE" "$FAILED_SLICES_S3_URI"
58+
echo "Some slices failed. Uploaded failure list to ${FAILED_SLICES_S3_URI}" >&2
59+
else
60+
echo "All slices exported successfully."
61+
fi

‎ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ WITH export_rows AS (
3030
AND f.checksum_value IS NOT NULL
3131
WHERE g.status = 'completed'
3232
AND c.name = :'collection'
33-
AND g.beginning_date_time >= to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') AT TIME ZONE 'UTC'
33+
34+
AND g.beginning_date_time >= to_date(:'yyyymmdd', 'YYYYMMDD')
3435
AND g.beginning_date_time < (
35-
to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') + interval '1 month'
36+
to_date(:'yyyymmdd', 'YYYYMMDD') + interval '1 day'
3637
) AT TIME ZONE 'UTC'
3738
GROUP BY
3839
g.granule_id,
@@ -64,4 +65,4 @@ SELECT jsonb_strip_nulls(
6465
)
6566
)
6667
FROM export_rows
67-
ORDER BY beginning_date_time, granule_id;
68+
ORDER BY beginning_date_time, granule_id;

‎ctorm/cumulus-db-md-extract/generate-slices-daily.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
2+
set -euo pipefail
3+
4+
5+
S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}"
6+
17
psql \
28
--host="$PGHOST" \
39
--port="${PGPORT:-5432}" \
@@ -22,10 +28,70 @@ psql \
2228
ON f.granule_cumulus_id = g.cumulus_id
2329
AND f.checksum_value IS NOT NULL
2430
WHERE g.status = 'completed'
31+
AND c.name in (
32+
'ANTPAT',
33+
'BFPQ',
34+
'CORNER_REFL',
35+
'DCOP',
36+
'DC_RADAR',
37+
'DSG_STATIC',
38+
'EA_L0B_L_CRSD',
39+
'EA_L0B_L_RRSD',
40+
'EA_L1_L_RIFG',
41+
'EA_L1_L_ROFF',
42+
'EA_L1_L_RSLC',
43+
'EA_L1_L_RUNW',
44+
'EA_L2_L_GCOV',
45+
'EA_L2_L_GOFF',
46+
'EA_L2_L_GSLC',
47+
'EA_L2_L_GUNW',
48+
'EA_L3_L_SME2',
49+
'FOE',
50+
'FRP',
51+
'FT_PARAM',
52+
'FT_WAVEFORM',
53+
'LRCLK_UTC',
54+
'L_CHAN_DATA',
55+
'MOE',
56+
'NISAR_DEM_TIF',
57+
'NISAR_DEM_VRT',
58+
'NISAR_L0A_RRST_BETA_V1',
59+
'NISAR_L0B_CRSD_BETA_V1',
60+
'NISAR_L0B_RRSD_BETA_V1',
61+
'NISAR_L1_RIFG_BETA_V1',
62+
'NISAR_L1_ROFF_BETA_V1',
63+
'NISAR_L1_RSLC_BETA_V1',
64+
'NISAR_L1_RUNW_BETA_V1',
65+
'NISAR_L2_GCOV_BETA_V1',
66+
'NISAR_L2_GOFF_BETA_V1',
67+
'NISAR_L2_GSLC_BETA_V1',
68+
'NISAR_L2_GUNW_BETA_V1',
69+
'NISAR_L3_SME2_BETA_V1',
70+
'NISAR_VWC',
71+
'NISAR_WATERMASK_TIF',
72+
'NISAR_WATERMASK_VRT',
73+
'NOE',
74+
'NRP',
75+
'OROST',
76+
'PA_L0B_L_RRSD',
77+
'POE',
78+
'PRP',
79+
'STUF',
80+
'TEC',
81+
'TFDB',
82+
'UR_L0B_L_RRSD',
83+
84+
'OPERA_L2_CSLC-S1_V1',
85+
'OPERA_L2_RTC-S1_V1',
86+
'OPERA_L3_DISP-S1_V1',
87+
'OPERA_L3_DIST-ALERT-S1_V1',
88+
'OPERA_L4_TROPO-ZENITH_V1'
89+
)
2590
GROUP BY
2691
c.name,
2792
to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD')
2893
ORDER BY
2994
c.name,
3095
to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD');
3196
" > slices_daily.tsv
97+
aws s3 cp slices_daily.tsv "$S3_URI/"

0 commit comments

Comments
 (0)