From 1781f0cf0c6dbea8076bace64078a91ff7e9c98d Mon Sep 17 00:00:00 2001 From: Joschuan Santana Date: Fri, 8 Mar 2024 10:33:42 -0300 Subject: [PATCH 1/7] Include CPU dockerfile --- src/ecr/roof-energy-inference-cpu/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/ecr/roof-energy-inference-cpu/Dockerfile diff --git a/src/ecr/roof-energy-inference-cpu/Dockerfile b/src/ecr/roof-energy-inference-cpu/Dockerfile new file mode 100644 index 0000000..4423703 --- /dev/null +++ b/src/ecr/roof-energy-inference-cpu/Dockerfile @@ -0,0 +1,12 @@ +# https://aws.amazon.com/cn/releasenotes/available-deep-learning-containers-images/ +# https://github.com/aws/deep-learning-containers/blob/master/available_images.md + +FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1-cpu-py310 + +RUN apt update && apt install -y binutils libproj-dev gdal-bin libgdal-dev + +RUN python3 -m pip install --upgrade pip --no-cache-dir && \ + pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 + +RUN pip3 install setuptools==57.5.0 && \ + pip3 install gdal==$(gdal-config --version) From 8f44fdf22783f922e4bd8209169caabbaa86b25a Mon Sep 17 00:00:00 2001 From: Joschuan Santana Date: Fri, 8 Mar 2024 17:08:15 -0300 Subject: [PATCH 2/7] Implement build-payload function --- src/lambda/build-payload/__init__.py | 0 src/lambda/build-payload/inference_builder.py | 48 +++++++++++++++++++ src/lambda/build-payload/lambda_function.py | 26 ++++++++++ src/lambda/build-payload/parameter_mappers.py | 33 +++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 src/lambda/build-payload/__init__.py create mode 100644 src/lambda/build-payload/inference_builder.py create mode 100644 src/lambda/build-payload/lambda_function.py create mode 100644 src/lambda/build-payload/parameter_mappers.py diff --git a/src/lambda/build-payload/__init__.py b/src/lambda/build-payload/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/lambda/build-payload/inference_builder.py b/src/lambda/build-payload/inference_builder.py new file mode 100644 index 0000000..2c3b2e1 --- /dev/null +++ b/src/lambda/build-payload/inference_builder.py @@ -0,0 +1,48 @@ +import parameter_mappers as mappers + +MODEL_NAME = 'sam_vit_h_4b8939.pth' + + +class InferenceBuilder(object): + def __init__(self, task_key: str, base_code_s3uri: str, attachments_bucket_name: str): + self.task_key = task_key + self.base_code_s3uri = base_code_s3uri + self.attachments_bucket_name = attachments_bucket_name + + def build_inference_parameters(self, payload): + return { + 'JobName': mappers.get_unique_job_name(self.task_key, 'roof-energy'), + 'ContainerArguments': payload['container_args'], + 'InputConfig': [ + mappers.get_processing_input( + 'code', + self.base_code_s3uri + '/code', + '/opt/ml/processing/code' + ), + mappers.get_processing_input( + 'models', + self.base_code_s3uri + f'/{MODEL_NAME}', + '/opt/ml/processing/models' + ) + ], + 'OutputConfig': [ + mappers.get_processing_output('outputs', + '/opt/ml/processing/outputs', payload['inference_outputs_uri']) + ] + } + + def run(self, inputs: dict, user_sub: str): + inference_container_args = [ + '--bounding-box', *[str(coord) for coord in inputs['BoundingBox']], + '--panel-size', str(inputs['PanelSize']), + '--available-area', str(inputs['AvailableArea']), + '--panel-power', str(inputs['PanelPower']) + ] + + payload = { + 'task_key': self.task_key, + 'container_args': inference_container_args, + 'inference_outputs_uri': f's3://{self.attachments_bucket_name}/sunscan/user:{user_sub}/roof_energy/{self.task_key}/outputs/' + } + + return self.build_inference_parameters(payload) diff --git a/src/lambda/build-payload/lambda_function.py b/src/lambda/build-payload/lambda_function.py new file mode 100644 index 0000000..94328cc --- /dev/null +++ b/src/lambda/build-payload/lambda_function.py @@ -0,0 +1,26 @@ +import os +import logging + +from inference_builder import InferenceBuilder + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logging.getLogger('botocore').setLevel(logging.CRITICAL) + +BASE_CODE_S3URI = os.getenv('BASE_CODE_S3URI') +ATTACHMENTS_BUCKET_NAME = os.getenv('ATTACHMENTS_BUCKET_NAME') + + +def lambda_handler(event: dict, context): + builder = InferenceBuilder( + event['TaskKey'], + BASE_CODE_S3URI, + ATTACHMENTS_BUCKET_NAME + ) + + payload = builder.run( + event['Inputs'], + user_sub=event['UserSub'] + ) + + return payload diff --git a/src/lambda/build-payload/parameter_mappers.py b/src/lambda/build-payload/parameter_mappers.py new file mode 100644 index 0000000..41f48af --- /dev/null +++ b/src/lambda/build-payload/parameter_mappers.py @@ -0,0 +1,33 @@ +import datetime as dt + +def get_unique_job_name(uuid_key: str, *references: list): + """ Returns a unique job name based on a given uuid_key + and the current timestamp """ + + return '-'.join([ + *uuid_key.split('-')[:2], + dt.datetime.utcnow().strftime('%Y%m%d-%H%M%S'), + *references + ])[:63] + +def get_processing_input(name: str, inputs_s3_uri: str, output_local_path: str): + return { + 'InputName': name, + 'S3Input': { + 'S3DataType': 'S3Prefix', + 'S3Uri': inputs_s3_uri, + 'LocalPath': output_local_path, + 'S3DataDistributionType': 'FullyReplicated', + 'S3InputMode': 'File' + } + } + +def get_processing_output(name: str, input_local_path: str, outputs_s3_uri: str): + return { + 'OutputName': name, + 'S3Output': { + 'LocalPath': input_local_path, + 'S3Uri': outputs_s3_uri, + 'S3UploadMode': 'EndOfJob' + } + } From c988416b824cd6edeb87019f09be6381c5d1e9cb Mon Sep 17 00:00:00 2001 From: Joschuan Santana Date: Fri, 8 Mar 2024 17:11:50 -0300 Subject: [PATCH 3/7] Implement cloudformation template --- .gitignore | 3 +- deployment/samconfig.yaml | 30 +++ deployment/template.yaml | 187 ++++++++++++++++++ events/roof-energy-event.json | 27 +++ .../RoofEnergyInferenceWorkflow.yaml | 45 +++++ 5 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 deployment/samconfig.yaml create mode 100644 deployment/template.yaml create mode 100644 events/roof-energy-event.json create mode 100644 src/stepfunctions/RoofEnergyInferenceWorkflow.yaml diff --git a/.gitignore b/.gitignore index c6fd4a7..bc2f197 100644 --- a/.gitignore +++ b/.gitignore @@ -94,4 +94,5 @@ _build/ # setuptools-scm/ src/*/_version.py data/ -models/ \ No newline at end of file +models/ +.aws-sam/ \ No newline at end of file diff --git a/deployment/samconfig.yaml b/deployment/samconfig.yaml new file mode 100644 index 0000000..21595bb --- /dev/null +++ b/deployment/samconfig.yaml @@ -0,0 +1,30 @@ +# More information about the configuration file can be found here: +# https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html +version: 0.1 + +default: + global: + parameters: + stack_name: sunscan-serverless-dev + build: + parameters: + cached: true + parallel: true + validate: + parameters: + lint: true + deploy: + parameters: + region: us-east-1 + s3_bucket: project-deployment-templates + s3_prefix: sunscan-serverless + capabilities: + - CAPABILITY_NAMED_IAM + - CAPABILITY_AUTO_EXPAND + tags: + - Product=Sunscan + - Environment=dev + parameter_overrides: + - Mode=dev + - AttachmentsBucketName=infra-attachments-dev + - ArtifactsBucketName=infra-artifacts-dev diff --git a/deployment/template.yaml b/deployment/template.yaml new file mode 100644 index 0000000..1c98209 --- /dev/null +++ b/deployment/template.yaml @@ -0,0 +1,187 @@ +AWSTemplateFormatVersion: 2010-09-09 +Transform: AWS::Serverless-2016-10-31 +Description: sunscan-serverless + +Parameters: + AttachmentsBucketName: + Type: String + Description: Attachments bucket name + ArtifactsBucketName: + Type: String + Description: Artifacts bucket name + Mode: + Type: String + Description: Environment context + AllowedValues: + - dev + - test + - prod + + +Globals: + Function: + Timeout: 30 + Runtime: python3.10 + Environment: + Variables: + LOG_LEVEL: INFO + POWERTOOLS_LOGGER_SAMPLE_RATE: 0.1 + POWERTOOLS_LOGGER_LOG_EVENT: true + + +Resources: + LambdaRole: + Type: AWS::IAM::Role + Properties: + Path: /service-role/ + RoleName: !Sub 'SunscanLambdaRole-${Mode}' + Description: Sunscan Lambda service role. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + + StatesRole: + Type: AWS::IAM::Role + Properties: + Path: /service-role/ + RoleName: !Sub 'SunscanStatesRole-${Mode}' + Description: Sunscan States service role. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: states.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: sunscan-states-policy + PolicyDocument: + Version: '2012-10-17' + Statement: + - + Effect: Allow + Action: + - xray:PutTraceSegments + - xray:PutTelemetryRecords + - xray:GetSamplingRules + - xray:GetSamplingTargets + Resource: '*' + - + Effect: Allow + Action: + - events:PutTargets + - events:PutRule + - events:DescribeRule + Resource: !Sub 'arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForSageMakerProcessingJobsRule' + - + Effect: Allow + Action: lambda:InvokeFunction + Resource: !GetAtt BuildPayloadFunction.Arn + - + Effect: Allow + Action: + - sagemaker:CreateProcessingJob + - sagemaker:DescribeProcessingJob + - sagemaker:StopProcessingJob + - sagemaker:AddTags + Resource: !Sub 'arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:processing-job/*' + - + Effect: Allow + Action: iam:PassRole + Resource: !GetAtt SageMakerRole.Arn + + SageMakerRole: + Type: AWS::IAM::Role + Properties: + Path: /service-role/ + RoleName: !Sub 'SunscanSageMakerRole-${Mode}' + Description: Sunscan SageMaker service role. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: sagemaker.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess + Policies: + - PolicyName: sunscan-lambda-policy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - cloudwatch:PutMetricData + - logs:CreateLogStream + - logs:CreateLogGroup + - logs:DescribeLogStreams + - logs:PutLogEvents + - ecr:GetAuthorizationToken + Resource: '*' + - Effect: Allow + Action: + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + Resource: !GetAtt RoofEnergyInferenceRepository.Arn + - Effect: Allow + Action: s3:ListBucket + Resource: + - !Sub 'arn:aws:s3:::${AttachmentsBucketName}' + - !Sub 'arn:aws:s3:::${ArtifactsBucketName}' + - Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + Resource: !Sub 'arn:aws:s3:::${AttachmentsBucketName}/sunscan/*' + - Effect: Allow + Action: s3:GetObject + Resource: !Sub 'arn:aws:s3:::${ArtifactsBucketName}/sunscan/sagemaker/*' + + RoofEnergyInferenceRepository: + Type: AWS::ECR::Repository + Properties: + ImageTagMutability: MUTABLE + RepositoryName: !Sub 'sunscan/roof-energy-inference-${Mode}' + + BuildPayloadFunction: + Type: AWS::Serverless::Function + Properties: + Role: !GetAtt LambdaRole.Arn + FunctionName: !Sub 'sunscan-build-payload-${Mode}' + Description: Prepare input data to parameters on workflow. + CodeUri: ../src/lambda/build-payload + Handler: lambda_function.lambda_handler + Environment: + Variables: + ATTACHMENTS_BUCKET_NAME: !Ref AttachmentsBucketName + BASE_CODE_S3URI: !Sub 's3://${ArtifactsBucketName}/sunscan/sagemaker/roof-energy' + + RoofEnergyInferenceWorkflow: + Type: AWS::Serverless::StateMachine + Properties: + Role: !GetAtt StatesRole.Arn + Name: !Sub 'sunscan-roof-energy-inference-workflow-${Mode}' + Definition: + Fn::Transform: + Name: AWS::Include + Parameters: + Location: ../src/stepfunctions/RoofEnergyInferenceWorkflow.yaml + DefinitionSubstitutions: + Mode: !Ref Mode + BuildPayloadFunctionArn: !GetAtt BuildPayloadFunction.Arn + InferenceRoleArn: !GetAtt SageMakerRole.Arn + InferenceImageUri: !GetAtt RoofEnergyInferenceRepository.RepositoryUri + InferenceInstanceType: ml.m5.xlarge + Tags: + Application: Sunscan + Product: Infradigital + Environment: !Ref Mode diff --git a/events/roof-energy-event.json b/events/roof-energy-event.json new file mode 100644 index 0000000..34db316 --- /dev/null +++ b/events/roof-energy-event.json @@ -0,0 +1,27 @@ +{ + "version": "0", + "id": "8728a076-d5d1-c624-124a-946b1b65e33b", + "detail-type": "Infradigital Application Task Requested", + "source": "infra.core", + "account": "123456789123", + "time": "2023-05-01T19:08:34Z", + "region": "us-east-1", + "resources": [ + "arn:aws:states:us-east-1:123456789123:stateMachine:infra-task-manager-dev", + "arn:aws:states:us-east-1:123456789123:execution:infra-task-manager-dev:05cd1ed4-6277-383d-a3ef-e36e63862e57" + ], + "detail": { + "TaskKey": "a4066ee9-09e3-4f58-895a-6ef9f0445456", + "AppServiceSlug": "sunscan#roof_energy_inference", + "UserSub": "2ecf2bb8-c700-4073-9d48-2745815dcd0d", + "TenantId": "iadb:ine:tsp", + "AccessLevel": "user", + "Name": "Miami neighbourhood", + "Inputs": { + "BoundingBox": [ -73.2316, 9.587, -73.2309, 9.5875 ], + "PanelSize": 4, + "AvailableArea": 0.5, + "PanelPower": 400 + } + } +} \ No newline at end of file diff --git a/src/stepfunctions/RoofEnergyInferenceWorkflow.yaml b/src/stepfunctions/RoofEnergyInferenceWorkflow.yaml new file mode 100644 index 0000000..8c3a08c --- /dev/null +++ b/src/stepfunctions/RoofEnergyInferenceWorkflow.yaml @@ -0,0 +1,45 @@ +Comment: sunscan#roof_energy_inference task inference workflow. +StartAt: BuildPayload + +States: + BuildPayload: + Type: Task + Resource: ${BuildPayloadFunctionArn} + Comment: Build input payload to execute SageMaker inference. + InputPath: $.detail + ResultPath: $.InferenceParameters + Next: InferenceRoofEnergy + + InferenceRoofEnergy: + Type: Task + Resource: arn:aws:states:::sagemaker:createProcessingJob.sync + Parameters: + RoleArn: ${InferenceRoleArn} + ProcessingJobName.$: $.InferenceParameters.JobName + ProcessingInputs.$: $.InferenceParameters.InputConfig + ProcessingOutputConfig: + Outputs.$: $.InferenceParameters.OutputConfig + AppSpecification: + ContainerEntrypoint: [python, /opt/ml/processing/code/entrypoint.py] + ContainerArguments.$: $.InferenceParameters.ContainerArguments + ImageUri: ${InferenceImageUri} + ProcessingResources: + ClusterConfig: + InstanceCount: 1 + InstanceType: ${InferenceInstanceType} + VolumeSizeInGB: 30 + Tags: + - Key: Application + Value: Sunscan + - Key: Product + Value: Infradigital + - Key: Environment + Value: ${Mode} + ResultSelector: + ProcessingJobArn.$: $.ProcessingJobArn + ProcessingJobName.$: $.ProcessingJobName + ProcessingJobStatus.$: $.ProcessingJobStatus + ProcessingStartTime.$: $.ProcessingStartTime + ProcessingEndTime.$: $.ProcessingEndTime + ResultPath: $.InferenceResult + End: True From c77b189bc5b2f86f73503c63daac1effec996773 Mon Sep 17 00:00:00 2001 From: Joschuan Santana Date: Fri, 8 Mar 2024 19:31:01 -0300 Subject: [PATCH 4/7] Upgrade inference to GPU --- deployment/template.yaml | 4 ++-- src/ecr/roof-energy-inference-cpu/Dockerfile | 2 +- src/ecr/roof-energy-inference-gpu/Dockerfile | 12 ++++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 src/ecr/roof-energy-inference-gpu/Dockerfile diff --git a/deployment/template.yaml b/deployment/template.yaml index 1c98209..d9e992b 100644 --- a/deployment/template.yaml +++ b/deployment/template.yaml @@ -179,8 +179,8 @@ Resources: Mode: !Ref Mode BuildPayloadFunctionArn: !GetAtt BuildPayloadFunction.Arn InferenceRoleArn: !GetAtt SageMakerRole.Arn - InferenceImageUri: !GetAtt RoofEnergyInferenceRepository.RepositoryUri - InferenceInstanceType: ml.m5.xlarge + InferenceImageUri: !Sub '${RoofEnergyInferenceRepository.RepositoryUri}:2.1.0-gpu-py310' + InferenceInstanceType: ml.p3.2xlarge Tags: Application: Sunscan Product: Infradigital diff --git a/src/ecr/roof-energy-inference-cpu/Dockerfile b/src/ecr/roof-energy-inference-cpu/Dockerfile index 4423703..156b53b 100644 --- a/src/ecr/roof-energy-inference-cpu/Dockerfile +++ b/src/ecr/roof-energy-inference-cpu/Dockerfile @@ -1,7 +1,7 @@ # https://aws.amazon.com/cn/releasenotes/available-deep-learning-containers-images/ # https://github.com/aws/deep-learning-containers/blob/master/available_images.md -FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1-cpu-py310 +FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1.0-cpu-py310-ubuntu20.04-sagemaker RUN apt update && apt install -y binutils libproj-dev gdal-bin libgdal-dev diff --git a/src/ecr/roof-energy-inference-gpu/Dockerfile b/src/ecr/roof-energy-inference-gpu/Dockerfile new file mode 100644 index 0000000..70ebeb3 --- /dev/null +++ b/src/ecr/roof-energy-inference-gpu/Dockerfile @@ -0,0 +1,12 @@ +# https://aws.amazon.com/cn/releasenotes/available-deep-learning-containers-images/ +# https://github.com/aws/deep-learning-containers/blob/master/available_images.md + +FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1.0-gpu-py310-cu121-ubuntu20.04-sagemaker + +RUN apt update && apt install -y binutils libproj-dev gdal-bin libgdal-dev + +RUN python3 -m pip install --upgrade pip --no-cache-dir && \ + pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 + +RUN pip3 install setuptools==57.5.0 && \ + pip3 install gdal==$(gdal-config --version) From e4fdaf4c66a68544a21edf640a9af704cb2ce651 Mon Sep 17 00:00:00 2001 From: Joschuan Santana Date: Fri, 8 Mar 2024 19:46:40 -0300 Subject: [PATCH 5/7] Fix sonarcloud issues 1 --- deployment/template.yaml | 25 ++++++++++---------- src/ecr/roof-energy-inference-cpu/Dockerfile | 12 ++++++---- src/ecr/roof-energy-inference-gpu/Dockerfile | 12 ++++++---- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/deployment/template.yaml b/deployment/template.yaml index d9e992b..377c8c7 100644 --- a/deployment/template.yaml +++ b/deployment/template.yaml @@ -64,35 +64,30 @@ Resources: PolicyDocument: Version: '2012-10-17' Statement: - - - Effect: Allow + - Effect: Allow Action: - xray:PutTraceSegments - xray:PutTelemetryRecords - xray:GetSamplingRules - xray:GetSamplingTargets - Resource: '*' - - - Effect: Allow + Resource: '*' # NOSONAR + - Effect: Allow Action: - events:PutTargets - events:PutRule - events:DescribeRule Resource: !Sub 'arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForSageMakerProcessingJobsRule' - - - Effect: Allow + - Effect: Allow Action: lambda:InvokeFunction Resource: !GetAtt BuildPayloadFunction.Arn - - - Effect: Allow + - Effect: Allow Action: - sagemaker:CreateProcessingJob - sagemaker:DescribeProcessingJob - sagemaker:StopProcessingJob - sagemaker:AddTags Resource: !Sub 'arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:processing-job/*' - - - Effect: Allow + - Effect: Allow Action: iam:PassRole Resource: !GetAtt SageMakerRole.Arn @@ -125,7 +120,7 @@ Resources: - logs:DescribeLogStreams - logs:PutLogEvents - ecr:GetAuthorizationToken - Resource: '*' + Resource: '*' # NOSONAR - Effect: Allow Action: - ecr:BatchCheckLayerAvailability @@ -165,6 +160,12 @@ Resources: ATTACHMENTS_BUCKET_NAME: !Ref AttachmentsBucketName BASE_CODE_S3URI: !Sub 's3://${ArtifactsBucketName}/sunscan/sagemaker/roof-energy' + BuildPayloadFunctionLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub '/aws/lambda/${BuildPayloadFunction}' + RetentionInDays: 30 + RoofEnergyInferenceWorkflow: Type: AWS::Serverless::StateMachine Properties: diff --git a/src/ecr/roof-energy-inference-cpu/Dockerfile b/src/ecr/roof-energy-inference-cpu/Dockerfile index 156b53b..2b284fa 100644 --- a/src/ecr/roof-energy-inference-cpu/Dockerfile +++ b/src/ecr/roof-energy-inference-cpu/Dockerfile @@ -3,10 +3,12 @@ FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1.0-cpu-py310-ubuntu20.04-sagemaker -RUN apt update && apt install -y binutils libproj-dev gdal-bin libgdal-dev +RUN apt update \ + && apt install --no-install-recommends -y binutils libproj-dev gdal-bin libgdal-dev \ + && apt clean -RUN python3 -m pip install --upgrade pip --no-cache-dir && \ - pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 +RUN python3 -m pip install --upgrade pip --no-cache-dir \ + && pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 -RUN pip3 install setuptools==57.5.0 && \ - pip3 install gdal==$(gdal-config --version) +RUN pip3 install setuptools==57.5.0 \ + && pip3 install gdal==$(gdal-config --version) diff --git a/src/ecr/roof-energy-inference-gpu/Dockerfile b/src/ecr/roof-energy-inference-gpu/Dockerfile index 70ebeb3..c013883 100644 --- a/src/ecr/roof-energy-inference-gpu/Dockerfile +++ b/src/ecr/roof-energy-inference-gpu/Dockerfile @@ -3,10 +3,12 @@ FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1.0-gpu-py310-cu121-ubuntu20.04-sagemaker -RUN apt update && apt install -y binutils libproj-dev gdal-bin libgdal-dev +RUN apt update \ + && apt install --no-install-recommends -y binutils libproj-dev gdal-bin libgdal-dev \ + && apt clean -RUN python3 -m pip install --upgrade pip --no-cache-dir && \ - pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 +RUN python3 -m pip install --upgrade pip --no-cache-dir \ + && pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 -RUN pip3 install setuptools==57.5.0 && \ - pip3 install gdal==$(gdal-config --version) +RUN pip3 install setuptools==57.5.0 \ + && pip3 install gdal==$(gdal-config --version) From f6d3824f23b80849b727019bc623c2b3476f4960 Mon Sep 17 00:00:00 2001 From: Joschuan Santana Date: Thu, 9 May 2024 12:08:57 -0300 Subject: [PATCH 6/7] Remove dockerfiles --- src/ecr/roof-energy-inference-cpu/Dockerfile | 14 -------------- src/ecr/roof-energy-inference-gpu/Dockerfile | 14 -------------- 2 files changed, 28 deletions(-) delete mode 100644 src/ecr/roof-energy-inference-cpu/Dockerfile delete mode 100644 src/ecr/roof-energy-inference-gpu/Dockerfile diff --git a/src/ecr/roof-energy-inference-cpu/Dockerfile b/src/ecr/roof-energy-inference-cpu/Dockerfile deleted file mode 100644 index 2b284fa..0000000 --- a/src/ecr/roof-energy-inference-cpu/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# https://aws.amazon.com/cn/releasenotes/available-deep-learning-containers-images/ -# https://github.com/aws/deep-learning-containers/blob/master/available_images.md - -FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1.0-cpu-py310-ubuntu20.04-sagemaker - -RUN apt update \ - && apt install --no-install-recommends -y binutils libproj-dev gdal-bin libgdal-dev \ - && apt clean - -RUN python3 -m pip install --upgrade pip --no-cache-dir \ - && pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 - -RUN pip3 install setuptools==57.5.0 \ - && pip3 install gdal==$(gdal-config --version) diff --git a/src/ecr/roof-energy-inference-gpu/Dockerfile b/src/ecr/roof-energy-inference-gpu/Dockerfile deleted file mode 100644 index c013883..0000000 --- a/src/ecr/roof-energy-inference-gpu/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# https://aws.amazon.com/cn/releasenotes/available-deep-learning-containers-images/ -# https://github.com/aws/deep-learning-containers/blob/master/available_images.md - -FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.1.0-gpu-py310-cu121-ubuntu20.04-sagemaker - -RUN apt update \ - && apt install --no-install-recommends -y binutils libproj-dev gdal-bin libgdal-dev \ - && apt clean - -RUN python3 -m pip install --upgrade pip --no-cache-dir \ - && pip3 install --upgrade --no-cache-dir astral==3.2 groundingdino-py==0.4.0 leafmap==0.31.3 openpyxl==3.1.2 segment-geospatial==0.10.2 - -RUN pip3 install setuptools==57.5.0 \ - && pip3 install gdal==$(gdal-config --version) From 2325c4b45c98bcd321feafbc8daac6658dda7a35 Mon Sep 17 00:00:00 2001 From: Joschuan Santana Date: Thu, 9 May 2024 12:18:15 -0300 Subject: [PATCH 7/7] Depricated datetime.utcnow --- src/lambda/build-payload/parameter_mappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda/build-payload/parameter_mappers.py b/src/lambda/build-payload/parameter_mappers.py index 41f48af..92c27d0 100644 --- a/src/lambda/build-payload/parameter_mappers.py +++ b/src/lambda/build-payload/parameter_mappers.py @@ -1,4 +1,4 @@ -import datetime as dt +from datetime import datetime, timezone def get_unique_job_name(uuid_key: str, *references: list): """ Returns a unique job name based on a given uuid_key @@ -6,7 +6,7 @@ def get_unique_job_name(uuid_key: str, *references: list): return '-'.join([ *uuid_key.split('-')[:2], - dt.datetime.utcnow().strftime('%Y%m%d-%H%M%S'), + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'), *references ])[:63]