Merge pull request #2549 from codeenigma/Updating-aws_admin_tools-role-with-new-option-and-setting-up-general-s3-bucket-name-PR-2.x

Updating-aws_admin_tools-role-with-new-option-and-setting-up-general-…

Author: matej5

roles/aws/aws_admin_tools/defaults/main.yml

@@ -45,3 +45,10 @@ aws_admin_tools:
         resource: "*"
         action:
           - "wafv2:GetWebACL"
+    - name: "get_infra_data_from_s3" # This is the bit that sets up the API Gateway to pull and share infra data over API. No Lambda function needed here though. How do we do this? See docs
+      resource: s3
+      type: GET
+      bucket: "codeenigma-{{ _aws_profile }}-general-storage-{{ _aws_region }}"
+      object: "lambda-results/infra-info"
+      url_params:
+        - date

roles/aws/aws_admin_tools/tasks/create.yml

@@ -37,17 +37,26 @@
     - _api_index | length > 0
     - _old_api_res_list is defined
 
-- name: Extract the scheduled functions from the list.
+- name: Extract the API Gateway functions from the list.
   ansible.builtin.set_fact:
     _api_functions: "{{ aws_admin_tools.functions | selectattr('resource', 'eq', 'api') | list }}"
 
-- name: Extract the API Gateway functions from the list.
+- name: Extract the scheduled functions from the list.
   ansible.builtin.set_fact:
     _schedule_functions: "{{ aws_admin_tools.functions | selectattr('resource', 'eq', 'schedule') | list }}"
 
+- name: Extract the API Gateway with S3 from the list.
+  ansible.builtin.set_fact:
+    _api_s3: "{{ aws_admin_tools.functions | selectattr('resource', 'eq', 's3') | list }}"
+
+- name: Remove old resources if API Gateway exists.
+  ansible.builtin.include_tasks: remove_old_reources.yml
+  loop: "{{ _api_functions + _api_s3 }}"
+  when: _api_index | length > 0
+
 - name: Create resources and set methods on API Gateway.
   ansible.builtin.include_tasks: create_methods.yml
-  loop: "{{ _api_functions }}"
+  loop: "{{ _api_functions + _api_s3 }}"
 
 - name: Create Event Bridge schedule.
   ansible.builtin.include_tasks: create_schedule.yml
@@ -113,3 +122,5 @@
     --source-arn arn:aws:execute-api:{{ _aws_region }}:{{ _acc_id }}:{{ _api_gate.id }}/*/{{ item.type }}/{{ item.name }}
     --region {{ _aws_region }}
   loop: "{{ _api_functions }}"
+
+# TODO Add scheduled triggers

roles/aws/aws_admin_tools/tasks/create_methods.yml

@@ -1,27 +1,3 @@
-- name: Get resources.
-  ansible.builtin.command: >-
-    aws apigateway get-resources
-    --rest-api-id "{{ _api_gate.id }}"
-    --region "{{ _aws_region }}"
-  register: _api_old_resource
-
-- name: Setting previous command output into variable.
-  ansible.builtin.set_fact:
-    _api_old_resource: "{{ _api_old_resource.stdout | from_json }}"
-
-- name: Find the index of root resource.
-  ansible.builtin.set_fact:
-    _api_old_resource_index: "{{ lookup('ansible.utils.index_of', _api_old_resource['items'], 'eq', '/' + item.name, 'path', wantlist=True) }}"
-
-- name: Delete resource.
-  ansible.builtin.command: >-
-    aws apigateway delete-resource
-    --rest-api-id "{{ _api_gate.id }}"
-    --resource-id "{{ _api_old_resource['items'][_api_old_resource_index[0]].id }}"
-    --region "{{ _aws_region }}"
-  register: _api_old_resource
-  when: _api_old_resource_index | length > 0
-
 - name: Create resource on API gateway.
   ansible.builtin.command: >-
     aws apigateway create-resource
@@ -64,17 +40,59 @@
     --region "{{ _aws_region }}"
   when: item.url_params is defined and item.url_params | length > 0
 
-- name: Add Lambda for method without URL params.
+- name: Build arn for S3 bucket.
+  ansible.builtin.set_fact:
+    _target_arn: "arn:aws:apigateway:{{ _aws_region }}:s3:path/{{ item.bucket }}/{{ item.object }}"
+  when: item.resource == "s3"
+
+- name: List Objects in S3 bucket.
+  amazon.aws.s3_object:
+    bucket: "{{ item.bucket }}"
+    mode: list
+  register: _objects_in_s3_bucket
+  when: item.resource == "s3"
+
+- name: Set _object_missing to false if object exist.
+  ansible.builtin.set_fact:
+    _object_missing: false
+  when: item.resource == "s3" and item.object in _objects_in_s3_bucket.s3_keys
+
+- name: Set _object_missing to true if object doesn't exist.
+  ansible.builtin.set_fact:
+    _object_missing: true
+  when: item.resource == "s3" and item.object not in _objects_in_s3_bucket.s3_keys
+
+- name: Create default file for S3.
+  ansible.builtin.template:
+    src: default_s3_object.j2
+    dest: default_s3_object
+  when: _object_missing is defined and _object_missing
+
+- name: Add default file to S3 bucket if it doesn't exist.
+  amazon.aws.s3_object:
+    bucket: "{{ item.bucket }}"
+    object: "{{ item.object }}"
+    src: "default_s3_object"
+    mode: put
+  when: _object_missing is defined and _object_missing
+
+- name: Build arn for Lambda.
+  ansible.builtin.set_fact:
+    _target_arn: "arn:aws:apigateway:{{ _aws_region }}:lambda:path/2015-03-31/functions/arn:aws:lambda:{{ _aws_region }}:{{ _acc_id }}:function:api_{{ item.name }}/invocations"
+  when: item.resource == "api"
+
+- name: Add Service for method without URL params.
   ansible.builtin.command: >-
     aws apigateway put-integration
     --rest-api-id "{{ _api_gate.id }}"
     --resource-id "{{ _api_resource.id }}"
     --http-method "{{ item.type }}"
     --type AWS
     --content-handling CONVERT_TO_TEXT
-    --integration-http-method POST
-    --uri "arn:aws:apigateway:{{ _aws_region }}:lambda:path/2015-03-31/functions/arn:aws:lambda:{{ _aws_region }}:{{ _acc_id }}:function:api_{{ item.name }}/invocations"
+    --integration-http-method {{ 'GET' if item.resource == 's3' else 'POST' }}
+    --uri "{{ _target_arn }}"
     --region {{ _aws_region }}
+    {{ '--credentials "arn:aws:iam::' + _acc_id + ':role/api_get_s3"' if item.resource == 's3' else '' }}
   when: item.url_params is not defined or item.url_params | length == 0
 
 - name: Generate template parts for each param
@@ -93,19 +111,20 @@
     dest: integration_template
   when: item.url_params is defined and item.url_params | length > 0
 
-- name: Add Lambda for method with URL params.
+- name: Add Service for method with URL params.
   ansible.builtin.command: >-
     aws apigateway put-integration
     --rest-api-id "{{ _api_gate.id }}"
     --resource-id "{{ _api_resource.id }}"
     --http-method "{{ item.type }}"
     --type AWS
     --content-handling CONVERT_TO_TEXT
-    --integration-http-method POST
+    --integration-http-method {{ 'GET' if item.resource == 's3' else 'POST' }}
     --passthrough-behavior WHEN_NO_TEMPLATES
     --request-template file://integration_template
-    --uri "arn:aws:apigateway:{{ _aws_region }}:lambda:path/2015-03-31/functions/arn:aws:lambda:{{ _aws_region }}:{{ _acc_id }}:function:api_{{ item.name }}/invocations"
+    --uri "{{ _target_arn }}"
     --region {{ _aws_region }}
+    {{ '--credentials "arn:aws:iam::' + _acc_id + ':role/api_get_s3"' if item.resource == 's3' else '' }}
   when: item.url_params is defined and item.url_params | length > 0
 
 - name: Add method response.

roles/aws/aws_admin_tools/tasks/lambda_functions.yml

@@ -14,6 +14,7 @@
       role: "{{ aws_iam_role._result[item.resource + '_' + item.name] }}"
       runtime: "{{ aws_admin_tools.runtime }}"
       function_file: "{{ lookup('template', item.resource + '_' + item.name + '.py.j2') }}"
-      s3_bucket: "ce-{{ _aws_profile }}-lambda-functions"
+      s3_bucket: "codeenigma-{{ _aws_profile }}-general-storage-{{ _aws_region }}"
+      s3_bucket_prefix: "lambda-functions"
       tags:
         Name: "{{ item.resource }}_{{ item.name }}"

roles/aws/aws_admin_tools/tasks/lambda_iam.yml

@@ -11,4 +11,4 @@
       aws_profile: "{{ _aws_profile }}"
       managed_policies: "{{ _policies }}"
       inline_policies: "{{ item.inline_policies | default(omit) }}"
-      policy_document: "{{ lookup('template', 'trusted_entitites.j2') }}"
+      policy_document: "{{ lookup('template', 'trust_lambda.j2') }}"

roles/aws/aws_admin_tools/tasks/main.yml

@@ -33,7 +33,7 @@
     _api_gate: "{{ _api_gate.stdout | from_json }}"
   when: _api_index | length == 0
 
-- name: Setting _api_index if API already exists.
+- name: Setting _api_gate if API already exists.
   ansible.builtin.set_fact:
     _api_gate: "{{ _api_gate_list.rest_apis[_api_index[0]] }}"
   when: _api_index | length > 0
@@ -72,13 +72,33 @@
   ansible.builtin.set_fact:
     _main_api_deploy: "{{ _main_api_deploy_tmp['items'] | last }}"
 
+- name: Extract the API Gateway without s3 for resource.
+  ansible.builtin.set_fact:
+    _api_without_s3: "{{ aws_admin_tools.functions | rejectattr('resource', 'eq', 's3') | list }}"
+
+- name: Set CloudWatch logs full access for apigateway.
+  ansible.builtin.set_fact:
+    _policies: "{{ ['arn:aws:iam::aws:policy/CloudWatchLogsFullAccess'] }}"
+
+- name: Create a policy for API Gateway to assume role.
+  ansible.builtin.include_role:
+    name: aws/aws_iam_role
+  vars:
+    aws_iam_role:
+      name: "api_get_s3"
+      aws_profile: "{{ _aws_profile }}"
+      managed_policies: "{{ _policies }}"
+      inline_policies:
+        - "s3:GetObject"
+      policy_document: "{{ lookup('template', 'trust_apigateway.j2') }}"
+
 - name: Configure Lambda IAM policies.
   ansible.builtin.include_tasks: lambda_iam.yml
-  loop: "{{ aws_admin_tools.functions }}"
+  loop: "{{ _api_without_s3 }}"
 
 - name: Configure Lambda functions.
   ansible.builtin.include_tasks: lambda_functions.yml
-  loop: "{{ aws_admin_tools.functions }}"
+  loop: "{{ _api_without_s3 }}"
 
 - name: Create WAF for API Gateway.
   ansible.builtin.include_role:

roles/aws/aws_admin_tools/tasks/remove_old_reources.yml

@@ -0,0 +1,23 @@
+- name: Get resources.
+  ansible.builtin.command: >-
+    aws apigateway get-resources
+    --rest-api-id "{{ _api_gate.id }}"
+    --region "{{ _aws_region }}"
+  register: _api_old_resource
+
+- name: Setting previous command output into variable.
+  ansible.builtin.set_fact:
+    _api_old_resource: "{{ _api_old_resource.stdout | from_json }}"
+
+- name: Find the index of root resource.
+  ansible.builtin.set_fact:
+    _api_old_resource_index: "{{ lookup('ansible.utils.index_of', _api_old_resource['items'], 'eq', '/' + item.name, 'path', wantlist=True) }}"
+
+- name: Delete resource.
+  ansible.builtin.command: >-
+    aws apigateway delete-resource
+    --rest-api-id "{{ _api_gate.id }}"
+    --resource-id "{{ _api_old_resource['items'][_api_old_resource_index[0]].id }}"
+    --region "{{ _aws_region }}"
+  register: _api_old_resource
+  when: _api_old_resource_index | length > 0

roles/aws/aws_admin_tools/templates/default_s3_object.j2

@@ -0,0 +1 @@
+Hello from S3!

roles/aws/aws_admin_tools/templates/get_infra_data_from_s3.py.j2

@@ -0,0 +1,12 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "apigateway.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}