diff --git a/README.md b/README.md index 20878418..279429a6 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,13 @@ Access to the "prod" environment is facilitated through a DNS URL, thanks to Rou `ssh ubuntu@[instance-id] -i keypair.pem -o ProxyCommand='aws ec2-instance-connect open-tunnel --instance-id %h'` -6. Now XC3 will run at 05:00AM UTC every day to generate data and populate Grafana. Few lambdas (Total Account Cost and Project spend) will run twice in a month. +6. Go to AWS Systems Manager, select Parameter Store, and create a new parameter named "/{namespace}/region_names". Set the value as a dictionary with region IDs as keys and region names as values. + +7. SSH into the private instance using EIC Endpoint to check if everything is working fine. Here replace [instance-id] needs to be replaced with ID + + `ssh ubuntu@[instance-id] -i keypair.pem -o ProxyCommand='aws ec2-instance-connect open-tunnel --instance-id %h'` + +8. Now XC3 will run at 05:00AM UTC every day to generate data and populate Grafana. Few lambdas (Total Account Cost and Project spend) will run twice in a month. Note : 1. If data is not available in Grafana UI then follow the troubleshooting guide at the last section of this page. @@ -172,6 +178,8 @@ case 1: If data is not showing into Grafana UI, there could be several reasons a 3. XC3 Budget Detail/IAM Role/User Workflow lambda may have failed to execute , please check Cloudwatch logs to address the issue. +4. Check if XC3's most expensive services data is missing, and if so, verify the existence of the corresponding SSM parameter in AWS Systems Manager. To address this issue, ensure you follow step 5 of the deployment instructions. + case 2: user not able to change/update/modify default dashboards in Grafana UI 1. You can't change/update default dashboards. diff --git a/User Manual Federated User.pdf b/User Manual Federated User.pdf new file mode 100644 index 00000000..fac02f86 Binary files /dev/null and b/User Manual Federated User.pdf differ diff --git a/custom_dashboard/dummy_local/data_push1.py b/custom_dashboard/dummy_local/data_push1.py new file mode 100755 index 00000000..0c1efb4b --- /dev/null +++ b/custom_dashboard/dummy_local/data_push1.py @@ -0,0 +1,50 @@ +import json +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway + +# Load the JSON data +with open('dummy_data1.json', 'r') as file: + data1 = json.load(file) + +def push_to_prometheus(data1): + registry = CollectorRegistry() + + # Create a single gauge with multiple labels + gauge = Gauge( + "FED_USER_Resource_Cost_List", + "FED USER Resource List And Cost", + labelnames=[ + "resource_id", + "resource", + "account_id", + "region", + "resource_name", + "month" + ], + registry=registry + ) + + # Iterate through the data and populate the gauge + for resource in data1['ec2']: + resource_id = resource['resource_id'] + service = resource['resource'] + cost = resource['cost'] + account_id = resource['account_id'] + region = resource['region'] + month = resource['month'] + resource_name = resource_id.split(':')[-1] + + # Set the gauge value + gauge.labels( + resource_id=resource_id, + resource=service, + account_id=account_id, + region=region, + resource_name=resource_name, + month=month + ).set(cost) + + # Push the metrics to Prometheus Pushgateway + push_to_gateway('localhost:9091', job='pushgateway', registry=registry) + +# Call the function to push data to Prometheus +push_to_prometheus(data1) \ No newline at end of file diff --git a/custom_dashboard/dummy_local/dummy.md b/custom_dashboard/dummy_local/dummy.md new file mode 100755 index 00000000..82096fea --- /dev/null +++ b/custom_dashboard/dummy_local/dummy.md @@ -0,0 +1 @@ +This is reference file for the dummy data that Team 12 used to showcase the Local Host Dashboard in the presentation \ No newline at end of file diff --git a/custom_dashboard/dummy_local/dummy_data1.json b/custom_dashboard/dummy_local/dummy_data1.json new file mode 100755 index 00000000..9c0af079 --- /dev/null +++ b/custom_dashboard/dummy_local/dummy_data1.json @@ -0,0 +1,150 @@ +{ + "ec2": [ + { + "resource_id": "arn:aws:lambda:ap-southeast-2:590183937261:function:xc3team12pradhumna-list_linked_accounts", + "cost": 120.50, + "region": "us-east-1", + "resource": "lambda", + "account_id": "Pradhumna-Dhungana", + "month": "January", + "resource_name": "xc3team12bhavabc-metadata-storage" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:123456789012:function:xc3team12pradhumna-iamrolesservicemapping", + "cost": 85.75, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Pradhumna-Dhungana", + "month": "February", + "resource_name": "xc3team12bhavabc-iamrolestografana" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:234567890123:function:xc3team12pradhumna-list_iam_users", + "cost": 67.30, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Pradhumna-Dhungana", + "month": "March", + "resource_name": "xc3team12bhavabc-iamrolesservicemapping" + }, + { + "resource_id": "arn:aws:ec2:ap-southeast-2:345678901234:instance/i-0a3ea6b780458b3f5", + "cost": 215.00, + "region": "us-east-1", + "resource": "ec2", + "account_id": "Pradhumna-Dhungana", + "month": "April", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:456789012345:function:xc3team12pradhumna-project-cost-breakdown", + "cost": 50.25, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Nilanshu-Basnet", + "month": "January", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:567890123456:function:xc3team12pradhumna-iamrolestografana", + "cost": 33.10, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Nilanshu-Basnet", + "month": "February", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:s3:::xc3team12pradhumna-metadata-storage", + "cost": 45.00, + "region": "us-west-1", + "resource": "s3", + "account_id": "Nilanshu-Basnet", + "month": "March", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + + }, + { + "resource_id": "arn:aws:s3:::xc3team12sb-metadata-storage", + "cost": 27.15, + "region": "us-west-1", + "resource": "s3", + "account_id": "Nilanshu-Basnet", + "month": "April", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:890123456789:function:xc3team12pradhumna-resource_parsing_lambda", + "cost": 95.00, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Nilanshu-Basnet", + "month": "January", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:901234567890:function:xc3team12pradhumna-list_iam_user_resources_cost", + "cost": 120.00, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Nilanshu-Basnet", + "month": "February", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:012345678901:function:xc3team12pradhumna-resource_list_lambda", + "cost": 60.40, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Nilanshu-Basnet", + "month": "March", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:590183937261:function:xc3team12pradhumna-total_account_cost", + "cost": 130.50, + "region": "ap-southeast-2", + "resource": "lambda", + "account_id": "Pradhumna-Dhungana", + "month": "April", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:s3:ap-southeast-2:590183937261:function:xc3team12pradhumna-most_expensive_service_lambda", + "cost": 175.75, + "region": "ap-southeast-2", + "resource": "s3", + "account_id": "Pradhumna-Dhungana", + "month": "January", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:590183937261:function:xc3team12pradhumna-cost_metrics_of_expensive_services", + "cost": 49.95, + "region": "eu-west-1", + "resource": "lambda", + "account_id": "Pradhumna-Dhungana", + "month": "February", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:ec2:ap-southeast-2:590183937261:function:xc3team12pradhumna-instancestatechange", + "cost": 23.20, + "region": "ap-southeast-2", + "resource": "ec2", + "account_id": "Nilanshu-Basnet", + "month": "March", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + }, + { + "resource_id": "arn:aws:lambda:ap-southeast-2:590183937261:function:xc3team12pradhumna-project-spend-cost", + "cost": 88.40, + "region": "eu-west-1", + "resource": "lambda", + "account_id": "Nilanshu-Basnet", + "month": "April", + "resource_name": "xc3team12bhavabc-cost_metrics_of_expensive_services" + } + ] + } + \ No newline at end of file diff --git a/custom_dashboard/grafana_dashboards/associated-services.json b/custom_dashboard/grafana_dashboards/associated-services.json old mode 100644 new mode 100755 diff --git a/custom_dashboard/grafana_dashboards/fed-iam-user.json b/custom_dashboard/grafana_dashboards/fed-iam-user.json new file mode 100755 index 00000000..5acd1d18 --- /dev/null +++ b/custom_dashboard/grafana_dashboards/fed-iam-user.json @@ -0,0 +1,741 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Account Wise Cost", + "type": "row" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(FED_USER_Resource_Cost_List{account_id=~\".*\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Spent on Federated IAM Users", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 5, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "count(count by (account_id) (FED_USER_Resource_Cost_List))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Fed Users", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 60 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 10, + "x": 8, + "y": 1 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Cost Details", + "url": "/d/adiesz3lsuq68b/federated-iam-user-cost?orgId=1&var-Account_ID=All&var-month=All&editPanel=5" + } + ], + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "text": { + "valueSize": 20 + }, + "valueMode": "color" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(account_id) (FED_USER_Resource_Cost_List{account_id=~\"$Account_ID\", month=~\"$month\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{account_id}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Cost of all the Federated IAM User", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(resource) (sum without(region, resource_id, resource_name) (FED_USER_Resource_Cost_List{account_id=~\"$Account_ID\", month=~\"$month\"}))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cost Split in AWS Services", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "text": { + "titleSize": 12, + "valueSize": 20 + }, + "valueMode": "color" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sort(sum by(month) (FED_USER_Resource_Cost_List{month=~\"$month\"}))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cost by Month", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "maxVizHeight": 236, + "minVizHeight": 6, + "minVizWidth": 10, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "text": { + "titleSize": 12, + "valueSize": 20 + }, + "valueMode": "color" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(region) (FED_USER_Resource_Cost_List{account_id=~\"$Account_ID\", month=~\"$month\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{Region}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Region wise Cost", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 12, + "panels": [], + "title": "Resource Wise Cost", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 17, + "x": 1, + "y": 17 + }, + "id": 10, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "FED_USER_Resource_Cost_List{resource_name=~\"$resource_name\", account_id=~\"$Account_ID\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{resource_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Resource Wise Cost", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 17 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "count(count by (resource_name) (FED_USER_Resource_Cost_List))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active resources", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": "Pradhumna-Dhungana", + "value": "Pradhumna-Dhungana" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(FED_USER_Resource_Cost_List,account_id)", + "hide": 0, + "includeAll": true, + "label": "Fed User", + "multi": false, + "name": "Account_ID", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(FED_USER_Resource_Cost_List,account_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(FED_USER_Resource_Cost_List,month)", + "hide": 0, + "includeAll": true, + "label": "Month", + "multi": false, + "name": "month", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(FED_USER_Resource_Cost_List,month)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(FED_USER_Resource_Cost_List,resource_name)", + "hide": 0, + "includeAll": true, + "label": "Resource Name", + "multi": false, + "name": "resource_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(FED_USER_Resource_Cost_List,resource_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Federated IAM User Cost", + "uid": "adiesz3lsuq68b", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/custom_dashboard/grafana_dashboards/grafana-dashboard.md b/custom_dashboard/grafana_dashboards/grafana-dashboard.md old mode 100644 new mode 100755 index 99a58e8a..a559f0fd --- a/custom_dashboard/grafana_dashboards/grafana-dashboard.md +++ b/custom_dashboard/grafana_dashboards/grafana-dashboard.md @@ -37,6 +37,17 @@ The IAM User Dashboard consists of the following panels: - Resource Cost (Last 14 days) - Total Services Cost of IAM User +## **Federated IAM User Dashboard** + +The Federated IAM User Dashboard consists of the following panels: +-Total Spent on Federated IAM Users +-Active Federated IAM Users +-Cost by Month +-Total Cost of all the Federated IAM Users +-Cost Split in AWS Services +-Region-wise Cost + + ## **Tagging Compliance** The tagging compliance dashboard consists of the following panel: @@ -56,4 +67,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. +limitations under the License. \ No newline at end of file diff --git a/custom_dashboard/grafana_dashboards/grafana_alerts.md b/custom_dashboard/grafana_dashboards/grafana_alerts.md new file mode 100755 index 00000000..bcc9b180 --- /dev/null +++ b/custom_dashboard/grafana_dashboards/grafana_alerts.md @@ -0,0 +1,55 @@ +# Setting Up Grafana Alerts to Slack + +## Introduction + +Keeping the team updated with critical alerts is crucial for the swift response to any issues that may arise. This guide will cover the setup of Grafana alerts to be automatically sent to a designated Slack channel. + +## Prerequisites + +- A Grafana server instance with admin access. +- A Slack workspace with permissions to add apps and create webhooks. + +## Configuring Grafana Alerts for Slack + +### 1. Create a Slack Webhook + +- Navigate to your Slack workspace and create an incoming webhook: + - Go to `https://.slack.com/apps`. + - Search for **Incoming Webhooks** and select "Add Configuration". + - Choose the channel where messages will be posted. + - Copy the Webhook URL provided. + +### 2. Add Slack Notification Channel in Grafana + +- Log in to your Grafana instance as an administrator. +- Go to "Alerting" and select "Notification channels". +- Click on "Add channel". +- Choose "Slack" as the type. +- Paste the Slack Webhook URL in the corresponding field. +- Name your notification channel for easy identification. +- Save the new notification channel. + +### 3. Configure Alert Rules + +- Go to the Grafana dashboard with the panel you want to set an alert for. +- Right on the title of the panel and select "Alerts". +- Create a new alert rule and define your conditions. +- Under "Send to", select the Slack notification channel you created. +- Save the alert rule. + +### 4. Test Your Notification + +- Once your alert is set up, simulate the condition that triggers the alert or use the "Test" function to send a test message to Slack. +- Confirm that the message appears in the Slack channel. + +## Automating Alert Notifications + +Grafana can be configured to periodically check alert rules and send notifications to Slack when conditions are met. Here are the steps for automation: + +- Navigate to the alert rule you've created. +- Under "Alert frequency", set how often Grafana should evaluate the alert condition. +- Ensure "Notifications" are enabled and configured properly. + +## Conclusion + +With Grafana alerts now set to deliver notifications to your Slack channel, your team can enjoy real-time updates, ensuring prompt responses to critical metrics. Remember to fine-tune alert conditions to avoid spamming the channel with notifications. diff --git a/custom_dashboard/grafana_dashboards/home-dashboard.json b/custom_dashboard/grafana_dashboards/home-dashboard.json old mode 100644 new mode 100755 index 49e6c1cd..1ece0e69 --- a/custom_dashboard/grafana_dashboards/home-dashboard.json +++ b/custom_dashboard/grafana_dashboards/home-dashboard.json @@ -415,7 +415,7 @@ }, "editorMode": "builder", "exemplar": false, - "expr": "Expensive_Services_Detail{job=\"Most_Expensive_Services\", region=\"$region\", account=\"$Account\"}", + "expr": "Expensive_Services_Detail{job=\"$Account\", region=\"$region\"}", "format": "table", "instant": true, "legendFormat": "__auto", diff --git a/custom_dashboard/grafana_dashboards/iam-role-dashboard.json b/custom_dashboard/grafana_dashboards/iam-role-dashboard.json old mode 100644 new mode 100755 diff --git a/custom_dashboard/grafana_dashboards/iam-user-dashboard.json b/custom_dashboard/grafana_dashboards/iam-user-dashboard.json old mode 100644 new mode 100755 diff --git a/custom_dashboard/grafana_dashboards/services-breakdown.json b/custom_dashboard/grafana_dashboards/services-breakdown.json old mode 100644 new mode 100755 diff --git a/custom_dashboard/grafana_dashboards/tagging-compliance-dashboard.json b/custom_dashboard/grafana_dashboards/tagging-compliance-dashboard.json old mode 100644 new mode 100755 diff --git a/custom_dashboard/grafana_panel_start_stop/grafana_panel_start_stop.md b/custom_dashboard/grafana_panel_start_stop/grafana_panel_start_stop.md old mode 100644 new mode 100755 diff --git a/custom_dashboard/grafana_panel_start_stop/start_stop_panel.html b/custom_dashboard/grafana_panel_start_stop/start_stop_panel.html old mode 100644 new mode 100755 diff --git a/custom_dashboard/grafana_panel_start_stop/start_stop_panel.js b/custom_dashboard/grafana_panel_start_stop/start_stop_panel.js old mode 100644 new mode 100755 diff --git a/infrastructure/backend.tf b/infrastructure/backend.tf index 6f2f0ab9..eff5f709 100644 --- a/infrastructure/backend.tf +++ b/infrastructure/backend.tf @@ -16,9 +16,9 @@ terraform { backend "s3" { - bucket = "terraform-state-xc3" // S3 bucket for terraform state management + bucket = "terraform-state-team12nilanshu" // S3 bucket for terraform state management key = "xc3/xc3.tfstate" // Specifies the S3 object key for storing the Terraform state file - region = "eu-west-1" + region = "ap-southeast-2" dynamodb_table = "terraform-lock" } } diff --git a/infrastructure/config.sh b/infrastructure/config.sh index 3c1aaf7f..e78b43a9 100644 --- a/infrastructure/config.sh +++ b/infrastructure/config.sh @@ -15,12 +15,12 @@ # limitations under the License. # Variables values that will be used in init script to create resources for XC3 infrastructure -export aws_region="eu-west-1" +export aws_region="ap-southeast-2" export dynamo_table_name="terraform-lock" -export bucket_name="terraform-state-xc3" -export project="example" +export bucket_name="terraform-state-team12nilanshu" +export project="xc3team12nilanshu" export domain="example.test.co" -export owner_email="admin@test.co" -export creator_email="admin@test.co" -export namespace="example" -export env="example" +export owner_email="104346575@student.swin.edu.au" +export creator_email="104346575@student.swin.edu.au" +export namespace="xc3team12nilanshu" +export env="dev" diff --git a/infrastructure/errored.tfstate b/infrastructure/errored.tfstate new file mode 100644 index 00000000..95332b88 --- /dev/null +++ b/infrastructure/errored.tfstate @@ -0,0 +1,265 @@ +{ + "version": 4, + "terraform_version": "1.5.7", + "serial": 31, + "lineage": "30a5748b-2212-3176-6e6c-d267d4d0fb13", + "outputs": {}, + "resources": [ + { + "module": "module.networking", + "mode": "managed", + "type": "aws_security_group", + "name": "serverless_sg", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:ap-southeast-2:590183937261:security-group/sg-02635dbb83ed1dba9", + "description": "XC3 serverless module access for updates", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "output from serverless sg", + "from_port": 0, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-02635dbb83ed1dba9", + "ingress": [ + { + "cidr_blocks": [ + "10.0.100.0/24" + ], + "description": "All Traffic", + "from_port": 0, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 65535 + } + ], + "name": "xc3team12nilanshu_serverless_security_group", + "name_prefix": "", + "owner_id": "590183937261", + "revoke_rules_on_delete": false, + "tags": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-Serverless-SG", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + }, + "tags_all": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-Serverless-SG", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + }, + "timeouts": null, + "vpc_id": "vpc-0be35a9f83d0a034d" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "module.networking.aws_subnet.private_subnet", + "module.networking.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.networking", + "mode": "managed", + "type": "aws_subnet", + "name": "private_subnet", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "ap-southeast-2a", + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:ap-southeast-2:590183937261:subnet/subnet-0bd6dbf349d8eb6b6", + "assign_ipv6_address_on_creation": false, + "availability_zone": "ap-southeast-2a", + "availability_zone_id": "apse2-az3", + "cidr_block": "10.0.100.0/24", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-0bd6dbf349d8eb6b6", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "590183937261", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-Private-Subnet-ap-southeast-2a", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + }, + "tags_all": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-Private-Subnet-ap-southeast-2a", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + }, + "timeouts": null, + "vpc_id": "vpc-0be35a9f83d0a034d" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.networking.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.networking", + "mode": "managed", + "type": "aws_vpc", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:ap-southeast-2:590183937261:vpc/vpc-0be35a9f83d0a034d", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-0d508465690f607ad", + "default_route_table_id": "rtb-0055304719580c8a6", + "default_security_group_id": "sg-038e88cc13bdee643", + "dhcp_options_id": "dopt-073c4c6800a962a07", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "enable_network_address_usage_metrics": false, + "id": "vpc-0be35a9f83d0a034d", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_cidr_block_network_border_group": "", + "ipv6_ipam_pool_id": "", + "ipv6_netmask_length": 0, + "main_route_table_id": "rtb-0055304719580c8a6", + "owner_id": "590183937261", + "tags": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-VPC", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + }, + "tags_all": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-VPC", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + } + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "module": "module.xc3", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::xc3team12nilanshu-metadata-storage", + "bucket": "xc3team12nilanshu-metadata-storage", + "bucket_domain_name": "xc3team12nilanshu-metadata-storage.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "xc3team12nilanshu-metadata-storage.s3.ap-southeast-2.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [ + { + "id": "2907c02c17f04ba6a28e6c4d72a44571ab54668383935c8b496119df3e7ac0b6", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z1WCIGYICN2BYD", + "id": "xc3team12nilanshu-metadata-storage", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "ap-southeast-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-Bucket", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + }, + "tags_all": { + "Creator": "104346575@student.swin.edu.au", + "Name": "xc3team12nilanshu-Bucket", + "Owner": "104346575@student.swin.edu.au", + "Project": "xc3team12nilanshu" + }, + "timeouts": null, + "versioning": [ + { + "enabled": true, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/infrastructure/main.tf b/infrastructure/main.tf index c52e1df1..693f39fd 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -83,4 +83,6 @@ module "serverless" { create_cloudtrail = var.create_cloudtrail create_cloudtrail_s3_bucket = var.create_cloudtrail_s3_bucket env = var.env + slack_webhook_url = var.slack_webhook_url + snsendpoint = var.snsendpoint } diff --git a/infrastructure/modules/serverless/cloudtrail.tf b/infrastructure/modules/serverless/cloudtrail.tf index 9b0a1f9d..32102b5a 100644 --- a/infrastructure/modules/serverless/cloudtrail.tf +++ b/infrastructure/modules/serverless/cloudtrail.tf @@ -16,11 +16,18 @@ resource "aws_s3_bucket" "this" { #ts:skip=AWS.S3Bucket.IAM.High.0370 We are aware of the risk and choose to skip this rule count = var.create_cloudtrail_s3_bucket ? 1 : 0 - bucket = "${var.namespace}-cloudtrail-logs-storage" tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-bucket" })) } +resource "aws_s3_bucket_versioning" "this" { + count = var.create_cloudtrail_s3_bucket ? 1 : 0 + bucket = aws_s3_bucket.this[count.index].id + versioning_configuration { + status = "Enabled" + } +} + # Create a CloudTrail trail and enable logging to the S3 bucket resource "aws_cloudtrail" "this" { count = var.create_cloudtrail ? 1 : 0 diff --git a/infrastructure/modules/serverless/eventbridge.tf b/infrastructure/modules/serverless/eventbridge.tf new file mode 100644 index 00000000..74128c6f --- /dev/null +++ b/infrastructure/modules/serverless/eventbridge.tf @@ -0,0 +1,25 @@ +# Define the EventBridge rule for the cron job +resource "aws_cloudwatch_event_rule" "federated_cron_job" { + name = "federated-cron-job" + description = "Cron job to invoke lambda to get all the federated users and resources provisioned by them." + #schedule_expression = "cron(0 0 */14 * ? *)" # Runs every 14 days + schedule_expression = "cron(0 0 * * ? *)" + + # Runs every minute + # schedule_expression = "cron(*/1 * * * ? *)" + +# Add tags for better organization and management + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-federatedCron-job" })) +} + + + + +data "aws_caller_identity" "current" {} + +# Define the target to invoke the Lambda function +resource "aws_cloudwatch_event_target" "invoke_lambda" { + rule = aws_cloudwatch_event_rule.federated_cron_job.name + target_id = "invoke-lambda-target" + arn = "arn:aws:lambda:${var.region}:${data.aws_caller_identity.current.account_id}:function:${var.namespace}-list_fed_users" +} diff --git a/infrastructure/modules/serverless/fed-user.tf b/infrastructure/modules/serverless/fed-user.tf new file mode 100644 index 00000000..6d3174f0 --- /dev/null +++ b/infrastructure/modules/serverless/fed-user.tf @@ -0,0 +1,198 @@ + +# Creating Inline policy +resource "aws_iam_role_policy" "fed_user_policy" { + name = "${var.namespace}-fed_user_policy" + role = aws_iam_role.fed-user_role.id + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Sid" : "Costexplorer", + "Effect" : "Allow", + "Action" : [ + "ce:GetCostAndUsage", + "ce:GetCostAndUsageWithResources", + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DetachNetworkInterface", + "ec2:AttachNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeRegions", + "SNS:Publish", + "iam:ListRoles", + "iam:ListUsers", + "lambda:ListTags", + "s3:PutObject", + "s3:GetObject", + "tag:GetResources", + "cloudtrail:LookupEvents" + ] + Effect = "Allow" + "Resource" : "*" + + }, + + ] + }) +} + + + +# Creating IAM Role for Lambda functions +resource "aws_iam_role" "fed-user_role" { + name = "${var.namespace}-fed-user_role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + managed_policy_arns = [ + "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess", + "arn:aws:iam::aws:policy/ResourceGroupsandTagEditorReadOnlyAccess", + "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole", + "arn:aws:iam::aws:policy/AWSLambda_FullAccess", + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-fed-user_role" })) +} + + +data "archive_file" "fed_user_resource_zip" { + type = "zip" + source_dir = "../src/federated_user" + output_path = "${path.module}/fed_user_resource.zip" + +} + + +data "archive_file" "list_fed_user_zip" { + type = "zip" + source_dir = "../src/federated_user" + output_path = "${path.module}/list_fed_user.zip" + +} + +resource "aws_lambda_function" "fed_user_resource" { + #ts:skip=AC_AWS_0485 We are aware of the risk and choose to skip this rule + #ts:skip=AC_AWS_0483 We are aware of the risk and choose to skip this rule + #ts:skip=AC_AWS_0484 We are aware of the risk and choose to skip this rule + function_name = "${var.namespace}-fed_user_resource" + role = aws_iam_role.fed-user_role.arn + runtime = "python3.9" + handler = "fed_user_resource.lambda_handler" + filename = data.archive_file.fed_user_resource_zip.output_path + environment { + variables = { + prometheus_ip = "${var.prometheus_ip}:9091" + region_names_path = "/${var.namespace}/region_names" + bucket_name = var.s3_xc3_bucket.bucket + } + } + memory_size = var.memory_size + timeout = var.timeout + layers = [var.prometheus_layer] + vpc_config { + subnet_ids = [var.subnet_id[0]] + security_group_ids = [var.security_group_id] + } + depends_on = [ + aws_lambda_function.list_fed_user + ] + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-fed_user_resource" })) + +} + + + + +resource "terraform_data" "list_fed_user_zip" { + triggers_replace = [aws_lambda_function.list_fed_user.arn] + provisioner "local-exec" { + command = "rm -r ${data.archive_file.list_fed_user_zip.output_path}" + } +} + + +resource "terraform_data" "fed_user_resource_zip" { + triggers_replace = [aws_lambda_function.fed_user_resource.arn] + provisioner "local-exec" { + command = "rm -r ${data.archive_file.fed_user_resource_zip.output_path}" + } +} + + +resource "aws_lambda_function" "list_fed_user" { + #ts:skip=AC_AWS_0484 We are aware of the risk and choose to skip this rule + #ts:skip=AC_AWS_0485 We are aware of the risk and choose to skip this rule + #ts:skip=AC_AWS_0483 We are aware of the risk and choose to skip this rule + function_name = "${var.namespace}-list_fed_users" + role = aws_iam_role.fed-user_role.arn + runtime = "python3.9" + handler = "list_fed_user.lambda_handler" + filename = data.archive_file.list_fed_user_zip.output_path + environment { + variables = { + prometheus_ip = "${var.prometheus_ip}:9091" + REGION = var.region + sns_topic = var.sns_topic_arn + bucket_name = var.s3_xc3_bucket.bucket + UNTAGGED_RESOURCE_LAMBDA_ARN = aws_lambda_function.resource_notification_lambda.arn + + } + } + memory_size = var.memory_size + timeout = var.timeout + layers = [var.prometheus_layer] + vpc_config { + subnet_ids = [var.subnet_id[0]] + security_group_ids = [var.security_group_id] + } + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-list_fed_user" })) +} + + +resource "aws_s3_object" "example_folder" { + bucket = var.s3_xc3_bucket.bucket + key = "fed-resources/" +} + + +resource "aws_s3_bucket_notification" "fed_user_trigger" { + bucket = var.s3_xc3_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.fed_user_resource.arn + filter_prefix = "fed-resources/" + events = ["s3:ObjectCreated:Put"] + filter_suffix = "resources.json" + } +} + + + +resource "aws_lambda_permission" "allow_buckets_for_trigger" { + statement_id = "AllowExecutionFromS3Bucket" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.fed_user_resource.arn + principal = "s3.amazonaws.com" + source_arn = var.s3_xc3_bucket.arn +} + + +# Grant permission to EventBridge to invoke the Lambda function +# ------------------------------------------------------------- +resource "aws_lambda_permission" "allow_eventbridge_invoke" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.list_fed_user.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.federated_cron_job.arn +} \ No newline at end of file diff --git a/infrastructure/modules/serverless/resource_notification.tf b/infrastructure/modules/serverless/resource_notification.tf new file mode 100644 index 00000000..c4da1817 --- /dev/null +++ b/infrastructure/modules/serverless/resource_notification.tf @@ -0,0 +1,74 @@ +# IAM Role for resource_notification Lambda Function +# ----------------------------------------- +# This resource block defines an IAM role for the Lambda function named resource_notification_lambda. + +resource "aws_iam_role" "resource_notification_lambda_role" { + name = "${var.namespace}-resource_notification_lambda_execution_role" # Name of the IAM role + + # Policy allowing Lambda service to assume the role + assume_role_policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Principal" : { + "Service" : "lambda.amazonaws.com" + }, + "Action" : "sts:AssumeRole" + } + ] + }) + + # Attach required managed policies to the role + managed_policy_arns = [ + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/AmazonSNSFullAccess", + "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess" + ] + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-resource-notification-lambda-execution-role" })) +} + + +# Archive Lambda function code +# ----------------------------- +# This data source archives the Lambda function code located in the "src/federated_user" directory. + +data "archive_file" "resource_notification_lambda_zip" { + type = "zip" + source_dir = "../src/federated_user" + output_path = "${path.module}/resource_notification_lambda.zip" +} + + +# Lambda Function - resource_notification +# ------------------------------ +# This resource block defines the Lambda function named resource_notification_lambda. + +resource "aws_lambda_function" "resource_notification_lambda" { + filename = data.archive_file.resource_notification_lambda_zip.output_path # Path to the Lambda function code zip archive + function_name = "${var.namespace}-resource_notification_lambda" # Name of the Lambda function + role = aws_iam_role.resource_notification_lambda_role.arn # IAM role ARN attached to the Lambda function + handler = "resource_notification.lambda_handler" # Entry point to the Lambda function + runtime = "python3.8" # Runtime environment for the Lambda function + + # Environment variables passed to the Lambda function + environment { + variables = { + SNS_TOPIC_ARN = aws_sns_topic.resource_alert.arn # Pass the ARN of the SNS topic as an environment variable + SLACK_WEBHOOK_URL = var.slack_webhook_url + NAME_SPACE = var.namespace + } + } + +# Add tags for better organization and management + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-resource-notification-lambda" })) + +} + + +resource "aws_lambda_permission" "allow_list_fed_user_invoke" { + statement_id = "AllowExecutionFromListFedUserLambda" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.resource_notification_lambda.arn + principal = "lambda.amazonaws.com" +} diff --git a/infrastructure/modules/serverless/sns_topic.tf b/infrastructure/modules/serverless/sns_topic.tf new file mode 100644 index 00000000..45495e4c --- /dev/null +++ b/infrastructure/modules/serverless/sns_topic.tf @@ -0,0 +1,38 @@ +# AWS SNS Topic creation +# ----------------------- +# This section creates an AWS SNS Topic named "resource_alert" to be used for resource alerts. + +resource "aws_sns_topic" "resource_alert" { + # Topic Name + # ---------- + # Specifies the name of the SNS topic. + name = "${var.namespace}-resource_alert" + + # Display Name + # ------------ + # Specifies the display name of the SNS topic. + display_name = "Resource Alert" +} + +# Subscription of an email address to the SNS topic +# ------------------------------------------------- +# This section subscribes an email address to the "resource_alert" SNS topic for receiving notifications. + +resource "aws_sns_topic_subscription" "email_subscription" { + # Topic ARN + # --------- + # Specifies the ARN of the SNS topic to which the subscription will be made. + topic_arn = aws_sns_topic.resource_alert.arn + + # Protocol + # -------- + # Specifies the protocol for the subscription. In this case, it's email. + protocol = "email" + + # Endpoint + # -------- + # Specifies the endpoint for the subscription, which is an email address. + endpoint = var.snsendpoint + + +} diff --git a/infrastructure/modules/serverless/variables.tf b/infrastructure/modules/serverless/variables.tf index 623bc392..822703cc 100644 --- a/infrastructure/modules/serverless/variables.tf +++ b/infrastructure/modules/serverless/variables.tf @@ -144,3 +144,12 @@ variable "env" { type = string } + variable "snsendpoint" { + type = string + description = "endpoint for email notification" +} + +variable "slack_webhook_url" { + type = string + description = "Webhook URL for slack notification." +} \ No newline at end of file diff --git a/infrastructure/modules/xc3/main.tf b/infrastructure/modules/xc3/main.tf index edaddcd0..0d67f2af 100644 --- a/infrastructure/modules/xc3/main.tf +++ b/infrastructure/modules/xc3/main.tf @@ -173,6 +173,13 @@ resource "aws_s3_bucket" "this" { tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-Bucket" })) } +resource "aws_s3_bucket_versioning" "this" { + bucket = aws_s3_bucket.this.id + versioning_configuration { + status = "Enabled" + } +} + # Uploading Cloud Custodian Policies and lambda layers in S3 bucket # tflint-ignore: terraform_required_providers resource "terraform_data" "upload_files_on_s3" { @@ -180,6 +187,8 @@ resource "terraform_data" "upload_files_on_s3" { provisioner "local-exec" { command = <