diff --git a/Group User Manual TAP Team 01.pdf b/Group User Manual TAP Team 01.pdf new file mode 100644 index 00000000..1f4c6c93 Binary files /dev/null and b/Group User Manual TAP Team 01.pdf differ diff --git a/README.md b/README.md index 20878418..279429a6 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,13 @@ Access to the "prod" environment is facilitated through a DNS URL, thanks to Rou `ssh ubuntu@[instance-id] -i keypair.pem -o ProxyCommand='aws ec2-instance-connect open-tunnel --instance-id %h'` -6. Now XC3 will run at 05:00AM UTC every day to generate data and populate Grafana. Few lambdas (Total Account Cost and Project spend) will run twice in a month. +6. Go to AWS Systems Manager, select Parameter Store, and create a new parameter named "/{namespace}/region_names". Set the value as a dictionary with region IDs as keys and region names as values. + +7. SSH into the private instance using EIC Endpoint to check if everything is working fine. Here replace [instance-id] needs to be replaced with ID + + `ssh ubuntu@[instance-id] -i keypair.pem -o ProxyCommand='aws ec2-instance-connect open-tunnel --instance-id %h'` + +8. Now XC3 will run at 05:00AM UTC every day to generate data and populate Grafana. Few lambdas (Total Account Cost and Project spend) will run twice in a month. Note : 1. If data is not available in Grafana UI then follow the troubleshooting guide at the last section of this page. @@ -172,6 +178,8 @@ case 1: If data is not showing into Grafana UI, there could be several reasons a 3. XC3 Budget Detail/IAM Role/User Workflow lambda may have failed to execute , please check Cloudwatch logs to address the issue. +4. Check if XC3's most expensive services data is missing, and if so, verify the existence of the corresponding SSM parameter in AWS Systems Manager. To address this issue, ensure you follow step 5 of the deployment instructions. + case 2: user not able to change/update/modify default dashboards in Grafana UI 1. You can't change/update default dashboards. diff --git a/custom_dashboard/grafana_dashboards/TAPTeam01.json b/custom_dashboard/grafana_dashboards/TAPTeam01.json new file mode 100644 index 00000000..49485950 --- /dev/null +++ b/custom_dashboard/grafana_dashboards/TAPTeam01.json @@ -0,0 +1,270 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Shows All Lambda Costs for 14 days", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Specific Resource", + "url": "http://3.25.208.25:3000/d/cdf8548d5p24gd/iam-role-costs?orgId=1&editPanel=2&var-resource_name=${__field.labels.resource_name}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_service_resource_cost", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Lambda Costs", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_service_resource_cost{resource_name=\"$resource_name\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Specific Lambda Only", + "type": "barchart" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "SagarTest", + "value": "SagarTest" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(resource_name)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "resource_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(resource_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "IAM Role Costs", + "uid": "cdf8548d5p24gd", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git "a/custom_dashboard/grafana_dashboards/TAPTeam01.json\357\200\272Zone.Identifier" "b/custom_dashboard/grafana_dashboards/TAPTeam01.json\357\200\272Zone.Identifier" new file mode 100644 index 00000000..3e92e270 --- /dev/null +++ "b/custom_dashboard/grafana_dashboards/TAPTeam01.json\357\200\272Zone.Identifier" @@ -0,0 +1,3 @@ +[ZoneTransfer] +ZoneId=3 +HostUrl=https://files.slack.com/files-pri/T06FJLPT25D-F06NSFZLTHR/download/tapteam01.json?origin_team=T06FJLPT25D diff --git a/custom_dashboard/grafana_dashboards/home-dashboard.json b/custom_dashboard/grafana_dashboards/home-dashboard.json index 49e6c1cd..1ece0e69 100644 --- a/custom_dashboard/grafana_dashboards/home-dashboard.json +++ b/custom_dashboard/grafana_dashboards/home-dashboard.json @@ -415,7 +415,7 @@ }, "editorMode": "builder", "exemplar": false, - "expr": "Expensive_Services_Detail{job=\"Most_Expensive_Services\", region=\"$region\", account=\"$Account\"}", + "expr": "Expensive_Services_Detail{job=\"$Account\", region=\"$region\"}", "format": "table", "instant": true, "legendFormat": "__auto", diff --git a/custom_dashboard/grafana_dashboards/newdashboard.json b/custom_dashboard/grafana_dashboards/newdashboard.json new file mode 100644 index 00000000..fa3fadb7 --- /dev/null +++ b/custom_dashboard/grafana_dashboards/newdashboard.json @@ -0,0 +1,621 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Breakdown to different Services", + "url": "d/edib44yiawqgwb/iam-cost-further-breakdown?editPanel=5&var-IAMRole=${__field.labels.role_arn}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"aws_total_cost_by_iam_role\", job=\"aws_costs_by_iam_role\", role_arn=\"arn:aws:iam::211125640160:role/service-role/ZA_Grafana_test-role-3cgu82zj\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_total_cost_by_iam_role", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total IAM Role Cost", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_lambda_cost_by_iam_role", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Lambda IAM Role Costs", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_cloudwatch_cost_by_iam_role", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Clouddwatch IAM Role Costs", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_sns_cost_by_iam_role", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "SNS IAM Roe Costs", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_cloudwatch_cost_by_iam_role{role_arn=\"$IAMRole\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_lambda_cost_by_iam_role{role_arn=\"$IAMRole\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aws_sns_cost_by_iam_role{role_arn=\"$IAMRole\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Breakdown for Specific IAM Role", + "type": "barchart" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "arn:aws:iam::211125640160:role/lambda_exec_role", + "value": "arn:aws:iam::211125640160:role/lambda_exec_role" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(aws_total_cost_by_iam_role,role_arn)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "IAMRole", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(aws_total_cost_by_iam_role,role_arn)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "IAM Cost Further Breakdown", + "uid": "edib44yiawqgwb", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git "a/custom_dashboard/grafana_dashboards/newdashboard.json\357\200\272Zone.Identifier" "b/custom_dashboard/grafana_dashboards/newdashboard.json\357\200\272Zone.Identifier" new file mode 100644 index 00000000..04f8fd57 --- /dev/null +++ "b/custom_dashboard/grafana_dashboards/newdashboard.json\357\200\272Zone.Identifier" @@ -0,0 +1,3 @@ +[ZoneTransfer] +ZoneId=3 +HostUrl=https://cdn.fbsbx.com/v/t59.2708-21/437497274_1359574258070611_5124526200910773114_n.json/newdashboard.json?_nc_cat=102&ccb=1-7&_nc_sid=2b0e22&_nc_ohc=C82Gb2t5KhoAb43h-Qv&_nc_ht=cdn.fbsbx.com&oh=03_AdWXJSQr3iSUiMIFHfU0_PwAdT9IJkmVk7E5zZniaOjyKw&oe=661988AA&dl=1 diff --git a/infrastructure/backend.tf b/infrastructure/backend.tf index 6f2f0ab9..d63d2f36 100644 --- a/infrastructure/backend.tf +++ b/infrastructure/backend.tf @@ -16,9 +16,9 @@ terraform { backend "s3" { - bucket = "terraform-state-xc3" // S3 bucket for terraform state management + bucket = "terraform-state-xc3-sagarpoudel" // S3 bucket for terraform state management key = "xc3/xc3.tfstate" // Specifies the S3 object key for storing the Terraform state file - region = "eu-west-1" + region = "ap-southeast-2" dynamodb_table = "terraform-lock" } } diff --git a/infrastructure/config.sh b/infrastructure/config.sh index 3c1aaf7f..bbfb5ab4 100644 --- a/infrastructure/config.sh +++ b/infrastructure/config.sh @@ -15,12 +15,12 @@ # limitations under the License. # Variables values that will be used in init script to create resources for XC3 infrastructure -export aws_region="eu-west-1" +export aws_region="ap-southeast-2" export dynamo_table_name="terraform-lock" -export bucket_name="terraform-state-xc3" -export project="example" -export domain="example.test.co" -export owner_email="admin@test.co" -export creator_email="admin@test.co" -export namespace="example" -export env="example" +export bucket_name="terraform-state-xc3-sagarpoudel" +export project="xc3sagarpoudel" +export domain="mailsagarpoudel@gmail.com" +export owner_email="mailsagarpoudel@gmail.com" +export creator_email="mailsagarpoudel@gmail.com" +export namespace="xc3sagarpoudel" +export env="dev" diff --git a/infrastructure/main.tf b/infrastructure/main.tf index c52e1df1..c25d09dd 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -42,11 +42,11 @@ module "xc3" { private_subnet_id = module.networking.private_subnet_id public_subnet_ids = module.networking.public_subnet_ids security_group_ids = module.networking.security_group_ids - ses_email_address = var.ses_email_address instance_type = var.instance_type namespace = var.namespace owner_email = var.owner_email creator_email = var.creator_email + ses_email_address = var.ses_email_address project = var.project region = var.region prometheus_layer = var.prometheus_layer @@ -75,12 +75,20 @@ module "serverless" { timeout = var.timeout memory_size = var.memory_size total_account_cost_lambda = var.total_account_cost_lambda + iam_role_cost_lambda = var.iam_role_cost_lambda account_id = var.account_id total_account_cost_cronjob = var.total_account_cost_cronjob cron_jobs_schedule = var.cron_jobs_schedule - slack_channel_url = var.slack_channel_url create_cloudtrail_kms = var.create_cloudtrail_kms create_cloudtrail = var.create_cloudtrail create_cloudtrail_s3_bucket = var.create_cloudtrail_s3_bucket env = var.env + CUR_s3_bucket_name = var.CUR_s3_bucket_name + CUR_s3_file_key = var.CUR_s3_file_key + slack_channel_url = var.slack_channel_url + slack_channel = var.slack_channel + slack_icon_emoji = var.slack_icon_emoji + slack_username = var.slack_username + ses_email_address = var.ses_email_address + receiver_email_address = var.receiver_email_address } diff --git a/infrastructure/modules/serverless/cloudtrail.tf b/infrastructure/modules/serverless/cloudtrail.tf index 9b0a1f9d..32102b5a 100644 --- a/infrastructure/modules/serverless/cloudtrail.tf +++ b/infrastructure/modules/serverless/cloudtrail.tf @@ -16,11 +16,18 @@ resource "aws_s3_bucket" "this" { #ts:skip=AWS.S3Bucket.IAM.High.0370 We are aware of the risk and choose to skip this rule count = var.create_cloudtrail_s3_bucket ? 1 : 0 - bucket = "${var.namespace}-cloudtrail-logs-storage" tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-bucket" })) } +resource "aws_s3_bucket_versioning" "this" { + count = var.create_cloudtrail_s3_bucket ? 1 : 0 + bucket = aws_s3_bucket.this[count.index].id + versioning_configuration { + status = "Enabled" + } +} + # Create a CloudTrail trail and enable logging to the S3 bucket resource "aws_cloudtrail" "this" { count = var.create_cloudtrail ? 1 : 0 diff --git a/infrastructure/modules/serverless/iam_role_cost.tf b/infrastructure/modules/serverless/iam_role_cost.tf new file mode 100644 index 00000000..cf74750b --- /dev/null +++ b/infrastructure/modules/serverless/iam_role_cost.tf @@ -0,0 +1,169 @@ +data "archive_file" "iam_role_cost" { + type = "zip" + source_file = "../src/iam_roles/iam_role_cost.py" + output_path = "${path.module}/iam_role_cost.zip" +} + +# Creating IAM Role for Lambda functions +resource "aws_iam_role" "iam_role_cost" { + name = "${var.namespace}-${var.iam_role_cost_lambda}-role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "IAMRoleSNSFunction" + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + managed_policy_arns = ["arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"] + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-iam_role_cost" })) +} + +resource "aws_iam_role_policy" "iam_role_cost" { + name = "${var.namespace}-${var.iam_role_cost_lambda}-ce-policy" + role = aws_iam_role.iam_role_cost.id + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Sid" : "SendEmail", + "Effect" : "Allow", + "Action" : [ + "ses:SendEmail" + ], + "Resource" : "*" + }, + { + "Sid" : "ListIAMRoles", + "Effect" : "Allow", + "Action" : [ + "iam:ListRoles" + ], + "Resource" : "*" + }, + { + "Sid": "IAMRoleCost", + "Effect": "Allow", + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DetachNetworkInterface", + "ec2:AttachNetworkInterface", + "ec2:DeleteNetworkInterface" + ], + "Resource": "*" + }, + { + "Sid": "SSMParameter", + "Effect": "Allow", + "Action": [ + "ssm:GetParameter" + ], + "Resource": "arn:aws:ssm:*:*:parameter/*" + }, + { + "Effect" : "Allow", + "Action" : [ + "s3:GetObject" + ], + "Resource" : "arn:aws:s3:::${var.CUR_s3_bucket_name}/*" // Update this with the correct ARN for your bucket + }, + { + "Effect": "Allow", + "Action": [ + "SNS:ListSubscriptionsByTopic"], + "Resource": "*" + }, + { + "Sid" : "ListLambdaFunctions", + "Effect" : "Allow", + "Action" : [ + "lambda:ListFunctions" + ], + "Resource" : "*" + } + ] + }) +} + +resource "aws_lambda_function" "iam_role_cost" { + function_name = "${var.namespace}-${var.iam_role_cost_lambda}" + role = aws_iam_role.iam_role_cost.arn + runtime = "python3.9" + handler = "${var.iam_role_cost_lambda}.lambda_handler" + filename = data.archive_file.iam_role_cost.output_path + environment { + variables = { + prometheus_ip = "${var.prometheus_ip}:9091" + account_detail = var.namespace + slack_channel_url = var.slack_channel_url + slack_channel = var.slack_channel + slack_icon_emoji = var.slack_icon_emoji + slack_username = var.slack_username + ses_email_address = var.ses_email_address + receiver_email_address = var.receiver_email_address + + CUR_s3_bucket_name = var.CUR_s3_bucket_name + CUR_s3_file_key = var.CUR_s3_file_key + } + } + memory_size = var.memory_size + timeout = var.timeout + layers = [var.prometheus_layer] + vpc_config { + subnet_ids = [var.subnet_id[0]] + security_group_ids = [var.security_group_id] + } + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-iam_role_cost" })) +} + +resource "aws_iam_policy" "iam_role_cost" { + name = "${var.namespace}-iam_role_cost_eventbridge_policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "lambda:InvokeFunction" + ] + Effect = "Allow" + Resource = aws_lambda_function.iam_role_cost.arn + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "iam_role_cost" { + policy_arn = aws_iam_policy.iam_role_cost.arn + role = aws_iam_role.iam_role_cost.name +} + +# Creating CloudWatch Event Rule for scheduling Lambda function +resource "aws_cloudwatch_event_rule" "lambda_schedule" { + name = "iam_role_cost_schedule" + description = "Schedule rule for triggering iam_role_cost Lambda" + schedule_expression = "cron(0 0 * * ? *)" # Runs every min "cron(0/1 * * * ? *)" +} + +# Creating CloudWatch Event Target for Lambda function +resource "aws_cloudwatch_event_target" "iam_role_cost_target" { + rule = aws_cloudwatch_event_rule.lambda_schedule.name + target_id = "invoke-iam_role_cost-lambda" + arn = aws_lambda_function.iam_role_cost.arn +} + +# Adding Lambda permission to allow CloudWatch Events to invoke the Lambda function +resource "aws_lambda_permission" "allow_cloudwatch" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.iam_role_cost.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.lambda_schedule.arn +} + diff --git a/infrastructure/modules/serverless/variables.tf b/infrastructure/modules/serverless/variables.tf index 623bc392..637e0699 100644 --- a/infrastructure/modules/serverless/variables.tf +++ b/infrastructure/modules/serverless/variables.tf @@ -17,16 +17,26 @@ variable "namespace" { description = "The namespace referring to an env" } -variable "owner_email" { +variable "receiver_email_address" { type = string - description = "Email address of a owner who is leading the team" + description = "The email address for receiver" } +variable "ses_email_address" { + type = string + description = "The email address for SES identity" +} variable "creator_email" { type = string - description = "Email of the the Creator who is provisioning the infrastructure" + description = "Email address of a person who is provisioning the infrastructure" +} + +variable "owner_email" { + type = string + description = "Email address of a person who is owner of the team/project" } + variable "project" { type = string description = "The name of the Project" @@ -67,6 +77,11 @@ variable "total_account_cost_lambda" { description = "The name of the lambda function that will be used to calculate cost metrics of provided AWS Account" } +variable "iam_role_cost_lambda" { + type = string + description = "Name of the Lambda function of Team 01 getting cost from IAM ROLE " +} + variable "account_id" { type = string description = "AWS Account ID" @@ -96,6 +111,7 @@ variable "lambda_names" { type = map(string) description = "The names of lambda functions in IAM Role Workflow" default = { + "iam_role_cost" = "../src/iam_roles/iam_role_cost.py" "iam_roles_all" = "../src/iam_roles/iam_roles_all.py" "iamrolesservice" = "../src/iam_roles/iamrolesservice.py" "iamrolesservicemapping" = "../src/iam_roles/iamrolesservicemapping.py" @@ -119,10 +135,6 @@ variable "s3_prefixes" { } } -variable "slack_channel_url" { - description = "Slack Channel URL" - type = string -} variable "create_cloudtrail_kms" { description = "Fetch the KMS if exist" @@ -144,3 +156,32 @@ variable "env" { type = string } +variable "CUR_s3_bucket_name"{ + description = "bucket name where CUR is located" + type = string +} + +variable "CUR_s3_file_key"{ + description = "path of csv file where cur is located" + type = string +} + +variable "slack_channel_url"{ + description = "slack web hook url to send notification via slack" + type = string +} +variable "slack_username"{ + description = "user_name" + type = string +} +variable "slack_icon_emoji"{ + description = "emoji" + type = string +} + +variable "slack_channel"{ + description = "Name of channel of slack" + type = string +} + + diff --git a/infrastructure/modules/xc3/main.tf b/infrastructure/modules/xc3/main.tf index edaddcd0..259fb515 100644 --- a/infrastructure/modules/xc3/main.tf +++ b/infrastructure/modules/xc3/main.tf @@ -173,6 +173,13 @@ resource "aws_s3_bucket" "this" { tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-Bucket" })) } +resource "aws_s3_bucket_versioning" "this" { + bucket = aws_s3_bucket.this.id + versioning_configuration { + status = "Enabled" + } +} + # Uploading Cloud Custodian Policies and lambda layers in S3 bucket # tflint-ignore: terraform_required_providers resource "terraform_data" "upload_files_on_s3" { @@ -209,12 +216,12 @@ resource "aws_lambda_layer_version" "lambda_layer_prometheus" { ## Cloudwatch Alarms # tflint-ignore: terraform_required_providers -resource "aws_cloudwatch_metric_alarm" "this" { - alarm_name = "${var.namespace}-xc3_cpu_usage_alarm" +resource "aws_cloudwatch_metric_alarm" "cpu_utilization_queue_alarm" { + alarm_name = "${var.namespace}-xc3-cpu-usage-alarm" comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = "2" - metric_name = "${var.namespace}-xc3_cpu_utilization" - namespace = var.namespace + evaluation_periods = "1" + metric_name = "CPUUtilization" + namespace = "AWS/EC2" period = "300" statistic = "Average" threshold = "80" @@ -227,6 +234,25 @@ resource "aws_cloudwatch_metric_alarm" "this" { tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-cpu_usage_alarm" })) } +resource "aws_cloudwatch_metric_alarm" "sqs_queue_alarm" { + alarm_name = "${var.namespace}-xc3-sqs-queue-alarm" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 1 + metric_name = "ApproximateAgeOfOldestMessage" + namespace = "AWS/SQS" + period = 300 + statistic = "Average" + threshold = 10 + alarm_description = "Alarm for SQS queue visibility threshold" + alarm_actions = [aws_sns_topic.this.arn] + + dimensions = { + QueueName = aws_sqs_queue.this.name + } + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-cpu_usage_alarm" })) +} + ## Creation of an WAFv2 # tflint-ignore: terraform_required_providers resource "aws_wafv2_web_acl" "this" { @@ -272,7 +298,7 @@ resource "aws_wafv2_web_acl" "this" { ## association code of webacl # tflint-ignore: terraform_required_providers resource "aws_wafv2_web_acl_association" "web_acl_association_with_lb" { - count = var.env == "prod" && var.domain_name == "" ? 1 : 0 + count = var.env == "prod" && var.domain_name != "" ? 1 : 0 resource_arn = aws_lb.this[0].arn web_acl_arn = aws_wafv2_web_acl.this.arn } diff --git a/infrastructure/pre_req.sh b/infrastructure/pre_req.sh old mode 100755 new mode 100644 index 229261bc..aabbcca7 --- a/infrastructure/pre_req.sh +++ b/infrastructure/pre_req.sh @@ -1,19 +1,14 @@ #!/usr/bin/env bash - # Copyright (c) 2023, Xgrid Inc, https://xgrid.co - # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 - # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - # This script creates an S3 bucket to store the Terraform state file, # enables public block access on the S3 bucket, creates a DynamoDB table # to maintain a lock on Terraform states, creates an EC2 key pair, and @@ -28,9 +23,7 @@ # - `domain`: The domain name to use when creating an ACM certificate that will be used in a Route 53 domain. # - `owner_email`: The email address of the owner of the team. # - `creator_email`: The email address of the creator who is spinning up the infrastructure. - CONFIG_FILE="./config.sh" - # Load the config.sh file if [ -f "$CONFIG_FILE" ]; then # shellcheck source=./config.sh @@ -40,7 +33,6 @@ else echo "Error: config.sh file not found" exit 1 fi - # shellcheck disable=SC2154 # AWS Region from config.sh to be used in rest of script echo "AWS Region: $aws_region" @@ -193,4 +185,4 @@ else fi else echo "Skiping due to ${env} environment." -fi +fi \ No newline at end of file diff --git a/infrastructure/terraform.auto.tfvars b/infrastructure/terraform.auto.tfvars index fe68b285..a8598a1f 100644 --- a/infrastructure/terraform.auto.tfvars +++ b/infrastructure/terraform.auto.tfvars @@ -12,36 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -namespace = "testing" -env = "prod" -region = "eu-west-1" -account_id = "123456789" +namespace = "xc3sagarpoudel" +env = "dev" +region = "ap-southeast-2" +account_id = "211125640160" vpc_cidr_block = "10.0.0.0/16" public_subnet_cidr_block = { - "eu-west-1a" = "10.0.0.0/24" - "eu-west-1b" = "10.0.1.0/24" + "ap-southeast-2a" = "10.0.0.0/24" + "ap-southeast-2b" = "10.0.1.0/24" } domain_name = "" hosted_zone_id = "Z053166920YP1STI0EK5X" private_subnet_cidr_block = { - "eu-west-1a" = "10.0.100.0/24" + "ap-southeast-2a" = "10.0.100.0/24" } # private_subnet_cidr_block = "10.0.100.0/24" allow_traffic = ["0.0.0.0/0"] // Use your own network CIDR -ses_email_address = "testing@testing.co" -creator_email = "testing@testing.co" -owner_email = "testing@testing.co" +ses_email_address = "mailtosagarpoudel@gmail.com" +receiver_email_address = "mailtosagarpoudel@gmail.com" +creator_email = "mailtosagarpoudel@gmail.com" +owner_email = "mailtosagarpoudel@gmail.com" instance_type = "t2.micro" total_account_cost_lambda = "total_account_cost" +iam_role_cost_lambda = "iam_role_cost" +CUR_s3_bucket_name = "team1reportbucket" +CUR_s3_file_key = "report/mycostreport/20240401-20240501/20240405T101631Z/mycostreport-00002.csv" +slack_channel_url ="https://hooks.slack.com/services/0011T06FJLPT25D0011/0011B06TQHHGHSQ0011/0011Rqmhc6wxZ1LtLx1NMXf1EtI10011" #omit 0011s +slack_channel = "C06NAMZR69E" +slack_icon_emoji = ":rocket:" +slack_username = "bot" total_account_cost_cronjob = "cron(0 0 1,15 * ? *)" // flexible can be set according to need prometheus_layer = "lambda_layers/python.zip" // s3 key for lambda layer memory_size = 128 timeout = 300 -project = "testing" -create_cloudtrail_kms = true -create_cloudtrail = true -create_cloudtrail_s3_bucket = true +project = "xc3sagarpoudel" +create_cloudtrail_kms = false +create_cloudtrail = false +create_cloudtrail_s3_bucket = false security_group_ingress = { "pushgateway" = { description = "PushGateway" diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index fd9738f6..c10cc52f 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -47,6 +47,11 @@ variable "ses_email_address" { description = "The email address for SES identity" } +variable "receiver_email_address" { + type = string + description = "The email address for receiver" +} + variable "instance_type" { type = string description = "The type of the EC2 instance" @@ -99,6 +104,11 @@ variable "total_account_cost_lambda" { description = "The name of the lambda function that will be used to calculate cost metrics of provided AWS Account" } +variable "iam_role_cost_lambda" { + type = string + description = "The name of the lambda function that will be used to calculate cost metrics of provided IAM ROLE " +} + variable "account_id" { type = string description = "AWS Account id in which infrastructure will be deployed" @@ -132,11 +142,6 @@ variable "cron_jobs_schedule" { } } -variable "slack_channel_url" { - description = "Slack Channel URL" - type = string - default = "" -} variable "create_cloudtrail_kms" { description = "Fetch the KMS if exist" @@ -158,3 +163,31 @@ variable "env" { type = string default = "dev" } + +variable "CUR_s3_bucket_name"{ + description = "bucket name where CUR is located" + type = string +} + +variable "CUR_s3_file_key"{ + description = "path of csv file where cur is located" + type = string +} + +variable "slack_channel_url"{ + description = "slack web hook url to send notification via slack" + type = string +} +variable "slack_username"{ + description = "user_name" + type = string +} +variable "slack_icon_emoji"{ + description = "emoji" + type = string +} + +variable "slack_channel"{ + description = "Name of channel of slack" + type = string +} \ No newline at end of file diff --git a/pre_requirement/terraform.auto.tfvars b/pre_requirement/terraform.auto.tfvars index f2d8dfe9..36a23671 100644 --- a/pre_requirement/terraform.auto.tfvars +++ b/pre_requirement/terraform.auto.tfvars @@ -13,8 +13,8 @@ # limitations under the License. -project = "example" -creator_email = "example@example.co" -owner_email = "example@example.co" -namespace = "example" -region = "eu-west-1" +project = "xc3sagar" +creator_email = "mailtosagarpoudel@gmail.com" +owner_email = "mailtosagarpoudel@gmail.com" +namespace = "xc3sagar" +region = "ap-southeast-2" \ No newline at end of file diff --git a/src/expensive_services_detail/cost_metrics_of_expensive_services.py b/src/expensive_services_detail/cost_metrics_of_expensive_services.py index 068dd740..3f234f6e 100644 --- a/src/expensive_services_detail/cost_metrics_of_expensive_services.py +++ b/src/expensive_services_detail/cost_metrics_of_expensive_services.py @@ -173,7 +173,7 @@ def lambda_handler(event, context): for resource in top_5_resources: resourcedata = { "Account": account_detail, - "Region": f"{region} ({region_names.get(region, 'unknown region name')})", + "Region": f"{region}-{region_names.get(region, 'unknown region name')}", "Service": resource["Keys"][0], "Cost": resource["Metrics"]["UnblendedCost"]["Amount"], } diff --git a/src/iam_roles/iam_role_cost.py b/src/iam_roles/iam_role_cost.py new file mode 100644 index 00000000..1cf2e890 --- /dev/null +++ b/src/iam_roles/iam_role_cost.py @@ -0,0 +1,325 @@ +import boto3 +import csv +from io import StringIO +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway +import os +import json +import urllib.request + + +def download_and_parse_csv(s3_client, bucket_name, file_key): + """Download and parse CSV file from S3.""" + response = s3_client.get_object(Bucket=bucket_name, Key=file_key) + csv_string = response["Body"].read().decode("utf-8") + return list(csv.DictReader(StringIO(csv_string))) + + +def fetch_lambda_to_role_mapping(lambda_client): + """Fetch mapping from Lambda ARNs to IAM Role ARNs.""" + mapping_lambda_arn_to_role_arn = {} + mapping_lambda_name_to_role_arn = {} + paginator = lambda_client.get_paginator("list_functions") + for page in paginator.paginate(): + for function in page["Functions"]: + mapping_lambda_arn_to_role_arn[function["FunctionArn"]] = function["Role"] + mapping_lambda_name_to_role_arn[function["FunctionName"]] = function["Role"] + return mapping_lambda_arn_to_role_arn, mapping_lambda_name_to_role_arn + + +def fetch_topic_subscriptions(sns_client, topic_arns): + """Fetch subscriptions for given SNS topics.""" + mapping = {} + for topic_arn in topic_arns: + if not topic_arn.startswith("arn:aws:sns:"): + print(f"Invalid ARN skipped: {topic_arn}") + continue + try: + subscriptions = sns_client.list_subscriptions_by_topic(TopicArn=topic_arn)[ + "Subscriptions" + ] + mapping[topic_arn] = [ + sub["Endpoint"] for sub in subscriptions if sub["Protocol"] == "lambda" + ] + except sns_client.exceptions.NotFoundException: + print(f"Topic {topic_arn} not found. Skipping...") + except Exception as e: + print(f"An error occurred: {e}") + return mapping + + +def fetch_cw_to_role_mapping(lambda_name_to_role_arn_mapping, cw_log_group_arns): + """Map IAM role ARNs for given CloudWatch log group ARNs.""" + mapping = {} + for cw_log_group_arn in cw_log_group_arns: + try: + parts = cw_log_group_arn.split(":") + if ( + len(parts) >= 7 + and parts[2] == "logs" + and parts[5] == "log-group" + and "/aws/lambda/" in parts[6] + ): + lambda_function_name = parts[6].split("/")[-1] + # Used function name to get the mapping, then + # access the IAM Role from the nested dictionary + role_arn = lambda_name_to_role_arn_mapping.get(lambda_function_name) + if role_arn: + mapping[cw_log_group_arn] = role_arn + else: + print(f"No role found for CloudWatch log ARN: {cw_log_group_arn}") + else: + print(f"Invalid CloudWatch log group ARN format: {cw_log_group_arn}") + except Exception as e: + print(f"Error processing CloudWatch log group ARN {cw_log_group_arn}: {e}") + + return mapping + + +def map_costs_to_roles( + csv_rows, + lambda_to_role_mapping, + topic_subscriptions, + name_to_role_mapping, + cw_to_role_mapping, +): + """Map costs to IAM roles based on Lambda functions and SNS subscriptions.""" + role_costs = {} + for row in csv_rows: + product_code = row["lineItem/ProductCode"] + resource_id = row["lineItem/ResourceId"] + cost = float(row["lineItem/UnblendedCost"]) + + if product_code == "AWSLambda" and resource_id in lambda_to_role_mapping: + role_arn = lambda_to_role_mapping[resource_id] + if role_arn not in role_costs: + role_costs[role_arn] = { + "LambdaCost": 0, + "SNSCost": 0, + "CloudWatchCost": 0, + } + role_costs[role_arn]["LambdaCost"] += cost + + elif product_code == "AmazonSNS" and resource_id in topic_subscriptions: + for lambda_arn in topic_subscriptions[resource_id]: + if lambda_arn in lambda_to_role_mapping: + role_arn = lambda_to_role_mapping[lambda_arn] + if role_arn not in role_costs: + role_costs[role_arn] = { + "LambdaCost": 0, + "SNSCost": 0, + "CloudWatchCost": 0, + } + role_costs[role_arn]["SNSCost"] += cost + + elif product_code == "AmazonCloudWatch" and resource_id in cw_to_role_mapping: + role_arn = cw_to_role_mapping[resource_id] + if role_arn not in role_costs: + role_costs[role_arn] = { + "LambdaCost": 0, + "SNSCost": 0, + "CloudWatchCost": 0, + } + role_costs[role_arn]["CloudWatchCost"] += cost + + return role_costs + + +def push_metrics_to_pushgateway(role_costs, prometheus_pushgateway): + """Push mapped costs by IAM roles to Prometheus Pushgateway.""" + registry = CollectorRegistry() + + # Gauge for Total costs by IAM role + total_cost_gauge = Gauge( + "aws_total_cost_by_iam_role", + "Total Cost by IAM Role", + ["role_arn"], + registry=registry, + ) + + # Gauge for Lambda costs by IAM role + lambda_cost_gauge = Gauge( + "aws_lambda_cost_by_iam_role", + "Cost of AWS Lambda by IAM Role", + ["role_arn"], + registry=registry, + ) + + # Gauge for SNS costs by IAM role + sns_cost_gauge = Gauge( + "aws_sns_cost_by_iam_role", + "Cost of Amazon SNS by IAM Role", + ["role_arn"], + registry=registry, + ) + + # Gauge for CloudWatch costs by IAM role + cloudwatch_cost_gauge = Gauge( + "aws_cloudwatch_cost_by_iam_role", + "Cost of Amazon CloudWatch by IAM Role", + ["role_arn"], + registry=registry, + ) + + for role_arn, costs in role_costs.items(): + # Calculate the total cost per IAM role + total_cost = sum(costs.values()) + total_cost_gauge.labels(role_arn=role_arn).set(total_cost) + lambda_cost_gauge.labels(role_arn=role_arn).set(costs.get("LambdaCost", 0)) + sns_cost_gauge.labels(role_arn=role_arn).set(costs.get("SNSCost", 0)) + cloudwatch_cost_gauge.labels(role_arn=role_arn).set( + costs.get("CloudWatchCost", 0) + ) + + push_to_gateway( + prometheus_pushgateway, job="aws_costs_by_iam_role", registry=registry + ) + + +# def main(bucket_name, file_key): +# s3_client = boto3.client("s3") +# lambda_client = boto3.client("lambda") +# sns_client = boto3.client("sns") + +# csv_rows = download_and_parse_csv(s3_client, bucket_name, file_key) +# lambda_to_role_mapping = fetch_lambda_to_role_mapping(lambda_client) + +# sns_topic_arns = { +# row["lineItem/ResourceId"] +# for row in csv_rows +# if row["lineItem/ProductCode"] == "AmazonSNS" +# } +# print("SNS Topic ARNs:", sns_topic_arns) +# topic_subscriptions = fetch_topic_subscriptions(sns_client, sns_topic_arns) + +# role_costs = map_costs_to_roles( +# csv_rows, lambda_to_role_mapping, topic_subscriptions +# ) + +# for role_arn, cost in role_costs.items(): +# print(f"Role ARN: {role_arn}, Total Cost: {cost}") + + +# # Example usage - replace 'your-bucket-name' and 'your-file-key' with actual values +# bucket_name = "team1reportbucket" +# file_key = ( +# "report/mycostreport/20240401-20240501/20240405T101631Z/mycostreport-00002.csv" +# ) +# main(bucket_name, file_key) + +def send_email_ses(role_costs, sender_email, recipient_email): + # Create the email message + subject = "AWS Costs by IAM Role" + body = "AWS Costs by IAM Role:\n\n" + for role_arn, costs in role_costs.items(): + body += f"Role ARN: {role_arn}\n" + body += f"Lambda Cost: {costs.get('LambdaCost', 0):.8f}\n" + body += f"SNS Cost: {costs.get('SNSCost', 0):.8f}\n" + body += f"CloudWatch Cost: {costs.get('CloudWatchCost', 0):.8f}\n" + body += f"Total Cost: {sum(costs.values()):.8f}\n\n" + + # Create an SES client + ses_client = boto3.client("ses") + + # Send the email using SES + response = ses_client.send_email( + Source=sender_email, + Destination={"ToAddresses": [recipient_email]}, + Message={ + "Subject": {"Data": subject}, + "Body": {"Text": {"Data": body}}, + }, + ) + print(f"Email sent! Message ID: {response['MessageId']}") + return body + +def send_to_slack(payload, webhook_url): + # Convert the payload to JSON + data = json.dumps(payload).encode('utf-8') + + # Send the payload to Slack + req = urllib.request.Request(webhook_url, data=data, headers={'Content-Type': 'application/json'}) + urllib.request.urlopen(req) + +def lambda_handler(event, context): + bucket_name = os.environ.get("CUR_s3_bucket_name") + file_key = os.environ.get("CUR_s3_file_key") + + # Clients initialization + s3_client = boto3.client("s3") + lambda_client = boto3.client("lambda") + sns_client = boto3.client("sns") + + # Download and parse the CSV file + csv_rows = download_and_parse_csv(s3_client, bucket_name, file_key) + + # Fetch Lambda to IAM role mapping + ( + lambda_arn_to_role_arn_mapping, + lambda_name_to_role_arn_mapping, + ) = fetch_lambda_to_role_mapping(lambda_client) + + # Fetch SNS Topic ARNs and corresponding subscriptions + sns_topic_arns = { + row["lineItem/ResourceId"] + for row in csv_rows + if row["lineItem/ProductCode"] == "AmazonSNS" + } + topic_subscriptions = fetch_topic_subscriptions(sns_client, sns_topic_arns) + + cw_log_group_arns = { + row["lineItem/ResourceId"] + for row in csv_rows + if row["lineItem/ProductCode"] == "AmazonCloudWatch" + } + + cw_to_role_mapping = fetch_cw_to_role_mapping( + lambda_name_to_role_arn_mapping, cw_log_group_arns + ) + + # # Map costs to roles + role_costs = map_costs_to_roles( + csv_rows, + lambda_arn_to_role_arn_mapping, + topic_subscriptions, + lambda_name_to_role_arn_mapping, + cw_to_role_mapping, + ) + + # Push metrics or process costs as needed + # For example, printing the role costs + # for role_arn, data in role_costs.items(): + # print(f"Role ARN: {role_arn}, Total Cost: {data}") + + # Push metrics to the Prometheus Pushgateway + prometheus_pushgateway = os.environ["prometheus_ip"] + push_metrics_to_pushgateway(role_costs, prometheus_pushgateway) + + print("Data successfully pushed to Prometheus Pushgateway.") + + #Slack + slack_channel = os.environ['slack_channel'] + slack_username = os.environ['slack_username'] + slack_icon_emoji = os.environ['slack_icon_emoji'] + slack_channel_url = os.environ['slack_channel_url'] + + + + # Send email with the role costs using SES + sender_email = os.environ["ses_email_address"] + recipient_email = os.environ["receiver_email_address"] + body = send_email_ses(role_costs, sender_email, recipient_email) + + # Format the payload for Slack + payload = { + 'text': body, + 'channel': slack_channel, + 'username': slack_username, + 'icon_emoji': slack_icon_emoji + } + + # Send the payload to Slack via webhook + send_to_slack(payload, slack_channel_url) + + #send_email_ses(role_costs, sender_email, recipient_email) + + print("Email sent successfully.") \ No newline at end of file diff --git a/terraform.tfstate b/terraform.tfstate new file mode 100644 index 00000000..7f25b89c --- /dev/null +++ b/terraform.tfstate @@ -0,0 +1,9 @@ +{ + "version": 4, + "terraform_version": "1.7.4", + "serial": 1, + "lineage": "33ff66b6-eedb-e8ad-9b8d-3956c305282e", + "outputs": {}, + "resources": [], + "check_results": null +} diff --git a/tests/unit_test/README.md b/tests/unit_test/README.md new file mode 100644 index 00000000..228dee01 --- /dev/null +++ b/tests/unit_test/README.md @@ -0,0 +1,64 @@ +## Introduction + +For XC3 we are using pytest for testing, and we have provisioned our infrastructure using Terraform. Additionally, we are using the Boto3 client library to interact with AWS services programmatically. + +In the following sections, we will explore how to use these tools to test our application effectively and efficiently. We will cover topics such as defining test cases, running tests, and analyzing test results. We will also discuss leveraging Terraform to create and manage the test environment and how Boto3 can help us interact with AWS services during testing. + +By the end of this guide, you should understand how to use pytest, Terraform, Boto3 and mock to test your application, and how these tools can help you achieve better software quality and reliability. + +## How will the tests be run? + +Tests will be run frequently throughout the development cycle, including during the build and deployment stages. It's a good practice to run tests after each code change and before each deployment to ensure that the changes don't introduce any new bugs or regressions. We have automated this process using GitHub Actions, which allows us to set up a workflow that automatically tests our code whenever changes are pushed to the repository. With GitHub Actions, you can define custom workflows in YAML files that specify the steps to run and the conditions under which they should be run. In our case, we have defined a workflow that runs the pytest command to execute the test suite whenever changes are pushed to the repository. The results of the test run are available in the GitHub Actions console, where developers can view logs and other diagnostic information. + +## How to run tests locally? +First, clone the repository [XC3](https://github.com/XgridInc/xc3.git) +and follow the [installation guide](https://docs.google.com/document/d/1YoaWoDSOjdOY2Wk3FXYHOni4wERCU7_WvsQamYkqS_o/edit) until you reach the 'terraform init' step within the 'XC3/infrastructure' directory. Once you've completed those initial steps, navigate to the 'XC3/test/unittests' directory. + +Run following command. +``` +python3 -m venv venv +source venv/bin/activate +pip3 install -r requirements.txt +``` +First run any test available to check all things are fine using this command. +``` +pytest -v "filename.py" +``` +This will show test failed/passed. if passed. +then now you can create new unit test file and test them by running same command +``` +pytest -v "fileName.py" +``` + +## Work Flow + +1. Code changes are pushed to the repository. +2. GitHub Actions detects the changes and triggers the workflow. +3. The workflow checks out the latest version of the code from the repository. +4. GitHub Actions provisions a runner and installs the necessary dependencies for the project. +5. The runner runs the pytest command to execute the test suite. +6. If any tests fail, the workflow is halted, and a notification is sent to the developers. +7. If all tests pass, the workflow moves on to the next stage, such as deploying the code to the production environment. +8. The results of the test run are available in the GitHub Actions console, where developers can view logs and other diagnostic information. + +## Testing Work Flow + + +![image](https://user-images.githubusercontent.com/105271892/234582908-831eb98c-8257-4bc7-8b51-a8a1b040b4b3.png) + +## Github Actions Workflow Overview + +![image](https://github.com/X-CBG/XC3/assets/122358742/ff37c233-7961-40ef-9759-bc642651ae22) + +Here is a possible flow for the case where the tests fail and the changes need to be fixed before the PR can be approved and merged: + +1. Create PR: The developer creates a new pull request in GitHub with their changes. +2. GitHub Action runs: When the PR is created, the GitHub Action is triggered automatically. +3. Run unit tests: The GitHub Action runs the unit tests defined in the project to verify the changes made in the PR. +4. Test failure: If the tests fail, the GitHub Action reports the failure and the PR is marked as failing the tests. +5. Fix the code: The developer needs to fix the code to make the tests pass. +6. Commit changes: Once the code is fixed, the developer commits the changes to the same branch used for the original PR. +7. GitHub Action runs again: As soon as the changes are committed, the GitHub Action is triggered again automatically. +8. Rerun unit tests: The GitHub Action runs the unit tests again to verify the changes made in the commit. +9. Test success: If the tests pass, the PR is marked as passing the tests and can be reviewed again. +10. Review: The code changes are reviewed by the team, and if approved, the PR is merged into the main branch. diff --git a/tests/unit_test/config.py b/tests/unit_test/config.py index 7988845d..ae73e37f 100644 --- a/tests/unit_test/config.py +++ b/tests/unit_test/config.py @@ -42,19 +42,278 @@ project_spend_cost = "project_spend_cost" resource_list = "resource_list" - -# Env -env = terraform_plan_json["variables"]["env"]["value"] -os.environ["ENV"] = env -os.system("export ENV") - - combined_list = [] for item in terraform_plan_json["planned_values"]["root_module"][ "child_modules" ]: # noqa: E501 combined_list.extend(item["resources"]) +kms_id = None +for data_item in combined_list: + if data_item["type"] == "aws_cloudtrail" and data_item["name"] == "this": + kms_id = data_item["values"]["kms_key_id"].split("/")[-1] + break + +s3_bucket = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_s3_bucket" + and data_item["name"] == "this" + and data_item["address"] == "module.xc3.aws_s3_bucket.this" + ): + s3_bucket = data_item["values"]["bucket"] + break + +s3_bucket_tags = None +for data_item in combined_list: + if data_item["type"] == "aws_s3_bucket" and data_item["name"] == "this": + s3_bucket_tags = data_item["values"]["tags"] + break + +vpc_cidr = None +for data_item in combined_list: + if data_item["type"] == "aws_vpc" and data_item["name"] == "this": + vpc_cidr = data_item["values"]["cidr_block"] + break + +vpc_public_subnet = None +for data_item in combined_list: + if data_item["type"] == "aws_subnet" and data_item["name"] == "public_subnet": + vpc_public_subnet = data_item["values"]["cidr_block"] + break + +vpc_private_subnet = None +for data_item in combined_list: + if data_item["type"] == "aws_subnet" and data_item["name"] == "private_subnet": + vpc_private_subnet = data_item["values"]["cidr_block"] + break + +cognito = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cognito_user_pool" + and data_item["address"] == "module.xc3.aws_cognito_user_pool.grafana_pool[0]" + ): + cognito = data_item["values"]["name"] + break + +cloudwatch_list_linked_account = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == list_linked_accounts + ): + cloudwatch_list_linked_account = data_item["values"]["name"] + break + +cloudwatch_list_linked_account_description = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == list_linked_accounts + ): + cloudwatch_list_linked_account_description = data_item["values"]["description"] + break + +cloudwatch_list_linked_account_cron = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == list_linked_accounts + ): + cloudwatch_list_linked_account_cron = data_item["values"]["schedule_expression"] + break + +cloudwatch_most_expensive_services = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == most_expensive_service + ): + cloudwatch_most_expensive_services = data_item["values"]["name"] + break + +cloudwatch_most_expensive_services_description = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == most_expensive_service + ): + cloudwatch_most_expensive_services_description = data_item["values"][ + "description" + ] + break + +cloudwatch_most_expensive_services_cron = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == most_expensive_service + ): + cloudwatch_most_expensive_services_cron = data_item["values"][ + "schedule_expression" + ] + break + +cloudwatch_project_spend_cost = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == project_spend_cost + ): + cloudwatch_project_spend_cost = data_item["values"]["name"] + break + +cloudwatch_project_spend_cost_description = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == project_spend_cost + ): + cloudwatch_project_spend_cost_description = data_item["values"]["description"] + break + +cloudwatch_project_spend_cost_cron = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == project_spend_cost + ): + cloudwatch_project_spend_cost_cron = data_item["values"]["schedule_expression"] + break + +cloudwatch_resource_list = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == resource_list + ): + cloudwatch_resource_list = data_item["values"]["name"] + break + +cloudwatch_resource_list_description = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == resource_list + ): + cloudwatch_resource_list_description = data_item["values"]["description"] + break + +cloudwatch_resource_list_cron = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_cloudwatch_event_rule" + and data_item["name"] == resource_list + ): + cloudwatch_resource_list_cron = data_item["values"]["schedule_expression"] + break + +ec2_ami = None +ec2_ami_count = 0 +for data_item in combined_list: + if data_item["type"] == "aws_instance" and data_item["name"] == "this": + ec2_ami = data_item["values"]["ami"] + ec2_ami_count += 1 + break + +ec2_az = None +for data_item in combined_list: + if data_item["type"] == "aws_subnet" and data_item["name"] == "public_subnet": + ec2_ami = data_item["values"]["availability_zone"] + break + +iam_role_project_spend_cost_name = None +for data_item in combined_list: + if data_item["type"] == "aws_iam_role" and data_item["name"] == "ProjectSpendCost": + iam_role_project_spend_cost_name = data_item["values"]["name"] + break + +iam_role_project_spend_cost_assumed_policy = None +for data_item in combined_list: + if data_item["type"] == "aws_iam_role" and data_item["name"] == "ProjectSpendCost": + iam_role_project_spend_cost_assumed_policy = data_item["values"][ + "assume_role_policy" + ] + break + +iam_role_resource_list_name = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_iam_role" + and data_item["name"] == "resource_list_service_role" + ): + iam_role_resource_list_name = data_item["values"]["name"] + break + +iam_role_resource_list_assumed_policy = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_iam_role" + and data_item["name"] == "resource_list_service_role" + ): + iam_role_resource_list_assumed_policy = data_item["values"][ + "assume_role_policy" + ] + break + +iam_role_total_account_cost_name = None +for data_item in combined_list: + if data_item["type"] == "aws_iam_role" and data_item["name"] == total_account_cost: + iam_role_total_account_cost_name = data_item["values"]["name"] + break + +iam_role_total_account_cost_assumed_policy = None +for data_item in combined_list: + if data_item["type"] == "aws_iam_role" and data_item["name"] == total_account_cost: + iam_role_total_account_cost_assumed_policy = data_item["values"][ + "assume_role_policy" + ] + break + +iam_role_most_expensive_service_role_name = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_iam_role" + and data_item["name"] == "most_expensive_service_role" + ): + iam_role_most_expensive_service_role_name = data_item["values"]["name"] + break + +mes_role_assumed_policy = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_iam_role" + and data_item["name"] == "most_expensive_service_role" + ): + mes_role_assumed_policy = data_item["values"]["assume_role_policy"] + break + +iam_role_list_linked_accounts_name = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_iam_role" + and data_item["name"] == list_linked_accounts + ): + iam_role_list_linked_accounts_name = data_item["values"]["name"] + break + +iam_role_list_linked_accounts_assumed_policy = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_iam_role" + and data_item["name"] == list_linked_accounts + ): + iam_role_list_linked_accounts_assumed_policy = data_item["values"][ + "assume_role_policy" + ] + break + +# Env +env = terraform_plan_json["variables"]["env"]["value"] +os.environ["ENV"] = env +os.system("export ENV") + # Region region = terraform_plan_json["variables"]["region"]["value"] @@ -73,3 +332,23 @@ if data_item["type"] == "apigw" and data_item["name"] == "this": api_gateway = data_item["values"]["name"] break +# SQS +sqs_name = None +for data_item in combined_list: + if data_item["type"] == "aws_sqs_queue" and data_item["name"] == "this": + sqs_name = data_item["values"]["name"] + break +# EIC endpoint +ec2_instance_connect_endpoint = None +for data_item in combined_list: + if ( + data_item["type"] == "aws_ec2_instance_connect_endpoint" + and data_item["name"] == "eicendpoint" + ): + ec2_instance_connect_endpoint = data_item["instances"]["arn"] + break +# AWS Lambda fucntions +lambda_function_names = [] +for data_item in combined_list: + if data_item["type"] == "aws_lambda_function": + lambda_function_names.append(data_item["name"]) diff --git a/tests/unit_test/requirements.txt b/tests/unit_test/requirements.txt new file mode 100644 index 00000000..da1aa6ff --- /dev/null +++ b/tests/unit_test/requirements.txt @@ -0,0 +1,31 @@ +boto3==1.28.7 +botocore==1.31.7 +certifi==2023.7.22 +cffi==1.15.1 +charset-normalizer==3.2.0 +cryptography==41.0.6 +ecdsa==0.18.0 +idna==3.4 +iniconfig==2.0.0 +Jinja2==3.1.3 +jmespath==1.0.1 +MarkupSafe==2.1.3 +moto==4.1.13 +packaging==23.1 +pluggy==1.2.0 +pyasn1==0.5.0 +pycparser==2.21 +pytest==7.4.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-jose==3.3.0 +PyYAML==6.0.1 +requests==2.31.0 +responses==0.23.1 +rsa==4.9 +s3transfer==0.6.1 +six==1.16.0 +types-PyYAML==6.0.12.10 +urllib3==1.26.18 +Werkzeug==2.3.6 +xmltodict==0.13.0 \ No newline at end of file diff --git a/tests/unit_test/unit-test-ce.py b/tests/unit_test/unit-test-ce.py new file mode 100644 index 00000000..7b061db0 --- /dev/null +++ b/tests/unit_test/unit-test-ce.py @@ -0,0 +1,99 @@ +""" +Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" +import unittest +import boto3 +from unittest.mock import patch, MagicMock +from config import region + +def function_to_get_cost_and_usage(): + """ + This function retrieves cost and usage information from AWS Cost Explorer for Amazon S3 services. + + Returns: + dict: A dictionary containing the total cost and currency, or None if no results are available. + """ + ce_client = boto3.client('ce', region_name=region) + + start_date = '2023-01-01' + end_date = '2023-01-31' + granularity = 'DAILY' + metrics = ['UnblendedCost'] + + filters = { + 'Dimensions': { + 'Key': 'SERVICE', + 'Values': ['Amazon S3'] + } + } + + response = ce_client.get_cost_and_usage( + TimePeriod={ + 'Start': start_date, + 'End': end_date + }, + Granularity=granularity, + Metrics=metrics, + Filter=filters + ) + + results = response['ResultsByTime'] + if results: + total_cost = results[0]['Total']['UnblendedCost']['Amount'] + currency = results[0]['Total']['UnblendedCost']['Unit'] + + return {'Total': float(total_cost), 'Currency': currency} + else: + return None + +class TestCostExplorer(unittest.TestCase): + """ + This class contains test cases for the `function_to_get_cost_and_usage` function. + """ + @patch('boto3.client') + def test_get_cost_and_usage(self, mock_boto3): + """ + Test the `function_to_get_cost_and_usage` function with mock data. + + The function should return the expected cost and currency information. + """ + # Mock the Boto3 client + ce_mock = MagicMock() + mock_boto3.return_value = ce_mock + + # Define the expected result + expected_result = {'Total': 100.0, 'Currency': 'USD'} + + # Configure the mock's return value + ce_mock.get_cost_and_usage.return_value = { + 'ResultsByTime': [ + { + 'Total': { + 'UnblendedCost': {'Amount': '100.0', 'Unit': 'USD'}, + } + } + ] + } + + # Call the function you want to test + result = function_to_get_cost_and_usage() + + # Assert that the result matches the expected value + self.assertEqual(result, expected_result) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_test/unit-test-cognito.py b/tests/unit_test/unit-test-cognito.py new file mode 100644 index 00000000..79133008 --- /dev/null +++ b/tests/unit_test/unit-test-cognito.py @@ -0,0 +1,101 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import random +import string +import unittest + +import boto3 +from config import cognito, region +from moto import mock_cognitoidp + + +class CognitoMockTests(unittest.TestCase): + @mock_cognitoidp + def test_create_user_and_verify_attributes(self): + region_name = region + + # Create a user pool + user_pool_name = cognito + user_pool_id = self.create_user_pool(user_pool_name, region_name) + + # Create a user + username = "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(8) + ) + email = ( + "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(8) + ) + + "@example.com" + ) + temporary_password = "".join( + random.choice(string.ascii_letters + string.digits + string.punctuation) + for _ in range(10) + ) + self.create_user(user_pool_id, username, temporary_password, email) + + # Verify user attributes + email_verified = self.get_user_attributes(user_pool_id, username) + logging.info("Verified Email: %s", email_verified) + + # Assert the user attributes + self.assertTrue(email_verified) + + def create_user_pool(self, user_pool_name, region_name): + cognito_client = boto3.client("cognito-idp", region_name=region_name) + user_pool_response = cognito_client.create_user_pool( + PoolName=user_pool_name, + Policies={ + "PasswordPolicy": { + "MinimumLength": 8, + "RequireLowercase": False, + "RequireNumbers": False, + "RequireSymbols": False, + "RequireUppercase": False, + } + }, + AutoVerifiedAttributes=["email"], + ) + return user_pool_response["UserPool"]["Id"] + + def create_user(self, user_pool_id, username, temporary_password, email): + cognito_client = boto3.client("cognito-idp", region_name=region) + cognito_client.admin_create_user( + UserPoolId=user_pool_id, + Username=username, + TemporaryPassword=temporary_password, + UserAttributes=[ + {"Name": "email", "Value": email}, + ], + ) + + def get_user_attributes(self, user_pool_id, username): + cognito_client = boto3.client("cognito-idp", region_name=region) + user_response = cognito_client.admin_get_user( + UserPoolId=user_pool_id, Username=username + ) + user_attributes = user_response["UserAttributes"] + email_verified = False + + for attribute in user_attributes: + if attribute["Name"] == "email": + email_verified = True + + return email_verified + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-cwe.py b/tests/unit_test/unit-test-cwe.py new file mode 100644 index 00000000..d5625016 --- /dev/null +++ b/tests/unit_test/unit-test-cwe.py @@ -0,0 +1,94 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest + +import boto3 +from config import ( + cloudwatch_list_linked_account, + cloudwatch_list_linked_account_cron, + cloudwatch_list_linked_account_description, + cloudwatch_most_expensive_services, + cloudwatch_most_expensive_services_cron, + cloudwatch_most_expensive_services_description, + cloudwatch_project_spend_cost, + cloudwatch_project_spend_cost_cron, + cloudwatch_project_spend_cost_description, + cloudwatch_resource_list, + cloudwatch_resource_list_cron, + cloudwatch_resource_list_description, + region, +) +from moto import mock_events + +try: + events_client = boto3.client("events", region_name=region) +except Exception as e: + logging.error("Error creating events client: " + str(e)) + + +class CloudWatchEventMockTests(unittest.TestCase): + @mock_events + def test_create_cloudwatch_events(self): + # Create a CloudWatch Event rule + rule_data = [ + { + "name": cloudwatch_list_linked_account, + "description": cloudwatch_list_linked_account_description, + "cron_expression": cloudwatch_list_linked_account_cron, + }, + { + "name": cloudwatch_most_expensive_services, + "description": cloudwatch_most_expensive_services_description, + "cron_expression": cloudwatch_most_expensive_services_cron, + }, + { + "name": cloudwatch_project_spend_cost, + "description": cloudwatch_project_spend_cost_description, + "cron_expression": cloudwatch_project_spend_cost_cron, + }, + { + "name": cloudwatch_resource_list, + "description": cloudwatch_resource_list_description, + "cron_expression": cloudwatch_resource_list_cron, + }, + ] + + for rule in rule_data: + rule_name = rule["name"] + rule_description = rule["description"] + rule_schedule_expression = rule["cron_expression"] + response = events_client.put_rule( + Name=rule_name, + ScheduleExpression=rule_schedule_expression, + State="ENABLED", + Description=rule_description, + ) + + # Verify the rule was created + self.assertEqual(response["ResponseMetadata"]["HTTPStatusCode"], 200) + + # Describe the CloudWatch Event rule + response = events_client.describe_rule(Name=rule_name) + + # Verify the rule attributes + self.assertEqual(response["Name"], rule_name) + self.assertEqual(response["State"], "ENABLED") + self.assertEqual(response["Description"], rule_description) + self.assertEqual(response["ScheduleExpression"], rule_schedule_expression) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-ec2.py b/tests/unit_test/unit-test-ec2.py new file mode 100644 index 00000000..703295ef --- /dev/null +++ b/tests/unit_test/unit-test-ec2.py @@ -0,0 +1,112 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +import unittest + +import boto3 +import pytest +from botocore.exceptions import ClientError +from config import ec2_ami, ec2_ami_count, ec2_az, region +from moto import mock_ec2, settings + + +class EC2Test(unittest.TestCase): + def setUp(self): + try: + self.client = boto3.client("ec2", region_name=region) + except Exception as e: + logging.error("Error creating EC2 client: " + str(e)) + + @mock_ec2 + def test_instance_launch_and_terminate(self): + + with pytest.raises(ClientError) as ex: + self.client.run_instances( + ImageId=ec2_ami, MinCount=1, MaxCount=ec2_ami_count, DryRun=True + ) + ex.value.response["ResponseMetadata"]["HTTPStatusCode"] == 412 + ex.value.response["Error"]["Code"] == "DryRunOperation" + ex.value.response["Error"][ + "Message" + ] == """An error occurred (DryRunOperation) + when calling the RunInstances operation: + Request would have succeeded, but DryRun flag is set""" + + reservation = self.client.run_instances( + ImageId=ec2_ami, MinCount=1, MaxCount=ec2_ami_count + ) + len(reservation["Instances"]) == ec2_ami_count + instance = reservation["Instances"][0] + instance["State"] == {"Code": 0, "Name": "pending"} + instance_id = instance["InstanceId"] + + reservations = self.client.describe_instances(InstanceIds=[instance_id])[ + "Reservations" + ] + len(reservations) == ec2_ami_count + reservations[0]["ReservationId"] == reservation["ReservationId"] + instances = reservations[0]["Instances"] + len(instances) == ec2_ami_count + instance = instances[0] + instance["InstanceId"] == instance_id + instance["State"] == {"Code": 16, "Name": "running"} + if settings.TEST_SERVER_MODE: + # Exact value can't be determined in ServerMode + instance.should.have.key("LaunchTime") + else: + launch_time = instance["LaunchTime"].strftime("%Y-%m-%dT%H:%M:%S.000Z") + logging.info("Launch Time: %s", launch_time) + if instance["VpcId"] is not None: + logging.info("EC2 in VPC") + instance["Placement"]["AvailabilityZone"] == ec2_az + + root_device_name = instance["RootDeviceName"] + mapping = instance["BlockDeviceMappings"][0] + mapping["DeviceName"] == root_device_name + mapping["Ebs"]["Status"] == "in-use" + volume_id = mapping["Ebs"]["VolumeId"] + re.match(r"vol-\w+", volume_id) + + volume = self.client.describe_volumes(VolumeIds=[volume_id])["Volumes"][0] + volume["Attachments"][0]["InstanceId"] == instance_id + volume["State"] == "in-use" + + with pytest.raises(ClientError) as ex: + self.client.terminate_instances(InstanceIds=[instance_id], DryRun=True) + ex.value.response["ResponseMetadata"]["HTTPStatusCode"] == 412 + ex.value.response["Error"]["Code"] == "DryRunOperation" + ex.value.response["Error"][ + "Message" + ] == """An error occurred (DryRunOperation) when calling + the TerminateInstances operation: + Request would have succeeded, but DryRun flag is set""" + + response = self.client.terminate_instances(InstanceIds=[instance_id]) + len(response["TerminatingInstances"]) == ec2_ami_count + instance = response["TerminatingInstances"][0] + instance["InstanceId"] == instance_id + instance["PreviousState"] == {"Code": 16, "Name": "running"} + instance["CurrentState"] == {"Code": 32, "Name": "shutting-down"} + + reservations = self.client.describe_instances(InstanceIds=[instance_id])[ + "Reservations" + ] + instance = reservations[0]["Instances"][0] + instance["State"] == {"Code": 48, "Name": "terminated"} + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-eic.py b/tests/unit_test/unit-test-eic.py new file mode 100644 index 00000000..e5266320 --- /dev/null +++ b/tests/unit_test/unit-test-eic.py @@ -0,0 +1,79 @@ +""" +Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" +import logging +import unittest + +import boto3 +from config import ec2_instance_connect_endpoint, region, env + +class EicMockTests(unittest.TestCase): + """ + Unit tests for EC2 Instance Connect Endpoint creation. + + These tests validate the existence of an EC2 Instance Connect Endpoint with the specified ARN when the environment is 'prod'. + """ + # Define a decorator to skip the test if the environment is "dev" + def skip_if_dev(func): + def wrapper(self, *args, **kwargs): + if env != "prod": + self.skipTest("Skipping test in 'dev' environment") + else: + func(self, *args, **kwargs) + + return wrapper + + @skip_if_dev + def setUp(self): + """ + Set up the test case by initializing the AWS EC2 client. + """ + # Initialize the AWS EC2 client + self.ec2_client = boto3.client('ec2', region_name=region) + + @skip_if_dev + def test_eni_endpoint_created(self): + """ + Test if the EC2 Instance Connect Endpoint with the specified ARN has been created. + + This test checks if the EC2 Instance Connect Endpoint exists and logs information for debugging. + + Raises: + AssertionError: If the number of EC2 Instance Connect Endpoints with the specified ARN is not equal to 1. + """ + # Log information for debugging + logging.info(f"Testing EC2 Instance Connect Endpoint: ARN: {ec2_instance_connect_endpoint}, Region: {region}, Environment: {env}") + # Check if the ENI endpoint exists + try: + # Use Boto3 to describe the network interfaces and find the one with the specified ARN + response = self.ec2_client.describe_network_interfaces(Filters=[{'Name': 'network-interface-id', 'Values': [ec2_instance_connect_endpoint]}]) + network_interfaces = response['NetworkInterfaces'] + + if len(network_interfaces) == 1: + logging.info(f"EC2 Instance Connect Endpoint with ARN '{ec2_instance_connect_endpoint}' has been created.") + elif len(network_interfaces) == 0: + logging.warning(f"EC2 Instance Connect Endpoint with ARN '{ec2_instance_connect_endpoint}' not found.") + else: + logging.warning(f"Multiple EC2 Instance Connect Endpoints with the same ARN '{ec2_instance_connect_endpoint}' found.") + + self.assertEqual(len(network_interfaces), 1, "Expected one EC2 Instance Connect Endpoint to be created.") + except Exception as e: + logging.warning(f"Error while checking EC2 Instance Connect Endpoint: {str(e)}") + +if __name__ == "__main__": + unittest.main() + logging.shutdown() diff --git a/tests/unit_test/unit-test-iam.py b/tests/unit_test/unit-test-iam.py new file mode 100644 index 00000000..9406090a --- /dev/null +++ b/tests/unit_test/unit-test-iam.py @@ -0,0 +1,119 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import unittest + +import boto3 +from config import ( + iam_role_list_linked_accounts_assumed_policy, + iam_role_list_linked_accounts_name, + iam_role_most_expensive_service_role_name, + iam_role_project_spend_cost_assumed_policy, + iam_role_project_spend_cost_name, + iam_role_resource_list_assumed_policy, + iam_role_resource_list_name, + iam_role_total_account_cost_assumed_policy, + iam_role_total_account_cost_name, + mes_role_assumed_policy, + region, +) +from moto import mock_iam + + +class IAMRoleMockTests(unittest.TestCase): + @mock_iam + def test_create_describe_delete_iam_roles(self): + iam_client = boto3.client("iam", region_name=region) + roles = [ + { + "role_name": iam_role_project_spend_cost_name, + "assume_role_policy": iam_role_project_spend_cost_assumed_policy, + }, + { + "role_name": iam_role_list_linked_accounts_name, + "assume_role_policy": iam_role_list_linked_accounts_assumed_policy, + }, + { + "role_name": iam_role_total_account_cost_name, + "assume_role_policy": iam_role_total_account_cost_assumed_policy, + }, + { + "role_name": iam_role_resource_list_name, + "assume_role_policy": iam_role_resource_list_assumed_policy, + }, + { + "role_name": iam_role_most_expensive_service_role_name, + "assume_role_policy": mes_role_assumed_policy, + }, + ] + + for roles_data in roles: + # Create an IAM role + response = iam_client.create_role( + RoleName=roles_data["role_name"], + AssumeRolePolicyDocument=json.dumps(roles_data["assume_role_policy"]), + ) + # Verify the role was created + self.assertEqual(response["ResponseMetadata"]["HTTPStatusCode"], 200) + + # Describe the IAM role + response = iam_client.get_role(RoleName=roles_data["role_name"]) + role = response["Role"] + + # Verify the role attributes + self.assertEqual(role["RoleName"], roles_data["role_name"]) + self.assertEqual( + role["AssumeRolePolicyDocument"], roles_data["assume_role_policy"] + ) + + # Delete the IAM role + response = iam_client.delete_role(RoleName=roles_data["role_name"]) + + # Verify the role was deleted + self.assertEqual(response["ResponseMetadata"]["HTTPStatusCode"], 200) + + @mock_iam + def test_create_describe_delete_iam_instance_profile(self): + iam_client = boto3.client("iam", region_name=region) + + # Create an IAM instance profile + instance_profile_name = "test-instance-profile" + response = iam_client.create_instance_profile( + InstanceProfileName=instance_profile_name + ) + + # Verify the instance profile was created + self.assertEqual(response["ResponseMetadata"]["HTTPStatusCode"], 200) + + # Describe the IAM instance profile + response = iam_client.get_instance_profile( + InstanceProfileName=instance_profile_name + ) + instance_profile = response["InstanceProfile"] + + # Verify the instance profile attributes + self.assertEqual(instance_profile["InstanceProfileName"], instance_profile_name) + + # Delete the IAM instance profile + response = iam_client.delete_instance_profile( + InstanceProfileName=instance_profile_name + ) + + # Verify the instance profile was deleted + self.assertEqual(response["ResponseMetadata"]["HTTPStatusCode"], 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-kms.py b/tests/unit_test/unit-test-kms.py new file mode 100644 index 00000000..cb40b926 --- /dev/null +++ b/tests/unit_test/unit-test-kms.py @@ -0,0 +1,54 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest + +import boto3 +from config import kms_id, region +from moto import mock_kms + +try: + kms_client = boto3.client("kms", region_name=region) +except Exception as e: + logging.error("Error creating KMS client: " + str(e)) + + +class KMSMockTests(unittest.TestCase): + @mock_kms + def test_encrypt_decrypt_data(self): + + # Create a KMS key + key_alias = f"alias/{kms_id}" + response = kms_client.create_key() + key_id = response["KeyMetadata"]["KeyId"] + kms_client.create_alias(AliasName=key_alias, TargetKeyId=key_id) + + # Encrypt data + plaintext = "Hello, World!" + encryption_response = kms_client.encrypt( + KeyId=key_alias, Plaintext=plaintext.encode() + ) + ciphertext = encryption_response["CiphertextBlob"] + + # Decrypt data + decryption_response = kms_client.decrypt(CiphertextBlob=ciphertext) + decrypted_plaintext = decryption_response["Plaintext"].decode() + + # Assert the original plaintext matches the decrypted plaintext + self.assertEqual(plaintext, decrypted_plaintext) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-lambda.py b/tests/unit_test/unit-test-lambda.py new file mode 100644 index 00000000..863ae5ce --- /dev/null +++ b/tests/unit_test/unit-test-lambda.py @@ -0,0 +1,65 @@ +""" +Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" +import logging +import unittest +import boto3 +from config import lambda_function_names, region, env + +class LambdaMockTests(unittest.TestCase): + """ + Unit tests for AWS Lambda functions. + + These tests validate the existence of Lambda functions with the specified names. + """ + def setUp(self): + """ + Set up the test case by initializing the AWS Lambda client. + """ + # Initialize the AWS Lambda client + self.lambda_client = boto3.client('lambda', region_name=region) + + def test_lambda_functions_created(self): + """ + Test if the expected Lambda functions have been created. + + This test checks if the Lambda functions exist and logs debugging information. + + Raises: + AssertionError: If any of the expected Lambda functions are not found. + """ + # Log information for debugging + logging.info(f"Testing Lambda Functions in Region: {region}, Environment: {env}") + + # Check if the Lambda functions exist + try: + # Use Boto3 to list Lambda functions + response = self.lambda_client.list_functions() + lambda_functions = response['Functions'] + + existing_function_names = [func['FunctionName'] for func in lambda_functions] + + for function_name in lambda_function_names: + self.assertIn(function_name, existing_function_names, f"Lambda Function '{function_name}' not found.") + + logging.info("All expected Lambda Functions have been created.") + except Exception as e: + logging.warning(f"Error while checking Lambda Functions: {str(e)}") + +if __name__ == "__main__": + unittest.main() + logging.shutdown() diff --git a/tests/unit_test/unit-test-loadbalancer.py b/tests/unit_test/unit-test-loadbalancer.py index b4a2581f..18daf431 100644 --- a/tests/unit_test/unit-test-loadbalancer.py +++ b/tests/unit_test/unit-test-loadbalancer.py @@ -16,55 +16,47 @@ SPDX-License-Identifier: Apache-2.0 """ -import logging import unittest - import boto3 -from config import ( # Import the 'env' variable and 'api_gateway_name' from your config # noqa: E501 - api_gateway, - env, - region, -) +import logging +from config import region, load_balancer, env -class TestApiGatewayCreation(unittest.TestCase): + +class TestLoadBalancerCreation(unittest.TestCase): def setUp(self): - # Initialize the AWS API Gateway client - self.apigateway_client = boto3.client("apigateway", region_name=region) + # Initialize the AWS ELB client + self.elb_client = boto3.client("elbv2", region_name=region) - @unittest.skipIf(env == "dev", "Skipping test in dev environment") - def test_api_gateway_created(self): - # Log the API Gateway name and other relevant information for debugging + @unittest.skipIf(env == "dev", "Skipping test in dev environment.") + def test_load_balancer_created(self): + # Log the load balancer name and other relevant information for debugging logging.info( - f"Testing API Gateway: {api_gateway}, Region: {region}, Environment: {env}" # noqa: E501 + f"Testing load balancer: {load_balancer}, Region: {region}, Environment: {env}" ) - # Check if the API Gateway exists + # Check if the load balancer exists try: - response = self.apigateway_client.get_rest_apis() - api_gateways = response["items"] - - # Filter API Gateways by name - matching_api_gateways = [ - api for api in api_gateways if api["name"] == api_gateway - ] + response = self.elb_client.describe_load_balancers(Names=[load_balancer]) + load_balancer_count = len(response["LoadBalancers"]) - if len(matching_api_gateways) == 1: - logging.info(f"API Gateway '{api_gateway}' has been created.") - elif len(matching_api_gateways) == 0: - logging.warning(f"API Gateway '{api_gateway}' not found.") + if load_balancer_count == 1: + logging.info(f"Load balancer '{load_balancer}' has been created.") + elif load_balancer_count == 0: + logging.warning(f"Load balancer '{load_balancer}' not found.") else: logging.warning( - f"Multiple API Gateways with the name '{api_gateway}' found." # noqa: E501 + f"Multiple load balancers with the name '{load_balancer}' found." ) self.assertEqual( - len(matching_api_gateways), - 1, - "Expected one API Gateway to be created.", # noqa: E501 + load_balancer_count, 1, "Expected one load balancer to be created." ) - except Exception as e: - logging.warning(f"Error while checking API Gateway: {str(e)}") + except self.elb_client.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "LoadBalancerNotFound": + logging.warning(f"Load balancer '{load_balancer}' not found.") + else: + raise if __name__ == "__main__": diff --git a/tests/unit_test/unit-test-nat.py b/tests/unit_test/unit-test-nat.py new file mode 100644 index 00000000..1a09a5d5 --- /dev/null +++ b/tests/unit_test/unit-test-nat.py @@ -0,0 +1,61 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest + +import boto3 +from config import region, vpc_cidr +from moto import mock_ec2 + +try: + ec2_client = boto3.client("ec2", region_name=region) +except Exception as e: + logging.error("Error creating EC2 client: " + str(e)) + + +class NATGatewayMockTests(unittest.TestCase): + @mock_ec2 + def test_create_describe_delete_nat_gateway(self): + + # Create a VPC + vpc_response = ec2_client.create_vpc(CidrBlock=vpc_cidr) + vpc_id = vpc_response["Vpc"]["VpcId"] + + # Create a subnet + subnet_response = ec2_client.create_subnet(VpcId=vpc_id, CidrBlock=vpc_cidr) + subnet_id = subnet_response["Subnet"]["SubnetId"] + + # Create an elastic IP + elastic_ip_response = ec2_client.allocate_address(Domain="vpc") + elastic_ip = elastic_ip_response["AllocationId"] + + # Create a NAT Gateway + nat_gateway_response = ec2_client.create_nat_gateway( + SubnetId=subnet_id, AllocationId=elastic_ip + ) + nat_gateway_id = nat_gateway_response["NatGateway"]["NatGatewayId"] + + # Describe NAT Gateway + nat_gateway_info = ec2_client.describe_nat_gateways( + NatGatewayIds=[nat_gateway_id] + ) + nat_gateway_state = nat_gateway_info["NatGateways"][0]["State"] + + # Assert NAT Gateway state + self.assertEqual(nat_gateway_state, "available") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-route53.py b/tests/unit_test/unit-test-route53.py new file mode 100644 index 00000000..3eedd670 --- /dev/null +++ b/tests/unit_test/unit-test-route53.py @@ -0,0 +1,93 @@ +""" +Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" +import logging +import unittest + +import boto3 +from moto import mock_route53 +from config import ec2_instance_connect_endpoint, region, env + +class TestRoute53(unittest.TestCase): + """ + Unit tests for Route 53 DNS record creation. + + These tests validate the creation of a DNS record in Route 53 when the environment is 'prod'. + """ + # Define a decorator to skip the test if the environment is "dev" + def skip_if_dev(func): + def wrapper(self, *args, **kwargs): + if env != "prod": + self.skipTest("Skipping test in 'dev' environment") + else: + func(self, *args, **kwargs) + + return wrapper + + @skip_if_dev + @mock_route53 + def test_create_dns_record(self): + """ + Test the creation of a DNS record in Route 53. + + This test creates a mock Route 53 client, a hosted zone, and a DNS record. It checks if the DNS record was created successfully. + + Raises: + AssertionError: If the DNS record creation does not return an HTTP status code of 200. + """ + # Create a mock Route 53 client + route53_client = boto3.client('route53', region_name=region) + + # Create a hosted zone + hosted_zone_name = 'xgrid.co' + response = route53_client.create_hosted_zone( + Name=hosted_zone_name, + CallerReference='test-123' + ) + hosted_zone_id = response['HostedZone']['Id'] + + # Define DNS record details + dns_name = 'xccc.xgrid.co' + dns_type = 'A' + dns_value = '192.168.1.1' # Random DNS value for testing. + + # Create a DNS record + dns_record_change = { + 'Changes': [ + { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'Name': dns_name, + 'Type': dns_type, + 'TTL': 300, + 'ResourceRecords': [{'Value': dns_value}] + } + } + ] + } + + response = route53_client.change_resource_record_sets( + HostedZoneId=hosted_zone_id, + ChangeBatch=dns_record_change + ) + + + # Check if the DNS record was created successfully + self.assertEqual(response['ResponseMetadata']['HTTPStatusCode'], 200) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-s3.py b/tests/unit_test/unit-test-s3.py new file mode 100644 index 00000000..67121291 --- /dev/null +++ b/tests/unit_test/unit-test-s3.py @@ -0,0 +1,75 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest + +import boto3 +from config import region, s3_bucket, s3_bucket_tags +from moto import mock_s3 + +try: + s3_client = boto3.client("s3") +except Exception as e: + logging.error("Error creating S3 client: " + str(e)) + + +class S3MockTests(unittest.TestCase): + @mock_s3 + def test_create_bucket(self): + bucket_name = s3_bucket + + s3_client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={"LocationConstraint": region}, + ) + logging.info("Bucket created successfully: %s", bucket_name) + + response = s3_client.list_buckets() + buckets = response["Buckets"] + self.assertEqual(len(buckets), 1) + self.assertEqual(buckets[0]["Name"], bucket_name) + + @mock_s3 + def test_check_bucket_tags(self): + bucket_name = s3_bucket + + s3_client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={"LocationConstraint": region}, + ) + + tags = s3_bucket_tags + + tag_set = [{"Key": key, "Value": value} for key, value in tags.items()] + s3_client.put_bucket_tagging( + Bucket=bucket_name, + Tagging={"TagSet": tag_set}, + ) + response = s3_client.get_bucket_tagging(Bucket=bucket_name) + tags_res = response["TagSet"] + if tags: + logging.info("Tags for bucket: %s", bucket_name) + else: + logging.debug("No tags found for bucket: %s", bucket_name) + + self.assertEqual(len(tags_res), 4) + + for index, tag_key in enumerate(tags.keys()): + self.assertEqual(tags_res[index]["Key"], tag_key) + self.assertEqual(tags_res[index]["Value"], tags[tag_key]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-sqs.py b/tests/unit_test/unit-test-sqs.py new file mode 100644 index 00000000..c41c50d1 --- /dev/null +++ b/tests/unit_test/unit-test-sqs.py @@ -0,0 +1,75 @@ +""" +Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" +import unittest +import boto3 +from moto import mock_sqs +from config import region, sqs_name + +class TestSQSService(unittest.TestCase): + """ + Unit tests for Amazon Simple Queue Service (SQS). + + These tests validate sending and receiving messages to and from an SQS queue, including error scenarios. + """ + + @mock_sqs + def test_send_and_receive_message(self): + """ + Test sending and receiving a message to/from an SQS queue, including error scenarios. + + This test creates a mock SQS client and a mock SQS queue, sends a message to the queue, and then receives the message. + It checks if a message was received and if it matches the sent message. It also covers error scenarios such as + sending a message to a non-existent queue and receiving a message from an empty queue. + + Raises: + AssertionError: If any part of the test fails, including error scenarios. + """ + # Create a mock SQS client + sqs_client = boto3.client("sqs", region_name=region) + + # Create a mock SQS queue + queue_name = sqs_name + queue_url = sqs_client.create_queue(QueueName=queue_name)["QueueUrl"] + + # Test sending a message to a non-existent queue + non_existent_queue_url = "https://sqs.us-east-1.amazonaws.com/123456789012/NonExistentQueue" + with self.assertRaises(boto3.exceptions.botocore.exceptions.ClientError): + sqs_client.send_message(QueueUrl=non_existent_queue_url, MessageBody="Test message to non-existent queue") + + # Send a message to the queue + message_body = "Test message" + sqs_client.send_message(QueueUrl=queue_url, MessageBody=message_body) + + # Receive a message from the empty queue + empty_queue_url = sqs_client.create_queue(QueueName="EmptyQueue")["QueueUrl"] + response = sqs_client.receive_message(QueueUrl=empty_queue_url, MaxNumberOfMessages=1) + self.assertNotIn("Messages", response) + + # Receive a message from the queue + response = sqs_client.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1) + + # Check if a message was received + self.assertIn("Messages", response) + self.assertEqual(len(response["Messages"]), 1) + + # Check if the received message matches the sent message + received_message = response["Messages"][0]["Body"] + self.assertEqual(received_message, message_body) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_test/unit-test-vpc.py b/tests/unit_test/unit-test-vpc.py new file mode 100644 index 00000000..bf8ea622 --- /dev/null +++ b/tests/unit_test/unit-test-vpc.py @@ -0,0 +1,163 @@ +# Copyright (c) 2023, Xgrid Inc, https://xgrid.co + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest + +import boto3 +from config import region, vpc_cidr, vpc_private_subnet, vpc_public_subnet +from moto import mock_ec2 + +try: + ec2_client = boto3.client("ec2", region_name=region) +except Exception as e: + logging.error("Error creating EC2 client: " + str(e)) + + +class MyVPCTest(unittest.TestCase): + @mock_ec2 + def test_create_vpc(self): + + # Create a mock VPC + response = ec2_client.create_vpc(CidrBlock=vpc_cidr) + vpc_id = response["Vpc"]["VpcId"] + + # Test VPC creation + self.assertEqual(response["Vpc"]["CidrBlock"], vpc_cidr) + + # Test VPC retrieval + vpc = ec2_client.describe_vpcs(VpcIds=[vpc_id])["Vpcs"][0] + self.assertEqual(vpc["VpcId"], vpc_id) + self.assertEqual(vpc["CidrBlock"], vpc_cidr) + + @mock_ec2 + def test_vpc_with_igw_and_subnet(self): + # Create a mock VPC + response = ec2_client.create_vpc(CidrBlock=vpc_cidr) + vpc_id = response["Vpc"]["VpcId"] + + # Create a mock internet gateway (IGW) + response = ec2_client.create_internet_gateway() + igw_id = response["InternetGateway"]["InternetGatewayId"] + + # Attach the IGW to the VPC + ec2_client.attach_internet_gateway(InternetGatewayId=igw_id, VpcId=vpc_id) + + # Create a mock subnet + response = ec2_client.create_subnet(CidrBlock=vpc_cidr, VpcId=vpc_id) + + # Perform your test assertions + self.assertEqual(response["Subnet"]["CidrBlock"], vpc_cidr) + self.assertEqual(response["Subnet"]["VpcId"], vpc_id) + + @mock_ec2 + def test_delete_vpc(self): + + # Create a mock VPC + response = ec2_client.create_vpc(CidrBlock=vpc_cidr) + vpc_id = response["Vpc"]["VpcId"] + + # Delete the VPC + response = ec2_client.delete_vpc(VpcId=vpc_id) + + # Test VPC deletion + self.assertEqual(response["ResponseMetadata"]["HTTPStatusCode"], 200) + + # Try to retrieve the deleted VPC + with self.assertRaises(ec2_client.exceptions.ClientError) as context: + ec2_client.describe_vpcs(VpcIds=[vpc_id]) + + self.assertEqual( + context.exception.response["Error"]["Code"], "InvalidVpcID.NotFound" + ) + + @mock_ec2 + def test_create_route_table(self): + # Create a mock VPC + response = ec2_client.create_vpc(CidrBlock=vpc_cidr) + vpc_id = response["Vpc"]["VpcId"] + + # Create a mock route table + response = ec2_client.create_route_table(VpcId=vpc_id) + route_table_id = response["RouteTable"]["RouteTableId"] + + # Perform your test assertions + self.assertEqual(response["RouteTable"]["VpcId"], vpc_id) + self.assertEqual(response["RouteTable"]["Associations"], []) + self.assertEqual(response["RouteTable"]["RouteTableId"], route_table_id) + + @mock_ec2 + def test_route_table_association(self): + # Create a mock VPC + response = ec2_client.create_vpc(CidrBlock=vpc_cidr) + vpc_id = response["Vpc"]["VpcId"] + + # Create a mock subnet + response = ec2_client.create_subnet(CidrBlock=vpc_cidr, VpcId=vpc_id) + subnet_id = response["Subnet"]["SubnetId"] + + # Create a mock route table + response = ec2_client.create_route_table(VpcId=vpc_id) + route_table_id = response["RouteTable"]["RouteTableId"] + + # Associate the route table with the subnet + response = ec2_client.associate_route_table( + RouteTableId=route_table_id, SubnetId=subnet_id + ) + + self.assertIsNotNone(response) + self.assertIn("AssociationId", response) + + @mock_ec2 + def test_create_vpc_public_private_subnets(self): + # Create a VPC + response = ec2_client.create_vpc( + CidrBlock=vpc_cidr, AmazonProvidedIpv6CidrBlock=True + ) + vpc_id = response["Vpc"]["VpcId"] + + # Create public subnet + public_subnet_response = ec2_client.create_subnet( + VpcId=vpc_id, + CidrBlock=vpc_public_subnet, + ) + public_subnet_id = public_subnet_response["Subnet"]["SubnetId"] + + # Create private subnet + private_subnet_response = ec2_client.create_subnet( + VpcId=vpc_id, + CidrBlock=vpc_private_subnet, + ) + private_subnet_id = private_subnet_response["Subnet"]["SubnetId"] + + # Verify VPC and subnets + vpcs = ec2_client.describe_vpcs(VpcIds=[vpc_id])["Vpcs"] + self.assertEqual(len(vpcs), 1) + self.assertEqual(vpcs[0]["VpcId"], vpc_id) + + subnets = ec2_client.describe_subnets( + SubnetIds=[public_subnet_id, private_subnet_id] + )["Subnets"] + self.assertEqual(len(subnets), 2) + self.assertIn(public_subnet_id, [subnet["SubnetId"] for subnet in subnets]) + self.assertIn(private_subnet_id, [subnet["SubnetId"] for subnet in subnets]) + + # Clean up resources + ec2_client.delete_subnet(SubnetId=public_subnet_id) + ec2_client.delete_subnet(SubnetId=private_subnet_id) + ec2_client.delete_vpc(VpcId=vpc_id) + + +if __name__ == "__main__": + unittest.main()