diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index a78ba467..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,85 +0,0 @@ -# Changelog - -All notable changes to the `XgridInc/xc3` repository will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [1.2.1] - 2023-10-08 - - -### Changed -- Now the KMS key is only create and use in prod env. - -### Fixed -- KMS datasource errors fixed. -- resolved conflicts. - -## [1.2.0] - 2023-09-20 - - -### Changed -- Created separate architecture diagrams for both Dev and Prod environements. -- Added cloudwatch alarms CPU utilization monitoring. -- Added WAFv2 and attached with loadbalancer. -- Integration testing for dev and prod environments respectively. -- Grafana region dashboard region ID readability implementation. - -### Fixed -- Loabalancer health and connection issues. -- Grafana Login Error with cognito (production environment). - - -## [1.1.0] - 2023-07-17 - - -### Changed -- Integrated EIC endpoint and removed bastion host -- Updated the infra-deployment based on environment variable -- Added the automation script to deploy XC3 infrastack -- Updated the Architecture Diagram of XC3 in the Readme file - - -### Fixed -- Fixed Linked List Lambda to run from a member account as well -- Added checks for the packages installation [AWS CLI, python, terraform] in the init.sh bash script. - - -## [1.0.1] - 2023-06-06 - - -### Changed -- Updated the Readme file -- Updated the Architecture Diagram in the Readme file - - -### Fixed -- SSL Certificate Validation issue -- Fixed the startup script of XC3's EC2 instance - - - - -## [1.0.0] - 2023-05-11 - -### Added -- Initial release of the `XgridInc/xc3` repository. -- `cloud_custodian_policies` folder having cloud custodian policies for tagging compliance, monitoring and alerting -- `Pre-requirements` folder to refer the IAM Permission Set required to setup XC3 -- `Infrastructure` folder with the necessary IaC to provision XC3 infrastack -- `src` folder having the XC3 backend -- `tests` folder with unit testing implementation v1.0.0 for terraform infrastructure. -- `workflows` folder which contains the diagram of all the XC3 workflows implemented -- `CODE_OF_CONDUCT.md` file for the Code of Conduct for the community -- `CONTRIBUTING.md` file which contains the detail of how to contribute to the Project -- `ROADMAP.md` file which has information about XC3's existing features and its future plans -- `UserGuide.md` file provides the guide on how to use XC3 Grafana Dashboards effectively - -### Changed -- Updated the startup script of XC3's EC2 instance - -[1.0.0]: https://github.com/XgridInc/xc3/releases/tag/v1.0.0 -[1.0.1]: https://github.com/XgridInc/xc3/releases/tag/v1.0.1 -[1.1.0]: https://github.com/XgridInc/xc3/releases/tag/v1.1.0 -[1.2.0]: https://github.com/XgridInc/xc3/releases/tag/v1.2.0 -[1.2.1]: https://github.com/XgridInc/xc3/releases/tag/v1.2.1 diff --git a/IMPORTANT_FOR_CLIENT.md b/IMPORTANT_FOR_CLIENT.md new file mode 100644 index 00000000..37303fc5 --- /dev/null +++ b/IMPORTANT_FOR_CLIENT.md @@ -0,0 +1,8 @@ +THIS BRANCH HAS CHANGES MADE TO IT IN: +1. infrastructure / modules / serverless directory, where we have changed the iam-role.tf file for adding three new lambda functions and their role and role policies + There are also changes made in the variables.tf file for adding the three lambda function file paths. There is also resource definitions for CloudWatch and EventBridge rules as well +2. infrastructure / variables.tf file, where we have added a variable for the lambda function cron job, namely 'roles_lambda_cron'. +3. src / iam-roles directory, where there are three new or extra python files in the directory, for the source code of the threee new Lambda fucntions. + +NOTE: THE CODE FOR THE THREE NEW LAMBDA FUNCTIONS HAS ALSO BEEN ZIPPED INTO A FILE NAMED SOURCE_CODE_FOR_IAM_ROLES ALONG WITH THE DASHBOARD FILE IN THE +ROOT DIRECOTORY WHICH CAN BE PASTED IN THE EXISTING LAMBDA FUNCTIONS IN THE IAM ROLE WORKFLOW TO BE TESTED AND INCORPORATED. diff --git a/README.md b/README.md index 279429a6..5917d787 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ +THIS BRANCH HAS CHANGES MADE TO IT IN: + +infrastructure / modules / serverless directory, where we have changed the iam-role.tf file for adding three new lambda functions and their role and role policies There are also changes made in the variables.tf file for adding the three lambda function file paths. There is also resource definitions for CloudWatch and EventBridge rules as well + +infrastructure / variables.tf file, where we have added a variable for the lambda function cron job, namely 'roles_lambda_cron'. + +src / iam-roles directory, where there are three new or extra python files in the directory, for the source code of the threee new Lambda fucntions. + +NOTE: THE CODE FOR THE THREE NEW LAMBDA FUNCTIONS HAS ALSO BEEN ZIPPED INTO A FILE NAMED SOURCE_CODE_FOR_IAM_ROLES ALONG WITH THE DASHBOARD FILE IN THE ROOT DIRECOTORY WHICH CAN BE PASTED IN THE EXISTING LAMBDA FUNCTIONS IN THE IAM ROLE WORKFLOW TO BE TESTED AND INCORPORATED.
[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/SOURCE_CODE_FOR_IAM_ROLES.zip b/SOURCE_CODE_FOR_IAM_ROLES.zip new file mode 100644 index 00000000..fd4d3c87 Binary files /dev/null and b/SOURCE_CODE_FOR_IAM_ROLES.zip differ diff --git a/UPDATED USER MANUAL (TEAM 14).pdf b/UPDATED USER MANUAL (TEAM 14).pdf new file mode 100644 index 00000000..3254a90c Binary files /dev/null and b/UPDATED USER MANUAL (TEAM 14).pdf differ diff --git a/User Manual.pdf b/User Manual.pdf deleted file mode 100644 index 9b5a0370..00000000 Binary files a/User Manual.pdf and /dev/null differ diff --git a/UserGuide.md b/UserGuide.md deleted file mode 100644 index 00853112..00000000 --- a/UserGuide.md +++ /dev/null @@ -1,88 +0,0 @@ -Welcome to the User Guide for XC3 Project. This guide provides introduction, instructions and information on how to use the XC3 grafana dashboards effectively. - -## What is XC3? - -Xgrid Cloud Cost Control is a cloud agnostic and risk-free package offering powered by Cloud Custodian that provides security enforcement, tagging, unused or invalid resources cleanup, account maintenance, cost control, and backups. It supports managing AWS public cloud environments and provides a visualization of the usage of resources in account with the support of managing resource utilization on a click. - -## How to Use XC3 - -Once XC3 infrastructure deployed in your AWS Account, Users needs to be added in Cognito pool with requested role (admin/editor/viewer) in respective cognito group. User get random username/password from cognito then they can set password on domain by sign in using random credentials. - -### User Role Guidelines - -Admin Role: - -Admins have full access to all features and settings in Grafana. They can create, edit, and delete users, dashboards, data sources, and alerts. They also have the ability to manage permissions and set up LDAP authentication. - -Editor Role: - -Editors can create and edit dashboards, but they cannot manage users or change system settings. They can also create and manage data sources and alerts that they have access to. However, they do not have access to system-level data sources or alerts. - -Viewer Role: - -Viewers have read-only access to Grafana. They can view dashboards and panels, but they cannot make any changes. They also do not have access to any administrative or editing functions. - -## Getting Started with XC3 - -Once you've logged in on XC3, you can start using dashboards. - -### Home - -On Home Page, you can get high-level detail of your AWS Account - -- To determine the AWS Account Cost, select the account ID and get total spend of AWS Account from last three months - ![image](https://github.com/XgridInc/xc3/assets/122358742/d494f7c8-0ee2-4630-8cac-6cd3fd00186b) - -- To determine the total spend on AWS for a particular project select the AWS account and project to get the cost of that specific project from the last 14 days. - ![image](https://github.com/XgridInc/xc3/assets/122358742/057f4937-6cd4-4e56-a785-b8b401342f2b) - -- To get the top 5 most expensive services in a specific AWS Region, select an AWS account and AWS region to get the top 5 most expensive services in that region from the last 14 days. - ![image](https://github.com/XgridInc/xc3/assets/122358742/c3c61f73-f22b-4f37-aac1-e806c383737b) - -### IAM Roles Dashboard -- By selecting IAM Role Dashboard you can view a list of IAM Roles by selecting AWS Account. - ![image](https://github.com/XgridInc/xc3/assets/122358742/f2d2490b-0539-4756-8a6a-653ae8370959) -- By selecting a specific IAM Role in a specific AWS Region you can view the resources, state of resources, and the cumulative cost of resources from the last 14 days associated with a specific IAM Role in the selected AWS Region. - ![image](https://github.com/XgridInc/xc3/assets/122358742/477d67c8-e87c-49c0-bb56-98d62077eb8e) -- By selecting a specific resource, you can view the detailed billing of that resource from the last 14 days. - !![image](https://github.com/XgridInc/xc3/assets/122358742/88460fb9-489e-4183-b267-1322ec3474bd) - -### IAM Users Dashboard -- By selecting IAM User Dashboard you can view a list of IAM Users by selecting AWS Account. -- By selecting a specific IAM User in a specific AWS Region you can view the resources, state of resources, and the cumulative cost of resources from the last 14 days spin up by a specific IAM User in the selected AWS Region. -- By selecting a specific resource, you can view the detailed billing of that resource from the last 14 days. -- By selecting the specific IAM User in a specific AWS Region you can get the total spend in that AWS Region by specific IAM User. - -### Tagging Compliance Dashboard -- By selecting Tagging Compliance Dashboard you can view the resource inventory that is non-conformant to tagging policies setup using following tags: - - Creator - - Owner - - Project - -- By selecting Tagging Compliance Dashboard you can view the resource inventory that is non-conformant to tagging policies setup using following tags: - Creator - Owner - Project - - in a specific AWS Region by selecting AWS Account and AWS Region. - - ![image](https://github.com/XgridInc/xc3/assets/122358742/4d75dd2d-cc3d-4129-ac4c-a030f2e8b5de) - -## Community Support - -If you have any questions or run into issues while using XC3, you can ask for help by reaching out on Slack XC3 - -We hope this guide helps you get started with XC3. If you have any feedback or suggestions for the project, please let us know! - -## License - -Copyright (c) 2023, Xgrid Inc, https://xgrid.co - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/custom_dashboard/grafana_dashboards/team14dashboard.json b/custom_dashboard/grafana_dashboards/team14dashboard.json new file mode 100644 index 00000000..cd84f8a0 --- /dev/null +++ b/custom_dashboard/grafana_dashboards/team14dashboard.json @@ -0,0 +1,833 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "hideTimeOverride": true, + "id": 9, + "interval": "5m", + "maxDataPoints": 986, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "iam_role_resource_costs_testing{}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "timeFrom": "24h", + "title": "IAM Roles Detail", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "byVariable": false, + "include": { + "names": [ + "role_name", + "rolearn" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "role_arn": 1, + "role_name": 0 + }, + "renameByName": { + "role_arn": "Role Arn", + "role_name": "Role ", + "rolearn": "Role ARN" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Cost", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 13, + "interval": "1h", + "options": { + "barRadius": 0, + "barWidth": 0.7, + "colorByField": "Service", + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xField": "Service", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "service_costs_testing{role_name=\"$Role\"}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Services and Cost", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "service", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Cost", + "service": "Service" + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 12, + "interval": "1h", + "options": { + "cellHeight": "md", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "iam_role_resource_costs_testing{role_name=\"$Role\", service=\"$Service\"}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Resource Detail", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "resource_name", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Cost", + "resource_name": "Resource Name" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + }, + { + "color": "#EAB839", + "value": 90 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 11, + "interval": "1h", + "options": { + "barRadius": 0, + "barWidth": 0.76, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "text": {}, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": -45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "topk(5, sum by(resource_name) (iam_role_resource_costs_testing{role_name=\"$Role\", service=\"$Service\"}))", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "IAM Roles top 5 Resources for service", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "resource_name", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Cost", + "resource_name": "Resource" + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "Cost ($)", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 10, + "interval": "45m", + "options": { + "barRadius": 0, + "barWidth": 0.73, + "colorByField": "Resource", + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xField": "Resource", + "xTickLabelMaxLength": 8, + "xTickLabelRotation": -60, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "topk(5, sum by(resource_name) (iam_role_resource_costs_testing{role_name=\"$Role\"}))", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Top 5 most expensive resources", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "byVariable": false, + "include": { + "names": [ + "resource_name", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Cost", + "resource_name": "Resource" + } + } + }, + { + "id": "limit", + "options": {} + } + ], + "transparent": true, + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "options": { + "afterRender": "", + "content": "
\n {{#each data}}\n \n \n \n \n \n \n
{{iam_role_service_resource_id}}{{iam_role_service_cost}}
\n {{/each}}\n
", + "defaultContent": "The query didn't return any results.", + "editor": { + "format": "auto", + "height": 428, + "language": "html" + }, + "editors": [], + "externalScripts": [], + "externalStyles": [], + "helpers": "$(\"#table-data\").on('click', '.editbtn', function (e) {\n e.preventDefault();\n var currentRow = $(this).closest(\"tr\");\n var $resource_id = currentRow.find(\".resourceid\").text();\n var $status = currentRow.find(\".status\").text();\n alert(\"Clicked!\")\n var res_id = $resource_id,\n status = $status;\n var resource_id = res_id.split(\"/\")[1];\n alert(resource_id);\n $.ajax({\n type: \"POST\",\n url: 'https://bz8p90fgzb.execute-api.eu-west-1.amazonaws.com/deploy/cost-details',\n contentType: 'application/json',\n data: JSON.stringify({\n 'resource_id': resource_id,\n 'status': status\n }),\n success: function (res) {\n alert(\"Success\");\n },\n error: function () {\n alert(\"Something went wrong!!\")\n }\n })\n .done(function (res) {\n console.log(res)\n });\n})", + "renderMode": "allRows", + "styles": "", + "wrap": true + }, + "pluginVersion": "4.5.0", + "targets": [ + { + "editorMode": "builder", + "expr": "IAM_Role_Service{job=\"IAM-roles-service-data\", iam_role_service_region=\"$Region\", iam_role_of_service=\"$IAMRole\"}", + "format": "table", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Start / Stop Action", + "type": "marcusolsson-dynamictext-panel" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values()", + "hide": 0, + "includeAll": true, + "label": "Account", + "multi": false, + "name": "Account", + "options": [], + "query": { + "qryType": 1, + "query": "label_values()", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values()", + "hide": 0, + "includeAll": true, + "label": "Region", + "multi": false, + "name": "Region", + "options": [], + "query": { + "qryType": 1, + "query": "label_values()", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "DummyRole1", + "value": "DummyRole1" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(iam_role_resource_costs_testing,role_name)", + "hide": 0, + "includeAll": false, + "label": "IAM Role", + "multi": false, + "name": "Role", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(iam_role_resource_costs_testing,role_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "lambda", + "value": "lambda" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(service_costs_testing{role_name=\"$Role\"},service)", + "hide": 0, + "includeAll": false, + "label": "Service", + "multi": false, + "name": "Service", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(service_costs_testing{role_name=\"$Role\"},service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Dummy Data", + "uid": "2711", + "version": 7, + "weekStart": "" + } \ No newline at end of file diff --git a/infrastructure/backend.tf b/infrastructure/backend.tf index 6f2f0ab9..6593cd72 100644 --- a/infrastructure/backend.tf +++ b/infrastructure/backend.tf @@ -16,9 +16,9 @@ terraform { backend "s3" { - bucket = "terraform-state-xc3" // S3 bucket for terraform state management + bucket = "terraform-state-team17sp" // S3 bucket for terraform state management key = "xc3/xc3.tfstate" // Specifies the S3 object key for storing the Terraform state file - region = "eu-west-1" + region = "ap-southeast-2" dynamodb_table = "terraform-lock" } } diff --git a/infrastructure/config.sh b/infrastructure/config.sh index 3c1aaf7f..f9b7624c 100644 --- a/infrastructure/config.sh +++ b/infrastructure/config.sh @@ -15,12 +15,12 @@ # limitations under the License. # Variables values that will be used in init script to create resources for XC3 infrastructure -export aws_region="eu-west-1" +export aws_region="ap-southeast-2" export dynamo_table_name="terraform-lock" -export bucket_name="terraform-state-xc3" -export project="example" +export bucket_name="terraform-state-team17sp" +export project="iamrrm" export domain="example.test.co" -export owner_email="admin@test.co" -export creator_email="admin@test.co" -export namespace="example" -export env="example" +export owner_email="spashra@gmail.com" +export creator_email="104202994@student.swin.edu.au" +export namespace="iamrrm" +export env="dev" diff --git a/infrastructure/modules/serverless/iam-role.tf b/infrastructure/modules/serverless/iam-role.tf index 5e3b0dfe..cc985137 100644 --- a/infrastructure/modules/serverless/iam-role.tf +++ b/infrastructure/modules/serverless/iam-role.tf @@ -391,6 +391,248 @@ resource "aws_iam_role" "lambda_execution_role_InstanceChangeState" { tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-instance-state-change" })) } +resource "aws_lambda_function" "RolesLambda" { + function_name = "${var.namespace}-roleslambda" + role = aws_iam_role.lambda_execution_role_RolesLambda.arn + runtime = "python3.9" # Adjust the runtime as needed + handler = "roleslambda.lambda_handler" # Adjust the handler as needed + filename = values(data.archive_file.lambda_function_zip)[6].output_path + + environment { + variables = { + # Add any environment variables specific to MappingLambda function + prometheus_ip = "${var.prometheus_ip}:9091" + region_names_path = "/${var.namespace}/region_names" + func_name_mapping_lambda = aws_lambda_function.MappingLambda.arn + bucket = "${var.namespace}-metadata-storage" + + } + } + + memory_size = var.memory_size + timeout = var.timeout + vpc_config { + subnet_ids = [var.subnet_id[0]] + security_group_ids = [var.security_group_id] + } + + layers = [var.prometheus_layer] + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-roles_lambda" })) +} + +resource "aws_iam_role_policy" "RolesLambda" { + name = "${var.namespace}-roleslambda-lambda-inline-policy" + role = aws_iam_role.lambda_execution_role_RolesLambda.id + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "s3:*", # Full S3 access + "iam:*", # Full IAM access + "ssm:Get*", + "ssm:List*", + "ce:GetCostAndUsageWithResources", # CloudWatch Cost Explorer + "ec2:*", + "lambda:InvokeFunction", # Allow invoking Lambda functions + "lambda:*" + ], + Resource = ["*"] + } + ] + }) +} + +# Creating IAM Role for RolesLambda function +resource "aws_iam_role" "lambda_execution_role_RolesLambda" { + name = "${var.namespace}-roleslambda" + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Effect = "Allow", + Sid = "roleslambdarole", + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + managed_policy_arns = [ + "arn:aws:iam::aws:policy/AmazonS3FullAccess", + "arn:aws:iam::aws:policy/IAMFullAccess", + "arn:aws:iam::aws:policy/CloudWatchReadOnlyAccess", + ] + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-IAM-Role-RolesLambda" })) +} + + +resource "aws_lambda_function" "MappingLambda" { + function_name = "${var.namespace}-mappinglambda" + role = aws_iam_role.lambda_execution_role_MappingLambda.arn + runtime = "python3.9" # Adjust the runtime as needed + handler = "mappinglambda.lambda_handler" # Adjust the handler as needed + filename = values(data.archive_file.lambda_function_zip)[5].output_path + + environment { + variables = { + # Add any environment variables specific to Mapping Lambda function + prometheus_ip = "${var.prometheus_ip}:9091" + region_names_path = "/${var.namespace}/region_names" + func_name_cost_lambda = aws_lambda_function.CostLambda.arn + bucket = "${var.namespace}-metadata-storage" + } + } + + memory_size = var.memory_size + timeout = var.timeout + vpc_config { + subnet_ids = [var.subnet_id[0]] + security_group_ids = [var.security_group_id] + } + + layers = [var.prometheus_layer] + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-mapping_lambda" })) +} + + +resource "aws_iam_role_policy" "MappingLambda" { + name = "${var.namespace}--mappinglambda-lambda-inline-policy" + role = aws_iam_role.lambda_execution_role_MappingLambda.id + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "s3:*", # Full S3 access + "iam:*", # Full IAM access + "ssm:Get*", + "ssm:List*", + "ce:GetCostAndUsageWithResources", # CloudWatch Cost Explorer + "ec2:*", + "lambda:*" + ], + Resource = ["*"] + } + ] + }) +} + +# Creating IAM Role for Mapping Lambda function +resource "aws_iam_role" "lambda_execution_role_MappingLambda" { + name = "${var.namespace}-Mappinglambda" + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Effect = "Allow", + Sid = "mappinglambdarole", + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + managed_policy_arns = [ + "arn:aws:iam::aws:policy/AmazonS3FullAccess", + "arn:aws:iam::aws:policy/IAMFullAccess", + "arn:aws:iam::aws:policy/CloudWatchReadOnlyAccess", + ] + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-IAM-Role-MappingLambda" })) +} + +resource "aws_lambda_function" "CostLambda" { + function_name = "${var.namespace}-costlambda" + role = aws_iam_role.lambda_execution_role_CostLambda.arn + runtime = "python3.9" # Adjust the runtime as needed + handler = "costlambda.lambda_handler" # Adjust the handler as needed + filename = values(data.archive_file.lambda_function_zip)[0].output_path + + environment { + variables = { + # Add any environment variables specific to CostLambda function + prometheus_ip = "${var.prometheus_ip}:9091" + region_names_path = "/${var.namespace}/region_names" + bucket = "${var.namespace}-metadata-storage" + + } + } + + memory_size = var.memory_size + timeout = var.timeout + vpc_config { + subnet_ids = [var.subnet_id[0]] + security_group_ids = [var.security_group_id] + } + + layers = [var.prometheus_layer] + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-cost_lambda" })) +} + +resource "aws_iam_role_policy" "CostLambda" { + name = "${var.namespace}-costlambda-lambda-inline-policy" + role = aws_iam_role.lambda_execution_role_CostLambda.id + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "s3:*", # Full S3 access + "iam:*", # Full IAM access + "ssm:Get*", + "ssm:List*", + "ce:GetCostAndUsageWithResources", # CloudWatch Cost Explorer + "ec2:*", + "lambda:InvokeFunction", # Allow invoking Lambda functions + "lambda:*" + ], + Resource = ["*"] + } + ] + }) +} + +# Creating IAM Role for CostLambda function +resource "aws_iam_role" "lambda_execution_role_CostLambda" { + name = "${var.namespace}-costlambda" + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Effect = "Allow", + Sid = "costlambdarole", + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + managed_policy_arns = [ + "arn:aws:iam::aws:policy/AmazonS3FullAccess", + "arn:aws:iam::aws:policy/IAMFullAccess", + "arn:aws:iam::aws:policy/CloudWatchReadOnlyAccess", + ] + + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-IAM-Role-CostLambda" })) +} + + + + resource "aws_lambda_permission" "allow_bucket_for_irtg" { statement_id = "AllowExecutionFromS3Bucket" action = "lambda:InvokeFunction" @@ -399,6 +641,27 @@ resource "aws_lambda_permission" "allow_bucket_for_irtg" { source_arn = var.s3_xc3_bucket.arn } + resource "aws_cloudwatch_event_rule" "RolesLambda" { + name = "${var.namespace}-RolesLambda-rule" + description = "Trigger the Lambda function every week on Monday" + schedule_expression = var.cron_jobs_schedule["roles_lambda_cron"] + tags = merge(local.tags, tomap({ "Name" = "${var.namespace}-RolesLambda" })) +} + +# Define the EventBridge target to invoke the Lambda function +resource "aws_cloudwatch_event_target" "RolesLambda" { + rule = aws_cloudwatch_event_rule.RolesLambda.name + arn = aws_lambda_function.RolesLambda.arn +} + +resource "aws_lambda_permission" "RolesLamnda" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.RolesLambda.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.RolesLambda.arn +} + resource "aws_s3_bucket_notification" "IamRolestoGrafana_trigger" { bucket = var.s3_xc3_bucket.id diff --git a/infrastructure/modules/serverless/variables.tf b/infrastructure/modules/serverless/variables.tf index 623bc392..f96a3dd9 100644 --- a/infrastructure/modules/serverless/variables.tf +++ b/infrastructure/modules/serverless/variables.tf @@ -100,6 +100,10 @@ variable "lambda_names" { "iamrolesservice" = "../src/iam_roles/iamrolesservice.py" "iamrolesservicemapping" = "../src/iam_roles/iamrolesservicemapping.py" "instancestatechange" = "../src/iam_roles/instancestatechange.py" + "mappinglambda" = "../src/iam_roles/mappinglambda.py" + "roleslambda" = "../src/iam_roles/roleslambda.py" + "costlambda" = "../src/iam_roles/costlambda.py" + } } diff --git a/infrastructure/terraform.auto.tfvars b/infrastructure/terraform.auto.tfvars index fe68b285..89ff9c72 100644 --- a/infrastructure/terraform.auto.tfvars +++ b/infrastructure/terraform.auto.tfvars @@ -12,36 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -namespace = "testing" -env = "prod" -region = "eu-west-1" -account_id = "123456789" +namespace = "iamrrm" +env = "dev" +region = "ap-southeast-2" +account_id = "851725378731" vpc_cidr_block = "10.0.0.0/16" public_subnet_cidr_block = { - "eu-west-1a" = "10.0.0.0/24" - "eu-west-1b" = "10.0.1.0/24" + "ap-southeast-2a" = "10.0.0.0/24" + "ap-southeast-2b" = "10.0.1.0/24" } domain_name = "" hosted_zone_id = "Z053166920YP1STI0EK5X" private_subnet_cidr_block = { - "eu-west-1a" = "10.0.100.0/24" + "ap-southeast-2a" = "10.0.100.0/24" } # private_subnet_cidr_block = "10.0.100.0/24" allow_traffic = ["0.0.0.0/0"] // Use your own network CIDR -ses_email_address = "testing@testing.co" -creator_email = "testing@testing.co" -owner_email = "testing@testing.co" +ses_email_address = "104202994@student.swin.edu.au" +creator_email = "104202994@student.swin.edu.au" +owner_email = "spashra@gmail.com" instance_type = "t2.micro" total_account_cost_lambda = "total_account_cost" total_account_cost_cronjob = "cron(0 0 1,15 * ? *)" // flexible can be set according to need prometheus_layer = "lambda_layers/python.zip" // s3 key for lambda layer memory_size = 128 timeout = 300 -project = "testing" -create_cloudtrail_kms = true -create_cloudtrail = true -create_cloudtrail_s3_bucket = true +project = "iamrrm" +create_cloudtrail_kms = false +create_cloudtrail = false +create_cloudtrail_s3_bucket = false security_group_ingress = { "pushgateway" = { description = "PushGateway" diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index fd9738f6..f5b82b4e 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -129,6 +129,7 @@ variable "cron_jobs_schedule" { list_linked_accounts_cron = "cron(0 0 1 * ? *)" most_expensive_service_cron = "cron(0 0 * * ? 1)" cost_report_notifier_cronjob = "cron(0 0 1,15 * ? *)" + roles_lambda_cron = "cron(0 0 1,15 * ? *)" } } diff --git a/pre_requirement/terraform.auto.tfvars b/pre_requirement/terraform.auto.tfvars index f2d8dfe9..7da5f852 100644 --- a/pre_requirement/terraform.auto.tfvars +++ b/pre_requirement/terraform.auto.tfvars @@ -13,8 +13,8 @@ # limitations under the License. -project = "example" -creator_email = "example@example.co" -owner_email = "example@example.co" -namespace = "example" -region = "eu-west-1" +project = "iamrrm" +creator_email = "spashra@gmail.com" +owner_email = "104202994@student.swin.edu.au" +namespace = "iamrrm" +region = "ap-southeast-2" diff --git a/src/iam_roles/costlambda.py b/src/iam_roles/costlambda.py new file mode 100644 index 00000000..9b570df7 --- /dev/null +++ b/src/iam_roles/costlambda.py @@ -0,0 +1,255 @@ +import json +import boto3 +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway +import datetime +import os + +def get_lambda_function_cost(lambda_function_arn, start_date, end_date): + cost_client = boto3.client('ce') + + try: + cost_response = cost_client.get_cost_and_usage( + TimePeriod={ + 'Start': start_date, + 'End': end_date + }, + Granularity='DAILY', + Metrics=['UnblendedCost'], + Filter={ + "Dimensions": { + "Key": "RESOURCE_ID", + "Values": [lambda_function_arn] + } + } + ) + + costs = cost_response['ResultsByTime'][0]['Total']['UnblendedCost']['Amount'] + costs = 0 if costs is None else float(costs) + + return costs + except Exception as e: + print(f"Error fetching cost for Lambda function '{lambda_function_arn}': {str(e)}") + return 0 + +def get_ec2_instance_cost(instance_id, start_date, end_date): + cost_client = boto3.client('ce') + + try: + cost_response = cost_client.get_cost_and_usage( + TimePeriod={ + 'Start': start_date, + 'End': end_date + }, + Granularity='DAILY', + Metrics=['UnblendedCost'], + Filter={ + "Dimensions": { + "Key": "INSTANCE_ID", + "Values": [instance_id] + } + } + ) + + # Extracting cost + costs = cost_response['ResultsByTime'][0]['Total']['UnblendedCost']['Amount'] + costs = 0 if costs is None else float(costs) + + return costs + + except Exception as e: + print(f"Error fetching cost for EC2 instance '{instance_id}': {str(e)}") + return 0 + + +def get_s3_bucket_cost(s3_bucket_name, start_date, end_date): + cost_client = boto3.client('ce') + + try: + cost_response = cost_client.get_cost_and_usage( + TimePeriod={ + 'Start': start_date, + 'End': end_date + }, + Granularity='DAILY', + Metrics=['UnblendedCost'], + Filter={ + "Dimensions": { + "Key": "LINKED_ACCOUNT", + "Values": [s3_bucket_name] + } + } + ) + + costs = cost_response['ResultsByTime'][0]['Total']['UnblendedCost']['Amount'] + costs = 0 if costs is None else float(costs) + + return costs + except Exception as e: + print(f"Error fetching cost for S3 bucket '{s3_bucket_name}': {str(e)}") + return 0 + +def get_costs_and_push_to_prometheus(parsed_roles, start_date, end_date): + s3 = boto3.client('s3') + registry = CollectorRegistry() + + # Gauge for IAM Role Information + iam_role_gauge = Gauge( + "iam_role_info", + "IAM Role Information", + labelnames=["role_name", "rolearn"], + registry=registry, + ) + + # Gauge for IAM Role and Associated Resource Costs + iam_role_resource_gauge = Gauge( + "iam_role_resource_costs", + "Role and Associated Resource Costs", + labelnames=["role_name", "service", "resource_name"], + registry=registry, + ) + + # Gauge for Service Costs + service_cost_gauge = Gauge( + "service_costs", + "Cost of Each Service for IAM Roles", + labelnames=["role_name", "service"], + registry=registry, + ) + + for role in parsed_roles: + role_name = role["RoleName"] + services = role["ServiceMapping"] + rolearn = role["RoleArn"] + + total_service_cost = 0 # Total cost for all services + + for service in services: + service_cost = 0 # Cost for each service + + if service == "lambda": + lambda_costs = [] + for lambda_function_arn in role["LambdaFunctions"]: + cost = get_lambda_function_cost(lambda_function_arn, start_date, end_date) + iam_role_resource_gauge.labels(role_name=role_name, service=service, resource_name=lambda_function_arn).set(cost) + lambda_costs.append({"LambdaFunctionArn": lambda_function_arn, "Cost": cost}) + service_cost += cost + + elif service == "s3": + s3_costs = [] + for s3_bucket_name in role["S3Buckets"]: + cost = get_s3_bucket_cost(s3_bucket_name, start_date, end_date) + iam_role_resource_gauge.labels(role_name=role_name, service=service, resource_name=s3_bucket_name).set(cost) + s3_costs.append({"S3BucketName": s3_bucket_name, "Cost": cost}) + service_cost += cost + + elif service == "ec2": + ec2_costs = [] + for ec2_instance_name in role["EC2Instances"]: + cost = get_ec2_instance_cost(ec2_instance_name, start_date, end_date) + iam_role_resource_gauge.labels(role_name=role_name, service=service, resource_name=ec2_instance_name).set(cost) + ec2_costs.append({"EC2InstanceName": ec2_instance_name, "Cost": cost}) + service_cost += cost + + elif service == "rds": + rds_costs = [] + for db_instance_identifier in role.get("RDSInstances", []): + cost = get_rds_instance_cost(db_instance_identifier, start_date, end_date) + iam_role_resource_gauge.labels(role_name=role_name, service=service, resource_name=db_instance_identifier).set(cost) + rds_costs.append({"DBInstanceIdentifier": db_instance_identifier, "Cost": cost}) + service_cost += cost + + elif service == "dynamodb": + dynamodb_costs = [] + for table_name in role.get("DynamoDBTables", []): + cost = get_dynamodb_table_cost(table_name, start_date, end_date) + iam_role_resource_gauge.labels(role_name=role_name, service=service, resource_name=table_name).set(cost) + dynamodb_costs.append({"TableName": table_name, "Cost": cost}) + service_cost += cost + + + # Set the total service cost as a Prometheus metric for service costs + service_cost_gauge.labels(role_name=role_name, service=service).set(service_cost) + + # Set IAM role info gauge + iam_role_gauge.labels(role_name=role_name, rolearn=rolearn).set(0) # Set value to 0 for each role + + # Append the total service cost for all services to the role dictionary + role["TotalServiceCost"] = total_service_cost + + try: + push_to_gateway(os.environ["prometheus_ip"], job="iam_roles_service_mapping", registry=registry) + print("Metrics pushed to Prometheus successfully.") + except Exception as e: + print(f"Error pushing metrics to Prometheus: {str(e)}") + + +def get_rds_instance_cost(db_instance_identifier, start_date, end_date): + cost_client = boto3.client('ce') + + try: + cost_response = cost_client.get_cost_and_usage( + TimePeriod={ + 'Start': start_date, + 'End': end_date + }, + Granularity='DAILY', + Metrics=['UnblendedCost'], + Filter={ + "Dimensions": { + "Key": "LINKED_ACCOUNT", + "Values": [db_instance_identifier] + } + } + ) + + costs = cost_response['ResultsByTime'][0]['Total']['UnblendedCost']['Amount'] + costs = 0 if costs is None else float(costs) + + return costs + except Exception as e: + print(f"Error fetching cost for RDS instance '{db_instance_identifier}': {str(e)}") + return 0 + +def get_dynamodb_table_cost(table_name, start_date, end_date): + cost_client = boto3.client('ce') + + try: + cost_response = cost_client.get_cost_and_usage( + TimePeriod={ + 'Start': start_date, + 'End': end_date + }, + Granularity='DAILY', + Metrics=['UnblendedCost'], + Filter={ + "Dimensions": { + "Key": "LINKED_ACCOUNT", + "Values": [table_name] + } + } + ) + + costs = cost_response['ResultsByTime'][0]['Total']['UnblendedCost']['Amount'] + costs = 0 if costs is None else float(costs) + + return costs + except Exception as e: + print(f"Error fetching cost for DynamoDB table '{table_name}': {str(e)}") + return 0 + +def lambda_handler(event, context): + payload = event + # Process payload and push metrics to Prometheus + parsed_roles = payload['iam_roles'] + # Set start date to the beginning of the current year + start_date = datetime.datetime(datetime.datetime.now().year, 1, 1).strftime('%Y-%m-%d') + # Set end date to the current date + end_date = datetime.datetime.now().strftime('%Y-%m-%d') + get_costs_and_push_to_prometheus(parsed_roles, start_date, end_date) + + return { + 'statusCode': 200, + 'body': json.dumps({"message": "Metrics pushed to Prometheus successfully."}) + } + + diff --git a/src/iam_roles/mappinglambda.py b/src/iam_roles/mappinglambda.py new file mode 100644 index 00000000..62f15427 --- /dev/null +++ b/src/iam_roles/mappinglambda.py @@ -0,0 +1,283 @@ +import json +import boto3 +import os + +def fetch_assume_role_policy_document(role_name): + iam = boto3.client('iam') + + try: + # Get the IAM role details, including AssumeRolePolicyDocument + role_details = iam.get_role(RoleName=role_name) + assume_role_policy_document = role_details['Role']['AssumeRolePolicyDocument'] + return assume_role_policy_document + except iam.exceptions.NoSuchEntityException: + print(f"IAM role '{role_name}' not found.") + except Exception as e: + print(f"Error fetching IAM role details: {str(e)}") + + return None + +def get_lambda_functions_for_role(role_name): + lambda_client = boto3.client('lambda') + + try: + # List Lambda functions associated with the IAM role + functions_response = lambda_client.list_functions( + FunctionVersion='ALL', # Use 'ALL' to list all versions + MaxItems=123 # Adjust MaxItems as needed + ) + + # Filter Lambda functions associated with the IAM role + lambda_functions = [func['FunctionArn'] for func in functions_response['Functions'] if role_name in func['Role']] + + return lambda_functions + except Exception as e: + print(f"Error fetching Lambda functions for role '{role_name}': {str(e)}") + + return [] + +def get_s3_buckets_for_role(role_name): + try: + # Create an IAM client + iam_client = boto3.client('iam') + + # Get the IAM policies attached to the role + role_policies_response = iam_client.list_attached_role_policies(RoleName=role_name) + role_policy_arns = [policy['PolicyArn'] for policy in role_policies_response['AttachedPolicies']] + + # Initialize a list to store bucket names associated with the role + s3_buckets = [] + + # Create an S3 client + s3_client = boto3.client('s3') + + # Iterate through each IAM policy attached to the role + for policy_arn in role_policy_arns: + # Get the policy document + policy_document_response = iam_client.get_policy_version( + PolicyArn=policy_arn, + VersionId='v1' # Assuming there's only one version + ) + policy_document = policy_document_response['PolicyVersion']['Document'] + + # Parse the policy document to find S3 related permissions + for statement in policy_document['Statement']: + if statement['Effect'] == 'Allow': + if 's3' in statement.get('Resource', '') or 's3' in statement.get('Resource', ''): + # If the statement has s3 in the resource, it's likely an S3 permission + # Add all S3 buckets to the list as we cannot infer bucket names directly from IAM policies + buckets_response = s3_client.list_buckets() + s3_buckets.extend(bucket['Name'] for bucket in buckets_response['Buckets']) + + return s3_buckets + + except Exception as e: + print(f"Error fetching S3 buckets for role '{role_name}': {str(e)}") + return [] + +def get_ec2_instances_for_role(role_name): + try: + iam = boto3.client('iam') + service_client = boto3.client('ec2') + + # Get instance profiles associated with the IAM role + instance_profile_detail = iam.list_instance_profiles_for_role(RoleName=role_name) + + # Initialize a list to store EC2 instance details + ec2_instances = [] + + # Iterate through instance profiles + for profile in instance_profile_detail['InstanceProfiles']: + instance_profile_arn = profile['Arn'] + + # Describe EC2 instances associated with the instance profile ARN + ec2_response = service_client.describe_instances( + Filters=[ + { + "Name": "iam-instance-profile.arn", + "Values": [instance_profile_arn], + } + ] + ) + + # Extract EC2 instance details + for reservation in ec2_response["Reservations"]: + for instance in reservation["Instances"]: + instance_id = instance["InstanceId"] + instance_region = instance["Placement"]["AvailabilityZone"][:-1] + instance_detail = { + "Instance_Region": instance_region, + "Instance": instance_id, + } + ec2_instances.append(instance_detail) + + return ec2_instances + + except Exception as e: + print(f"Error fetching EC2 instances for role '{role_name}': {str(e)}") + return [] + +def get_rds_instances_for_role(role_name): + try: + iam = boto3.client('iam') + rds_client = boto3.client('rds') + + # Get instance profiles associated with the IAM role + instance_profile_detail = iam.list_instance_profiles_for_role(RoleName=role_name) + + # Initialize a list to store RDS instance identifiers + rds_instances = [] + + # Iterate through instance profiles + for profile in instance_profile_detail['InstanceProfiles']: + instance_profile_arn = profile['Arn'] + + # Describe RDS instances associated with the instance profile ARN + rds_response = rds_client.describe_db_instances() + + # Extract RDS instance identifiers + for db_instance in rds_response['DBInstances']: + rds_instances.append(db_instance['DBInstanceIdentifier']) + + return rds_instances + + except Exception as e: + print(f"Error fetching RDS instances for role '{role_name}': {str(e)}") + return [] + +def get_dynamodb_tables_for_role(role_name): + try: + # Create an IAM client + iam_client = boto3.client('iam') + + # Get the IAM policies attached to the role + role_policies_response = iam_client.list_attached_role_policies(RoleName=role_name) + role_policy_arns = [policy['PolicyArn'] for policy in role_policies_response['AttachedPolicies']] + + # Create a DynamoDB client + dynamodb_client = boto3.client('dynamodb') + + # Initialize a list to store table names associated with the role + dynamodb_tables = [] + + # Iterate through each IAM policy attached to the role + for policy_arn in role_policy_arns: + # Get the policy document + policy_document_response = iam_client.get_policy_version( + PolicyArn=policy_arn, + VersionId='v1' # Assuming there's only one version + ) + policy_document = policy_document_response['PolicyVersion']['Document'] + + # Parse the policy document to find DynamoDB related permissions + for statement in policy_document['Statement']: + if statement['Effect'] == 'Allow': + if 'dynamodb' in statement['Resource']: + # Extract DynamoDB table ARN + table_arn = statement['Resource'] + + # Extract table name from ARN + table_name = table_arn.split(':table/')[1] + + # Add the table name to the list if it's not already there + if table_name not in dynamodb_tables: + dynamodb_tables.append(table_name) + + return dynamodb_tables + + except Exception as e: + print(f"Error fetching DynamoDB tables for role '{role_name}': {str(e)}") + return [] + + +def parse_roles_from_payload(iam_roles): + parsed_roles = [] + + for role_info in iam_roles: + role = { + "RoleName": role_info.get("RoleName", ""), + "RoleArn": role_info.get("RoleArn", ""), + "CreationDate": role_info.get("CreationDate", ""), + "Description": role_info.get("Description", ""), + "Path": role_info.get("Path", ""), + "ServiceMapping": [] + } + + # Get AssumeRolePolicyDocument using the fetch_assume_role_policy_document function + assume_role_policy_document = fetch_assume_role_policy_document(role["RoleName"]) + + # Check if AssumeRolePolicyDocument is present + if assume_role_policy_document: + statement_service = assume_role_policy_document.get("Statement", []) + + # Parsing services attached to IAM role + service_mapping = [] + for statement in statement_service: + data_principal = statement.get("Principal", {}) + for key, value in data_principal.items(): + if key == "Service": + if isinstance(value, list): + service_list = [item.split(".")[0] for item in value] + else: + service_list = [value.split(".")[0]] + + for resource in service_list: + if resource == "lambda": + role["LambdaFunctions"] = [] + lambda_functions = get_lambda_functions_for_role(role["RoleName"]) + role["LambdaFunctions"] = lambda_functions + service_mapping.append(resource) + elif resource == "s3": + role["S3Buckets"] = [] + s3_buckets = get_s3_buckets_for_role(role["RoleName"]) + role["S3Buckets"] = s3_buckets + service_mapping.append(resource) + elif resource == "ec2": + role["EC2Instances"] =[] + ec2_instances = get_ec2_instances_for_role(role["RoleName"]) + role["EC2Instances"] = ec2_instances + service_mapping.append(resource) + elif resource == "rds": + role["RDSInstances"] =[] + rds_instances = get_rds_instances_for_role(role["RoleName"]) + role["RDSInstances"] = rds_instances + service_mapping.append(resource) + elif resource == "dynamodb": + role["DynamoDBTables"] =[] + dynamodb_tables = get_dynamodb_tables_for_role(role["RoleName"]) + role["DynamoDBTables"] = dynamodb_tables + service_mapping.append(resource) + else: + service_mapping.append(resource) + + role["ServiceMapping"] = service_mapping if service_mapping else ["No Affiliated Services"] + + parsed_roles.append(role) + + return parsed_roles + +def lambda_handler(event, context): + payload = event + iam_roles = payload['iam_roles'] + # Fetch roles, Lambda functions, and S3 buckets associated with roles + parsed_roles = parse_roles_from_payload(iam_roles) + + payload = { + "iam_roles": parsed_roles + } + + # Convert the payload to JSON + json_payload = json.dumps(payload) + + # Invoke lambda3 + lambda_client = boto3.client('lambda') + response = lambda_client.invoke( + FunctionName= os.environ["func_name_cost_lambda"], + InvocationType='Event', # Synchronous invocation + Payload=json_payload + ) + + return { + 'statusCode': 200, + 'body': json.dumps(parsed_roles, indent=2) + } diff --git a/src/iam_roles/roleslambda.py b/src/iam_roles/roleslambda.py new file mode 100644 index 00000000..aadae755 --- /dev/null +++ b/src/iam_roles/roleslambda.py @@ -0,0 +1,59 @@ +import boto3 +import json +import time +import os +def lambda_handler(event, context): + # Retrieve IAM role details + iam_roles = get_iam_role_details() + + # Prepare the payload to send to lambda2 + payload = { + "iam_roles": iam_roles + } + + # Convert the payload to JSON + json_payload = json.dumps(payload) + + # Invoke lambda2 + lambda_client = boto3.client('lambda') + response = lambda_client.invoke( + FunctionName= os.environ["func_name_mapping_lambda"], + InvocationType='Event', # Synchronous invocation + Payload=json_payload + ) + + + + # Extract and parse the response + + return 0 + +def get_iam_role_details(): + iam_client = boto3.client('iam') + + # List all IAM roles in the current AWS account + response = iam_client.list_roles() + + # Extract details for each IAM role + iam_roles = [] + for role in response['Roles']: + role_name = role['RoleName'] + role_arn = role['Arn'] + role_creation_date = role['CreateDate'].strftime('%Y-%m-%d %H:%M:%S') + + # Additional details for each IAM role + role_description = role.get('Description', '') + role_path = role.get('Path', '') + + # You can add more details based on your requirements + + iam_roles.append({ + 'RoleName': role_name, + 'RoleArn': role_arn, + 'CreationDate': role_creation_date, + 'Description': role_description, + 'Path': role_path, + # Add more fields as needed + }) + + return iam_roles