Migrate latest code from Stash to GitHub

Dockerfile

@@ -9,8 +9,8 @@ RUN go mod download
 COPY code/* /code/
 
 FROM base as build
-RUN go build -o main *.go \
-    && zip lambda.zip main
+RUN go build -o bootstrap *.go \
+    && zip lambda.zip bootstrap
 
 FROM base as test
 RUN go test
@@ -21,4 +21,4 @@ RUN golangci-lint run .
 FROM alpine as release
 WORKDIR /code
 COPY --from=build /code/lambda.zip /code/
-VOLUME /code
+VOLUME /code

cloudformation.yaml

@@ -1,25 +1,116 @@
 AWSTemplateFormatVersion: "2010-09-09"
 Transform: AWS::Serverless-2016-10-31
-Description: Pushes logs from AWS to Logic Monitor logs endpoint.
+Description: Pushes logs from AWS to LogicMonitor logs endpoint.
 Mappings:
   Constants:
     LMForwarder:
-      Version: 1.0.0
+      Version: 1.8.0
+  RegionToLayerArnMap:
+    us-east-1:
+      "LayerArn": "arn:aws:lambda:us-east-1:177933569100:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    us-east-2:
+      "LayerArn": "arn:aws:lambda:us-east-2:590474943231:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    eu-west-1:
+      "LayerArn": "arn:aws:lambda:eu-west-1:015030872274:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    eu-west-2:
+      "LayerArn": "arn:aws:lambda:eu-west-2:133256977650:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    eu-west-3:
+      "LayerArn": "arn:aws:lambda:eu-west-3:780235371811:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    us-west-2:
+      "LayerArn": "arn:aws:lambda:us-west-2:345057560386:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    us-west-1:
+      "LayerArn": "arn:aws:lambda:us-west-1:997803712105:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ca-central-1:
+      "LayerArn": "arn:aws:lambda:ca-central-1:200266452380:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    eu-central-1:
+      "LayerArn": "arn:aws:lambda:eu-central-1:187925254637:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    eu-central-2:
+      "LayerArn": "arn:aws:lambda:eu-central-2:772501565639:layer:AWS-Parameters-and-Secrets-Lambda-Extension:1"
+    eu-north-1:
+      "LayerArn": "arn:aws:lambda:eu-north-1:427196147048:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    eu-south-1:
+      "LayerArn": "arn:aws:lambda:eu-south-1:325218067255:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    eu-south-2:
+      "LayerArn": "arn:aws:lambda:eu-south-2:524103009944:layer:AWS-Parameters-and-Secrets-Lambda-Extension:1"
+    cn-north-1:
+      "LayerArn": "arn:aws-cn:lambda:cn-north-1:287114880934:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    cn-northwest-1:
+      "LayerArn": "arn:aws-cn:lambda:cn-northwest-1:287310001119:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-east-1:
+      "LayerArn": "arn:aws:lambda:ap-east-1:768336418462:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-south-2:
+      "LayerArn": "arn:aws:lambda:ap-south-2:070087711984:layer:AWS-Parameters-and-Secrets-Lambda-Extension:1"
+    ap-northeast-1:
+      "LayerArn": "arn:aws:lambda:ap-northeast-1:133490724326:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-northeast-3:
+      "LayerArn": "arn:aws:lambda:ap-northeast-3:576959938190:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-northeast-2:
+      "LayerArn": "arn:aws:lambda:ap-northeast-2:738900069198:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-southeast-1:
+      "LayerArn": "arn:aws:lambda:ap-southeast-1:044395824272:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-southeast-2:
+      "LayerArn": "arn:aws:lambda:ap-southeast-2:665172237481:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-southeast-3:
+      "LayerArn": "arn:aws:lambda:ap-southeast-3:490737872127:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    ap-south-1:
+      "LayerArn": "arn:aws:lambda:ap-south-1:176022468876:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    sa-east-1:
+      "LayerArn": "arn:aws:lambda:sa-east-1:933737806257:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    af-south-1:
+      "LayerArn": "arn:aws:lambda:af-south-1:317013901791:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    me-central-1:
+      "LayerArn": "arn:aws:lambda:me-central-1:858974508948:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    me-south-1:
+      "LayerArn": "arn:aws:lambda:me-south-1:832021897121:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    us-gov-east-1:
+      "LayerArn": "arn:aws-us-gov:lambda:us-gov-east-1:129776340158:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
+    us-gov-west-1:
+      "LayerArn": "arn:aws-us-gov:lambda:us-gov-west-1:127562683043:layer:AWS-Parameters-and-Secrets-Lambda-Extension:4"
 Parameters:
   LMCompanyName:
     Type: String
     Default: ""
     Description: The LogicMonitor acccount name.
+  LMCompanyDomain:
+    Type: String
+    Description: The LogicMonitor acccount domain.
+    AllowedValues: [ logicmonitor.com, lmgov.us, qa-lmgov.us ]
+    Default: "logicmonitor.com"
   LMAccessId:
     Type: String
     NoEcho: true
     Default: ""
-    Description: The LM API tokens access ID
+    Description: The LogicMonitor API tokens access ID
   LMAccessKey:
     Type: String
     NoEcho: true
     Default: ""
-    Description: The LM API tokens access key
+    Description: The LogicMonitor API tokens access key
+  LMBearerToken:
+    Type: String
+    NoEcho: true
+    Default: ""
+    Description: The LogicMonitor API Bearer token. (You must specify LMBearerToken if not providing LMAccessId, LMAccessKey. In case you provide all, LMAccessId and LMAccessKey will be used to authenticate with Logicmonitor.  )
+  Metadata:
+    Type: CommaDelimitedList
+    Description: Metadata if any
+    Default: awsRegion, arn
+    AllowedValues:
+      - awsRegion
+      - arn
+  MetadataKeysForJsonLogs:
+    Type: CommaDelimitedList
+    Description: Json keys to include in event metadata for cloudtaril logs and cloudwatch logs of json type
+    Default: ""
+  LMTenantIdentifier:
+    Type: String
+    Description: LogicMonitor Tenant Identifier send in the metadata with logs to LogicMonitor portal
+    Default: ""
+  UseSecretManagerForStoringSecrets:
+    Type: String
+    Description: If true it will use secrets manager and cache the credentials which can incur cost to secrets manager.If false it will use enviromental variables
+    AllowedValues: [true, false]
+    Default: "false"
   FunctionName:
     Type: String
     Default: LMLogsForwarder
@@ -36,12 +127,36 @@ Parameters:
     Description:  The memory size for the LM Forwarder lambda function
   FunctionTimeoutInSeconds:
     Type: Number
-    Default: 120
+    Default: 360
     Description: The timeout for the LM Forwarder lambda function
   PermissionsBoundaryArn:
     Type: String
     Default: ""
     Description: ARN for the Permissions Boundary Policy
+  AccountLevelSubscription:
+    Type: String
+    AllowedValues: [true, false]
+    Default: "false"
+    Description: If true it will enable account level subscription to forward logs to the LMLogsForwarder lambda.
+  FilterPattern:
+    Type: String
+    Description: Specify the log event structure and any filter conditions to apply on your log data as it gets streamed to LMLogsForwarder lambda. Only applicable if AccountLevelSubscription is true.
+    Default: ""
+  SelectionCriteria:
+    Type: String
+    Description: Use this parameter to apply a subscription filter policy to a subset of log groups in the account. The string can be up to 25KB in length. Example LogGroupName NOT IN ["MyLogGroup", "MyAnotherLogGroup"]
+    Default: ""
+
+Rules:
+  NoAccessIdKeyProvided:
+    RuleCondition: !Or [!Equals [!Ref LMAccessId, ""], !Equals [!Ref LMAccessKey, ""]]
+    Assertions:
+      - AssertDescription: Please specify either LMAccessId, LMAccessKey both or LMBearerToken to authenticate with Logicmonitor.
+        Assert: !Not
+          - !Equals
+            - !Ref LMBearerToken
+            - ""
+
 Resources:
   Forwarder:
     Type: AWS::Serverless::Function
@@ -50,40 +165,55 @@ Resources:
       FunctionName:
         Ref: FunctionName
       Description: Pushes aws logs to Logic Monitor log endpoint.
-      Handler: main
+      Handler: bootstrap
       MemorySize:
         Ref: FunctionMemorySize
-      Runtime: go1.x
-      CodeUri: 
+      Runtime: provided.al2
+      CodeUri:
         Bucket: !Ref ForwarderZipsBucket
         Key: main.zip
       Timeout:
         Ref: FunctionTimeoutInSeconds
+      Layers:
+        - Fn::If:
+            - CreateSecretManager
+            - !FindInMap [RegionToLayerArnMap, !Ref "AWS::Region", LayerArn]
+            - Ref: AWS::NoValue
       Environment:
         Variables:
-          LM_COMPANY_NAME:
+          LM_ACCOUNT:
             Ref: LMCompanyName
-          LM_ACCESS_ID_ARN:
-            Ref: AccessIdSecret
-          LM_ACCESS_KEY_ARN:
-            Ref: AccessKeySecret
+          LM_ACCOUNT_DOMAIN:
+            Ref: LMCompanyDomain
+          METADATA: !Join [",", !Ref Metadata]
+          LOGICMONITOR_ACCESS_ID: !If [UseLMV1ForAuth, !If [CreateSecretManager, !Ref AccessIdSecret, !Ref LMAccessId],  ""] #!Ref AccessIdSecret, !If [UseLMV1ForAuth, !Ref LMAccessId, "" ]
+          LOGICMONITOR_ACCESS_KEY: !If [UseLMV1ForAuth, !If [CreateSecretManager, !Ref AccessKeySecret, !Ref LMAccessKey],  ""]
+          LOGICMONITOR_BEARER_TOKEN: !If [MustUseBearerTokenForAuth, !If [CreateSecretManager, !Ref BearerTokenSecret, !Ref LMBearerToken],  ""]
           LM_SCRUB_REGEX:
             Ref: LMRegexScrub
+          USE_SECRET_MANAGER:
+            Ref: UseSecretManagerForStoringSecrets
+          JSON_METADATA_KEYS: !Join [",", !Ref MetadataKeysForJsonLogs]
+          ADD_CLOUDWATCH_METADATA: "true"
+          LM_TENANT_IDENTIFIER:
+            Ref: LMTenantIdentifier
       Policies:
         - Version: "2012-10-17"
           Statement:
             - Effect: Allow
               Action:
                 - s3:Get*
               Resource: "*"
-        - Version: "2012-10-17"
-          Statement:
-            - Effect: Allow
-              Action:
-                - secretsmanager:GetSecretValue
-              Resource:
-                - Ref: AccessKeySecret
-                - Ref: AccessIdSecret
+            - Fn::If:
+                - CreateSecretManager
+                - Effect: Allow
+                  Action:
+                    - secretsmanager:GetSecretValue
+                  Resource:
+                    - !If [UseLMV1ForAuth, !Ref AccessIdSecret, !Ref AWS::NoValue]
+                    - !If [UseLMV1ForAuth, !Ref AccessKeySecret, !Ref AWS::NoValue]
+                    - !If [MustUseBearerTokenForAuth, !Ref BearerTokenSecret, !Ref AWS::NoValue]
+                - Ref: AWS::NoValue
   CloudWatchLogsPermission:
     Type: AWS::Lambda::Permission
     Properties:
@@ -102,32 +232,27 @@ Resources:
     Type: AWS::S3::Bucket
     Properties:
       BucketEncryption:
-          ServerSideEncryptionConfiguration:
+        ServerSideEncryptionConfiguration:
           - ServerSideEncryptionByDefault:
               SSEAlgorithm: AES256
-      PublicAccessBlockConfiguration:
-        BlockPublicAcls: true
-        BlockPublicPolicy: true
-        IgnorePublicAcls: true
-        RestrictPublicBuckets: true
   ForwarderZip:
     Type: Custom::ForwarderZip
     Properties:
       ServiceToken: !GetAtt "ForwarderZipCopier.Arn"
       DestZipsBucket: !Ref "ForwarderZipsBucket"
       SourceZipUrl:
         Fn::Sub:
-          - "https://logicmonitor-logs-forwarder.s3.amazonaws.com/source/${LMForwarderVersion}/lambda.zip"
+          - "https://logicmonitor-logs-forwarder.s3.amazonaws.com/source/code/${LMForwarderVersion}/lambda.zip"
           - {
-              LMForwarderVersion:
-                !FindInMap [Constants, LMForwarder, Version],
-            }
+            LMForwarderVersion:
+              !FindInMap [Constants, LMForwarder, Version],
+          }
   ForwarderZipCopier:
     Type: AWS::Serverless::Function
     Properties:
       Description: Copies Logic Monitor Forwarder zip to the destination S3 bucket
       Handler: index.handler
-      Runtime: python3.7
+      Runtime: python3.11
       Timeout: 300
       InlineCode: |
         import json
@@ -136,10 +261,10 @@ Resources:
         import boto3
         import urllib.request
         import os
-        
+
         logger = logging.getLogger()
         logger.setLevel(logging.INFO)
-        
+
         def send_cfn_resp(event, context, response_status):
             resp_body = json.dumps({
                 'Status': response_status,
@@ -161,7 +286,7 @@ Resources:
             s3 = boto3.client('s3')
             filename = "main.zip"
             with urllib.request.urlopen(source_zip_url) as data:
-                s3.upload_fileobj(data, dest_zips_bucket, filename)
+                s3.upload_fileobj(data, dest_zips_bucket, filename, ExtraArgs={'ServerSideEncryption':'AES256'})
         def timeout(event, context):
             logger.error('Execution is about to time out, sending failure response to CloudFormation')
             send_cfn_resp(event, context, 'FAILED')
@@ -213,25 +338,63 @@ Resources:
           LM_FORWARDER_VERSION: !FindInMap [Constants, LMForwarder, Version]
   AccessKeySecret:
     Type: AWS::SecretsManager::Secret
+    Condition: CreateSecretAccess
     Properties:
-      Description: Logic Monitor Access Key
+      Description: LogicMonitor Access Key
       SecretString:
         Ref: LMAccessKey
   AccessIdSecret:
     Type: AWS::SecretsManager::Secret
+    Condition: CreateSecretAccess
     Properties:
-      Description: Logic Monitor Access Id
+      Description: LogicMonitor Access Id
       SecretString:
         Ref: LMAccessId
+  BearerTokenSecret:
+    Type: AWS::SecretsManager::Secret
+    Condition: CreateSecretBearerToken
+    Properties:
+      Description: LogicMonitor Bearer Token
+      SecretString:
+        Ref: LMBearerToken
+  AccountPolicy:
+    Condition: IsAccountLevel
+    Type: AWS::Logs::AccountPolicy
+    DependsOn: [Forwarder, CloudWatchLogsPermission, S3Permission]
+    Properties:
+      PolicyName: account-subscriptionfilter-policy
+      PolicyDocument: !Sub "{\"DestinationArn\":\"arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${FunctionName}\", \"FilterPattern\": \"${FilterPattern}\"}"
+      PolicyType: "SUBSCRIPTION_FILTER_POLICY"
+      Scope: "ALL"
+      SelectionCriteria:
+        Fn::If:
+          - HasSelectionCriteria
+          - Ref: SelectionCriteria
+          - Ref: AWS::NoValue
 Conditions:
   SetPermissionsBoundary:
     Fn::Not:
       - Fn::Equals:
           - Ref: PermissionsBoundaryArn
           - ""
+  AccessIdEmpty: !Equals [!Ref LMAccessId, ""]
+  AccessKeyEmpty: !Equals [!Ref LMAccessKey, ""]
+  MustUseBearerTokenForAuth: !Or
+    - !Condition AccessIdEmpty
+    - !Condition AccessKeyEmpty
+  UseLMV1ForAuth: !Not [!Condition MustUseBearerTokenForAuth]
+  CreateSecretManager: !Equals [!Ref UseSecretManagerForStoringSecrets, true]
+  CreateSecretAccess: !And
+    - !Condition UseLMV1ForAuth
+    - !Condition CreateSecretManager
+  CreateSecretBearerToken: !And
+    - !Condition MustUseBearerTokenForAuth
+    - !Condition CreateSecretManager
+  IsAccountLevel: !Equals [!Ref AccountLevelSubscription, 'true']
+  HasSelectionCriteria:  !Not [!Equals [!Ref SelectionCriteria, ""]]
 Outputs:
   LMForwarderArn:
-    Description: Logic Monitor Forwarder Lambda Function ARN
+    Description: LogicMonitor Forwarder Lambda Function ARN
     Value:
       Fn::GetAtt:
         - Forwarder
@@ -247,11 +410,21 @@ Metadata:
         Parameters:
           - FunctionName
           - LMCompanyName
+          - LMCompanyDomain
           - LMAccessId
           - LMAccessKey
+          - LMBearerToken
           - LMRegexScrub
+          - UseSecretManagerForStoringSecrets
       - Label:
           default: Lambda Function (Optional)
         Parameters:
+          - Metadata
+          - MetadataKeysForJsonLogs
+          - LMTenantIdentifier
           - FunctionMemorySize
           - FunctionTimeoutInSeconds
+          - AccountLevelSubscription
+          - FilterPattern
+          - SelectionCriteria
+