diff --git a/.gitignore b/.gitignore index 4e395d0..df855de 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,6 @@ packaged.yml # SAM .aws-sam/ #VS Code -.vscode/ \ No newline at end of file +.vscode/ +#JetBrains +.idea \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d97afe3..f99669c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Version 1.3.0 (2025-12-22) +- Upgraded runtime from Python 3.11 to Python 3.12 +- Updated to use Amazon Linux 2023 base (via Python 3.12 runtime) +- Python 3.11 deprecation date: June 30, 2026 +- Python 3.12 support extends until October 31, 2028 +- No code changes required - all dependencies compatible with Python 3.12 + +## Previous Versions - updated to last release 1.1.1 - updated semantic version - updated urls and lables diff --git a/DEVELOPER.md b/DEVELOPER.md new file mode 100644 index 0000000..b4007fa --- /dev/null +++ b/DEVELOPER.md @@ -0,0 +1,66 @@ +# Developer + +This project uses [SAM](https://aws.amazon.com/serverless/sam/) tool for build, +run locally, package and publish. + +There is a serverless file to play locally as well, but here we will focus on +the [SAM](https://aws.amazon.com/serverless/sam/) way of doing it. + +## Requirements + +- Make + +### Local run / build + +- Docker +- AWS Account +- AWS Profile configured on your computer, remember that AWS uses the `default` + profile unless you specify it. You can specify usign `AWS_PROFILE` enviroment + variable. +- [AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-reference.html#serverless-sam-cli) + +### Package + +- An AWS S3 bucket where the package will be uploaded + +### Deploy + +- AWS Account and enough permissions to do the deploy, generated template will + need IAM capabilities. + +### Publishing + +- For personal publishing is mandatory to update the function name on the + `template.yml` file to avoid collide with the New Relic official release of + this application. + +## Building it locally + +This will generate an image of the lambda application using a docker container +that you will be able to run locally. + +Just run `make build`. + +## Running locally + +You should have built the image locally as mentioned in the previous step. + +Then you need to have a "sample" event of a file in an S3 bucket so we can use +it, we provide a sample one in the test/mock.json but it wouldn't work if you +haven't access to the given S3 bucket. + +Then just run `LICENSE_KEY= TEST_FILE="./test/mock.json" make run` to run it locally. + +## Packaging + +Run `BUCKET= REGION= make package` + +## Deploying + +Run `REGION= STACK_NAME= make deploy` + +## Publishing + +Run `REGION= make publish` to publish your package. Remember to +update the function name before publishing it to do not collide with the New +Relic official application. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..94f4e67 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +build: + sam build --use-container + +run: check-run-env + sam local invoke "NewRelicLogIngestionFunction" -e $(TEST_FILE) + +package: check-package-env + sam package --output-template-file packaged.yml --s3-bucket $(BUCKET) --region $(REGION) + +deploy: check-deploy-env + sam deploy --template-file packaged.yml --stack-name $(STACK_NAME) --region $(REGION) + +publish: check-publish-env + sam publish --template packaged.yml --region $(REGION) + +check-run-env: +ifndef LICENSE_KEY + $(error LICENSE_KEY is undefined) +endif +ifndef TEST_FILE + $(error TEST_FILE is undefined) +endif + +check-package-env: +ifndef REGION + $(error REGION is undefined) +endif +ifndef BUCKET + $(error BUCKET is undefined) +endif + +check-deploy-env: +ifndef REGION + $(error REGION is undefined) +endif +ifndef STACK_NAME + $(error STACK_NAME is undefined) +endif + +check-publish-env: +ifndef REGION + $(error REGION is undefined) +endif diff --git a/README.md b/README.md index b082562..9cf811c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Community Project header](https://github.com/newrelic/open-source-office/raw/master/examples/categories/images/Community_Project.png)](https://github.com/newrelic/open-source-office/blob/master/examples/categories/index.md#community-project) +[![Community Plus header](https://github.com/newrelic/opensource-website/raw/master/src/images/categories/Community_Plus.png)](https://opensource.newrelic.com/oss-category/#community-plus) # AWS Lambda for sending logs from S3 to New Relic @@ -22,5 +22,11 @@ Contributions to improve s3-log-ingestion-lambda are encouraged! Keep in mind wh To execute our corporate CLA, which is required if your contribution is on behalf of a company, or if you have any questions, please drop us an email at opensource@newrelic.com. +## Developers + +For more information about how to contribute from the developer point of view, +we recommend you to take a look to the [DEVELOPER.md](./DEVELOPER.md) that +contains most of the info you'll need. + ## License -`s3-log-ingestion-lambda` is licensed under the [Apache 2.0](http://apache.org/licenses/LICENSE-2.0.txt) License. The s3-log-ingestion-lambda also uses source code from third party libraries. Full details on which libraries are used and the terms under which they are licensed can be found in the third party notices docume \ No newline at end of file +`s3-log-ingestion-lambda` is licensed under the [Apache 2.0](http://apache.org/licenses/LICENSE-2.0.txt) License. The s3-log-ingestion-lambda also uses source code from third party libraries. Full details on which libraries are used and the terms under which they are licensed can be found in the third party notices document \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 245282d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -aiohttp==3.6.2 -async-timeout==3.0.1 -attrs==19.3.0 -boto==2.49.0 -boto3==1.14.40 -botocore==1.17.40 -certifi==2020.6.20 -chardet==3.0.4 -docutils==0.15.2 -idna==2.10 -jmespath==0.10.0 -multidict==4.7.6 -Pympler==0.9 -python-dateutil==2.8.1 -requests==2.24.0 -s3transfer==0.3.3 -six==1.15.0 -smart-open==2.1.0 -typing-extensions==3.7.4.2 -urllib3==1.25.10 -yarl==1.5.1 diff --git a/serverless.yml b/serverless.yml index d2bbb57..bd2d7ff 100644 --- a/serverless.yml +++ b/serverless.yml @@ -22,32 +22,37 @@ service: ${env:SERVICE_NAME} provider: name: aws - runtime: python3.8 + runtime: python3.12 iamRoleStatements: - Effect: "Allow" Action: - - "s3:GetObject" + - "s3:GetObject" Resource: "arn:aws:s3:::${env:S3_BUCKET_NAME}/*" plugins: - serverless-python-requirements - + custom: pythonRequirements: + fileName: ./src/requirements.txt dockerizePip: non-linux functions: NewRelic-s3-log-ingestion: handler: src/handler.lambda_handler - environment: + environment: LICENSE_KEY: ${env:LICENSE_KEY} LOG_TYPE: ${env:LOG_TYPE} DEBUG_ENABLED: ${env:DEBUG_ENABLED} + S3_CLOUD_TRAIL_LOG_PATTERN: ${env:S3_CLOUD_TRAIL_LOG_PATTERN} + S3_IGNORE_PATTERN: ${env:S3_IGNORE_PATTERN} + BATCH_SIZE_FACTOR: ${env:BATCH_SIZE_FACTOR} ADDITIONAL_ATTRIBUTES: ${env:ADDITIONAL_ATTRIBUTES} + events: - s3: bucket: ${env:S3_BUCKET_NAME} event: s3:ObjectCreated:* rules: - prefix: ${env:S3_PREFIX, ""} - existing: true \ No newline at end of file + existing: true diff --git a/src/handler.py b/src/handler.py index 3d758b0..25c0b09 100644 --- a/src/handler.py +++ b/src/handler.py @@ -11,7 +11,6 @@ import logging from smart_open import open import re -from pympler import asizeof from dateutil import parser @@ -19,7 +18,8 @@ US_LOGGING_INGEST_HOST = "https://log-api.newrelic.com/log/v1" EU_LOGGING_INGEST_HOST = 'https://log-api.eu.newrelic.com/log/v1' -LOGGING_LAMBDA_VERSION = '1.1.1' +JP_LOGGING_INGEST_HOST = 'https://log-api.jp.nr-data.net/log/v1' +LOGGING_LAMBDA_VERSION = '1.4.3' LOGGING_PLUGIN_METADATA = { 'type': "s3-lambda", 'version': LOGGING_LAMBDA_VERSION @@ -33,6 +33,11 @@ class InvalidArgumentException(Exception): def _format_error(e, text): return "{}. {}".format(e, text) +def _get_optional_env(key, default): + """ + Returns the default value even if the environment variable is set but empty + """ + return os.getenv(key, default) or default def _get_additional_attributes(attributes=None): """ @@ -46,7 +51,7 @@ def _get_additional_attributes(attributes=None): """ if attributes: return attributes - env_attributes = os.getenv("ADDITIONAL_ATTRIBUTES", "{}") + env_attributes = _get_optional_env("ADDITIONAL_ATTRIBUTES", "{}") try: return json.loads(env_attributes) except json.JSONDecodeError as e: @@ -70,6 +75,7 @@ def _get_additional_attributes(attributes=None): # Max batch size for sending requests (1MB) MAX_BATCH_SIZE = 1000 * 1024 BATCH_SIZE_FACTOR = 1.5 + REQUEST_BATCH_SIZE = 25 completed_requests = 0 @@ -83,20 +89,63 @@ class BadRequestException(Exception): pass +def _is_ignore_log_file(key=None, regex_pattern=None): + """ + This functions checks whether this log file should be ignored based on regex pattern. + """ + if not regex_pattern: + regex_pattern = _get_optional_env("S3_IGNORE_PATTERN", "$^") + + return bool(re.search(regex_pattern, key)) + + +def _isCloudTrail(key=None, regex_pattern=None): + """ + This functions checks whether this log file is a CloudTrail log based on regex pattern. + """ + if not regex_pattern: + regex_pattern = os.getenv("S3_CLOUD_TRAIL_LOG_PATTERN", "") + if regex_pattern: + return bool(re.search(regex_pattern, key)) + # Default: string-based check avoids polynomial backtracking + return bool(key and key.endswith('.json.gz') and '_CloudTrail_' in key + and '_CloudTrail-Digest_' not in key) + +def _isCloudTrailDigest(key=None): + """ + This functions checks whether this log file is a CloudTrail-Digest based on regex pattern. + """ + return bool(key and key.endswith('.json.gz') and '_CloudTrail-Digest_' in key) + +def _convert_float(s): + try: + f = float(s) + except ValueError: + f = 1.5 + return f + +def _get_batch_size_factor(batch_size_factor=None): + """ + This functions gets BATCH_SIZE_FACTOR from env vars. + """ + if batch_size_factor: + return batch_size_factor + return _convert_float(_get_optional_env("BATCH_SIZE_FACTOR", BATCH_SIZE_FACTOR)) + def _get_license_key(license_key=None): """ This functions gets New Relic's license key from env vars. """ if license_key: return license_key - return os.getenv("LICENSE_KEY", "") + return _get_optional_env("LICENSE_KEY", "") def _get_log_type(log_type=None): """ This functions gets the New Relic logtype from env vars. """ - return log_type or os.getenv("LOG_TYPE") or os.getenv("LOGTYPE", "") + return log_type or _get_optional_env("LOG_TYPE", "") def _setting_console_logging_level(): @@ -104,7 +153,7 @@ def _setting_console_logging_level(): Determines whether or not debug logging should be enabled based on the env var. Defaults to false. """ - if os.getenv("DEBUG_ENABLED", "false").lower() == "true": + if _get_optional_env("DEBUG_ENABLED", "false").lower() == "true": print("enabling debug mode") logger.setLevel(logging.DEBUG) else: @@ -120,11 +169,12 @@ def _get_logging_endpoint(ingest_url=None): return ingest_url if "NR_LOGGING_ENDPOINT" in os.environ: return os.environ["NR_LOGGING_ENDPOINT"] - return ( - EU_LOGGING_INGEST_HOST - if _get_license_key().startswith("eu") - else US_LOGGING_INGEST_HOST - ) + license_key = _get_license_key() + if license_key.startswith("eu"): + return EU_LOGGING_INGEST_HOST + elif license_key.startswith("jp"): + return JP_LOGGING_INGEST_HOST + return US_LOGGING_INGEST_HOST def _compress_payload(data): @@ -133,7 +183,9 @@ def _compress_payload(data): This method usually returns a list of one element, but can be bigger if the payload size is too big """ + logger.debug(f"uncompressed size: {sys.getsizeof(json.dumps(data).encode())}") payload = gzip.compress(json.dumps(data).encode()) + logger.debug(f"compressed size: {sys.getsizeof(payload)}") return payload @@ -235,40 +287,46 @@ async def _fetch_data_from_s3(bucket, key, context): logger.error( "The log file uploaded to S3 is larger than the supported max size of 400MB") return - + BATCH_SIZE_FACTOR = _get_batch_size_factor() s3MetaData = { "invoked_function_arn": context.invoked_function_arn, "s3_bucket_name": bucket, "s3_key": key } log_file_url = "s3://{}/{}".format(bucket, key) + if _isCloudTrailDigest(key): + # CloudTrail-Digest will not have any logs in it. Hence, no need to continue further + return async with aiohttp.ClientSession() as session: log_batches = [] batch_request = [] batch_counter = 1 + log_batch_size = 0 start = time.time() - isCloudTrail = bool(re.search(".*CloudTrail.*\.json.gz$", key)) with open(log_file_url, encoding='utf-8') as log_lines: + if _isCloudTrail(key): + # This is a CloudTrail log - we need to apply special preprocessing + cloudtrail_events=json.loads(log_lines.read())["Records"] + for this_event in cloudtrail_events: + # Convert the eventTime to Posix time and pass it to New Relic as a timestamp attribute + this_event['timestamp']=time.mktime((parser.parse(this_event['eventTime'])).timetuple()) + log_lines = cloudtrail_events + for index, log in enumerate(log_lines): - if isCloudTrail: - # This is a CloudTrail log - we need to apply special preprocessing - cloudtrail_events=json.loads(log)["Records"] - for this_event in cloudtrail_events: - # Convert the eventTime to Posix time and pass it to New Relic as a timestamp attribute - this_event['timestamp']=time.mktime((parser.parse(this_event['eventTime'])).timetuple()) - log_batches.extend(cloudtrail_events) - else: - if index % 500 == 0: - logger.debug(f"index: {index}") - log_batches.append(log) - if asizeof.asizeof(log_batches) > (MAX_BATCH_SIZE * BATCH_SIZE_FACTOR): - logger.debug(f"sending batch: {batch_counter}") + log_batch_size += sys.getsizeof(str(log)) + if index % 500 == 0: + logger.debug(f"index: {index}") + logger.debug(f"log_batch_size: {log_batch_size}") + log_batches.append(log) + if log_batch_size > (MAX_BATCH_SIZE * BATCH_SIZE_FACTOR): + logger.debug(f"sending batch: {batch_counter} log_batch_size: {log_batch_size}") data = {"context": s3MetaData, "entry": log_batches} batch_request.append(create_log_payload_request(data, session)) if len(batch_request) >= REQUEST_BATCH_SIZE: await asyncio.gather(*batch_request) batch_request = [] log_batches = [] + log_batch_size = 0 batch_counter += 1 data = {"context": s3MetaData, "entry": log_batches} batch_request.append(create_log_payload_request(data, session)) @@ -288,6 +346,12 @@ def lambda_handler(event, context): bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.parse.unquote_plus( event['Records'][0]['s3']['object']['key'], encoding='utf-8') + + # Allow user to skip log file using regex pattern set in env variable: S3_IGNORE_PATTERN + if _is_ignore_log_file(key): + logger.debug(f"Ignore log file based on S3_IGNORE_PATTERN: {key}") + return {'statusCode': 200, 'message': 'ignored this log'} + try: asyncio.run(_fetch_data_from_s3(bucket, key, context)) except KeyError as e: diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..08f93d4 --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1,18 @@ +aiohttp>=3.14.3 +async-timeout>=4.0.2 +attrs>=22.1.0 +boto3>=1.34.31 +botocore>=1.34.31 +certifi>=2024.7.4 +chardet>=5.0.0 +idna>=3.8 +jmespath>=1.0.0 +multidict>=6.0.0 +python-dateutil>=2.8.2 +requests>=2.32.4 +s3transfer>=0.10.0 +six>=1.16.0 +smart_open>=5.2.0 +typing-extensions>=4.5.0 +urllib3>=2.2.0 +yarl>=1.9.1 \ No newline at end of file diff --git a/template.yml b/template.yml index 311b0d2..3008cf6 100644 --- a/template.yml +++ b/template.yml @@ -1,6 +1,31 @@ AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: Sends log data from S3 to New Relic Logging. +Metadata: + AWS::ServerlessRepo::Application: + Name: NewRelic-log-ingestion-s3 + Description: Send log data from a S3 bucket to New Relic Logging. + Author: New Relic + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE + ReadmeUrl: README.md + Labels: ['newrelic', 'logs', 'logging', 'ingestion', 'lambda', 's3'] + HomePageUrl: https://github.com/newrelic/aws_s3_log_ingestion_lambda + SemanticVersion: 1.4.3 + SourceCodeUrl: https://github.com/newrelic/aws_s3_log_ingestion_lambda + + AWS::CloudFormation::Interface: + ParameterGroups: + - + Label: + default: "Deployment Settings" + Parameters: + - NRLicenseKey + - FunctionRole + - PermissionsBoundary + - NRLogType + - DebugEnabled + Parameters: NRLicenseKey: Type: String @@ -14,29 +39,82 @@ Parameters: Type: String Description: A boolean to determine if you want to output debug messages in the CloudWatch console Default: "false" - AdditionalTags: + AdditionalAttributes: + Type: String + Description: "(Optional) A string containing json object(string,string). These attributes will be added to New Relic payload." + Default: "" + S3CloudTrailLogPattern: + Type: String + Description: "(Optional) Regex pattern to check if the file is from CloudTrail" + Default: "" + S3IgnorePattern: Type: String - Description: A string containing json object(string,string). These attributes will be added to New Relic payload. - Default: "{}" + Description: "(Optional) Regex pattern to ignore files" + Default: "" + BatchSizeFactor: + Type: String + Description: "(Optional) Indicates the expected compression factor of your logs. Used to check if logs could be sent to the API (that is limited to 1Mb of compressed data)" + Default: "" + FunctionRole: + Type: String + Description: | + (Optional) The ARN of an IAM role to use as this function's execution role. Should provide the AWSLambdaBasicExecutionRole policy. + If not specified, an appropriate Role will be created, which will require CAPABILITY_IAM to be acknowledged. + Default: "" + PermissionsBoundary: + Type: String + Description: | + (Optional) The ARN of a permissions boundary to use for this function's execution role. This property works only if the role is generated for you. + Default: "" -Metadata: - AWS::ServerlessRepo::Application: - Name: NewRelic-log-ingestion-s3 - Description: Send log data from a S3 bucket to New Relic Logging. - Author: New Relic - SpdxLicenseId: Apache-2.0 - LicenseUrl: LICENSE - ReadmeUrl: README.md - Labels: ['newrelic', 'logs', 'logging', 'ingestion', 'lambda', 's3'] - HomePageUrl: https://github.com/newrelic/aws_s3_log_ingestion_lambda - SemanticVersion: 1.1.1 - SourceCodeUrl: https://github.com/newrelic/aws_s3_log_ingestion_lambda +Conditions: + NoRole: !Equals ['', !Ref FunctionRole] + NoCap: !Not [ !Equals ['', !Ref FunctionRole] ] + HasPermissionBoundary: !Not [ !Equals ['', !Ref PermissionsBoundary] ] Resources: + NewRelicLogIngestion: + Type: 'AWS::Serverless::Function' + Condition: NoCap + Properties: + Runtime: python3.12 + CodeUri: src/ + Handler: handler.lambda_handler + FunctionName: NewRelic-s3-log-ingestion + Timeout: 900 + MemorySize: 256 + Environment: + Variables: + LICENSE_KEY: !Ref NRLicenseKey + LOG_TYPE: !Ref NRLogType + DEBUG_ENABLED: !Ref DebugEnabled + S3_CLOUD_TRAIL_LOG_PATTERN: !Ref S3CloudTrailLogPattern + S3_IGNORE_PATTERN: !Ref S3IgnorePattern + BATCH_SIZE_FACTOR: !Ref BatchSizeFactor + ADDITIONAL_ATTRIBUTES: !Ref AdditionalAttributes + PermissionsBoundary: !If [ HasPermissionBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue ] + Role: !Ref FunctionRole + Policies: + - Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - 's3:GetObject' + Resource: 'arn:aws:s3:::*' + Events: + BucketEvent1: + Type: S3 + Properties: + Bucket: + Ref: SourceLogBucket + Events: + - 's3:ObjectCreated:*' + NewRelicLogIngestionFunction: Type: 'AWS::Serverless::Function' + Condition: NoRole Properties: - Runtime: python3.8 + Runtime: python3.12 CodeUri: src/ Handler: handler.lambda_handler FunctionName: NewRelic-s3-log-ingestion @@ -47,7 +125,11 @@ Resources: LICENSE_KEY: !Ref NRLicenseKey LOG_TYPE: !Ref NRLogType DEBUG_ENABLED: !Ref DebugEnabled - ADDITIONAL_ATTRIBUTES: !Ref AdditionalTags + S3_CLOUD_TRAIL_LOG_PATTERN: !Ref S3CloudTrailLogPattern + S3_IGNORE_PATTERN: !Ref S3IgnorePattern + BATCH_SIZE_FACTOR: !Ref BatchSizeFactor + ADDITIONAL_ATTRIBUTES: !Ref AdditionalAttributes + PermissionsBoundary: !If [ HasPermissionBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue ] Policies: - Version: '2012-10-17' Statement: diff --git a/test/mock.json b/test/mock.json new file mode 100644 index 0000000..ed796d0 --- /dev/null +++ b/test/mock.json @@ -0,0 +1,20 @@ +{ + "Records": [ + { + "eventVersion": "2.1", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventName": "ObjectCreated:Put", + "s3": { + "s3SchemaVersion": "1.0", + "bucket": { + "name": "s3-nr-log-test-bucket", + "arn": "arn:aws:s3:::s3-nr-log-test-bucket" + }, + "object": { + "key": "cloudflare-test-data.json.gz" + } + } + } + ] +}