From 5d334fca903ae23b5531401f231ae5c83e63494b Mon Sep 17 00:00:00 2001 From: Vishal Charpe Date: Tue, 1 Jul 2025 12:16:20 +0530 Subject: [PATCH] DEV-203529: DEV-203529-logstash-customisable-value-for-_resource.type-field --- CHANGELOG.md | 8 ++++++++ README.md | 22 +++++++++++----------- lib/logstash/outputs/lmlogs.rb | 8 +++++++- lib/logstash/outputs/version.rb | 2 +- spec/outputs/metadata_spec.rb | 3 +-- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 584cdde..b933e56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,3 +18,11 @@ - Dont send event metadata by default. Add config include_metadata_keys for including custom metadata ## 2.0.1 - Fix incomplete json metadata error. +## 2.0.2 +- Added domain support for GovCloud +## 2.0.3 +- Added validation for domain support +## 2.0.4 +- Added resource type in metadata. +## 2.0.5 +- Added support for customizable resource type. \ No newline at end of file diff --git a/README.md b/README.md index 16c6111..c041722 100644 --- a/README.md +++ b/README.md @@ -29,17 +29,17 @@ The allowed values for portal_domain are ["logicmonitor.com", "lmgov.us", "qa-lm ## Important options -| Option | Description| Default | -| --- | --- | --- | -| batch_size | Event batch size to send to LM Logs.| 100 | -| message_key | Key that will be used by the plugin as the system key | "message" | -| lm_property | Key that will be used by LM to match resource based on property | "system.hostname" | -| keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true | -| timestamp_is_key | If true, LM Logs will use a specified key as the event timestamp | false | -| timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp" | -| include_metadata | If true, all metadata fields will be sent to LM Logs | false | -| include_metadata_keys | Array of json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson. If config `include_metadata` is set to true, all metadata will be sent regardless of this config. | [] | - +| Option | Description | Default | +|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------| +| batch_size | Event batch size to send to LM Logs. | 100 | +| message_key | Key that will be used by the plugin as the system key | "message" | +| lm_property | Key that will be used by LM to match resource based on property | "system.hostname" | +| keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true | +| timestamp_is_key | If true, LM Logs will use a specified key as the event timestamp | false | +| timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp" | +| include_metadata | If true, all metadata fields will be sent to LM Logs | false | +| include_metadata_keys | Array of json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson. If config `include_metadata` is set to true, all metadata will be sent regardless of this config. | [] | +| resource_type | If a Resource Type is explicitly specified, that value will be statically applied to all ingested logs. If set to `predef.externalResourceType`, the Resource Type will be assigned dynamically based on the `predef.externalResourceType` property set for the specific resource the logs are mapped to in LM. If left blank, the Resource Type field will remain unset in the ingested logs. | "" | See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options The syntax for `message_key` and `source_key` values are available in the [Logstash Event API Documentation](https://www.elastic.co/guide/en/logstash/current/event-api.html) diff --git a/lib/logstash/outputs/lmlogs.rb b/lib/logstash/outputs/lmlogs.rb index d1b0adc..16a3fea 100644 --- a/lib/logstash/outputs/lmlogs.rb +++ b/lib/logstash/outputs/lmlogs.rb @@ -110,6 +110,9 @@ class InvalidHTTPConfigError < StandardError; end # json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson. config :include_metadata_keys, :validate => :array, :required => false, :default => [] + # Customised resource_type to map logs to existing LM resources + config :resource_type, :validate => :string, :required => false + @@MAX_PAYLOAD_SIZE = 8*1024*1024 # For developer debugging. @@ -308,7 +311,10 @@ def processEvent(event) if @include_metadata lmlogs_event = event_json lmlogs_event.delete("@timestamp") # remove redundant timestamp field - lmlogs_event["_resource.type"]="Logstash" + unless @resource_type.to_s.strip.empty? + lmlogs_event["_resource.type"] = @resource_type + end + if lmlogs_event.dig("event", "original") != nil lmlogs_event["event"].delete("original") # remove redundant log field end diff --git a/lib/logstash/outputs/version.rb b/lib/logstash/outputs/version.rb index 73c1065..0272127 100644 --- a/lib/logstash/outputs/version.rb +++ b/lib/logstash/outputs/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module LmLogsLogstashPlugin - VERSION = '2.0.4' + VERSION = '2.0.5' end diff --git a/spec/outputs/metadata_spec.rb b/spec/outputs/metadata_spec.rb index 847d2d4..26b0837 100644 --- a/spec/outputs/metadata_spec.rb +++ b/spec/outputs/metadata_spec.rb @@ -76,8 +76,7 @@ def check_same_hash(h1,h2) "nested" => {"nested2" => {"nested3" => "value", "nested3b" => "value"}, "nested_ignored" => "somevalue" - }, - "_resource.type"=>"Logstash" + } } puts " actual : #{constructed_event} \n expected : #{expected_event}" puts " hash diff : #{Hashdiff.diff(constructed_event,expected_event)}"