diff --git a/.gitignore b/.gitignore index 16768962..7d8e1442 100644 --- a/.gitignore +++ b/.gitignore @@ -362,6 +362,9 @@ MigrationBackup/ # Fody - auto-generated XML schema FodyWeavers.xsd +# Environment variables +.env + # appsettings.json appsettings.json diff --git a/Directory.Packages.props b/Directory.Packages.props index 6d8c3c25..4c3e4ef6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -25,6 +25,12 @@ + + + + + + diff --git a/SnakeAid.Api/DI/DependencyInjection.cs b/SnakeAid.Api/DI/DependencyInjection.cs index dd6045fa..9fd1b7a0 100644 --- a/SnakeAid.Api/DI/DependencyInjection.cs +++ b/SnakeAid.Api/DI/DependencyInjection.cs @@ -15,6 +15,11 @@ using Serilog; using Doppler.Extensions.Configuration; // Ensure this is available +using OpenTelemetry.Trace; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Exporter; +using Npgsql; namespace SnakeAid.Api.DI; @@ -281,4 +286,37 @@ public static IServiceCollection AddSwagger(this IServiceCollection services) } #endregion + + #region OpenTelemetry + + public static IServiceCollection AddOpenTelemetryServices(this IServiceCollection services, IConfiguration configuration) + { + var otlpEndpoint = configuration["OpenTelemetry:Endpoint"] ?? "http://localhost:4317"; // fallback for local without docker + + services.AddOpenTelemetry() + .WithTracing(tracerProviderBuilder => + { + tracerProviderBuilder + .AddSource("SnakeAid.Api") + .SetResourceBuilder( + ResourceBuilder.CreateDefault() + .AddService("SnakeAid.Api")) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddNpgsql() + .AddOtlpExporter(opts => opts.Endpoint = new Uri(otlpEndpoint)); + }) + .WithMetrics(metricsProviderBuilder => + { + metricsProviderBuilder + .AddMeter("SnakeAid.Api") + .AddAspNetCoreInstrumentation() + .AddRuntimeInstrumentation() + .AddOtlpExporter(opts => opts.Endpoint = new Uri(otlpEndpoint)); + }); + + return services; + } + + #endregion } diff --git a/SnakeAid.Api/Program.cs b/SnakeAid.Api/Program.cs index d76b4fec..a5a5ce48 100644 --- a/SnakeAid.Api/Program.cs +++ b/SnakeAid.Api/Program.cs @@ -105,6 +105,7 @@ public static async Task Main(string[] args) builder.Services.AddScoped(); builder.Services.AddServices(builder.Configuration); + builder.Services.AddOpenTelemetryServices(builder.Configuration); // Register services using Scrutor builder.Services.Scan(scan => scan diff --git a/SnakeAid.Api/SnakeAid.Api.csproj b/SnakeAid.Api/SnakeAid.Api.csproj index 63f99af7..2b30b8b5 100644 --- a/SnakeAid.Api/SnakeAid.Api.csproj +++ b/SnakeAid.Api/SnakeAid.Api.csproj @@ -11,7 +11,13 @@ - + + + + + + + diff --git a/SnakeAid.Api/appsettings.Example.json b/SnakeAid.Api/appsettings.Example.json index 7ba7a771..4f2ad573 100644 --- a/SnakeAid.Api/appsettings.Example.json +++ b/SnakeAid.Api/appsettings.Example.json @@ -10,6 +10,7 @@ "Serilog", "Serilog.Sinks.Console", "Serilog.Sinks.SQLite", + "Serilog.Sinks.Grafana.Loki", "Serilog.Enrichers.Environment", "Serilog.Enrichers.Span" ], @@ -36,6 +37,26 @@ "outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] {Application} {Message:lj} {Properties:j}{NewLine}{Exception}" } }, + { + "Name": "GrafanaLoki", + "Args": { + "requestUri": "http://loki:3100", + "labels": [ + { + "key": "app", + "value": "SnakeAid.API" + }, + { + "key": "environment", + "value": "Development" + } + ], + "propertiesAsLabels": ["Application", "Environment"], + "queueLimit": 10000, + "batchPostingLimit": 1000, + "period": "00:00:02" + } + }, { "Name": "SQLite", "Args": { @@ -46,6 +67,9 @@ } ] }, + "OpenTelemetry": { + "Endpoint": "http://otel-collector:4317" + }, "Jwt": { "SecretKey": "mysuperlongsecretkeythatishard2guess123!", "RefreshSecretKey": "anotherultrasecretkeythatmustbe32chars!!", diff --git a/SnakeAid.Docs b/SnakeAid.Docs index 1976c6f4..13a83208 160000 --- a/SnakeAid.Docs +++ b/SnakeAid.Docs @@ -1 +1 @@ -Subproject commit 1976c6f4e921976550de5fba46f33197db52923c +Subproject commit 13a83208ec40201b8f5ab8cae171ef7e4bd87be7 diff --git a/docker-compose.grafana.yml b/docker-compose.grafana.yml new file mode 100644 index 00000000..51e66d78 --- /dev/null +++ b/docker-compose.grafana.yml @@ -0,0 +1,90 @@ +services: + snakeaid-api: + image: thekhiem7/snakeaid-api:latest + pull_policy: always + build: + context: . + dockerfile: Dockerfile + ports: + - "8080:8080" + environment: + - DOPPLER_TOKEN=${DOPPLER_TOKEN} + - DOPPLER_CONFIG=snake-aid/dev + + restart: unless-stopped + networks: + - snakeaid-net + + loki: + image: grafana/loki:latest + command: -config.file=/etc/loki/local-config.yaml + ports: + - "3100:3100" + volumes: + - ./observability/loki-config.yaml:/etc/loki/local-config.yaml + networks: + - snakeaid-net + + tempo: + image: grafana/tempo:latest + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - ./observability/tempo-config.yaml:/etc/tempo.yaml + - ./observability/tempo-data:/var/tempo + ports: + - "14268" # jaeger ingest + - "3200" # tempo + - "4317" # otlp grpc + - "4318" # otlp http + - "9411" # zipkin + networks: + - snakeaid-net + + prometheus: + image: prom/prometheus:latest + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-lifecycle + volumes: + - ./observability/prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + networks: + - snakeaid-net + + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" + volumes: + - ./observability/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + networks: + - snakeaid-net + + otel-collector: + image: otel/opentelemetry-collector:latest + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./observability/otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "1888:1888" # pprof extension + - "8888:8888" # Prometheus metrics exposed by the collector + - "8889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP http receiver + - "55679:55679" # zpages extension + depends_on: + - loki + - tempo + - prometheus + networks: + - snakeaid-net + +networks: + snakeaid-net: + driver: bridge \ No newline at end of file diff --git a/observability/grafana-datasources.yaml b/observability/grafana-datasources.yaml new file mode 100644 index 00000000..5c47aebb --- /dev/null +++ b/observability/grafana-datasources.yaml @@ -0,0 +1,37 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: false + jsonData: + graphiteVersion: '1.1' + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true + + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogs: + datasourceUid: 'Loki' + tags: ['job', 'instance', 'pod', 'namespace'] + mappedTags: [{ key: 'service.name', value: 'app' }] + mapTagNamesEnabled: false + spanStartTimeShift: '0' + spanEndTimeShift: '0' + filterByTraceID: true + filterBySpanID: true + tracesToMetrics: + datasourceUid: 'Prometheus' + tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }] + queries: + - name: 'Sample query' + query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))' diff --git a/observability/loki-config.yaml b/observability/loki-config.yaml new file mode 100644 index 00000000..290e7d4d --- /dev/null +++ b/observability/loki-config.yaml @@ -0,0 +1,31 @@ +auth_enabled: false + +limits_config: + allow_structured_metadata: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 diff --git a/observability/otel-collector-config.yaml b/observability/otel-collector-config.yaml new file mode 100644 index 00000000..9ad27472 --- /dev/null +++ b/observability/otel-collector-config.yaml @@ -0,0 +1,28 @@ +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + +exporters: + otlp: + endpoint: tempo:4317 + tls: + insecure: true + prometheus: + endpoint: "0.0.0.0:8889" + namespace: "snakeaid_api" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] # To Tempo (via OTLP) + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] # Expose via Prometheus scrape endpoint diff --git a/observability/prometheus.yaml b/observability/prometheus.yaml new file mode 100644 index 00000000..36300dc8 --- /dev/null +++ b/observability/prometheus.yaml @@ -0,0 +1,8 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['otel-collector:8889'] diff --git a/observability/tempo-config.yaml b/observability/tempo-config.yaml new file mode 100644 index 00000000..8219e237 --- /dev/null +++ b/observability/tempo-config.yaml @@ -0,0 +1,25 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + http: + +ingester: + trace_idle_period: 10s + max_block_bytes: 1_000_000 + max_block_duration: 5m + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +overrides: + metrics_generator_processors: [service-graphs, span-metrics]