Skip to content

Commit 638657b

Browse files
akasheakashe
committed
Added prometheus and graphana monitoring
1 parent 8200e81 commit 638657b

13 files changed

Lines changed: 498 additions & 9 deletions

File tree

‎infrastructure/kubernetes/base/grafana.yaml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: grafana
5+
labels:
6+
app: grafana
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: grafana
12+
template:
13+
metadata:
14+
labels:
15+
app: grafana
16+
spec:
17+
containers:
18+
- name: grafana
19+
image: grafana/grafana:latest
20+
ports:
21+
- containerPort: 3000
22+
env:
23+
- name: GF_SECURITY_ADMIN_PASSWORD
24+
value: "admin123"
25+
volumeMounts:
26+
- name: grafana-datasource
27+
mountPath: /etc/grafana/provisioning/datasources
28+
- name: grafana-dashboards
29+
mountPath: /etc/grafana/provisioning/dashboards
30+
- name: graphrag-dashboard
31+
mountPath: /var/lib/grafana/dashboards
32+
volumes:
33+
- name: grafana-datasource
34+
configMap:
35+
name: grafana-datasource
36+
- name: grafana-dashboards
37+
configMap:
38+
name: grafana-dashboards
39+
- name: graphrag-dashboard
40+
configMap:
41+
name: graphrag-dashboard
42+
---
43+
apiVersion: v1
44+
kind: Service
45+
metadata:
46+
name: grafana
47+
spec:
48+
selector:
49+
app: grafana
50+
ports:
51+
- port: 3000
52+
targetPort: 3000
53+
type: LoadBalancer
54+
---
55+
apiVersion: v1
56+
kind: ConfigMap
57+
metadata:
58+
name: grafana-datasource
59+
data:
60+
datasource.yaml: |
61+
apiVersion: 1
62+
datasources:
63+
- name: Prometheus
64+
type: prometheus
65+
access: proxy
66+
url: http://prometheus:9090
67+
isDefault: true
68+
---
69+
apiVersion: v1
70+
kind: ConfigMap
71+
metadata:
72+
name: grafana-dashboards
73+
data:
74+
dashboard.yaml: |
75+
apiVersion: 1
76+
providers:
77+
- name: 'default'
78+
orgId: 1
79+
folder: ''
80+
type: file
81+
disableDeletion: false
82+
editable: true
83+
options:
84+
path: /var/lib/grafana/dashboards
85+
---
86+
apiVersion: v1
87+
kind: ConfigMap
88+
metadata:
89+
name: graphrag-dashboard
90+
data:
91+
graphrag-dashboard.json: |
92+
{
93+
"id": null,
94+
"title": "GraphRAG System Metrics - Auto Generated",
95+
"tags": ["graphrag", "microservices"],
96+
"timezone": "browser",
97+
"panels": [
98+
{
99+
"id": 1,
100+
"title": "Service Health",
101+
"type": "stat",
102+
"targets": [{"expr": "up", "legendFormat": "{{job}}"}],
103+
"gridPos": {"h": 6, "w": 8, "x": 0, "y": 0}
104+
},
105+
{
106+
"id": 2,
107+
"title": "Neo4j Database Stats",
108+
"type": "stat",
109+
"targets": [
110+
{"expr": "neo4j_service_nodes_loaded_total", "legendFormat": "Nodes"},
111+
{"expr": "neo4j_service_relationships_loaded_total", "legendFormat": "Relationships"}
112+
],
113+
"gridPos": {"h": 6, "w": 8, "x": 8, "y": 0}
114+
},
115+
{
116+
"id": 3,
117+
"title": "LLM Costs (USD)",
118+
"type": "stat",
119+
"targets": [{"expr": "generation_service_llm_cost_usd_total", "legendFormat": "Total Cost"}],
120+
"gridPos": {"h": 6, "w": 8, "x": 16, "y": 0}
121+
},
122+
{
123+
"id": 4,
124+
"title": "API Request Rate",
125+
"type": "timeseries",
126+
"targets": [
127+
{"expr": "rate(query_service_requests_total[5m])", "legendFormat": "Query - {{endpoint}}"},
128+
{"expr": "rate(retrieval_service_requests_total[5m])", "legendFormat": "Retrieval - {{endpoint}}"}
129+
],
130+
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 6}
131+
}
132+
],
133+
"time": {"from": "now-1h", "to": "now"},
134+
"refresh": "10s"
135+
}

‎infrastructure/kubernetes/base/graphrag-dashboard.json‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
{
2+
"dashboard": {
3+
"id": null,
4+
"title": "GraphRAG System Metrics",
5+
"tags": ["graphrag", "microservices"],
6+
"timezone": "browser",
7+
"panels": [
8+
{
9+
"id": 1,
10+
"title": "Service Health (Up/Down)",
11+
"type": "stat",
12+
"targets": [
13+
{
14+
"expr": "up",
15+
"legendFormat": "{{job}}"
16+
}
17+
],
18+
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
19+
"fieldConfig": {
20+
"defaults": {
21+
"mappings": [
22+
{"options": {"0": {"text": "DOWN"}}, "type": "value"},
23+
{"options": {"1": {"text": "UP"}}, "type": "value"}
24+
],
25+
"thresholds": {
26+
"steps": [
27+
{"color": "red", "value": 0},
28+
{"color": "green", "value": 1}
29+
]
30+
}
31+
}
32+
}
33+
},
34+
{
35+
"id": 2,
36+
"title": "API Request Rate (req/sec)",
37+
"type": "timeseries",
38+
"targets": [
39+
{
40+
"expr": "rate(query_service_requests_total[5m])",
41+
"legendFormat": "Query Service - {{endpoint}}"
42+
},
43+
{
44+
"expr": "rate(retrieval_service_requests_total[5m])",
45+
"legendFormat": "Retrieval Service - {{endpoint}}"
46+
}
47+
],
48+
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
49+
},
50+
{
51+
"id": 3,
52+
"title": "Neo4j Database Stats",
53+
"type": "stat",
54+
"targets": [
55+
{
56+
"expr": "neo4j_service_nodes_loaded_total",
57+
"legendFormat": "Nodes Loaded"
58+
},
59+
{
60+
"expr": "neo4j_service_relationships_loaded_total",
61+
"legendFormat": "Relationships Loaded"
62+
}
63+
],
64+
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}
65+
},
66+
{
67+
"id": 4,
68+
"title": "LLM Token Usage",
69+
"type": "timeseries",
70+
"targets": [
71+
{
72+
"expr": "rate(generation_service_llm_tokens_total[5m])",
73+
"legendFormat": "{{type}} tokens/sec"
74+
}
75+
],
76+
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 8}
77+
},
78+
{
79+
"id": 5,
80+
"title": "LLM Costs (USD)",
81+
"type": "stat",
82+
"targets": [
83+
{
84+
"expr": "generation_service_llm_cost_usd_total",
85+
"legendFormat": "Total Cost"
86+
}
87+
],
88+
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 8},
89+
"fieldConfig": {
90+
"defaults": {
91+
"unit": "currencyUSD"
92+
}
93+
}
94+
},
95+
{
96+
"id": 6,
97+
"title": "Papers Retrieved by Topic",
98+
"type": "timeseries",
99+
"targets": [
100+
{
101+
"expr": "rate(retrieval_service_papers_retrieved_total[5m])",
102+
"legendFormat": "{{topic}} - {{operation}}"
103+
}
104+
],
105+
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}
106+
},
107+
{
108+
"id": 7,
109+
"title": "Neo4j Query Performance",
110+
"type": "timeseries",
111+
"targets": [
112+
{
113+
"expr": "retrieval_service_neo4j_query_duration_seconds",
114+
"legendFormat": "{{query_type}}"
115+
}
116+
],
117+
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}
118+
},
119+
{
120+
"id": 8,
121+
"title": "Data Loading Progress",
122+
"type": "gauge",
123+
"targets": [
124+
{
125+
"expr": "neo4j_service_data_loading_progress_percent",
126+
"legendFormat": "Loading Progress"
127+
}
128+
],
129+
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 24},
130+
"fieldConfig": {
131+
"defaults": {
132+
"max": 100,
133+
"min": 0,
134+
"unit": "percent"
135+
}
136+
}
137+
},
138+
{
139+
"id": 9,
140+
"title": "Generation Service Response Times",
141+
"type": "timeseries",
142+
"targets": [
143+
{
144+
"expr": "generation_service_request_duration_seconds",
145+
"legendFormat": "Response Time"
146+
}
147+
],
148+
"gridPos": {"h": 8, "w": 18, "x": 6, "y": 24}
149+
}
150+
],
151+
"time": {
152+
"from": "now-1h",
153+
"to": "now"
154+
},
155+
"refresh": "10s"
156+
}
157+
}

‎infrastructure/kubernetes/base/kustomization.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,6 @@ resources:
1313
- neo4j-database/service.yaml
1414
- neo4j-service/deployment.yaml
1515
- neo4j-service/service.yaml
16-
- neo4j-service/data-loader-job.yaml
16+
- neo4j-service/data-loader-job.yaml
17+
- prometheus.yaml
18+
- grafana.yaml

‎infrastructure/kubernetes/base/prometheus.yaml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: prometheus
5+
labels:
6+
app: prometheus
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: prometheus
12+
template:
13+
metadata:
14+
labels:
15+
app: prometheus
16+
spec:
17+
containers:
18+
- name: prometheus
19+
image: prom/prometheus:latest
20+
ports:
21+
- containerPort: 9090
22+
volumeMounts:
23+
- name: prometheus-config
24+
mountPath: /etc/prometheus/prometheus.yml
25+
subPath: prometheus.yml
26+
args:
27+
- '--config.file=/etc/prometheus/prometheus.yml'
28+
- '--storage.tsdb.path=/prometheus/'
29+
- '--web.console.libraries=/etc/prometheus/console_libraries'
30+
- '--web.console.templates=/etc/prometheus/consoles'
31+
- '--web.enable-lifecycle'
32+
volumes:
33+
- name: prometheus-config
34+
configMap:
35+
name: prometheus-config
36+
---
37+
apiVersion: v1
38+
kind: Service
39+
metadata:
40+
name: prometheus
41+
spec:
42+
selector:
43+
app: prometheus
44+
ports:
45+
- port: 9090
46+
targetPort: 9090
47+
type: LoadBalancer
48+
---
49+
apiVersion: v1
50+
kind: ConfigMap
51+
metadata:
52+
name: prometheus-config
53+
data:
54+
prometheus.yml: |
55+
global:
56+
scrape_interval: 15s
57+
scrape_configs:
58+
- job_name: 'kubernetes-services'
59+
kubernetes_sd_configs:
60+
- role: endpoints
61+
relabel_configs:
62+
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
63+
action: keep
64+
regex: true
65+
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
66+
action: replace
67+
target_label: __metrics_path__
68+
regex: (.+)
69+
- job_name: 'query-service'
70+
static_configs:
71+
- targets: ['query-service:8000']
72+
- job_name: 'retrieval-service'
73+
static_configs:
74+
- targets: ['retrieval-service:8001']
75+
- job_name: 'generation-service'
76+
static_configs:
77+
- targets: ['generation-service:8002']
78+
- job_name: 'neo4j-service'
79+
static_configs:
80+
- targets: ['neo4j-service:8003']

0 commit comments

Comments
 (0)