diff --git a/elisa-logbook/elisa-logbook-deployment.yaml b/elisa-logbook/elisa-logbook-deployment.yaml index 9dfacc5f..29b813e0 100644 --- a/elisa-logbook/elisa-logbook-deployment.yaml +++ b/elisa-logbook/elisa-logbook-deployment.yaml @@ -71,20 +71,29 @@ spec: name: http protocol: TCP startupProbe: - tcpSocket: + httpGet: + path: /ready port: http initialDelaySeconds: 15 periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 livenessProbe: - tcpSocket: + httpGet: + path: /live port: http - initialDelaySeconds: 15 - periodSeconds: 20 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 readinessProbe: - tcpSocket: + httpGet: + path: /ready port: http - initialDelaySeconds: 15 - periodSeconds: 20 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 resources: limits: memory: 1Gi diff --git a/elisa-logbook/logbook.py b/elisa-logbook/logbook.py index 419cb021..5cdea4c5 100644 --- a/elisa-logbook/logbook.py +++ b/elisa-logbook/logbook.py @@ -83,6 +83,31 @@ def sanitize_run_type(run_type: str) -> str: # The first type of logging is fileLogbook, which writes logs to a given file in the current working directory. +@app.route("/ready") +def ready(): + status = {"file_system": "healthy"} + all_healthy = True + + try: + if not Path(app.config["PATH"]).exists(): + status["file_system"] = "unreachable" + all_healthy = False + + except Exception: + status["file_system"] = "error" + all_healthy = False + + if all_healthy: + return jsonify({"status": "ready", **status}), 200 + else: + return jsonify({"status": "not ready", **status}), 503 + + +@app.route("/live") +def live(): + return jsonify({"status": "live"}), 200 + + @app.route("/") def index(): return "

Welcome to the logbook API!

" diff --git a/ers-protobuf-dbwriter/dbwriter.py b/ers-protobuf-dbwriter/dbwriter.py index f6e02108..8e546fbf 100644 --- a/ers-protobuf-dbwriter/dbwriter.py +++ b/ers-protobuf-dbwriter/dbwriter.py @@ -4,6 +4,7 @@ # received with this code. # +import json import logging import re import sys @@ -13,6 +14,7 @@ import click import erskafka.ERSSubscriber as erssub import google.protobuf.json_format as pb_json +import sqlalchemy from sqlalchemy import ( BigInteger, Column, @@ -25,9 +27,76 @@ ) from sqlalchemy.exc import OperationalError, ProgrammingError, SQLAlchemyError +import socket as _socket +from threading import Thread + CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) MAX_RETRIES = 3 logger = logging.getLogger(__name__) +engine = None + +_health_server_socket = None + + +def _handle_health_client(conn): + try: + data = b"" + conn.settimeout(5) + while b"\r\n\r\n" not in data: + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + + if b"GET /ready" in data: + status = {"database": "healthy"} + all_healthy = True + try: + if engine is not None: + with engine.connect() as c: + c.execute(sqlalchemy.text("SELECT 1")) + except Exception: + status["database"] = "unreachable" + all_healthy = False + code, phrase = (200, "OK") if all_healthy else (503, "Service Unavailable") + body = json.dumps({"status": "ready" if all_healthy else "not ready", **status}).encode() + elif b"GET /live" in data: + code, phrase = 200, "OK" + body = json.dumps({"status": "live"}).encode() + else: + code, phrase = 404, "Not Found" + body = b"Not Found" + + response = ( + f"HTTP/1.0 {code} {phrase}\r\nContent-Type: application/json\r\nContent-Length: {len(body)}\r\n\r\n" + ).encode() + body + conn.sendall(response) + except Exception: + pass + finally: + try: + conn.close() + except Exception: + pass + + +def _health_accept_loop(sock): + while True: + try: + conn, _ = sock.accept() + Thread(target=_handle_health_client, args=(conn,), daemon=True).start() + except OSError: + break + + +def start_health_server(port: int): + global _health_server_socket + _health_server_socket = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + _health_server_socket.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) + _health_server_socket.bind(("0.0.0.0", port)) + _health_server_socket.listen(10) + Thread(target=_health_accept_loop, args=(_health_server_socket,), daemon=True).start() + logger.info("Health server started on port %d", port) @click.command(context_settings=CONTEXT_SETTINGS) @@ -62,6 +131,12 @@ help="name of table used in the database", ) @click.option("--debug", type=click.BOOL, default=True, help="Set debug print levels") +@click.option( + "--health-port", + type=click.INT, + default=None, + help="Port for HTTP health endpoint (if not set, no health endpoint)", +) def cli( subscriber_bootstrap, subscriber_group, @@ -69,6 +144,7 @@ def cli( db_uri, db_table, debug, + health_port, ): logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", @@ -82,6 +158,7 @@ def cli( metadata = MetaData() try: + global engine engine = create_engine( db_uri, pool_size=5, @@ -97,6 +174,9 @@ def cli( check_tables(engine=engine) + if health_port is not None: + start_health_server(health_port) + subscriber_conf = {} subscriber_conf["bootstrap"] = subscriber_bootstrap subscriber_conf["timeout"] = subscriber_timeout diff --git a/ers-protobuf-dbwriter/entrypoint.sh b/ers-protobuf-dbwriter/entrypoint.sh index db3bd3fd..67560775 100755 --- a/ers-protobuf-dbwriter/entrypoint.sh +++ b/ers-protobuf-dbwriter/entrypoint.sh @@ -8,11 +8,12 @@ if [[ ! -f ../entrypoint_functions.sh ]]; then fi source ../entrypoint_functions.sh -ensure_required_variables "ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER ERS_DBWRITER_KAFKA_TIMEOUT_MS ERS_DBWRITER_KAFKA_GROUP DATABASE_URI ERS_DBWRITER_DB_TABLENAME" +ensure_required_variables "ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER ERS_DBWRITER_KAFKA_TIMEOUT_MS ERS_DBWRITER_KAFKA_GROUP DATABASE_URI ERS_DBWRITER_DB_TABLENAME HEALTH_PORT" exec python3 ./dbwriter.py --subscriber-bootstrap "${ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER}" \ --subscriber-group "${ERS_DBWRITER_KAFKA_GROUP}" \ --subscriber-timeout "${ERS_DBWRITER_KAFKA_TIMEOUT_MS}" \ --db-uri "${DATABASE_URI}" \ --db-table "${ERS_DBWRITER_DB_TABLENAME}" \ + --health-port "${HEALTH_PORT}" \ --debug False diff --git a/ers-protobuf-dbwriter/ers-dbwriter-deployment.yaml b/ers-protobuf-dbwriter/ers-dbwriter-deployment.yaml index 1bf40ed3..e28e6e4a 100644 --- a/ers-protobuf-dbwriter/ers-dbwriter-deployment.yaml +++ b/ers-protobuf-dbwriter/ers-dbwriter-deployment.yaml @@ -50,6 +50,10 @@ spec: - image: ghcr.io/dune-daq/microservices:9685 imagePullPolicy: Always name: ers-protobuf-dbwriter + ports: + - containerPort: 8080 + protocol: TCP + name: health env: - name: MICROSERVICE value: ers-protobuf-dbwriter @@ -66,6 +70,8 @@ spec: secretKeyRef: key: uri name: ers-postgresql-svcbind-custom-user + - name: HEALTH_PORT + value: "8080" resources: limits: memory: 1Gi @@ -86,6 +92,30 @@ spec: volumeMounts: - name: tmp-volume mountPath: /tmp + startupProbe: + httpGet: + path: /ready + port: health + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /live + port: health + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: health + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 volumes: - name: tmp-volume emptyDir: diff --git a/opmon-protobuf-dbwriter/dbwriter.py b/opmon-protobuf-dbwriter/dbwriter.py index 2ffe6d61..e9171054 100644 --- a/opmon-protobuf-dbwriter/dbwriter.py +++ b/opmon-protobuf-dbwriter/dbwriter.py @@ -4,6 +4,7 @@ # received with this code. # +import json import logging import queue import threading @@ -16,8 +17,74 @@ import opmonlib.opmon_entry_pb2 as opmon_schema from influxdb import InfluxDBClient +import socket as _socket +from threading import Thread + CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) logger = logging.getLogger(__name__) +influx = None + +_health_server_socket = None + + +def _handle_health_client(conn): + try: + data = b"" + conn.settimeout(5) + while b"\r\n\r\n" not in data: + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + + if b"GET /ready" in data: + status = {"influxdb": "healthy"} + all_healthy = True + try: + if influx is not None: + influx.ping() + except Exception: + status["influxdb"] = "unreachable" + all_healthy = False + code, phrase = (200, "OK") if all_healthy else (503, "Service Unavailable") + body = json.dumps({"status": "ready" if all_healthy else "not ready", **status}).encode() + elif b"GET /live" in data: + code, phrase = 200, "OK" + body = json.dumps({"status": "live"}).encode() + else: + code, phrase = 404, "Not Found" + body = b"Not Found" + + response = ( + f"HTTP/1.0 {code} {phrase}\r\nContent-Type: application/json\r\nContent-Length: {len(body)}\r\n\r\n" + ).encode() + body + conn.sendall(response) + except Exception: + pass + finally: + try: + conn.close() + except Exception: + pass + + +def _health_accept_loop(sock): + while True: + try: + conn, _ = sock.accept() + Thread(target=_handle_health_client, args=(conn,), daemon=True).start() + except OSError: + break + + +def start_health_server(port: int): + global _health_server_socket + _health_server_socket = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + _health_server_socket.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) + _health_server_socket.bind(("0.0.0.0", port)) + _health_server_socket.listen(10) + Thread(target=_health_accept_loop, args=(_health_server_socket,), daemon=True).start() + logger.info("Health server started on port %d", port) @click.command(context_settings=CONTEXT_SETTINGS) @@ -67,6 +134,12 @@ help="Size in ms of the batches sent to influx", ) @click.option("--debug", type=click.BOOL, default=True, help="Set debug print levels") +@click.option( + "--health-port", + type=click.INT, + default=None, + help="Port for HTTP health endpoint (if not set, no health endpoint)", +) def cli( subscriber_bootstrap, subscriber_group, @@ -76,6 +149,7 @@ def cli( influxdb_create, influxdb_timeout, debug, + health_port, ): logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", @@ -85,6 +159,7 @@ def cli( # Create InfluxDB client using from_dsn for URI-based connection # The from_dsn method extracts database name from the URI path + global influx influx = InfluxDBClient.from_dsn(influxdb_uri) # Extract database name from URI using urlparse @@ -115,6 +190,9 @@ def cli( callback_function = partial(process_entry, q=q) + if health_port is not None: + start_health_server(health_port) + sub.add_callback(name="to_influx", function=callback_function) thread = threading.Thread( diff --git a/opmon-protobuf-dbwriter/entrypoint.sh b/opmon-protobuf-dbwriter/entrypoint.sh index b0875c0c..c3e69293 100755 --- a/opmon-protobuf-dbwriter/entrypoint.sh +++ b/opmon-protobuf-dbwriter/entrypoint.sh @@ -8,7 +8,7 @@ if [[ ! -f ../entrypoint_functions.sh ]]; then fi source ../entrypoint_functions.sh -ensure_required_variables "OPMON_DBWRITER_KAFKA_BOOTSTRAP_SERVER OPMON_DBWRITER_KAFKA_GROUP OPMON_DBWRITER_SUBSCRIBER_TIMEOUT_MS OPMON_DBWRITER_TOPIC DATABASE_URI OPMON_DBWRITER_BATCH_SIZE_MS" +ensure_required_variables "OPMON_DBWRITER_KAFKA_BOOTSTRAP_SERVER OPMON_DBWRITER_KAFKA_GROUP OPMON_DBWRITER_SUBSCRIBER_TIMEOUT_MS OPMON_DBWRITER_TOPIC DATABASE_URI OPMON_DBWRITER_BATCH_SIZE_MS HEALTH_PORT" exec python3 ./dbwriter.py --subscriber-bootstrap "${OPMON_DBWRITER_KAFKA_BOOTSTRAP_SERVER}" \ --subscriber-group "${OPMON_DBWRITER_KAFKA_GROUP}" \ @@ -17,4 +17,5 @@ exec python3 ./dbwriter.py --subscriber-bootstrap "${OPMON_DBWRITER_KAFKA_BOOTST --influxdb-uri "${DATABASE_URI}" \ --influxdb-timeout "${OPMON_DBWRITER_BATCH_SIZE_MS}" \ --influxdb-create True \ + --health-port "${HEALTH_PORT}" \ --debug False diff --git a/opmon-protobuf-dbwriter/opmon-dbwriter-deployment.yaml b/opmon-protobuf-dbwriter/opmon-dbwriter-deployment.yaml index c09c4431..13fd2ba1 100644 --- a/opmon-protobuf-dbwriter/opmon-dbwriter-deployment.yaml +++ b/opmon-protobuf-dbwriter/opmon-dbwriter-deployment.yaml @@ -48,6 +48,10 @@ spec: - image: ghcr.io/dune-daq/microservices:mroda-opmon_protobuf ## TBC imagePullPolicy: Always name: opmon-protobuf-dbwriter + ports: + - containerPort: 8080 + protocol: TCP + name: health env: - name: MICROSERVICE value: opmon-protobuf-dbwriter @@ -66,6 +70,8 @@ spec: secretKeyRef: key: uri name: influxdbv1-readwrite-password + - name: HEALTH_PORT + value: "8080" resources: limits: memory: 1Gi @@ -86,6 +92,30 @@ spec: volumeMounts: - name: tmp-volume mountPath: /tmp + startupProbe: + httpGet: + path: /ready + port: health + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /live + port: health + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: health + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 volumes: - name: tmp-volume emptyDir: diff --git a/runnumber-rest/api.py b/runnumber-rest/api.py index dd976bcc..b70eed90 100644 --- a/runnumber-rest/api.py +++ b/runnumber-rest/api.py @@ -119,6 +119,28 @@ def get(self, runNum): return flask.make_response(flask.jsonify({"Exception": f"{err_obj}"})) +@app.route("/ready") +def ready(): + status = {"database": "healthy"} + all_healthy = True + + try: + db.session.execute(db.select(1)) + except Exception: + status["database"] = "unreachable" + all_healthy = False + + if all_healthy: + return flask.jsonify({"status": "ready", **status}), 200 + else: + return flask.jsonify({"status": "not ready", **status}), 503 + + +@app.route("/live") +def live(): + return flask.jsonify({"status": "live"}), 200 + + @app.route("/") def index(): return f""" diff --git a/runnumber-rest/runnumber-rest-deployment.yaml b/runnumber-rest/runnumber-rest-deployment.yaml index 15d3b262..fd66650d 100644 --- a/runnumber-rest/runnumber-rest-deployment.yaml +++ b/runnumber-rest/runnumber-rest-deployment.yaml @@ -62,20 +62,29 @@ spec: protocol: TCP name: http startupProbe: - tcpSocket: + httpGet: + path: /ready port: http initialDelaySeconds: 15 periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 livenessProbe: - tcpSocket: + httpGet: + path: /live port: http - initialDelaySeconds: 15 - periodSeconds: 20 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 readinessProbe: - tcpSocket: + httpGet: + path: /ready port: http - initialDelaySeconds: 15 - periodSeconds: 20 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 resources: limits: memory: 1Gi diff --git a/runregistry-rest/api.py b/runregistry-rest/api.py index 87cdef91..2f2ca112 100644 --- a/runregistry-rest/api.py +++ b/runregistry-rest/api.py @@ -305,6 +305,28 @@ def get(self, runNum): return flask.make_response(flask.jsonify({"Exception": f"{err_obj}"}), 500) +@app.route("/ready") +def ready(): + status = {"database": "healthy"} + all_healthy = True + + try: + db.session.execute(db.select(1)) + except Exception: + status["database"] = "unreachable" + all_healthy = False + + if all_healthy: + return flask.jsonify({"status": "ready", **status}), 200 + else: + return flask.jsonify({"status": "not ready", **status}), 503 + + +@app.route("/live") +def live(): + return flask.jsonify({"status": "live"}), 200 + + @app.route("/") def index(): return f""" diff --git a/runregistry-rest/runregistry-rest-deployment.yaml b/runregistry-rest/runregistry-rest-deployment.yaml index a6dea226..ecfa5053 100644 --- a/runregistry-rest/runregistry-rest-deployment.yaml +++ b/runregistry-rest/runregistry-rest-deployment.yaml @@ -62,20 +62,29 @@ spec: protocol: TCP name: http startupProbe: - tcpSocket: + httpGet: + path: /ready port: http initialDelaySeconds: 15 periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 livenessProbe: - tcpSocket: + httpGet: + path: /live port: http - initialDelaySeconds: 15 - periodSeconds: 20 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 readinessProbe: - tcpSocket: + httpGet: + path: /ready port: http - initialDelaySeconds: 15 - periodSeconds: 20 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 resources: limits: memory: 1Gi