diff --git a/Dockerfile b/Dockerfile index d24b75c..2ef046b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,26 @@ -FROM openjdk:21 -COPY target/sqlite-spring-boot.jar /opt/assets/sqlite-spring-boot.jar +# Use a reliable base image that avoids root permission issues +FROM maven:3.9.5-eclipse-temurin-21 AS builder + +# Set working directory +WORKDIR /opt + +# Copy project files +COPY pom.xml . +COPY src ./src + +# Run build +RUN mvn clean package + +# Final image: just the JAR +FROM openjdk:21 + +COPY --from=builder /opt/target/*.jar /opt/assets/sqlite-spring-boot.jar WORKDIR /opt/assets + ENV JAVA_OPTS_GC="-server -XX:+AlwaysPreTouch -XX:+UseG1GC -XX:+ScavengeBeforeFullGC -XX:+DisableExplicitGC" ENV JAVA_OPTS="-Xms256m -Xmx256m" ENV HOSTNAME=client -EXPOSE 50000 -EXPOSE 50001 -EXPOSE 29000 -EXPOSE 10800 11211 47100 47500 49112 8080 + +EXPOSE 50000 50001 29000 10800 11211 47100 47500 49112 8080 + CMD java $JAVA_OPTS_GC $JAVA_OPTS -Dhostname=$HOSTNAME -jar sqlite-spring-boot.jar diff --git a/README.md b/README.md index bb7720b..0964756 100644 --- a/README.md +++ b/README.md @@ -17,3 +17,13 @@ ## More Details Μore details along with evaluation experiments can be found in the MSc dissertation (file: StorageFabricForEdgeDevices.pdf). + + +## Execution updated +1) docker rmi -f edge-sqlite-db:1.0.0 +2) docker compose -f docker-kafka-compose.yml up +3) docker build -f Dockerfile . -t edge-sqlite-db:1.0.0 +4) docker compose -f docker-compose-node1.yml up + +## Kafka Url +http://localhost:9007 diff --git a/Storage fabric Demo.postman_collection.json b/Storage fabric Demo.postman_collection.json new file mode 100644 index 0000000..c690bbb --- /dev/null +++ b/Storage fabric Demo.postman_collection.json @@ -0,0 +1,456 @@ +{ + "info": { + "_postman_id": "784159dd-a37e-4042-9a94-18d4942b1e1a", + "name": "Storage fabric Demo", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "8982722" + }, + "item": [ + { + "name": "node 1", + "item": [ + { + "name": "PUT metric1 MEMORY_USAGE", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"entityType\": \"MEMORY_USAGE\",\n \"unit\": \"PLAIN\",\n \"minVal\": 10,\n \"maxVal\": 10000,\n \"higherIsBetter\": false,\n \"val\": 1000,\n \"timestamp\": 20240305142500,\n \"container\": {\n \"name\": \"node 1\",\n \"label\": \"node one memory\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29000/metrics/add", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "add" + ] + } + }, + "response": [] + }, + { + "name": "PUT metric2 MEMORY_USAGE", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"entityType\": \"MEMORY_USAGE\",\n \"unit\": \"PLAIN\",\n \"minVal\": 10,\n \"maxVal\": 10000,\n \"higherIsBetter\": false,\n \"val\": 1000,\n \"timestamp\": 20240305142500,\n \"container\": {\n \"name\": \"node 1\",\n \"label\": \"node one memory\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29000/metrics/add", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "add" + ] + } + }, + "response": [] + }, + { + "name": "PUT metric3 CPU_USAGE", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"entityType\": \"CPU_USAGE\",\n \"unit\": \"PERCENTAGE\",\n \"minVal\": 0,\n \"maxVal\": 100,\n \"higherIsBetter\": false,\n \"val\": 10,\n \"timestamp\": 20240305142500,\n \"container\": {\n \"name\": \"node 1\",\n \"label\": \"node one memory\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29000/metrics/add", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "add" + ] + } + }, + "response": [] + }, + { + "name": "DELETE metric", + "request": { + "method": "DELETE", + "header": [], + "body": { + "mode": "raw", + "raw": " { \n \"entityType\": \"MEMORY_USAGE\",\n \"higherIsBetter\": true,\n \"val\": 36,\n \"timestamp\": 20240219142500,\n \"container\": {\n \"name\": \"tdsf\",\n \"label\": \"first container\"\n }\n }", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29000/metrics/delete?id=3", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "delete" + ], + "query": [ + { + "key": "id", + "value": "3" + } + ] + } + }, + "response": [] + }, + { + "name": "GET metrics", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST}}:29000/metrics/get?type=CPU_USAGE&node=client", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "get" + ], + "query": [ + { + "key": "type", + "value": "CPU_USAGE" + }, + { + "key": "node", + "value": "client" + } + ] + } + }, + "response": [] + }, + { + "name": "GET all metrics", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST}}:29000/metrics/getByEntityType?type=MEMORY_USAGE", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "getByEntityType" + ], + "query": [ + { + "key": "type", + "value": "MEMORY_USAGE" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "node 2", + "item": [ + { + "name": "PUT metric1 MEMORY_USAGE", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"entityType\": \"MEMORY_USAGE\",\n \"unit\": \"PLAIN\",\n \"minVal\": 10,\n \"maxVal\": 10000,\n \"higherIsBetter\": false,\n \"val\": 1400,\n \"timestamp\": 20240305142600,\n \"container\": {\n \"name\": \"node 2\",\n \"label\": \"node two memory\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29001/metrics/add", + "host": [ + "{{HOST}}" + ], + "port": "29001", + "path": [ + "metrics", + "add" + ] + } + }, + "response": [] + }, + { + "name": "DELETE metric", + "request": { + "method": "DELETE", + "header": [], + "body": { + "mode": "raw", + "raw": " { \n \"entityType\": \"MEMORY_USAGE\",\n \"higherIsBetter\": true,\n \"val\": 36,\n \"timestamp\": 20240219142500,\n \"container\": {\n \"name\": \"tdsf\",\n \"label\": \"first container\"\n }\n }", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29000/metrics/delete?id=3", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "delete" + ], + "query": [ + { + "key": "id", + "value": "3" + } + ] + } + }, + "response": [] + }, + { + "name": "GET metrics", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST}}:29000/metrics/get?type=CPU_USAGE&node=client", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "get" + ], + "query": [ + { + "key": "type", + "value": "CPU_USAGE" + }, + { + "key": "node", + "value": "client" + } + ] + } + }, + "response": [] + }, + { + "name": "GET all metrics", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST}}:29000/metrics/getByEntityType?type=MEMORY_USAGE", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "getByEntityType" + ], + "query": [ + { + "key": "type", + "value": "MEMORY_USAGE" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "node 3", + "item": [ + { + "name": "PUT metric1 MEMORY_USAGE", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"entityType\": \"MEMORY_USAGE\",\n \"unit\": \"PLAIN\",\n \"minVal\": 10,\n \"maxVal\": 10000,\n \"higherIsBetter\": false,\n \"val\": 1400,\n \"timestamp\": 20240305142600,\n \"container\": {\n \"name\": \"node 3\",\n \"label\": \"node three memory\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29001/metrics/add", + "host": [ + "{{HOST}}" + ], + "port": "29001", + "path": [ + "metrics", + "add" + ] + } + }, + "response": [] + }, + { + "name": "DELETE metric", + "request": { + "method": "DELETE", + "header": [], + "body": { + "mode": "raw", + "raw": " { \n \"entityType\": \"MEMORY_USAGE\",\n \"higherIsBetter\": true,\n \"val\": 36,\n \"timestamp\": 20240219142500,\n \"container\": {\n \"name\": \"tdsf\",\n \"label\": \"first container\"\n }\n }", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST}}:29000/metrics/delete?id=3", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "delete" + ], + "query": [ + { + "key": "id", + "value": "3" + } + ] + } + }, + "response": [] + }, + { + "name": "GET metrics", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST}}:29000/metrics/get?type=CPU_USAGE&node=client", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "get" + ], + "query": [ + { + "key": "type", + "value": "CPU_USAGE" + }, + { + "key": "node", + "value": "client" + } + ] + } + }, + "response": [] + }, + { + "name": "GET all metrics", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST}}:29000/metrics/getByEntityType?type=MEMORY_USAGE", + "host": [ + "{{HOST}}" + ], + "port": "29000", + "path": [ + "metrics", + "getByEntityType" + ], + "query": [ + { + "key": "type", + "value": "MEMORY_USAGE" + } + ] + } + }, + "response": [] + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ], + "variable": [ + { + "key": "HOST", + "value": "localhost", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/__pycache__/addData.cpython-39.pyc b/__pycache__/addData.cpython-39.pyc new file mode 100644 index 0000000..25d0166 Binary files /dev/null and b/__pycache__/addData.cpython-39.pyc differ diff --git a/addData.py b/addData.py new file mode 100644 index 0000000..7ce9536 --- /dev/null +++ b/addData.py @@ -0,0 +1,116 @@ +import requests +import random +from datetime import datetime, timedelta +import time + + +# Map each node to its metric count +# NODE_METRICS = { +# "localhost:29000": 5, +# "localhost:29001": 5, +# "localhost:29002": 5, +# "localhost:29003": 5, +# "localhost:29004": 5, +# } + +NODE_METRICS = { + "localhost:29000": 5, +} + + +ENTITY_TYPES = ["CPU_USAGE", "MEMORY_USAGE", "GPU_USAGE"] +UNITS = ["PLAIN", "PERCENTAGE"] +HEADERS = {"Content-Type": "application/json"} + +CONTAINERS = 5 + +def generate_containers(n): + return [ + { + "name": f"container-{i + 1}", + "label": "sensor", + "location_x": random.uniform(-180, 180), + "location_y": random.uniform(-90, 90), + "boot_timestamp": (datetime.utcnow() - timedelta(days=random.randint(0, 90))).strftime('%Y%m%d%H%M') + + } + for i in range(n) + ] + + + +def generate_metrics(n, container_ids): + now = datetime.utcnow() + + return [ + { + "val": round(random.uniform(0, 10000), 2), + "timestamp": int((now - timedelta(minutes=random.randint(0, 1000))).strftime('%Y%m%d%H%M%S')), + "entityType": random.choice(ENTITY_TYPES), + "unit": random.choice(UNITS), + "minVal": random.uniform(0, 100), + "maxVal": random.uniform(100, 1000), + "container_id": random.choice(container_ids), + "higherIsBetter": random.choice([True, False]) + } + for _ in range(n) + ] + + + +def post_data(node, endpoint, data): + url = f"http://{node}/{endpoint}/add" + try: + method = "put" if endpoint == "metrics" else "post" + response = getattr(requests, method)(url, json=data, headers=HEADERS) + + if response.status_code not in (200, 201): + print(f"[{node}] Failed POST to /{endpoint}: {response.status_code} {response.text}") + return None + raw = response.text.strip() + if not raw: + print(f"[{node}] Empty response from /{endpoint}/add") + return None + print(f"[{node}] Response from /{endpoint}/add: '{raw}'") + return int(raw) + except Exception as e: + print(f"[{node}] Error posting to /{endpoint}: {e}") + return None + + + + +def populate_node(node, metrics_count): + print(f"\n🔄 Populating {node}...") + + containers = [] + count = 1 + for container in generate_containers(CONTAINERS): + res = post_data(node, "containers", container) + container["id"] = res # Save the actual ID returned by backend + containers.append(count) + count += 1 + + + + print(f"[{node}] Adding metrics...") + for metric in generate_metrics(metrics_count, containers): + res = post_data(node, "metrics", metric) + + print(f"Done with {node}.") + + + + +def main(): + for node, metrics_count in NODE_METRICS.items(): + populate_node(node, metrics_count) + time.sleep(1) # Sleep to avoid overwhelming the server + + print("\nAll nodes populated!") + +if __name__ == "__main__": + main() + + + diff --git a/docker-client-compose.yaml b/docker-client-compose.yaml deleted file mode 100644 index 81622fb..0000000 --- a/docker-client-compose.yaml +++ /dev/null @@ -1,31 +0,0 @@ -version: "3.2" - -services: - sqlite-client: - image: edge-sqlite-db:1.0.0 - hostname: client - networks: - - sqlite - ports: - - 29001:29000 - environment: - DISCOVERY: "server" - HOSTNAME: "client" - CLUSTER_HEAD: "true" #There should be only 1 cluster head that takes care of data placement. Default value is false. - PLACEMENT: "false" #Enable/disable data placement algorithms - JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size - SIZE: "512" #Size for total off-heap storage (1/2 for persistence and 1/2 for in-memory caches) - volumes: - - sqlite_databases:/mnt/databases - -networks: - sqlite: - - -volumes: - sqlite_databases: - driver: local - driver_opts: - type: none - o: bind - device: C:\Users\30695\Desktop\thesis\sqlite diff --git a/docker-compose-node1.yml b/docker-compose-node1.yml new file mode 100644 index 0000000..8bb3d82 --- /dev/null +++ b/docker-compose-node1.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node1: + image: edge-sqlite-db:1.0.0 + hostname: node1 + networks: + - sqlite + ports: + - 29000:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node1" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node10.yml b/docker-compose-node10.yml new file mode 100644 index 0000000..b9baeba --- /dev/null +++ b/docker-compose-node10.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node10: + image: edge-sqlite-db:1.0.0 + hostname: node10 + networks: + - sqlite + ports: + - 29009:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node10" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node2.yml b/docker-compose-node2.yml new file mode 100644 index 0000000..327e86e --- /dev/null +++ b/docker-compose-node2.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node2: + image: edge-sqlite-db:1.0.0 + hostname: node2 + networks: + - sqlite + ports: + - 29001:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node2" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node3.yml b/docker-compose-node3.yml new file mode 100644 index 0000000..4840f89 --- /dev/null +++ b/docker-compose-node3.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node3: + image: edge-sqlite-db:1.0.0 + hostname: node3 + networks: + - sqlite + ports: + - 29002:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node3" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node4.yml b/docker-compose-node4.yml new file mode 100644 index 0000000..de3f90a --- /dev/null +++ b/docker-compose-node4.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node4: + image: edge-sqlite-db:1.0.0 + hostname: node4 + networks: + - sqlite + ports: + - 29003:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node4" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node5.yml b/docker-compose-node5.yml new file mode 100644 index 0000000..e762960 --- /dev/null +++ b/docker-compose-node5.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node5: + image: edge-sqlite-db:1.0.0 + hostname: node5 + networks: + - sqlite + ports: + - 29004:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node5" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node6.yml b/docker-compose-node6.yml new file mode 100644 index 0000000..4bb0e1c --- /dev/null +++ b/docker-compose-node6.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node6: + image: edge-sqlite-db:1.0.0 + hostname: node6 + networks: + - sqlite + ports: + - 29005:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node6" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node7.yml b/docker-compose-node7.yml new file mode 100644 index 0000000..183fb68 --- /dev/null +++ b/docker-compose-node7.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node7: + image: edge-sqlite-db:1.0.0 + hostname: node7 + networks: + - sqlite + ports: + - 29006:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node7" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node8.yml b/docker-compose-node8.yml new file mode 100644 index 0000000..84b52c7 --- /dev/null +++ b/docker-compose-node8.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node8: + image: edge-sqlite-db:1.0.0 + hostname: node8 + networks: + - sqlite + ports: + - 29007:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node8" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose-node9.yml b/docker-compose-node9.yml new file mode 100644 index 0000000..f82fe4c --- /dev/null +++ b/docker-compose-node9.yml @@ -0,0 +1,30 @@ +version: "3.2" + +services: + node9: + image: edge-sqlite-db:1.0.0 + hostname: node9 + networks: + - sqlite + ports: + - 29008:29000 + environment: + DISCOVERY: "node1" + HOSTNAME: "node9" + DHT_DATA_SYNC: 30 + DHT_REMOVAL_SYNC: 60 + JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size + volumes: + - sqlite_databases:/mnt/databases + +networks: + sqlite: + + +volumes: + sqlite_databases: + driver: local + driver_opts: + type: none + o: bind + device: C:\Users\Konstaninos\IdeaProjects\StorageFabricForEdgeDevices\databases diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index efae3b6..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: "3.2" - -services: - sqlite-server: - image: edge-sqlite-db:1.0.0 - hostname: server - networks: - - sqlite - ports: - - 29000:29000 - environment: - DISCOVERY: "server" - HOSTNAME: "server" - CLUSTER_HEAD: "true" #There should be only 1 cluster head that takes care of data placement. Default value is false. - PLACEMENT: "false" #Enable/disable data placement algorithms - JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size - SIZE: "512" #Size for total off-heap storage (1/2 for persistence and 1/2 for in-memory caches) - volumes: - - sqlite_databases:/mnt/databases - -networks: - sqlite: - - -volumes: - sqlite_databases: - driver: local - driver_opts: - type: none - o: bind - device: C:\Users\30695\Desktop\thesis\sqlite diff --git a/docker-kafka-compose.yml b/docker-kafka-compose.yml index bc2ac1b..12a5746 100644 --- a/docker-kafka-compose.yml +++ b/docker-kafka-compose.yml @@ -35,7 +35,7 @@ services: KAFKA_JMX_PORT: 9101 KAFKA_JMX_HOSTNAME: localhost kafdrop: - image: obsidiandynamics/kafdrop + image: obsidiandynamics/kafdrop:4.0.2 restart: "no" networks: - sqlite @@ -43,7 +43,10 @@ services: - "9007:9000" environment: KAFKA_BROKERCONNECT: "kafka:29092" - JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify" + #JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify" + JVM_OPTS: "-Xms32M -Xmx64M" + SERVER_SERVLET_CONTEXTPATH: "/" + depends_on: - "kafka" diff --git a/docker-second-client-compose.yaml b/docker-second-client-compose.yaml deleted file mode 100644 index a70e063..0000000 --- a/docker-second-client-compose.yaml +++ /dev/null @@ -1,31 +0,0 @@ -version: "3.2" - -services: - sqlite-second: - image: edge-sqlite-db:1.0.0 - hostname: second - networks: - - sqlite - ports: - - 29002:29000 - environment: - DISCOVERY: "server" - HOSTNAME: "second" - CLUSTER_HEAD: "true" #There should be only 1 cluster head that takes care of data placement. Default value is false. - PLACEMENT: "false" #Enable/disable data placement algorithms - JAVA_OPTS: "-Xms256m -Xmx256m" #Java heap size - SIZE: "512" #Size for total off-heap storage (1/2 for persistence and 1/2 for in-memory caches) - volumes: - - sqlite_databases:/mnt/databases - -networks: - sqlite: - - -volumes: - sqlite_databases: - driver: local - driver_opts: - type: none - o: bind - device: C:\Users\30695\Desktop\thesis\sqlite diff --git a/graphs/combined-fetch-metrics.png b/graphs/combined-fetch-metrics.png new file mode 100644 index 0000000..428e68f Binary files /dev/null and b/graphs/combined-fetch-metrics.png differ diff --git a/graphs/fetch_metrics_analysis.py b/graphs/fetch_metrics_analysis.py new file mode 100644 index 0000000..ef7199a --- /dev/null +++ b/graphs/fetch_metrics_analysis.py @@ -0,0 +1,50 @@ +import matplotlib.pyplot as plt + + +benchmark_data_by_nodes = { + 2: {"Method": ["fetchAll", "fetchByEntityType"], "Time (ms)": [67, 45], "Size (KB)": [12.8, 4.37]}, + 4: {"Method": ["fetchAll", "fetchByEntityType"], "Time (ms)": [104, 82], "Size (KB)": [12.9, 14.8]}, + 5: {"Method": ["fetchAll", "fetchByEntityType"], "Time (ms)": [133, 94], "Size (KB)": [12.94, 19.2]}, + 7: {"Method": ["fetchAll", "fetchByEntityType"], "Time (ms)": [165, 106], "Size (KB)": [31.3, 23.8]}, + 10: {"Method": ["fetchAll", "fetchByEntityType"], "Time (ms)": [190, 132], "Size (KB)": [52.47, 26.3]}, +} + +# Extract data into separate lists +node_counts = sorted(benchmark_data_by_nodes.keys()) +methods = ["fetchAll", "fetchByEntityType"] + +time_data = {method: [] for method in methods} +size_data = {method: [] for method in methods} + +for node in node_counts: + data = benchmark_data_by_nodes[node] + for i, method in enumerate(data["Method"]): + time_data[method].append(data["Time (ms)"][i]) + size_data[method].append(data["Size (KB)"][i]) + +# Plotting combined time and size graph +fig, ax1 = plt.subplots(figsize=(10, 6)) + +# Plot Time (ms) on primary y-axis +colors = {"fetchAll": 'tab:blue', "fetchByEntityType": 'tab:green'} +for method in methods: + ax1.plot(node_counts, time_data[method], label=f"{method} Time", marker='o', color=colors[method]) +ax1.set_xlabel('Number of Nodes') +ax1.set_ylabel('Time (ms)', color='black') +ax1.tick_params(axis='y') +ax1.legend(loc='upper left') + +# Plot Size (KB) on secondary y-axis +ax2 = ax1.twinx() +colors = {"fetchAll": 'tab:red', "fetchByEntityType": 'tab:orange'} +for method in methods: + ax2.plot(node_counts, size_data[method], label=f"{method} Size", linestyle='--', marker='x', color=colors[method]) +ax2.set_ylabel('Size (KB)', color='black') +ax2.tick_params(axis='y') +ax2.legend(loc='upper right') + +plt.title("Fetch Method Comparison Across Node Counts") +plt.grid(True, linestyle='--', alpha=0.5) +plt.tight_layout() +plt.savefig("combined-fetch-metrics.png") +plt.show() diff --git a/insertData.py b/insertData.py new file mode 100644 index 0000000..245f23d --- /dev/null +++ b/insertData.py @@ -0,0 +1,73 @@ +import random +from datetime import datetime, timedelta + +NUM_CONTAINERS = 2 +NUM_METRICS = 10 + +ENTITY_TYPES = [0,1,2] +UNITS = [0,1] + + +def generate_containers(n): + containers = [] + for i in range(n): + name = f"container-{i + 1}" + label = "sensor" + location_x = round(random.uniform(-180, 180), 6) + location_y = round(random.uniform(-90, 90), 6) + boot_timestamp = (datetime.utcnow() - timedelta(days=random.randint(0, 365))).strftime('%Y%m%d%H%M') + containers.append((name, label, location_x, location_y, boot_timestamp)) + return containers + + +def generate_metrics(n, container_ids): + metrics = [] + now = datetime.utcnow() + for _ in range(n): + val = round(random.uniform(0, 10000), 2) + timestamp = int((now - timedelta(minutes=random.randint(0, 1000))).strftime('%Y%m%d%H%M%S')) + entity_type = int(random.choice(ENTITY_TYPES)) + unit = random.choice(UNITS) + min_val = round(random.uniform(0, 100), 2) + max_val = round(random.uniform(100, 1000), 2) + higher_is_better = random.choice([True, False]) + container_id = random.choice(container_ids) + metrics.append((val, timestamp, entity_type, unit, min_val, max_val, higher_is_better, container_id)) + return metrics + + +def generate_sql(): + container_sql = [] + metric_sql = [] + + # Generate containers and insert statements + containers = generate_containers(NUM_CONTAINERS) + for i, (name, label, x, y, boot_ts) in enumerate(containers): + container_sql.append( + f"INSERT INTO ContainerEntity (name, label, location_x, location_y, boot_timestamp) VALUES " + f"('{name}', '{label}', {x}, {y}, '{boot_ts}');" + ) + + # Assuming the DB will assign container IDs 1..N in order + container_ids = list(range(1, NUM_CONTAINERS + 1)) + metrics = generate_metrics(NUM_METRICS, container_ids) + for m in metrics: + val, ts, etype, unit, minval, maxval, hib, cid = m + metric_sql.append( + f"INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES " + f"({val}, {ts}, '{etype}', '{unit}', {minval}, {maxval}, {'TRUE' if hib else 'FALSE'}, {cid});" + ) + + return container_sql + [""] + metric_sql + + +def save_sql(filename="populate_data.sql"): + statements = generate_sql() + with open(filename, "w") as f: + for stmt in statements: + f.write(stmt + "\n") + print(f"✅ SQL script saved to '{filename}'") + + +if __name__ == "__main__": + save_sql() diff --git a/json_clean/cleaned_metrics.json b/json_clean/cleaned_metrics.json new file mode 100644 index 0000000..f934427 --- /dev/null +++ b/json_clean/cleaned_metrics.json @@ -0,0 +1,179 @@ +[ + { + "container_id": "4", + "timestamp": "20250619140100", + "node1@DISK_USAGE": 9800.04 + }, + { + "container_id": "4", + "timestamp": "20250619113000", + "node1@MEMORY_USAGE": 9102.25 + }, + { + "container_id": "2", + "timestamp": "20250619022659", + "node4@DISK_USAGE": 5330.69 + }, + { + "container_id": "4", + "timestamp": "20250619123600", + "node4@CPU_USAGE": 263.8, + "node1@CPU_USAGE": 9231.62 + }, + { + "container_id": "2", + "timestamp": "20250618230600", + "node4@CPU_USAGE": 9801.4, + "node1@MEMORY_USAGE": 5585.62 + }, + { + "container_id": "2", + "timestamp": "20250619001600", + "node1@CPU_USAGE": 8696.97 + }, + { + "container_id": "5", + "timestamp": "20250618225200", + "node4@MEMORY_USAGE": 1218.86, + "node1@MEMORY_USAGE": 2499.24 + }, + { + "container_id": "1", + "timestamp": "20250619141859", + "node4@DISK_USAGE": 7739.96 + }, + { + "container_id": "5", + "timestamp": "20250618232500", + "node1@CPU_USAGE": 7058.74 + }, + { + "container_id": "5", + "timestamp": "20250619023459", + "node4@MEMORY_USAGE": 8632.19, + "node4@CPU_USAGE": 4593.33 + }, + { + "container_id": "5", + "timestamp": "20250619031259", + "node4@CPU_USAGE": 1470.3 + }, + { + "container_id": "3", + "timestamp": "20250619050459", + "node4@DISK_USAGE": 4075.31 + }, + { + "container_id": "3", + "timestamp": "20250619073259", + "node4@MEMORY_USAGE": 7668.28, + "node4@DISK_USAGE": 4357.53 + }, + { + "container_id": "1", + "timestamp": "20250618232059", + "node4@DISK_USAGE": 3385.85 + }, + { + "container_id": "3", + "timestamp": "20250619100259", + "node4@CPU_USAGE": 1806.89 + }, + { + "container_id": "4", + "timestamp": "20250619094500", + "node1@DISK_USAGE": 9650.98 + }, + { + "container_id": "4", + "timestamp": "20250619062200", + "node1@MEMORY_USAGE": 9420.12 + }, + { + "container_id": "4", + "timestamp": "20250619071200", + "node1@CPU_USAGE": 2000.16 + }, + { + "container_id": "4", + "timestamp": "20250619054559", + "node4@DISK_USAGE": 8107.42 + }, + { + "container_id": "4", + "timestamp": "20250619022000", + "node1@MEMORY_USAGE": 4546.73 + }, + { + "container_id": "4", + "timestamp": "20250619032700", + "node1@MEMORY_USAGE": 4300.64 + }, + { + "container_id": "4", + "timestamp": "20250619002859", + "node4@MEMORY_USAGE": 4223.78 + }, + { + "container_id": "4", + "timestamp": "20250619010559", + "node4@DISK_USAGE": 2162.09 + }, + { + "container_id": "2", + "timestamp": "20250619123559", + "node4@DISK_USAGE": 5139.56 + }, + { + "container_id": "4", + "timestamp": "20250618222900", + "node1@CPU_USAGE": 9840.42 + }, + { + "container_id": "5", + "timestamp": "20250619104700", + "node1@MEMORY_USAGE": 3882.55 + }, + { + "container_id": "5", + "timestamp": "20250619133859", + "node4@CPU_USAGE": 3477.85 + }, + { + "container_id": "1", + "timestamp": "20250619052000", + "node1@CPU_USAGE": 9557.23 + }, + { + "container_id": "5", + "timestamp": "20250619093759", + "node4@CPU_USAGE": 6348.95 + }, + { + "container_id": "3", + "timestamp": "20250619030500", + "node1@CPU_USAGE": 3141.89 + }, + { + "container_id": "3", + "timestamp": "20250619044259", + "node4@CPU_USAGE": 4150.27, + "node4@DISK_USAGE": 4245.5 + }, + { + "container_id": "1", + "timestamp": "20250619133900", + "node4@MEMORY_USAGE": 465.82, + "node1@MEMORY_USAGE": 8077.88 + }, + { + "container_id": "1", + "timestamp": "20250619124700", + "node1@CPU_USAGE": 9003.2 + }, + { + "container_id": "2", + "timestamp": "20250619065900", + "node1@CPU_USAGE": 2540.35 + } +] \ No newline at end of file diff --git a/json_clean/input.json b/json_clean/input.json new file mode 100644 index 0000000..0859a93 --- /dev/null +++ b/json_clean/input.json @@ -0,0 +1,513 @@ +[ + { + "node1@DISK_USAGE_higherIsBetter": true, + "boot_timestamp": 202412121442, + "node1@DISK_USAGE_unit": "PERCENTAGE", + "node1@DISK_USAGE_minVal": 70.15, + "container_name": "container-4", + "location_x": -60.61942, + "location_y": 70.16838, + "node1@DISK_USAGE": 9800.04, + "node1@DISK_USAGE_maxVal": 657.6, + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619140100" + }, + { + "node1@MEMORY_USAGE_minVal": 30.91, + "boot_timestamp": 202412121442, + "node1@MEMORY_USAGE_higherIsBetter": true, + "container_name": "container-4", + "node1@MEMORY_USAGE_maxVal": 234.09, + "location_x": -60.61942, + "location_y": 70.16838, + "node1@MEMORY_USAGE_unit": "PERCENTAGE", + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619113000", + "node1@MEMORY_USAGE": 9102.25 + }, + { + "boot_timestamp": 202410021447, + "container_name": "container-2", + "node4@DISK_USAGE_higherIsBetter": false, + "location_x": -18.835371, + "node4@DISK_USAGE_unit": "PLAIN", + "location_y": -71.91453, + "node4@DISK_USAGE_minVal": 84.44, + "container_id": "2", + "container_label": "sensor", + "timestamp": "20250619022659", + "node4@DISK_USAGE": 5330.69, + "node4@DISK_USAGE_maxVal": 233.1 + }, + { + "node4@CPU_USAGE_minVal": 27.38, + "boot_timestamp": 202503211447, + "node4@CPU_USAGE_maxVal": 178.34, + "node1@CPU_USAGE_higherIsBetter": true, + "location_x": -58.682648, + "location_y": 66.24983, + "node1@CPU_USAGE_minVal": 96.11, + "node4@CPU_USAGE": 263.8, + "container_label": "sensor", + "node4@CPU_USAGE_unit": "PERCENTAGE", + "node1@CPU_USAGE_maxVal": 845.32, + "container_name": "container-4", + "node1@CPU_USAGE_unit": "PLAIN", + "node4@CPU_USAGE_higherIsBetter": false, + "node1@CPU_USAGE": 9231.62, + "container_id": "4", + "timestamp": "20250619123600" + }, + { + "node4@CPU_USAGE_minVal": 89.34, + "node1@MEMORY_USAGE_minVal": 46.27, + "boot_timestamp": 202410021447, + "node4@CPU_USAGE_maxVal": 205.15, + "node1@MEMORY_USAGE_maxVal": 542.56, + "location_x": -18.835371, + "location_y": -71.91453, + "node4@CPU_USAGE": 9801.4, + "container_label": "sensor", + "node4@CPU_USAGE_unit": "PERCENTAGE", + "node1@MEMORY_USAGE": 5585.62, + "node1@MEMORY_USAGE_higherIsBetter": false, + "container_name": "container-2", + "node4@CPU_USAGE_higherIsBetter": false, + "node1@MEMORY_USAGE_unit": "PLAIN", + "container_id": "2", + "timestamp": "20250618230600" + }, + { + "boot_timestamp": 202503021442, + "node1@CPU_USAGE_maxVal": 393.87, + "container_name": "container-2", + "node1@CPU_USAGE_higherIsBetter": false, + "node1@CPU_USAGE_unit": "PLAIN", + "location_x": 136.7923, + "node1@CPU_USAGE_minVal": 83.38, + "location_y": 21.38392, + "node1@CPU_USAGE": 8696.97, + "container_id": "2", + "container_label": "sensor", + "timestamp": "20250619001600" + }, + { + "node4@MEMORY_USAGE_higherIsBetter": true, + "node1@MEMORY_USAGE_minVal": 57.97, + "boot_timestamp": 202408141447, + "node4@MEMORY_USAGE": 1218.86, + "node4@MEMORY_USAGE_unit": "PERCENTAGE", + "node1@MEMORY_USAGE_maxVal": 711.76, + "location_x": -174.74579, + "location_y": 4.327451, + "container_label": "sensor", + "node1@MEMORY_USAGE": 2499.24, + "node4@MEMORY_USAGE_maxVal": 369.1, + "node1@MEMORY_USAGE_higherIsBetter": false, + "container_name": "container-5", + "node4@MEMORY_USAGE_minVal": 36.13, + "node1@MEMORY_USAGE_unit": "PLAIN", + "container_id": "5", + "timestamp": "20250618225200" + }, + { + "boot_timestamp": 202406271447, + "container_name": "container-1", + "node4@DISK_USAGE_higherIsBetter": false, + "location_x": -98.829185, + "node4@DISK_USAGE_unit": "PERCENTAGE", + "location_y": -51.079384, + "node4@DISK_USAGE_minVal": 5.88, + "container_id": "1", + "container_label": "sensor", + "timestamp": "20250619141859", + "node4@DISK_USAGE": 7739.96, + "node4@DISK_USAGE_maxVal": 141.93 + }, + { + "boot_timestamp": 202412091442, + "node1@CPU_USAGE_maxVal": 630.06, + "container_name": "container-5", + "node1@CPU_USAGE_higherIsBetter": false, + "node1@CPU_USAGE_unit": "PLAIN", + "location_x": 86.3264, + "node1@CPU_USAGE_minVal": 18.77, + "location_y": 31.96573, + "node1@CPU_USAGE": 7058.74, + "container_id": "5", + "container_label": "sensor", + "timestamp": "20250618232500" + }, + { + "node4@CPU_USAGE_minVal": 8.24, + "node4@MEMORY_USAGE_higherIsBetter": true, + "boot_timestamp": 202408141447, + "node4@MEMORY_USAGE": 8632.19, + "node4@CPU_USAGE_maxVal": 726.29, + "node4@MEMORY_USAGE_unit": "PERCENTAGE", + "location_x": -174.74579, + "location_y": 4.327451, + "node4@CPU_USAGE": 4593.33, + "container_label": "sensor", + "node4@CPU_USAGE_unit": "PERCENTAGE", + "node4@MEMORY_USAGE_maxVal": 269.3, + "container_name": "container-5", + "node4@MEMORY_USAGE_minVal": 4.59, + "node4@CPU_USAGE_higherIsBetter": false, + "container_id": "5", + "timestamp": "20250619023459" + }, + { + "node4@CPU_USAGE_minVal": 53.32, + "boot_timestamp": 202408141447, + "container_name": "container-5", + "node4@CPU_USAGE_maxVal": 337.38, + "location_x": -174.74579, + "node4@CPU_USAGE_higherIsBetter": true, + "location_y": 4.327451, + "node4@CPU_USAGE": 1470.3, + "container_id": "5", + "container_label": "sensor", + "timestamp": "20250619031259", + "node4@CPU_USAGE_unit": "PERCENTAGE" + }, + { + "boot_timestamp": 202408161447, + "container_name": "container-3", + "node4@DISK_USAGE_higherIsBetter": true, + "location_x": -107.979904, + "node4@DISK_USAGE_unit": "PLAIN", + "location_y": -43.41263, + "node4@DISK_USAGE_minVal": 25.94, + "container_id": "3", + "container_label": "sensor", + "timestamp": "20250619050459", + "node4@DISK_USAGE": 4075.31, + "node4@DISK_USAGE_maxVal": 313.81 + }, + { + "node4@MEMORY_USAGE_higherIsBetter": false, + "boot_timestamp": 202408161447, + "node4@MEMORY_USAGE": 7668.28, + "node4@MEMORY_USAGE_unit": "PERCENTAGE", + "node4@DISK_USAGE_higherIsBetter": false, + "location_x": -107.979904, + "location_y": -43.41263, + "node4@DISK_USAGE_minVal": 46.88, + "container_label": "sensor", + "node4@DISK_USAGE_maxVal": 813.82, + "node4@MEMORY_USAGE_maxVal": 776.48, + "container_name": "container-3", + "node4@MEMORY_USAGE_minVal": 48.01, + "node4@DISK_USAGE_unit": "PERCENTAGE", + "container_id": "3", + "timestamp": "20250619073259", + "node4@DISK_USAGE": 4357.53 + }, + { + "boot_timestamp": 202406271447, + "container_name": "container-1", + "node4@DISK_USAGE_higherIsBetter": true, + "location_x": -98.829185, + "node4@DISK_USAGE_unit": "PERCENTAGE", + "location_y": -51.079384, + "node4@DISK_USAGE_minVal": 47.09, + "container_id": "1", + "container_label": "sensor", + "timestamp": "20250618232059", + "node4@DISK_USAGE": 3385.85, + "node4@DISK_USAGE_maxVal": 335.46 + }, + { + "node4@CPU_USAGE_minVal": 31.66, + "boot_timestamp": 202408161447, + "container_name": "container-3", + "node4@CPU_USAGE_maxVal": 181.16, + "location_x": -107.979904, + "node4@CPU_USAGE_higherIsBetter": true, + "location_y": -43.41263, + "node4@CPU_USAGE": 1806.89, + "container_id": "3", + "container_label": "sensor", + "timestamp": "20250619100259", + "node4@CPU_USAGE_unit": "PLAIN" + }, + { + "node1@DISK_USAGE_higherIsBetter": true, + "boot_timestamp": 202412121442, + "node1@DISK_USAGE_unit": "PERCENTAGE", + "node1@DISK_USAGE_minVal": 50.88, + "container_name": "container-4", + "location_x": -60.61942, + "location_y": 70.16838, + "node1@DISK_USAGE": 9650.98, + "node1@DISK_USAGE_maxVal": 618.11, + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619094500" + }, + { + "node1@MEMORY_USAGE_minVal": 86.86, + "boot_timestamp": 202412121442, + "node1@MEMORY_USAGE_higherIsBetter": true, + "container_name": "container-4", + "node1@MEMORY_USAGE_maxVal": 671.51, + "location_x": -60.61942, + "location_y": 70.16838, + "node1@MEMORY_USAGE_unit": "PERCENTAGE", + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619062200", + "node1@MEMORY_USAGE": 9420.12 + }, + { + "boot_timestamp": 202412121442, + "node1@CPU_USAGE_maxVal": 810.21, + "container_name": "container-4", + "node1@CPU_USAGE_higherIsBetter": true, + "node1@CPU_USAGE_unit": "PLAIN", + "location_x": -60.61942, + "node1@CPU_USAGE_minVal": 88.48, + "location_y": 70.16838, + "node1@CPU_USAGE": 2000.16, + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619071200" + }, + { + "boot_timestamp": 202503211447, + "container_name": "container-4", + "node4@DISK_USAGE_higherIsBetter": true, + "location_x": -58.682648, + "node4@DISK_USAGE_unit": "PLAIN", + "location_y": 66.24983, + "node4@DISK_USAGE_minVal": 26.15, + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619054559", + "node4@DISK_USAGE": 8107.42, + "node4@DISK_USAGE_maxVal": 694.35 + }, + { + "node1@MEMORY_USAGE_minVal": 13.92, + "boot_timestamp": 202412121442, + "node1@MEMORY_USAGE_higherIsBetter": false, + "container_name": "container-4", + "node1@MEMORY_USAGE_maxVal": 503.07, + "location_x": -60.61942, + "location_y": 70.16838, + "node1@MEMORY_USAGE_unit": "PERCENTAGE", + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619022000", + "node1@MEMORY_USAGE": 4546.73 + }, + { + "node1@MEMORY_USAGE_minVal": 91.9, + "boot_timestamp": 202412121442, + "node1@MEMORY_USAGE_higherIsBetter": true, + "container_name": "container-4", + "node1@MEMORY_USAGE_maxVal": 314.18, + "location_x": -60.61942, + "location_y": 70.16838, + "node1@MEMORY_USAGE_unit": "PERCENTAGE", + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619032700", + "node1@MEMORY_USAGE": 4300.64 + }, + { + "node4@MEMORY_USAGE_higherIsBetter": false, + "boot_timestamp": 202503211447, + "node4@MEMORY_USAGE_maxVal": 142.68, + "container_name": "container-4", + "node4@MEMORY_USAGE": 4223.78, + "node4@MEMORY_USAGE_unit": "PLAIN", + "node4@MEMORY_USAGE_minVal": 66.9, + "location_x": -58.682648, + "location_y": 66.24983, + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619002859" + }, + { + "boot_timestamp": 202503211447, + "container_name": "container-4", + "node4@DISK_USAGE_higherIsBetter": true, + "location_x": -58.682648, + "node4@DISK_USAGE_unit": "PLAIN", + "location_y": 66.24983, + "node4@DISK_USAGE_minVal": 1.25, + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250619010559", + "node4@DISK_USAGE": 2162.09, + "node4@DISK_USAGE_maxVal": 124.63 + }, + { + "boot_timestamp": 202410021447, + "container_name": "container-2", + "node4@DISK_USAGE_higherIsBetter": true, + "location_x": -18.835371, + "node4@DISK_USAGE_unit": "PERCENTAGE", + "location_y": -71.91453, + "node4@DISK_USAGE_minVal": 41.25, + "container_id": "2", + "container_label": "sensor", + "timestamp": "20250619123559", + "node4@DISK_USAGE": 5139.56, + "node4@DISK_USAGE_maxVal": 574.92 + }, + { + "boot_timestamp": 202412121442, + "node1@CPU_USAGE_maxVal": 695.17, + "container_name": "container-4", + "node1@CPU_USAGE_higherIsBetter": true, + "node1@CPU_USAGE_unit": "PLAIN", + "location_x": -60.61942, + "node1@CPU_USAGE_minVal": 56.09, + "location_y": 70.16838, + "node1@CPU_USAGE": 9840.42, + "container_id": "4", + "container_label": "sensor", + "timestamp": "20250618222900" + }, + { + "node1@MEMORY_USAGE_minVal": 79.52, + "boot_timestamp": 202412091442, + "node1@MEMORY_USAGE_higherIsBetter": true, + "container_name": "container-5", + "node1@MEMORY_USAGE_maxVal": 269.75, + "location_x": 86.3264, + "location_y": 31.96573, + "node1@MEMORY_USAGE_unit": "PLAIN", + "container_id": "5", + "container_label": "sensor", + "timestamp": "20250619104700", + "node1@MEMORY_USAGE": 3882.55 + }, + { + "node4@CPU_USAGE_minVal": 15.4, + "boot_timestamp": 202408141447, + "container_name": "container-5", + "node4@CPU_USAGE_maxVal": 903.54, + "location_x": -174.74579, + "node4@CPU_USAGE_higherIsBetter": false, + "location_y": 4.327451, + "node4@CPU_USAGE": 3477.85, + "container_id": "5", + "container_label": "sensor", + "timestamp": "20250619133859", + "node4@CPU_USAGE_unit": "PERCENTAGE" + }, + { + "boot_timestamp": 202407051442, + "node1@CPU_USAGE_maxVal": 639.09, + "container_name": "container-1", + "node1@CPU_USAGE_higherIsBetter": true, + "node1@CPU_USAGE_unit": "PERCENTAGE", + "location_x": -74.31161, + "node1@CPU_USAGE_minVal": 72.79, + "location_y": -47.90809, + "node1@CPU_USAGE": 9557.23, + "container_id": "1", + "container_label": "sensor", + "timestamp": "20250619052000" + }, + { + "node4@CPU_USAGE_minVal": 18.86, + "boot_timestamp": 202408141447, + "container_name": "container-5", + "node4@CPU_USAGE_maxVal": 436.99, + "location_x": -174.74579, + "node4@CPU_USAGE_higherIsBetter": true, + "location_y": 4.327451, + "node4@CPU_USAGE": 6348.95, + "container_id": "5", + "container_label": "sensor", + "timestamp": "20250619093759", + "node4@CPU_USAGE_unit": "PERCENTAGE" + }, + { + "boot_timestamp": 202411011442, + "node1@CPU_USAGE_maxVal": 359.08, + "container_name": "container-3", + "node1@CPU_USAGE_higherIsBetter": true, + "node1@CPU_USAGE_unit": "PLAIN", + "location_x": 140.49405, + "node1@CPU_USAGE_minVal": 60.09, + "location_y": 34.734318, + "node1@CPU_USAGE": 3141.89, + "container_id": "3", + "container_label": "sensor", + "timestamp": "20250619030500" + }, + { + "node4@CPU_USAGE_minVal": 82.69, + "boot_timestamp": 202408161447, + "node4@CPU_USAGE_maxVal": 777.35, + "node4@DISK_USAGE_higherIsBetter": true, + "location_x": -107.979904, + "location_y": -43.41263, + "node4@CPU_USAGE": 4150.27, + "node4@DISK_USAGE_minVal": 1.97, + "container_label": "sensor", + "node4@CPU_USAGE_unit": "PLAIN", + "node4@DISK_USAGE_maxVal": 507.49, + "container_name": "container-3", + "node4@CPU_USAGE_higherIsBetter": false, + "node4@DISK_USAGE_unit": "PLAIN", + "container_id": "3", + "timestamp": "20250619044259", + "node4@DISK_USAGE": 4245.5 + }, + { + "node4@MEMORY_USAGE_higherIsBetter": true, + "node1@MEMORY_USAGE_minVal": 35.65, + "boot_timestamp": 202406271447, + "node4@MEMORY_USAGE": 465.82, + "node4@MEMORY_USAGE_unit": "PERCENTAGE", + "node1@MEMORY_USAGE_maxVal": 120.32, + "location_x": -98.829185, + "location_y": -51.079384, + "container_label": "sensor", + "node1@MEMORY_USAGE": 8077.88, + "node4@MEMORY_USAGE_maxVal": 263.05, + "node1@MEMORY_USAGE_higherIsBetter": false, + "container_name": "container-1", + "node4@MEMORY_USAGE_minVal": 13.41, + "node1@MEMORY_USAGE_unit": "PLAIN", + "container_id": "1", + "timestamp": "20250619133900" + }, + { + "boot_timestamp": 202407051442, + "node1@CPU_USAGE_maxVal": 558.73, + "container_name": "container-1", + "node1@CPU_USAGE_higherIsBetter": false, + "node1@CPU_USAGE_unit": "PLAIN", + "location_x": -74.31161, + "node1@CPU_USAGE_minVal": 62.63, + "location_y": -47.90809, + "node1@CPU_USAGE": 9003.2, + "container_id": "1", + "container_label": "sensor", + "timestamp": "20250619124700" + }, + { + "boot_timestamp": 202503021442, + "node1@CPU_USAGE_maxVal": 433.65, + "container_name": "container-2", + "node1@CPU_USAGE_higherIsBetter": false, + "node1@CPU_USAGE_unit": "PLAIN", + "location_x": 136.7923, + "node1@CPU_USAGE_minVal": 93.37, + "location_y": 21.38392, + "node1@CPU_USAGE": 2540.35, + "container_id": "2", + "container_label": "sensor", + "timestamp": "20250619065900" + } +] diff --git a/json_clean/test.py b/json_clean/test.py new file mode 100644 index 0000000..952a66b --- /dev/null +++ b/json_clean/test.py @@ -0,0 +1,23 @@ +import json + +with open("input.json", "r") as f: + data = json.load(f) + +filtered = [] +for item in data: + entry = { + "container_id": item.get("container_id"), + "timestamp": item.get("timestamp"), + } + + # Extract only metrics like nodeX@CPU_USAGE, nodeX@MEMORY_USAGE, nodeX@DISK_USAGE + for key, value in item.items(): + if any(metric in key for metric in ["@CPU_USAGE", "@MEMORY_USAGE", "@DISK_USAGE"]): + if not any(key.endswith(suffix) for suffix in ["_minVal", "_maxVal", "_unit", "_higherIsBetter"]): + entry[key] = value + + filtered.append(entry) + +# Save the cleaned result +with open("cleaned_metrics.json", "w") as f: + json.dump(filtered, f, indent=2) diff --git a/nulls/compute_nulls.py b/nulls/compute_nulls.py new file mode 100644 index 0000000..ad3b5dd --- /dev/null +++ b/nulls/compute_nulls.py @@ -0,0 +1,214 @@ +import json + +data = [ + { + "container_id": "4", + "timestamp": "20250619140100", + "node1@DISK_USAGE": 9800.04 + }, + { + "container_id": "4", + "timestamp": "20250619113000", + "node1@MEMORY_USAGE": 9102.25 + }, + { + "container_id": "2", + "timestamp": "20250619022659", + "node4@DISK_USAGE": 5330.69 + }, + { + "container_id": "4", + "timestamp": "20250619123600", + "node4@CPU_USAGE": 263.8, + "node1@CPU_USAGE": 9231.62 + }, + { + "container_id": "2", + "timestamp": "20250618230600", + "node4@CPU_USAGE": 9801.4, + "node1@MEMORY_USAGE": 5585.62 + }, + { + "container_id": "2", + "timestamp": "20250619001600", + "node1@CPU_USAGE": 8696.97 + }, + { + "container_id": "5", + "timestamp": "20250618225200", + "node4@MEMORY_USAGE": 1218.86, + "node1@MEMORY_USAGE": 2499.24 + }, + { + "container_id": "1", + "timestamp": "20250619141859", + "node4@DISK_USAGE": 7739.96 + }, + { + "container_id": "5", + "timestamp": "20250618232500", + "node1@CPU_USAGE": 7058.74 + }, + { + "container_id": "5", + "timestamp": "20250619023459", + "node4@MEMORY_USAGE": 8632.19, + "node4@CPU_USAGE": 4593.33 + }, + { + "container_id": "5", + "timestamp": "20250619031259", + "node4@CPU_USAGE": 1470.3 + }, + { + "container_id": "3", + "timestamp": "20250619050459", + "node4@DISK_USAGE": 4075.31 + }, + { + "container_id": "3", + "timestamp": "20250619073259", + "node4@MEMORY_USAGE": 7668.28, + "node4@DISK_USAGE": 4357.53 + }, + { + "container_id": "1", + "timestamp": "20250618232059", + "node4@DISK_USAGE": 3385.85 + }, + { + "container_id": "3", + "timestamp": "20250619100259", + "node4@CPU_USAGE": 1806.89 + }, + { + "container_id": "4", + "timestamp": "20250619094500", + "node1@DISK_USAGE": 9650.98 + }, + { + "container_id": "4", + "timestamp": "20250619062200", + "node1@MEMORY_USAGE": 9420.12 + }, + { + "container_id": "4", + "timestamp": "20250619071200", + "node1@CPU_USAGE": 2000.16 + }, + { + "container_id": "4", + "timestamp": "20250619054559", + "node4@DISK_USAGE": 8107.42 + }, + { + "container_id": "4", + "timestamp": "20250619022000", + "node1@MEMORY_USAGE": 4546.73 + }, + { + "container_id": "4", + "timestamp": "20250619032700", + "node1@MEMORY_USAGE": 4300.64 + }, + { + "container_id": "4", + "timestamp": "20250619002859", + "node4@MEMORY_USAGE": 4223.78 + }, + { + "container_id": "4", + "timestamp": "20250619010559", + "node4@DISK_USAGE": 2162.09 + }, + { + "container_id": "2", + "timestamp": "20250619123559", + "node4@DISK_USAGE": 5139.56 + }, + { + "container_id": "4", + "timestamp": "20250618222900", + "node1@CPU_USAGE": 9840.42 + }, + { + "container_id": "5", + "timestamp": "20250619104700", + "node1@MEMORY_USAGE": 3882.55 + }, + { + "container_id": "5", + "timestamp": "20250619133859", + "node4@CPU_USAGE": 3477.85 + }, + { + "container_id": "1", + "timestamp": "20250619052000", + "node1@CPU_USAGE": 9557.23 + }, + { + "container_id": "5", + "timestamp": "20250619093759", + "node4@CPU_USAGE": 6348.95 + }, + { + "container_id": "3", + "timestamp": "20250619030500", + "node1@CPU_USAGE": 3141.89 + }, + { + "container_id": "3", + "timestamp": "20250619044259", + "node4@CPU_USAGE": 4150.27, + "node4@DISK_USAGE": 4245.5 + }, + { + "container_id": "1", + "timestamp": "20250619133900", + "node4@MEMORY_USAGE": 465.82, + "node1@MEMORY_USAGE": 8077.88 + }, + { + "container_id": "1", + "timestamp": "20250619124700", + "node1@CPU_USAGE": 9003.2 + }, + { + "container_id": "2", + "timestamp": "20250619065900", + "node1@CPU_USAGE": 2540.35 + } +] + +# Define all expected keys (excluding container_id and timestamp which are always present) +expected_keys = [ + "node1@DISK_USAGE", + "node1@MEMORY_USAGE", + "node4@DISK_USAGE", + "node4@CPU_USAGE", + "node1@CPU_USAGE", + "node4@MEMORY_USAGE" +] + +# Initialize null counter for each key +null_counts = {key: 0 for key in expected_keys} + +# Go through each record and count missing fields +for entry in data: + for key in expected_keys: + if key not in entry: + null_counts[key] += 1 + +# Print results +total_entries = len(data) +total_possible_fields = total_entries * len(expected_keys) +total_nulls = sum(null_counts.values()) + +print("Nulls per column:") +for key, count in null_counts.items(): + print(f"{key}: {count} nulls ({(count/total_entries)*100:.1f}%)") + +print(f"\nTotal entries: {total_entries}") +print(f"Total possible fields: {total_possible_fields}") +print(f"Total null values: {total_nulls}") +print(f"Overall null percentage: {(total_nulls / total_possible_fields) * 100:.2f}%") diff --git a/populate_data.sql b/populate_data.sql new file mode 100644 index 0000000..d5b0106 --- /dev/null +++ b/populate_data.sql @@ -0,0 +1,13 @@ +INSERT INTO ContainerEntity (name, label, location_x, location_y, boot_timestamp) VALUES ('container-1', 'sensor', 96.372249, 35.963685, '202505271209'); +INSERT INTO ContainerEntity (name, label, location_x, location_y, boot_timestamp) VALUES ('container-2', 'sensor', -113.834002, -4.175589, '202409181209'); + +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (3819.17, 20250620072325, '2', '0', 11.74, 833.19, TRUE, 1); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (5043.89, 20250620092525, '2', '0', 65.1, 843.1, TRUE, 2); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (2079.01, 20250620045625, '2', '1', 93.56, 521.92, FALSE, 1); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (205.73, 20250620114625, '2', '0', 68.86, 558.84, TRUE, 1); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (2348.32, 20250620101025, '1', '1', 65.61, 237.84, FALSE, 2); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (2687.45, 20250620055425, '1', '1', 36.09, 427.38, FALSE, 1); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (6750.52, 20250620070425, '2', '1', 97.82, 143.0, TRUE, 2); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (4433.28, 20250619214025, '0', '0', 41.56, 415.34, TRUE, 1); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (460.36, 20250620114725, '1', '1', 63.63, 445.25, FALSE, 2); +INSERT INTO MetricEntity (val, timestamp, entityType, unit, minVal, maxVal, higherIsBetter, container_id) VALUES (9296.05, 20250620021925, '2', '0', 7.73, 789.59, FALSE, 2); diff --git a/src/main/java/com/thesis/sqlite/controllers/JoinController.java b/src/main/java/com/thesis/sqlite/controllers/JoinController.java new file mode 100644 index 0000000..91881d2 --- /dev/null +++ b/src/main/java/com/thesis/sqlite/controllers/JoinController.java @@ -0,0 +1,54 @@ +package com.thesis.sqlite.controllers; +import com.thesis.sqlite.entities.MetricEntity; +import com.thesis.sqlite.enumerations.EntityTypeE; +import com.thesis.sqlite.repositories.MetricRepository; +import com.thesis.sqlite.service.JoinService; +import lombok.AllArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@AllArgsConstructor +@RequestMapping("/join") + +public class JoinController { + private final JoinService joinService; + private final MetricRepository metricRepository; + + @GetMapping("/fetchAllTables") + List> fetchAllTables() { + return joinService.fetchAllTables(); + } + + @GetMapping("/fetchByEntityType") + public List> fetchByEntityType( + @RequestParam List entityTypes) { + return joinService.fetchByEntityType(entityTypes); + } + + @GetMapping("/nodeJoin") + public List> getJoinedMetrics( + @RequestParam(required = false) List types + ) { + List metrics; + + if (types != null && !types.isEmpty()) { + metrics = metricRepository.findAllByEntityTypeIn(types); + } else { + metrics = metricRepository.findAll(); + } + + return joinService.joinLocally(metrics); + } + + + + + + +} diff --git a/src/main/java/com/thesis/sqlite/dht/DhtService.java b/src/main/java/com/thesis/sqlite/dht/DhtService.java index c663e64..472b5e1 100644 --- a/src/main/java/com/thesis/sqlite/dht/DhtService.java +++ b/src/main/java/com/thesis/sqlite/dht/DhtService.java @@ -26,6 +26,7 @@ import lombok.Getter; import lombok.NonNull; + @Service public class DhtService { private final Map> dataMap; @@ -175,7 +176,7 @@ private void syncData(@NonNull String destination, SyncRequest syncRequest) { } // Execute this job every 5 minutes - @Scheduled(fixedRate = 5, initialDelay = 5, timeUnit = TimeUnit.MINUTES) + @Scheduled(fixedRateString = "${dht.rate.data.sync}", initialDelayString = "${dht.rate.data.sync}", timeUnit = TimeUnit.SECONDS) private void syncScheduledJob() { final var startTime = System.nanoTime(); Utils.LOGGER.info("Now syncing node '{}'", Utils.HOSTNAME); @@ -206,7 +207,7 @@ private void syncScheduledJob() { } // Execute this job every 1 minutes - @Scheduled(fixedRate = 1, initialDelay = 6, timeUnit = TimeUnit.MINUTES) + @Scheduled(fixedRateString = "${dht.rate.removal.sync}", initialDelayString = "${dht.rate.removal.sync}", timeUnit = TimeUnit.SECONDS) private void removeNodesFromDeletionMap() { Utils.LOGGER.info("Try delete lock in remove"); synchronized (deleteLock) { diff --git a/src/main/java/com/thesis/sqlite/entities/ContainerEntity.java b/src/main/java/com/thesis/sqlite/entities/ContainerEntity.java index b0c4f47..ae92f99 100644 --- a/src/main/java/com/thesis/sqlite/entities/ContainerEntity.java +++ b/src/main/java/com/thesis/sqlite/entities/ContainerEntity.java @@ -26,4 +26,11 @@ public class ContainerEntity { public String name; @Column(nullable = false) public String label; + @Column(nullable = false) + public float location_x; + @Column(nullable = false) + public float location_y; + @Column(nullable = false) + public Long boot_timestamp; + } diff --git a/src/main/java/com/thesis/sqlite/repositories/MetricRepository.java b/src/main/java/com/thesis/sqlite/repositories/MetricRepository.java index 05de6df..945d80a 100644 --- a/src/main/java/com/thesis/sqlite/repositories/MetricRepository.java +++ b/src/main/java/com/thesis/sqlite/repositories/MetricRepository.java @@ -8,6 +8,7 @@ import com.thesis.sqlite.entities.MetricEntity; import com.thesis.sqlite.enumerations.EntityTypeE; +import org.springframework.data.repository.query.Param; public interface MetricRepository extends CrudRepository { List findAll(); @@ -18,4 +19,7 @@ public interface MetricRepository extends CrudRepository { @Query("SELECT DISTINCT m.entityType FROM MetricEntity m") Set findDistinctEntityTypes(); + + List findAllByEntityTypeIn(List types); + } diff --git a/src/main/java/com/thesis/sqlite/service/JoinService.java b/src/main/java/com/thesis/sqlite/service/JoinService.java new file mode 100644 index 0000000..1422ff5 --- /dev/null +++ b/src/main/java/com/thesis/sqlite/service/JoinService.java @@ -0,0 +1,252 @@ +package com.thesis.sqlite.service; + +import com.thesis.sqlite.dht.DhtService; +import com.thesis.sqlite.entities.ContainerEntity; +import com.thesis.sqlite.entities.MetricEntity; +import com.thesis.sqlite.entities.PodEntity; +import com.thesis.sqlite.enumerations.EntityTypeE; +import com.thesis.sqlite.utils.Utils; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import java.util.*; +import java.util.stream.Collectors; + +@Service +@AllArgsConstructor +public class JoinService { + private static final RestTemplate REST_TEMPLATE = new RestTemplate(); + private DhtService dhtService; + private final int t = 20; // Threshold for tuple ratio rule + private final PodService podService; + private final ContainerService containerService; + + + // Method to fetch all tables from all nodes, using a coordinator if specified + public List> fetchAllTables() { + Set nodeNames = dhtService.getExistingNodes(); // Get nodes dynamically + List> allJoined = new ArrayList<>(); + String endpoint = "join/nodeJoin"; + + for (String node : nodeNames) { + try { + String url = String.format("http://%s:29000/%s", node, endpoint); + List> joinedFromNode = fetchDataFromServer(url); // Fetch from each node + + if (joinedFromNode != null) { + for (Map row : joinedFromNode) { + row.put("node", node); // Tag each row with the node name + allJoined.add(row); + } + } + } catch (Exception e) { + System.out.println("Error fetching joined data from node " + node + ": " + e.getMessage()); + } + } + + + return groupByContainerAndTimestamp(allJoined); + } + + + + private List> groupByContainerAndTimestamp(List> rows) { + Map> pivoted = new HashMap<>(); + + for (Map row : rows) { + Object containerIdObj = row.get("container_id"); + Object timestampObj = row.get("timestamp"); + if (containerIdObj == null || timestampObj == null) continue; + if (Boolean.TRUE.equals(row.get("skip_join"))) { + + Utils.LOGGER.warn("Skipping join for container_id={} on node={}", containerIdObj, row.get("node")); + continue; // Skip rows from nodes that didn't do the container join + } + + + String containerId = String.valueOf(containerIdObj); + String timestamp = String.valueOf(timestampObj); + String node = String.valueOf(row.get("node")); + String entityType = String.valueOf(row.get("entityType")); + Object value = row.get("val"); + + long timestampRaw = Long.parseLong(String.valueOf(row.get("timestamp"))); + long bucketedTimestamp = timestampRaw / 10000; // group every 60 sec + // Use container + timestamp as key + String key = containerId + "_" + bucketedTimestamp; + + + // Init group + pivoted.putIfAbsent(key, new HashMap<>()); + Map group = pivoted.get(key); + + // Group keys + group.put("container_id", containerId); + group.put("timestamp", timestamp); + + // Add metric + metadata + String columnPrefix = node + "@" + entityType; + group.put(columnPrefix, value); + copyMetricDetail(group, row, "unit", columnPrefix); + copyMetricDetail(group, row, "higherIsBetter", columnPrefix); + copyMetricDetail(group, row, "minVal", columnPrefix); + copyMetricDetail(group, row, "maxVal", columnPrefix); + + // Add container metadata once + copyIfAbsent(group, row, "container_name"); + copyIfAbsent(group, row, "container_label"); + copyIfAbsent(group, row, "location_x"); + copyIfAbsent(group, row, "location_y"); + copyIfAbsent(group, row, "boot_timestamp"); + } + + return new ArrayList<>(pivoted.values()); + } + + + private void copyMetricDetail(Map target, Map source, String detailKey, String columnPrefix) { + Object value = source.get(detailKey); + if (value != null) { + target.put(columnPrefix + "_" + detailKey, value); + } + } + + + + + + + + + // Method to fetch tables with specific metric in them + public List> fetchByEntityType(List metricTypes) { + try { + Set nodeNames = new HashSet<>(); + for (EntityTypeE type : metricTypes) { + nodeNames.addAll(dhtService.get(type)); // union of all nodes that store any type + } + + List> allJoined = new ArrayList<>(); + + for (String node : nodeNames) { + // Build the URL with all types (e.g., types=A&types=B) + String joinedTypes = metricTypes.stream() + .map(Enum::name) + .collect(Collectors.joining("&types=")); + + String url = String.format("http://%s:29000/join/nodeJoin?types=%s", node, joinedTypes); + + // Fetch joined data from node + List> joinedFromNode = fetchDataFromServer(url); + if (joinedFromNode != null) { + for (Map row : joinedFromNode) { + row.put("node", node); // tag origin node + allJoined.add(row); + } + } + } + + return groupByContainerAndTimestamp(allJoined); + } catch (Exception e) { + System.out.println("Error fetching joined data: " + e.getMessage()); + return Collections.emptyList(); + } + } + + + + // Helper method to fetch data from server via HTTP + private List> fetchDataFromServer(String url) { + try { + return REST_TEMPLATE.getForObject(url, List.class); + } catch (Exception e) { + System.out.println("Error fetching data from " + url + ": " + e.getMessage()); + return Collections.emptyList(); + } + } + + + // Method to check if a join can be skipped based on the Tuple Ratio rule TR = nS / nR (nS = number of training examples in entity table + // S, nR = number of distinct values of the foreign key attribute in the relation R) + public boolean canSkipJoin(List metrics, String fieldName) { + int nS = metrics.size(); + Set distinctFKs = new HashSet<>(); + + for (MetricEntity metric : metrics) { + if ("container".equals(fieldName) && metric.getContainer() != null) { + distinctFKs.add(metric.getContainer().getId()); + } else if ("pod".equals(fieldName) && metric.getPod() != null) { + distinctFKs.add(metric.getPod().getId()); + } + } + + int nR = distinctFKs.size(); + double TR = nR == 0 ? 0 : (double) nS / nR; + + Utils.LOGGER.warn("Tuple Ratio for {} = {} / {} = {}", fieldName, nS, nR, TR); + return TR >= 20; + } + + + public List> joinLocally(List metrics) { + boolean skipContainerJoin = canSkipJoin(metrics, "container"); + boolean skipPodJoin = canSkipJoin(metrics, "pod"); + + List> joined = new ArrayList<>(); + + for (MetricEntity metric : metrics) { + Map row = new HashMap<>(); + + // Basic metric fields + row.put("id", metric.getId()); + row.put("val", metric.getVal()); + row.put("timestamp", metric.getTimestamp()); + row.put("entityType", metric.getEntityType()); + row.put("unit", metric.getUnit()); + row.put("minVal", metric.getMinVal()); + row.put("maxVal", metric.getMaxVal()); + row.put("higherIsBetter", metric.isHigherIsBetter()); + + // Always include container_id if available + if (metric.getContainer() != null) { + row.put("container_id", metric.getContainer().getId()); + row.put("skip_join", skipContainerJoin); + if (!skipContainerJoin) { + row.put("container_name", metric.getContainer().getName()); + row.put("container_label", metric.getContainer().getLabel()); + row.put("location_x", metric.getContainer().getLocation_x()); + row.put("location_y", metric.getContainer().getLocation_y()); + row.put("boot_timestamp", metric.getContainer().getBoot_timestamp()); + } + } + + // Always include pod_id if available + if (metric.getPod() != null) { + row.put("pod_id", metric.getPod().getId()); + if (!skipPodJoin) { + row.put("pod_name", metric.getPod().getName()); + row.put("pod_namespace", metric.getPod().getNamespace()); + row.put("pod_uuid", metric.getPod().getUuid()); + + } + } + + joined.add(row); + } + + return joined; + } + + private void copyIfAbsent(Map target, Map source, String key) { + if (!target.containsKey(key) && source.containsKey(key)) { + target.put(key, source.get(key)); + } + } + + + + + + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 01fdd52..209d0ac 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -4,6 +4,8 @@ url=jdbc:sqlite:/mnt/databases/${hostname}.db spring.jpa.database-platform=org.hibernate.community.dialect.SQLiteDialect hibernate.hbm2ddl.auto=update # hibernate.show_sql=true +dht.rate.data.sync=${DHT_DATA_SYNC:300} +dht.rate.removal.sync=${DHT_REMOVAL_SYNC:60} spring.kafka.consumer.bootstrap-servers=kafka:29092 spring.kafka.consumer.group-id=${hostname} diff --git a/tempCodeRunnerFile.py b/tempCodeRunnerFile.py new file mode 100644 index 0000000..19ec956 --- /dev/null +++ b/tempCodeRunnerFile.py @@ -0,0 +1 @@ +CONTAINERS \ No newline at end of file