From 333da030067249014b8bf4843e135ec7f6445b01 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 11:38:52 +0300 Subject: [PATCH 01/25] added .idea folder --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a93d3e2..000caec 100644 --- a/.gitignore +++ b/.gitignore @@ -157,7 +157,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Ignore the default SQLite database. *.sqlite3 From fc01d0640c07a667e5ef38f2c13513299ef1ea67 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 11:56:59 +0300 Subject: [PATCH 02/25] added graph path settings to default config --- docs/supergraph.conf.example | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/supergraph.conf.example b/docs/supergraph.conf.example index 52bb57e..1bfba0a 100644 --- a/docs/supergraph.conf.example +++ b/docs/supergraph.conf.example @@ -8,5 +8,10 @@ port = 7687 user = neo4j password = password +[graph] +base_path = /opt/otp/complex_rest/plugins/supergraph/graphs +tmp_path = /opt/otp/complex_rest/plugins/supergraph/tmp +id_name_map_path = /opt/otp/complex_rest/plugins/supergraph/id_name_map + [schema] default_root_name = ROOT From 00c23075b9856ba1a1a3e3b58f87f5615fe7856b Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 11:58:59 +0300 Subject: [PATCH 03/25] added graph path settings to settings.py --- complex_rest_dtcd_supergraph/settings.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/complex_rest_dtcd_supergraph/settings.py b/complex_rest_dtcd_supergraph/settings.py index 0c882ec..3c22d23 100644 --- a/complex_rest_dtcd_supergraph/settings.py +++ b/complex_rest_dtcd_supergraph/settings.py @@ -1,4 +1,5 @@ import configparser +import os import uuid from pathlib import Path from types import SimpleNamespace @@ -59,6 +60,21 @@ "Have you forgot to initialize the database with initialize.py?" ) +# graphs configuration + +GRAPH_BASE_PATH = ini_config['graph']['base_path'] +GRAPH_TMP_PATH = ini_config['graph']['tmp_path'] +GRAPH_ID_NAME_MAP_PATH = ini_config['graph']['id_name_map_path'] + +if not os.path.isdir(GRAPH_BASE_PATH): + os.mkdir(Path(GRAPH_BASE_PATH)) + +if not os.path.isdir(GRAPH_TMP_PATH): + os.mkdir(Path(GRAPH_TMP_PATH)) + +if not os.path.isdir(GRAPH_ID_NAME_MAP_PATH): + os.mkdir(Path(GRAPH_ID_NAME_MAP_PATH)) + # TODO users can delete default root node, so uid in text file becomes old # TODO during the testing we reset the database after each test with open(path) as f: From 2adb50f4de29c558091671f219a249fc7bac41b4 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 12:14:29 +0300 Subject: [PATCH 04/25] added graph filesystem manager class --- .../utils/abc_graphmanager.py | 24 +++++ .../utils/filesystem_graphmanager.py | 101 ++++++++++++++++++ .../utils/graphmanager_exception.py | 16 +++ 3 files changed, 141 insertions(+) create mode 100644 complex_rest_dtcd_supergraph/utils/abc_graphmanager.py create mode 100644 complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py create mode 100644 complex_rest_dtcd_supergraph/utils/graphmanager_exception.py diff --git a/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py b/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py new file mode 100644 index 0000000..ed23ec2 --- /dev/null +++ b/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py @@ -0,0 +1,24 @@ +from abc import ABC, abstractmethod + + +class AbstractGraphManager(ABC): + + @abstractmethod + def read(self, graph_id: str) -> None: + pass + + @abstractmethod + def read_all(self) -> None: + pass + + @abstractmethod + def write(self, graph: dict) -> None: + pass + + @abstractmethod + def update(self, graph: dict) -> None: + pass + + @abstractmethod + def remove(self, graph_id: str) -> None: + pass diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py new file mode 100644 index 0000000..8b2ab13 --- /dev/null +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -0,0 +1,101 @@ +import os +import shutil +import uuid +import json + +from ..utils.graphmanager_exception import GraphManagerException, NO_GRAPH, NO_ID, NAME_EXISTS +from ..utils.abc_graphmanager import AbstractGraphManager +from pathlib import Path + + +class FilesystemGraphManager(AbstractGraphManager): + + def __init__(self, path, tmp_path, map_path): # or better import it from settings here? + self.final_path = path + self.tmp_path = tmp_path + '/tmp.graphml' + self.map_path = map_path + '/graph_map.json' # not empty, at least {} + if not os.path.isfile(self.map_path): + with open(self.map_path, 'w') as map_file: + map_file.write('{}') + self.map_backup_path = map_path + '/graph_backup.json' + self.map_tmp_path = map_path + '/graph_tmp.json' + self.default_filename = 'graph.graphml' + + def read(self, graph_id): + graph_data = {} + try: + with open(Path(self.final_path) / graph_id / self.default_filename, 'r') as graph: + graph_data['content'] = graph.read() + shutil.copyfile(self.map_path, self.map_backup_path) + with open(self.map_path, 'r') as map_file: + id_map = json.load(map_file) + graph_data['name'] = id_map[graph_id]['name'] + return graph_data + except OSError: + raise GraphManagerException(NO_GRAPH, graph_id) + + def read_all(self): + graph_list = [] + with open(self.map_path, 'r') as map_file: + id_map = json.load(map_file) + for k, v in id_map.items(): + graph_list.append({'graph_id': k, 'name': v['name']}) + return graph_list + + def write(self, graph: dict): + shutil.copyfile(self.map_path, self.map_backup_path) + with open(self.map_path, 'r') as map_file: + id_map = json.load(map_file) + name = graph["name"] + for _, v in id_map.items(): + if v['name'] == name: + raise GraphManagerException(NAME_EXISTS, name) + unique_id = str(uuid.uuid4()) + id_map[unique_id] = {'name': graph["name"]} + with open(self.map_tmp_path, 'w') as map_tmp_file: + json.dump(id_map, map_tmp_file) + os.rename(self.map_tmp_path, self.map_path) # atomic operation + with open(self.tmp_path, 'w') as file: + file.write(graph["content"]) + graph_dir = Path(self.final_path) / unique_id + os.mkdir(graph_dir) + os.rename(self.tmp_path, Path(graph_dir / self.default_filename)) # atomic operation + + def update(self, graph: dict): + if 'graph_id' not in graph: + raise GraphManagerException(NO_ID) + if 'name' in graph: + shutil.copyfile(self.map_path, self.map_backup_path) + with open(self.map_path, 'r') as map_file: + id_map = json.load(map_file) + name = graph["name"] + for k, v in id_map.items(): + if v['name'] == name and k != graph['graph_id']: + raise GraphManagerException(NAME_EXISTS, name) + try: + id_map[graph['graph_id']]['name'] = name + except KeyError: + raise GraphManagerException(NO_GRAPH, graph['graph_id']) + with open(self.map_tmp_path, 'w') as map_tmp_file: + json.dump(id_map, map_tmp_file) + os.rename(self.map_tmp_path, self.map_path) # atomic operation + if 'content' in graph: + graph_dir = Path(self.final_path) / graph['graph_id'] + if not os.path.isdir(graph_dir): + raise GraphManagerException(NO_GRAPH, graph['graph_id']) + with open(self.tmp_path, 'w') as file: + file.write(graph["content"]) + os.rename(self.tmp_path, Path(graph_dir) / self.default_filename) # atomic operation + + def remove(self, graph_id): + graph_dir = Path(self.final_path) / graph_id + if not os.path.isdir(graph_dir): + raise GraphManagerException(NO_GRAPH, graph_id) + shutil.rmtree(Path(self.final_path) / graph_id) # delete directory and it's content + shutil.copyfile(self.map_path, self.map_backup_path) + with open(self.map_path, 'r') as map_file: + id_map = json.load(map_file) + del id_map[graph_id] + with open(self.map_tmp_path, 'w') as map_tmp_file: + json.dump(id_map, map_tmp_file) + os.rename(self.map_tmp_path, self.map_path) # atomic operation diff --git a/complex_rest_dtcd_supergraph/utils/graphmanager_exception.py b/complex_rest_dtcd_supergraph/utils/graphmanager_exception.py new file mode 100644 index 0000000..eab075f --- /dev/null +++ b/complex_rest_dtcd_supergraph/utils/graphmanager_exception.py @@ -0,0 +1,16 @@ +NO_ID = 0 +NO_GRAPH = 1 +NAME_EXISTS = 2 + + +class GraphManagerException(Exception): + + def __init__(self, problem, *args): + msg = 'no message' + if problem == NO_ID: + msg = 'No id provided' + elif problem == NO_GRAPH: + msg = f"No graph found with id -> {args[0]}" + elif problem == NAME_EXISTS: + msg = f"Name -> {args[0]} already exists" + super().__init__(msg) From 6e8414023d8be9eb4a5c977088b353050d02974c Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 12:34:31 +0300 Subject: [PATCH 05/25] added minor type annotation --- complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index 8b2ab13..0619651 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -21,7 +21,7 @@ def __init__(self, path, tmp_path, map_path): # or better import it from settin self.map_tmp_path = map_path + '/graph_tmp.json' self.default_filename = 'graph.graphml' - def read(self, graph_id): + def read(self, graph_id) -> dict: graph_data = {} try: with open(Path(self.final_path) / graph_id / self.default_filename, 'r') as graph: From 9509b75af0302ffd0fdaa0e195362c9061478df4 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 12:35:25 +0300 Subject: [PATCH 06/25] added all 4 methods for graphs --- complex_rest_dtcd_supergraph/views/graphs.py | 198 ++++++------------- 1 file changed, 65 insertions(+), 133 deletions(-) diff --git a/complex_rest_dtcd_supergraph/views/graphs.py b/complex_rest_dtcd_supergraph/views/graphs.py index 5210165..b407698 100644 --- a/complex_rest_dtcd_supergraph/views/graphs.py +++ b/complex_rest_dtcd_supergraph/views/graphs.py @@ -1,138 +1,70 @@ -""" -Views for graph management operations for roots and fragments. -""" - -import uuid - -import neomodel -from rest_framework.request import Request - -from rest.permissions import AllowAny -from rest.response import SuccessResponse from rest.views import APIView +from rest.response import Response, status +from rest.permissions import AllowAny +from rest_framework.request import Request +from ..utils.filesystem_graphmanager import FilesystemGraphManager +from ..settings import GRAPH_BASE_PATH, GRAPH_TMP_PATH, GRAPH_ID_NAME_MAP_PATH +import logging -from .. import settings -from ..converters import GraphDataConverter -from ..managers import Manager -from ..models import Root -from ..serializers import ContentSerializer, GraphSerializer -from .fragments import get_fragment_from_root_or_404 -from .mixins import ContainerManagementMixin -from .shortcuts import get_node_or_404 - - -class RootGraphView(ContainerManagementMixin, APIView): - """Retrieve, replace or delete graph content of a root.""" - - http_method_names = ["get", "put", "delete"] - permission_classes = (AllowAny,) - converter = GraphDataConverter() - manager = Manager() - - @neomodel.db.transaction - def get(self, request: Request, pk: uuid.UUID): - """Read graph content of a root.""" - - root = get_node_or_404(Root.nodes, uid=pk.hex) - payload = self.read(root) - serializer = ContentSerializer(instance=payload) - - return SuccessResponse(data={"graph": serializer.data}) - - @neomodel.db.transaction - def put(self, request: Request, pk: uuid.UUID): - """Replace graph content of a root.""" - - root = get_node_or_404(Root.nodes, uid=pk.hex) - serializer = GraphSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - self.replace(root, serializer.data["graph"]) - - return SuccessResponse() - - @neomodel.db.transaction - def delete(self, request: Request, pk: uuid.UUID): - """Delete graph content of a root.""" - - root = get_node_or_404(Root.nodes, uid=pk.hex) - root.clear(content_only=True) - - return SuccessResponse() - - -class DefaultRootGraphView(RootGraphView): - """Retrieve, replace or delete graph content of the default root.""" - - pk = settings.DEFAULT_ROOT_UUID - - def get(self, request: Request): - return super().get(request, self.pk) - - def put(self, request: Request): - return super().put(request, self.pk) - - def delete(self, request: Request): - return super().delete(request, self.pk) - +logger = logging.getLogger('supergraph') -class RootFragmentGraphView(ContainerManagementMixin, APIView): - """Retrieve, replace or delete graph content of this root's fragment.""" - http_method_names = ["get", "put", "delete"] +class Graph(APIView): permission_classes = (AllowAny,) - converter = GraphDataConverter() - manager = Manager() - - @neomodel.db.transaction - def get(self, request: Request, root_pk: uuid.UUID, fragment_pk: uuid.UUID): - """Read graph content of the given root's fragment.""" - - fragment = get_fragment_from_root_or_404(root_pk, fragment_pk) - payload = self.read(fragment) - serializer = ContentSerializer(instance=payload) - - return SuccessResponse(data={"graph": serializer.data}) - - @neomodel.db.transaction - def put(self, request: Request, root_pk: uuid.UUID, fragment_pk: uuid.UUID): - """Replace graph content of this root's fragment.""" - - # validate incoming graph content - serializer = GraphSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - # query root and fragment - root = get_node_or_404(Root.nodes, uid=root_pk.hex) - fragment = get_node_or_404(root.fragments, uid=fragment_pk.hex) - # convert to domain classes, update fragment's content - self.replace(fragment, serializer.data["graph"]) - # re-connect root to content - self.manager.reconnect(root, fragment) - - return SuccessResponse() - - @neomodel.db.transaction - def delete(self, request: Request, root_pk: uuid.UUID, fragment_pk: uuid.UUID): - """Delete graph content this root's fragment.""" - - fragment = get_fragment_from_root_or_404(root_pk, fragment_pk) - fragment.clear() - - return SuccessResponse() - - -class DefaultRootFragmentGraphView(RootFragmentGraphView): - """Retrieve, replace or delete graph content of default root's fragment.""" - - root_pk = settings.DEFAULT_ROOT_UUID - - def get(self, request: Request, pk: uuid.UUID): - fragment_pk = pk - return super().get(request, self.root_pk, fragment_pk) - - def put(self, request: Request, pk: uuid.UUID): - fragment_pk = pk - return super().put(request, self.root_pk, fragment_pk) - - def delete(self, request: Request, pk: uuid.UUID): - fragment_pk = pk - return super().delete(request, self.root_pk, fragment_pk) + http_method_names = ['get', 'post', 'put', 'delete'] + graph_manager = FilesystemGraphManager(GRAPH_BASE_PATH, GRAPH_TMP_PATH, GRAPH_ID_NAME_MAP_PATH) + + def post(self, request) -> Response: + graphs = request.data + for graph in graphs: + try: + self.graph_manager.write(graph) + except Exception as e: + return Response( + {"status": "ERROR", "msg": str(e)}, + status.HTTP_400_BAD_REQUEST + ) + return Response( + {"status": "SUCCESS"}, + status.HTTP_200_OK + ) + + def put(self, request: Request) -> Response: + graphs = request.data + for graph in graphs: + try: + self.graph_manager.update(graph) + except Exception as e: + return Response( + {"status": "ERROR", "msg": str(e)}, + status.HTTP_400_BAD_REQUEST + ) + return Response( + {"status": "SUCCESS"}, + status.HTTP_200_OK + ) + + def delete(self, request) -> Response: + ids = request.data + for id in ids: + try: + self.graph_manager.remove(id) + except Exception as e: + return Response( + {"status": "ERROR", "msg": str(e)}, + status.HTTP_400_BAD_REQUEST + ) + return Response( + {"status": "SUCCESS"}, + status.HTTP_200_OK + ) + + def get(self, request: Request) -> Response: + qs = dict(request.query_params) + if 'id' not in qs: + return Response(self.graph_manager.read_all(), status.HTTP_200_OK) + try: + graph_content = self.graph_manager.read(qs['id'][0]) + except Exception as e: + return Response({"status": "ERROR", "msg": str(e)}, status.HTTP_400_BAD_REQUEST) + return Response(graph_content, status.HTTP_200_OK) From 045484fecd4395fe021fc718dd51b54b97e5253e Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 12:40:38 +0300 Subject: [PATCH 07/25] added minor type annotation --- .../utils/filesystem_graphmanager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index 0619651..16bf73d 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -34,7 +34,7 @@ def read(self, graph_id) -> dict: except OSError: raise GraphManagerException(NO_GRAPH, graph_id) - def read_all(self): + def read_all(self) -> list: graph_list = [] with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) @@ -42,7 +42,7 @@ def read_all(self): graph_list.append({'graph_id': k, 'name': v['name']}) return graph_list - def write(self, graph: dict): + def write(self, graph: dict) -> None: shutil.copyfile(self.map_path, self.map_backup_path) with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) @@ -61,7 +61,7 @@ def write(self, graph: dict): os.mkdir(graph_dir) os.rename(self.tmp_path, Path(graph_dir / self.default_filename)) # atomic operation - def update(self, graph: dict): + def update(self, graph: dict) -> None: if 'graph_id' not in graph: raise GraphManagerException(NO_ID) if 'name' in graph: @@ -87,7 +87,7 @@ def update(self, graph: dict): file.write(graph["content"]) os.rename(self.tmp_path, Path(graph_dir) / self.default_filename) # atomic operation - def remove(self, graph_id): + def remove(self, graph_id) -> None: graph_dir = Path(self.final_path) / graph_id if not os.path.isdir(graph_dir): raise GraphManagerException(NO_GRAPH, graph_id) From bcddc78c21fa470025e845b1e94ec5ee4ae5d354 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 12:48:35 +0300 Subject: [PATCH 08/25] added one url pattern for graph --- complex_rest_dtcd_supergraph/urls.py | 68 ++-------------------------- 1 file changed, 5 insertions(+), 63 deletions(-) diff --git a/complex_rest_dtcd_supergraph/urls.py b/complex_rest_dtcd_supergraph/urls.py index 90decc2..6ddbdda 100644 --- a/complex_rest_dtcd_supergraph/urls.py +++ b/complex_rest_dtcd_supergraph/urls.py @@ -1,67 +1,9 @@ -from django.urls import path - -from .views import ( - DefaultRootFragmentDetailView, - DefaultRootFragmentGraphView, - DefaultRootFragmentListView, - DefaultRootGraphView, - ResetNeo4jView, - RootDetailView, - RootFragmentDetailView, - RootFragmentGraphView, - RootFragmentListView, - RootGraphView, - RootListView, -) +from django.urls import re_path +from .views import GraphView app_name = "supergraph" -urlpatterns = [ - # root management - path("roots", RootListView.as_view(), name="roots"), - path("roots/", RootDetailView.as_view(), name="root-detail"), - path("roots//graph", RootGraphView.as_view(), name="root-graph"), - # root fragments - path( - "roots//fragments", - RootFragmentListView.as_view(), - name="root-fragments", - ), - path( - "roots//fragments/", - RootFragmentDetailView.as_view(), - name="root-fragment-detail", - ), - path( - "roots//fragments//graph", - RootFragmentGraphView.as_view(), - name="root-fragment-graph", - ), - # services - path("reset", ResetNeo4jView.as_view(), name="reset"), -] -# backward API compatibility: fragment management for default root -# this will be deprecated soon -urlpatterns += [ - path( - "fragments", - DefaultRootFragmentListView.as_view(), - name="default-root-fragments", - ), - path( - "fragments/", - DefaultRootFragmentDetailView.as_view(), - name="default-root-fragment-detail", - ), - path( - "fragments//graph", - DefaultRootFragmentGraphView.as_view(), - name="default-root-fragment-graph", - ), - path( - "fragments/root/graph", - DefaultRootGraphView.as_view(), - name="default-root-graph", - ), -] +urlpatterns = [ + re_path(r'^graphContent/object/?$', GraphView.as_view()), +] \ No newline at end of file From c67efde159a474f24d558fbbe0c7ac3dffa9c440 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 12:48:56 +0300 Subject: [PATCH 09/25] updated init of the views --- complex_rest_dtcd_supergraph/views/__init__.py | 16 +--------------- complex_rest_dtcd_supergraph/views/graphs.py | 2 +- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/complex_rest_dtcd_supergraph/views/__init__.py b/complex_rest_dtcd_supergraph/views/__init__.py index 4f07c36..4566980 100644 --- a/complex_rest_dtcd_supergraph/views/__init__.py +++ b/complex_rest_dtcd_supergraph/views/__init__.py @@ -1,17 +1,3 @@ -from .fragments import ( - DefaultRootFragmentDetailView, - DefaultRootFragmentListView, - RootFragmentDetailView, - RootFragmentListView, -) from .graphs import ( - DefaultRootFragmentGraphView, - DefaultRootGraphView, - RootFragmentGraphView, - RootGraphView, -) -from .roots import ( - RootDetailView, - RootListView, + GraphView, ) -from .service import ResetNeo4jView diff --git a/complex_rest_dtcd_supergraph/views/graphs.py b/complex_rest_dtcd_supergraph/views/graphs.py index b407698..8872fd4 100644 --- a/complex_rest_dtcd_supergraph/views/graphs.py +++ b/complex_rest_dtcd_supergraph/views/graphs.py @@ -9,7 +9,7 @@ logger = logging.getLogger('supergraph') -class Graph(APIView): +class GraphView(APIView): permission_classes = (AllowAny,) http_method_names = ['get', 'post', 'put', 'delete'] graph_manager = FilesystemGraphManager(GRAPH_BASE_PATH, GRAPH_TMP_PATH, GRAPH_ID_NAME_MAP_PATH) From 28d8005ff7513987f1c42aef9d52f8e13c3c0461 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 13:14:06 +0300 Subject: [PATCH 10/25] removed neo4j support and usage --- complex_rest_dtcd_supergraph/converters.py | 149 ------ .../create_default_root.py | 55 -- complex_rest_dtcd_supergraph/exceptions.py | 19 - complex_rest_dtcd_supergraph/fields.py | 78 --- complex_rest_dtcd_supergraph/managers.py | 272 ---------- .../models/__init__.py | 10 +- complex_rest_dtcd_supergraph/models/nodes.py | 176 ------ .../models/relations.py | 21 - .../reinstall_labels.py | 90 ---- complex_rest_dtcd_supergraph/serializers.py | 156 ------ complex_rest_dtcd_supergraph/settings.py | 37 -- complex_rest_dtcd_supergraph/structures.py | 96 ---- complex_rest_dtcd_supergraph/urls.py | 2 +- complex_rest_dtcd_supergraph/utils.py | 112 ---- .../views/fragments.py | 135 ----- complex_rest_dtcd_supergraph/views/mixins.py | 38 -- complex_rest_dtcd_supergraph/views/roots.py | 84 --- complex_rest_dtcd_supergraph/views/service.py | 20 - .../views/shortcuts.py | 64 --- docs/scripts/database_init.sh | 18 - requirements/base.txt | 2 - tests/test_converters.py | 57 -- tests/test_fields.py | 46 -- tests/test_managers.py | 29 - tests/test_utils.py | 74 --- tests/test_views.py | 500 +----------------- 26 files changed, 4 insertions(+), 2336 deletions(-) delete mode 100644 complex_rest_dtcd_supergraph/converters.py delete mode 100644 complex_rest_dtcd_supergraph/create_default_root.py delete mode 100644 complex_rest_dtcd_supergraph/exceptions.py delete mode 100644 complex_rest_dtcd_supergraph/fields.py delete mode 100644 complex_rest_dtcd_supergraph/managers.py delete mode 100644 complex_rest_dtcd_supergraph/models/nodes.py delete mode 100644 complex_rest_dtcd_supergraph/models/relations.py delete mode 100644 complex_rest_dtcd_supergraph/reinstall_labels.py delete mode 100644 complex_rest_dtcd_supergraph/serializers.py delete mode 100644 complex_rest_dtcd_supergraph/structures.py delete mode 100644 complex_rest_dtcd_supergraph/utils.py delete mode 100644 complex_rest_dtcd_supergraph/views/fragments.py delete mode 100644 complex_rest_dtcd_supergraph/views/mixins.py delete mode 100644 complex_rest_dtcd_supergraph/views/roots.py delete mode 100644 complex_rest_dtcd_supergraph/views/service.py delete mode 100644 complex_rest_dtcd_supergraph/views/shortcuts.py delete mode 100755 docs/scripts/database_init.sh delete mode 100644 tests/test_converters.py delete mode 100644 tests/test_fields.py delete mode 100644 tests/test_managers.py diff --git a/complex_rest_dtcd_supergraph/converters.py b/complex_rest_dtcd_supergraph/converters.py deleted file mode 100644 index 7159170..0000000 --- a/complex_rest_dtcd_supergraph/converters.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -This module contains converter classes. -""" - -from copy import deepcopy -from operator import itemgetter -from typing import Iterable, Dict - -from .settings import KEYS -from .structures import Content, Edge, Group, Port, Vertex -from .utils import savable_as_property - - -class GraphDataConverter: - """Supports conversion between front-end data and internal classes.""" - - @staticmethod - def _extract_savable_properties(properties: Dict[str, dict]): - result = {} - - for name in properties: - data = properties[name] - value = data.get(KEYS.value) - - if value is not None and savable_as_property(value): - result[name] = data.pop(KEYS.value) - - return result - - @staticmethod - def _restore_properties(original: dict, properties: dict): - for name, value in properties.items(): - if name in original: # FIXME handle this elsewhere - original[name][KEYS.value] = value - - @staticmethod - def _get_ports(nodes: Iterable[dict]): - for node in nodes: - for port in node.get(KEYS.init_ports, []): - yield port - - def _to_vertex(self, data: dict): - meta = deepcopy(data) - uid = meta.pop(KEYS.yfiles_id) - properties = self._extract_savable_properties(meta.get(KEYS.properties, {})) - ports = meta.pop(KEYS.init_ports, []) # save only ids - port_ids = set(map(itemgetter(KEYS.yfiles_id), ports)) - - return Vertex(uid=uid, properties=properties, meta=meta, ports=port_ids) - - def _from_vertex(self, vertex: Vertex, id2port: dict): - data = deepcopy(vertex.meta) - data[KEYS.yfiles_id] = vertex.uid - - # FIXME workaround to handle stale properties on nodes; find better way - if KEYS.properties in data: - self._restore_properties(data[KEYS.properties], vertex.properties) - - ports = [id2port[port_id] for port_id in vertex.ports] - if ports: - data[KEYS.init_ports] = ports - - return data - - def _to_port(self, data: dict): - meta = deepcopy(data) - uid = meta.pop(KEYS.yfiles_id) - properties = self._extract_savable_properties(meta.get(KEYS.properties, {})) - - return Port(uid=uid, properties=properties, meta=meta) - - def _from_port(self, port: Port): - data = deepcopy(port.meta) - data[KEYS.yfiles_id] = port.uid - - # FIXME workaround to handle stale properties on nodes; find better way - if KEYS.properties in data: - self._restore_properties(data[KEYS.properties], port.properties) - - return data - - @staticmethod - def _to_edge(data: dict): - meta = deepcopy(data) - start = meta.pop(KEYS.source_port) - end = meta.pop(KEYS.target_port) - - return Edge(start=start, end=end, meta=meta) - - @staticmethod - def _from_edge(edge: Edge): - data = deepcopy(edge.meta) - data[KEYS.source_port] = edge.start - data[KEYS.target_port] = edge.end - - return data - - @staticmethod - def _to_group(data: dict): - meta = deepcopy(data) - uid = meta.pop(KEYS.yfiles_id) - - return Group(uid=uid, meta=meta) - - @staticmethod - def _from_group(group: Group): - data = deepcopy(group.meta) - data[KEYS.yfiles_id] = group.uid - - return data - - def _from_vertices_and_ports(self, content: Content): - ports = list(map(self._from_port, content.ports)) - id2port = {p[KEYS.yfiles_id]: p for p in ports} - nodes = [self._from_vertex(v, id2port) for v in content.vertices] - - return nodes - - def to_content(self, data: dict) -> Content: - """Convert graph data in specified format to content.""" - - # pre-condition: data is valid - nodes = data[KEYS.nodes] - vertices = list(map(self._to_vertex, nodes)) - ports = list(map(self._to_port, self._get_ports(nodes))) - edges = list(map(self._to_edge, data.get(KEYS.edges, []))) - groups = list(map(self._to_group, data.get(KEYS.groups, []))) - - return Content( - vertices=vertices, - ports=ports, - edges=edges, - groups=groups, - ) - - def to_data(self, content: Content) -> dict: - """Convert content to graph data in specified exchange format.""" - - nodes = self._from_vertices_and_ports(content) - edges = list(map(self._from_edge, content.edges)) - groups = list(map(self._from_group, content.groups)) - - result = { - KEYS.nodes: nodes, - KEYS.edges: edges, - KEYS.groups: groups, - } - - return result diff --git a/complex_rest_dtcd_supergraph/create_default_root.py b/complex_rest_dtcd_supergraph/create_default_root.py deleted file mode 100644 index 40f9b40..0000000 --- a/complex_rest_dtcd_supergraph/create_default_root.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Helper script creates the initial Root node and saves its UID into a -file "default_root_uid.txt". -""" - - -# FIXME make this relative? -from models import Root - - -def create_default_root_and_save_uid(data: dict, path) -> Root: - """Create Root node with the data and write its uid to given file.""" - - root = Root(**data).save() - - with open(path, "w") as f: - f.write(str(root.uid)) - - return root - - -if __name__ == "__main__": - import configparser - from pathlib import Path - import neomodel - - PROJECT_DIR = Path(__file__).parent - - # FIXME a little ugly - duplication - # main config - config_parser = configparser.ConfigParser(allow_no_value=True) - config_parser.read(PROJECT_DIR / "supergraph.conf") - ini_config = config_parser - # neomodel - # https://neomodel.readthedocs.io/en/latest/configuration.html - protocol = ini_config["neo4j"]["protocol"] - address = ini_config["neo4j"]["address"] - port = ini_config["neo4j"]["port"] - user = ini_config["neo4j"]["user"] - password = ini_config["neo4j"]["password"] - bolt_url = f"{protocol}://{user}:{password}@{address}:{port}" - - # Connect after to override any code in the module that may set the connection - print("Connecting to {}\n".format(bolt_url)) - neomodel.db.set_connection(bolt_url) - - # create default Root and save its UID - # FIXME same directory? /var/opt/complex_rest/plugins/supergraph? - filename = "default_root_uid.txt" - - root = create_default_root_and_save_uid( - data={"name": ini_config["schema"]["default_root_name"]}, - path=PROJECT_DIR / filename, - ) - print(f"Created {root} and saved its uid to {filename}.") diff --git a/complex_rest_dtcd_supergraph/exceptions.py b/complex_rest_dtcd_supergraph/exceptions.py deleted file mode 100644 index 20b6f15..0000000 --- a/complex_rest_dtcd_supergraph/exceptions.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Custom exceptions. -""" - -from rest_framework.exceptions import APIException - - -class LoadingError(APIException): - """Failed to convert data to content.""" - - status_code = 400 - default_detail = "Cannot convert data to content." - default_code = "error" - - -class ManagerError(APIException): - status_code = 400 - default_detail = "Manager error." - default_code = "error" diff --git a/complex_rest_dtcd_supergraph/fields.py b/complex_rest_dtcd_supergraph/fields.py deleted file mode 100644 index b795809..0000000 --- a/complex_rest_dtcd_supergraph/fields.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Custom fields. -""" - -import uuid - -from django.utils.translation import gettext_lazy as _ -from rest_framework.serializers import DictField, UUIDField - -from .settings import KEYS - - -class ContainsOrFailMixin: - default_error_messages = { - "key_error": _("Key '{value}' is missing."), - } - - def _contains_or_fail(self, data: dict, key): - if key not in data: - self.fail("key_error", value=key) - - -class CustomUUIDFIeld(UUIDField): - """A field that ensures the input is a valid UUID string. - - Overloads `to_representation` to work with UUIDs in hex string format. - """ - - def to_representation(self, value): - # if string, try to convert to UUID first - if isinstance(value, str): - try: - value = uuid.UUID(hex=value) - except: - self.fail("invalid", value=value) - - return super().to_representation(value) - - -class VertexField(ContainsOrFailMixin, DictField): - """A vertex dictionary representation. - - Validates the vertex to have an ID field. - """ - - id_key = KEYS.yfiles_id - - def to_internal_value(self, data: dict): - data = super().to_internal_value(data) - self._contains_or_fail(data, self.id_key) - - return data - - -class GroupField(VertexField): - """A group representation.""" - - -class EdgeField(ContainsOrFailMixin, DictField): - """An edge dictionary representation. - - Validates the edge to have start and end vertices and ports. - """ - - keys = ( - KEYS.source_node, - KEYS.target_node, - KEYS.source_port, - KEYS.target_port, - ) - - def to_internal_value(self, data): - data = super().to_internal_value(data) - - for key in self.keys: - self._contains_or_fail(data, key) - - return data diff --git a/complex_rest_dtcd_supergraph/managers.py b/complex_rest_dtcd_supergraph/managers.py deleted file mode 100644 index 09509cf..0000000 --- a/complex_rest_dtcd_supergraph/managers.py +++ /dev/null @@ -1,272 +0,0 @@ -""" -Custom managers designed to work with Neo4j database. - -These help us abstract management operations for graphs, fragments, etc. -and isolate details and complexity. -""" - -from collections import defaultdict -from dataclasses import dataclass -from itertools import chain -from typing import Iterable, List, Mapping, Sequence - -import neomodel - -from . import models -from . import structures -from .models.relations import RELATION_TYPES -from .utils import connect_if_not_connected, free_properties - - -def reconnect_to_container( - container: models.Container, - vertices: Iterable[models.Vertex], - groups: Iterable[models.Group], -): - """Reconnect merged entities to parent container.""" - - for vertex in vertices: - connect_if_not_connected(container.vertices, vertex) - - for group in groups: - connect_if_not_connected(container.groups, group) - - -class _Reader: - """Read operations on a container.""" - - def __init__(self) -> None: - self._foreign_key_mapping = defaultdict(set) # parent:children uid pairs - - def _query_ports(self, vertices: Iterable[models.Vertex]): - ports: List[models.Port] = [] - - for vertex in vertices: - for port in vertex.ports.all(): - ports.append(port) - self._foreign_key_mapping[vertex.uid].add(port.uid) - - return ports - - @staticmethod - def _to_primitive(node, subclass): - uid = node.uid - properties = free_properties(node) - meta = node.meta_ - - return subclass(uid=uid, properties=properties, meta=meta) - - def _to_vertex(self, node: models.Vertex): - vertex = self._to_primitive(node, subclass=structures.Vertex) - - # populate ports mappings - for child_id in self._foreign_key_mapping.get(vertex.uid, []): - vertex.ports.add(child_id) - - return vertex - - def read(self, container: models.Container) -> structures.Content: - self._foreign_key_mapping.clear() - - # step 1 - get the insides - vertices = container.vertices.all() - ports = self._query_ports(vertices) - edges = container.edges - groups = container.groups.all() - - # step 2 - map to content - vertices = [self._to_vertex(node) for node in vertices] - ports = [self._to_primitive(node, structures.Port) for node in ports] - edges = [ - structures.Edge(start=start.uid, end=end.uid, meta=edge.meta_) - for (start, edge, end) in edges - ] - groups = [self._to_primitive(node, structures.Group) for node in groups] - - return structures.Content( - vertices=vertices, - ports=ports, - edges=edges, - groups=groups, - ) - - -class _Deprecator: - """Deletes deprecated content of a container.""" - - @staticmethod - def _delete_deprecated_vertices_groups_ports( - container: models.Container, content: structures.Content - ): - """Delete vertices, groups and ports from the container not in the content.""" - - # query uids of primitive nodes (vertices, groups, ports) in the container - uid2node = {} - - for vertex in container.vertices.all(): - uid2node[vertex.uid] = vertex - - for port in vertex.ports.all(): - uid2node[port.uid] = port - - for group in container.groups.all(): - uid2node[group.uid] = group - - new_uids = set( - item.uid - for item in chain( - content.vertices, - content.ports, - content.groups, - ) - ) - deprecated_uids = set(uid2node) - new_uids - - for uid in deprecated_uids: - uid2node[uid].delete() - - return deprecated_uids - - @staticmethod - def _delete_deprecated_edges( - container: models.Container, content: structures.Content - ): - """Delete edges from the container not in the content.""" - - current_uids = set((op.uid, ip.uid) for op, _, ip in container.edges) - new_uids = set(edge.uid for edge in content.edges) - deprecated_uids = current_uids - new_uids - - neomodel.db.cypher_query( - query=( - "UNWIND $list AS pair " - "MATCH ({uid: pair[0]}) " - f" -[r:{RELATION_TYPES.edge}]-> " - "({uid: pair[1]}) " - "DELETE r" - ), - params={"list": list(map(list, deprecated_uids))}, - ) - - return deprecated_uids - - def delete_difference( - self, container: models.Container, content: structures.Content - ): - """Delete entities from the container that are not in the content.""" - - self._delete_deprecated_vertices_groups_ports(container, content) - self._delete_deprecated_edges(container, content) - - -class _Merger: - """Merges content entities.""" - - @dataclass(frozen=True) - class MergedResult: - """Lightweight container for merged entities.""" - - __slots__ = ["vertices", "ports", "edges", "groups"] - vertices: Sequence[models.Vertex] - ports: Sequence[models.Port] - edges: Sequence[models.EdgeRel] - groups: Sequence[models.Group] - - @staticmethod - def _merge_ports(ports: Iterable[structures.Port]): - # FIXME possible clash between user-defined property name and uid/meta_ key - data = [ - dict(uid=port.uid, meta_=port.meta, **port.properties) for port in ports - ] - - return models.Port.create_or_update(*data) - - @staticmethod - def _merge_edges( - edges: Iterable[structures.Edge], - uid2port: Mapping[structures.ID, models.Port], - ) -> List[models.EdgeRel]: - relations = [] - - for edge in edges: - output_port = uid2port[edge.start] - input_port = uid2port[edge.end] - rel = connect_if_not_connected(output_port.neighbor, input_port) - rel.meta_ = edge.meta # over-write metadata - rel.save() - relations.append(rel) - - return relations - - @staticmethod - def _merge_vertices( - vertices: Iterable[structures.Vertex], - uid2port: Mapping[structures.ID, models.Port], - ) -> List[models.Vertex]: - nodes = [] - - for vertex in vertices: - # FIXME possible clash between user-defined property name and uid/meta_ key - node = models.Vertex.create_or_update( - dict(uid=vertex.uid, meta_=vertex.meta, **vertex.properties), - lazy=True, - )[0] - nodes.append(node) - - # connect this vertex to ports - for uid in vertex.ports: - port = uid2port[uid] - connect_if_not_connected(node.ports, port) - - return nodes - - @staticmethod - def _merge_groups(groups: Iterable[structures.Group]): - data = [dict(uid=group.uid, meta_=group.meta) for group in groups] - - return models.Group.create_or_update(*data, lazy=True) - - def merge(self, content: structures.Content): - """Merge content entities.""" - - ports = self._merge_ports(content.ports) - uid2port = {port.uid: port for port in ports} - edges = self._merge_edges(content.edges, uid2port) - vertices = self._merge_vertices(content.vertices, uid2port) - groups = self._merge_groups(content.groups) - - return self.MergedResult( - vertices=vertices, - ports=ports, - edges=edges, - groups=groups, - ) - - -class Manager: - """Handles read and write operations on the container's content.""" - - def __init__(self) -> None: - self._reader = _Reader() - self._deprecator = _Deprecator() - self._merger = _Merger() - - def read(self, container: models.Container): - """Return the content of a given container.""" - - return self._reader.read(container) - - def replace(self, container: models.Container, content: structures.Content): - """Replace the content of a given container.""" - - # content pre-conditions (referential integrity within the content): - # - for each edge, start (output) & end (input) ports exist in content - # - for each vertex, all ports exist in content - self._deprecator.delete_difference(container, content) - result = self._merger.merge(content) # TODO does not replace old properties - reconnect_to_container(container, result.vertices, result.groups) - - def reconnect(self, parent: models.Container, child: models.Container): - """Reconnect the content of a child container to parent.""" - - reconnect_to_container(parent, child.vertices, child.groups) diff --git a/complex_rest_dtcd_supergraph/models/__init__.py b/complex_rest_dtcd_supergraph/models/__init__.py index 05a8ecc..8b13789 100644 --- a/complex_rest_dtcd_supergraph/models/__init__.py +++ b/complex_rest_dtcd_supergraph/models/__init__.py @@ -1,9 +1 @@ -from .nodes import ( - Container, - Fragment, - Group, - Port, - Root, - Vertex, -) -from .relations import EdgeRel + diff --git a/complex_rest_dtcd_supergraph/models/nodes.py b/complex_rest_dtcd_supergraph/models/nodes.py deleted file mode 100644 index d401943..0000000 --- a/complex_rest_dtcd_supergraph/models/nodes.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -Node classes for neomodel. - -The models here closely mirror classes from `structures` module. -""" - -from typing import List, Tuple - -from neomodel import ( - db, - JSONProperty, - Relationship, - RelationshipTo, - StringProperty, - StructuredNode, - UniqueIdProperty, -) -from neomodel.contrib import SemiStructuredNode - -from .relations import EdgeRel, RELATION_TYPES - - -# type aliases -CustomUniqueIdProperty = StringProperty # TODO better validation for IDs - - -class AbstractPrimitive(SemiStructuredNode): - """Abstract entity. - - May contain nested metadata in its `meta_` property, as well as - ad-hoc properties not defined here. - - We assume that ad-hoc properties are user-defined valid Neo4j - properties. Invalid ones are stored in `meta_`. - """ - - __abstract_node__ = True - - uid = CustomUniqueIdProperty(unique_index=True, required=True) - meta_ = JSONProperty() - - -class Port(AbstractPrimitive): - """A vertex port. - - An output port connects to an input port via the edge relationship. - """ - - neighbor = Relationship("Port", RELATION_TYPES.edge, model=EdgeRel) - - -class Vertex(AbstractPrimitive): - """A vertex coming from Y-files. - - Vertices have ports, through which they connect to other vertices. - """ - - # TODO explicit input and output ports? - ports = RelationshipTo(Port, RELATION_TYPES.default) - - def delete(self, cascade=True): - """Delete this vertex. - - If cascade is enabled, also delete all connected ports. - """ - - if cascade: - self.clear() - - return super().delete() - - def clear(self): - """Delete all connected ports.""" - - for port in self.ports.all(): - port.delete() - - -class Group(AbstractPrimitive): - """A group is a container for vertices or other groups. - - Front-end needs it to group objects. Currently it has no backend use. - """ - - -class Container(StructuredNode): - """A container for the content. - - May include vertices and groups. - """ - - uid = UniqueIdProperty() - name = StringProperty(max_length=255, required=True) # TODO settings - - vertices = RelationshipTo(Vertex, RELATION_TYPES.contains) - groups = RelationshipTo(Group, RELATION_TYPES.contains) - - def delete(self, cascade=True): - """Delete this container. - - If cascade is enabled, delete all related vertices and groups in - a cascading fashion. - """ - - if cascade: - self.clear() - - return super().delete() - - def clear(self): - """Delete all related vertices and groups in a cascading fashion.""" - - for vertex in self.vertices.all(): - vertex.delete(cascade=True) - - for group in self.groups.all(): - group.delete() - - @property - def edges(self) -> List[Tuple[Port, EdgeRel, Port]]: - """Return a list of tuples (start, edge, end) inside this container.""" - - q = ( - f"MATCH (this) WHERE id(this)={self.id} " - "MATCH (this) -- (:Vertex) " - f" -- (src:Port) -[r:{RELATION_TYPES.edge}]-> (dst:Port) " - "MATCH (dst) -- (:Vertex) -- (this) " - "RETURN src, r, dst" - ) - results, _ = db.cypher_query(q, resolve_objects=True) - - return [(r[0], r[1], r[2]) for r in results] - - -class Fragment(Container): - """Fragment is a container for primitives. - - A fragment may include vertices and groups. - We use fragments to partition the graph into regions for security - control and ease of work. - """ - - -class Root(Container): - """A root is a collection of fragments and content. - - Roots partition global Neo4j graph into non-overlapping subgraphs.""" - - fragments = RelationshipTo(Fragment, RELATION_TYPES.contains) - - def delete(self, cascade=True): - """Delete this root. - - Deletes all related fragments, vertices and groups in a cascading - fashion. - """ - - if cascade: - self.clear() - - return super().delete(cascade=False) - - def clear(self, content_only=False): - """Delete all related fragments, vertices and groups in a cascading fashion. - - If `content_only` is True, then only delete the content: - vertices and groups. - """ - - super().clear() - - if content_only: - return - - for fragment in self.fragments.all(): - fragment.delete(cascade=False) # already cleared related content diff --git a/complex_rest_dtcd_supergraph/models/relations.py b/complex_rest_dtcd_supergraph/models/relations.py deleted file mode 100644 index f383130..0000000 --- a/complex_rest_dtcd_supergraph/models/relations.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -Relationship classes for neomodel. -""" - -from types import SimpleNamespace - -from neomodel import JSONProperty, StructuredRel - - -# settings -RELATION_TYPES = SimpleNamespace() -RELATION_TYPES.contains = "CONTAINS" -RELATION_TYPES.default = "CONN" # TODO better name? -RELATION_TYPES.edge = "EDGE" - - -class EdgeRel(StructuredRel): - """An edge between the ports of vertices.""" - - # TODO this must be semi-structured too - meta_ = JSONProperty() diff --git a/complex_rest_dtcd_supergraph/reinstall_labels.py b/complex_rest_dtcd_supergraph/reinstall_labels.py deleted file mode 100644 index d10f354..0000000 --- a/complex_rest_dtcd_supergraph/reinstall_labels.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Helper script to re-install Neo4j constraints and indexes from current -models. -""" - -import argparse -import configparser -import sys -from importlib import import_module -from os import path -from pathlib import Path - -from neomodel import db, install_all_labels, remove_all_labels - - -def load_python_module_or_file(name): - # taken from neomodel source - # Is a file - if name.lower().endswith(".py"): - basedir = path.dirname(path.abspath(name)) - # Add base directory to pythonpath - sys.path.append(basedir) - module_name = path.basename(name)[:-3] - - else: # A module - # Add current directory to pythonpath - sys.path.append(path.abspath(path.curdir)) - - module_name = name - - if module_name.startswith("."): - pkg = module_name.split(".")[1] - else: - pkg = None - - import_module(module_name, package=pkg) - print("Loaded {}.".format(name)) - - -def main(): - parser = argparse.ArgumentParser( - description=""" - Setup indexes and constraints on labels in Neo4j for your neomodel schema. - - Reads database credentials from configuration file. - """ - ) - - parser.add_argument( - "apps", - type=str, - nargs="+", - help="python modules or files to load schema from.", - ) - - parser.add_argument( - "config", - type=Path, - help="path to configuration file", - ) - - args = parser.parse_args() - - # read database connection settings - config = configparser.ConfigParser(allow_no_value=True) - config.read(args.config) - protocol = config["neo4j"]["protocol"] - address = config["neo4j"]["address"] - port = config["neo4j"]["port"] - user = config["neo4j"]["user"] - password = config["neo4j"]["password"] - bolt_url = f"{protocol}://{user}:{password}@{address}:{port}" - - for app in args.apps: - load_python_module_or_file(app) - - # Connect after to override any code in the module that may set the connection - print("Connecting to {}\n".format(bolt_url)) - db.set_connection(bolt_url) - - try: - remove_all_labels() - except Exception: - pass - - install_all_labels() - - -if __name__ == "__main__": - main() diff --git a/complex_rest_dtcd_supergraph/serializers.py b/complex_rest_dtcd_supergraph/serializers.py deleted file mode 100644 index a4301f6..0000000 --- a/complex_rest_dtcd_supergraph/serializers.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Custom DRF serializers. -""" - -from itertools import chain -from operator import itemgetter -from types import SimpleNamespace - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from .fields import CustomUUIDFIeld, EdgeField, GroupField, VertexField -from .models import Container, Fragment, Root -from .settings import KEYS - - -class ContainerSerializer(serializers.Serializer): - id = CustomUUIDFIeld(read_only=True, source="uid") - name = serializers.CharField(max_length=255) # TODO value in settings - - container_class = Container - - def create(self, validated_data): - """Construct an instance and save it to the database.""" - - return self.container_class(**validated_data).save() - - def update(self, instance: container_class, validated_data: dict): - """Update the instance in the database.""" - - instance.name = validated_data["name"] - - return instance.save() - - def save(self, **kwargs) -> container_class: - """Create or update an instance in the database.""" - - return super().save(**kwargs) - - -class FragmentSerializer(ContainerSerializer): - container_class = Fragment - - -class RootSerializer(ContainerSerializer): - container_class = Root - - fragments = FragmentSerializer(read_only=True, source="fragments.all", many=True) - - -class ContentSerializer(serializers.Serializer): - default_error_messages = { - "does_not_exist": _("An entity with id [{value}] does not exist."), - "not_unique": _("Data contains non-unique, duplicated IDs."), - "self_reference": _("A group with id [{value}] has a self-reference."), - } - - keys = SimpleNamespace( - id=KEYS.yfiles_id, - src_node=KEYS.source_node, - tgt_node=KEYS.target_node, - src_port=KEYS.source_port, - tgt_port=KEYS.target_port, - parent_id=KEYS.parent_id, - init_ports=KEYS.init_ports, - ) - - nodes = serializers.ListField(child=VertexField()) - edges = serializers.ListField(child=EdgeField()) - groups = serializers.ListField(child=GroupField()) - - # TODO validate uniqueness of all IDs - - def validate_nodes(self, value): - ids = set(map(itemgetter(self.keys.id), value)) - - if len(ids) != len(value): - self.fail("not_unique") - - # TODO validate ports - - return value - - def validate_edges(self, value): - keys = ( - self.keys.src_node, - self.keys.tgt_node, - self.keys.src_port, - self.keys.tgt_port, - ) - ids = set(map(itemgetter(*keys), value)) - - if len(ids) != len(value): - self.fail("not_unique") - - return value - - def validate_groups(self, value): - # unique IDs - value = self.validate_nodes(value) - - # no self-reference - for obj in value: - id_ = obj[self.keys.id] - parent_id = obj.get(self.keys.parent_id) - - if parent_id == id_: - self.fail("self_reference", value=id_) - - return value - - def validate(self, data: dict): - self._validate_references(data) - self._validate_parent_groups_exist(data) - - return data - - def _validate_references(self, data: dict): - nodes = data["nodes"] - edges = data["edges"] - - # for each edge, make sure referred nodes really exist - node_ids = set(map(itemgetter(self.keys.id), nodes)) - for id_ in chain.from_iterable( - map(itemgetter(self.keys.src_node, self.keys.tgt_node), edges) - ): - if id_ not in node_ids: - self.fail("does_not_exist", value=id_) - - # for each edge, make sure referred ports really exist - port_ids = set() - for node in nodes: - for port in node.get(self.keys.init_ports, []): - port_ids.add(port.get(self.keys.id)) - - for id_ in chain.from_iterable( - map(itemgetter(self.keys.src_port, self.keys.tgt_port), edges) - ): - if id_ not in port_ids: - self.fail("does_not_exist", value=id_) - - def _validate_parent_groups_exist(self, data: dict): - # make sure parent groups exist for vertices and groups - groups = data.get("groups", []) - group_ids = set(map(itemgetter(self.keys.id), groups)) - nodes = data["nodes"] - - for obj in chain(groups, nodes): - parent_id = obj.get(self.keys.parent_id) - - if parent_id is not None and parent_id not in group_ids: - self.fail("does_not_exist", value=parent_id) - - -class GraphSerializer(serializers.Serializer): - graph = ContentSerializer() diff --git a/complex_rest_dtcd_supergraph/settings.py b/complex_rest_dtcd_supergraph/settings.py index 3c22d23..4323fdc 100644 --- a/complex_rest_dtcd_supergraph/settings.py +++ b/complex_rest_dtcd_supergraph/settings.py @@ -4,8 +4,6 @@ from pathlib import Path from types import SimpleNamespace -import neomodel - from core.settings.ini_config import merge_ini_config_with_defaults @@ -15,13 +13,6 @@ "logging": { "level": "INFO", }, - "neo4j": { - "protocol": "bolt", - "address": "localhost", - "port": 7687, - "user": "neo4j", - "password": "password", - }, } # main config @@ -43,23 +34,6 @@ KEYS.value = "value" KEYS.yfiles_id = "primitiveID" -# neomodel -# https://neomodel.readthedocs.io/en/latest/configuration.html -protocol = ini_config["neo4j"]["protocol"] -address = ini_config["neo4j"]["address"] -port = ini_config["neo4j"]["port"] -user = ini_config["neo4j"]["user"] -password = ini_config["neo4j"]["password"] -neomodel.config.DATABASE_URL = f"{protocol}://{user}:{password}@{address}:{port}" - -# DB schema -filename = "default_root_uid.txt" -path = PROJECT_DIR / filename -assert path.exists(), ( - f"Cannot find '{filename}' in this plugin's directory. " - "Have you forgot to initialize the database with initialize.py?" -) - # graphs configuration GRAPH_BASE_PATH = ini_config['graph']['base_path'] @@ -74,14 +48,3 @@ if not os.path.isdir(GRAPH_ID_NAME_MAP_PATH): os.mkdir(Path(GRAPH_ID_NAME_MAP_PATH)) - -# TODO users can delete default root node, so uid in text file becomes old -# TODO during the testing we reset the database after each test -with open(path) as f: - try: - DEFAULT_ROOT_UUID = uuid.UUID(f.read()) - except ValueError: - raise ValueError( - "Cannot convert Root uid to UUID. " - f"Please make sure ID in '{filename}' is a correct UUID." - ) diff --git a/complex_rest_dtcd_supergraph/structures.py b/complex_rest_dtcd_supergraph/structures.py deleted file mode 100644 index e760141..0000000 --- a/complex_rest_dtcd_supergraph/structures.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -This module contains classes from domain/business layer. - -For now, classes here represent intermediary step between presentation -layer and database-related activities. -""" - -from dataclasses import dataclass, field -from typing import Any, MutableMapping, MutableSet, MutableSequence - - -# custom types / aliases -ID = str - - -@dataclass -class Primitive: - """A primitive entity. - - Primitive entities have unique IDs and may contain user-defined - properties and metadata. - """ - - # TODO ABC? - uid: ID - properties: MutableMapping[str, Any] = field(default_factory=dict) - meta: MutableMapping[str, Any] = field(default_factory=dict) - - -@dataclass -class Port(Primitive): - """A vertex port. - - Vertices connect to one another through the ports. - """ - - -@dataclass -class Vertex(Primitive): - """A vertex coming from Y-files. - - Vertices contain user-defined properties and additional metadata - for front-end. - Vertices may have multiple ports, through which they connect - to other vertices. - """ - - ports: MutableSet[ID] = field(default_factory=set) - - -@dataclass -class Group(Primitive): - """A group is a container for vertices or other groups. - - Front-end needs it to group objects. Currently it has no backend use. - """ - - -@dataclass -class Edge: - """An edge between an output and an input ports of two vertices.""" - - start: ID # output port ID - end: ID # input port ID - meta: MutableMapping[str, Any] = field(default_factory=dict) - - def __post_init__(self): - self.uid = (self.start, self.end) - - -@dataclass -class Content: - """ - Represents graph content. - - We assume that incoming data is valid: referential integrity is ok, - all ids are unique, etc. - """ - - vertices: MutableSequence[Vertex] - ports: MutableSequence[Port] - edges: MutableSequence[Edge] - groups: MutableSequence[Group] - - @property - def info(self): - """Print basic statistics about the content.""" - - return ", ".join( - ( - f"{len(self.vertices)} vertices", - f"{len(self.ports)} ports", - f"{len(self.edges)} edges", - f"{len(self.groups)} groups", - ) - ) diff --git a/complex_rest_dtcd_supergraph/urls.py b/complex_rest_dtcd_supergraph/urls.py index 6ddbdda..523a548 100644 --- a/complex_rest_dtcd_supergraph/urls.py +++ b/complex_rest_dtcd_supergraph/urls.py @@ -5,5 +5,5 @@ app_name = "supergraph" urlpatterns = [ - re_path(r'^graphContent/object/?$', GraphView.as_view()), + re_path(r'^complex_rest_dtcd_supergraph/v1/graph/?$', GraphView.as_view()), ] \ No newline at end of file diff --git a/complex_rest_dtcd_supergraph/utils.py b/complex_rest_dtcd_supergraph/utils.py deleted file mode 100644 index 2905029..0000000 --- a/complex_rest_dtcd_supergraph/utils.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -This module provides custom utility functions. -""" - -import uuid -from typing import Sequence - -import neomodel -from neomodel import contrib - - -# allowed property types in neo4j -# see https://neo4j.com/docs/cypher-manual/current/syntax/values/#property-types -PROPERTY_TYPES = (int, float, str, bool) - - -# custom Django path converters -# https://docs.djangoproject.com/en/4.0/topics/http/urls/#registering-custom-path-converters -class HexUUIDConverter: - """Matches/converts a UUID from/to a string of 32 hexadecimal digits.""" - - regex = "[0-9a-f]{32}" - - def to_python(self, value: str): - return uuid.UUID(value) - - def to_url(self, value: str): - return str(value) - - -def free_properties(node: contrib.SemiStructuredNode): - """Return a dictionary with ad-hoc properties for a given node. - - Ad-hoc properties are those not specified at node's definition. - """ - - # see SemiStructuredNode.inflate, NodeMeta and PropertyManager - defined: dict = node.defined_properties(aliases=False, rels=False) - existing: dict = node.__properties__ - free = set(existing) - set(defined) - {"id"} - - return {key: existing[key] for key in free} - - -def save_properties(properties: dict, node: contrib.SemiStructuredNode): - """Save properties dictionary on a given node.""" - - for key, val in properties.items(): - setattr(node, key, val) - - return node - - -def connect_if_not_connected( - manager: neomodel.RelationshipManager, - node: neomodel.StructuredNode, - properties: dict = None, -) -> neomodel.StructuredRel: - """Use the relationship manager to connect a node. - - If the connection exists, return it. Otherwise, create new relation - with the given properties and return it. - """ - - if not manager.is_connected(node): - return manager.connect(node, properties) - else: - return manager.relationship(node) - - -def valid_property(value) -> bool: - """ - Return `True` if the value is a valid Neo4j property, `False` otherwise. - """ - - return isinstance(value, PROPERTY_TYPES) - - -def homogeneous(seq: Sequence) -> bool: - """Return `True` if all items in a sequence have the same type, - `False` otherwise.""" - - if len(seq) == 0: - return True - - t = type(seq[0]) - - return all(type(item) is t for item in seq) - - -def savable_as_property(value) -> bool: - """Return `True` if the value can be stored as Neo4j property, - `False` otherwise. - - See https://neo4j.com/docs/cypher-manual/current/syntax/values/ for - more information on property types. - """ - - # valid property - if valid_property(value): - return True - - # homogeneous lists of valid properties - if ( - isinstance(value, list) - and all(map(valid_property, value)) - and homogeneous(value) - ): - return True - - # anything else is invalid - return False diff --git a/complex_rest_dtcd_supergraph/views/fragments.py b/complex_rest_dtcd_supergraph/views/fragments.py deleted file mode 100644 index 85d0942..0000000 --- a/complex_rest_dtcd_supergraph/views/fragments.py +++ /dev/null @@ -1,135 +0,0 @@ -""" -Views for fragment management operations. - -Fragments belong to roots. -""" - -import uuid - -import neomodel -from rest_framework import status -from rest_framework.request import Request - -from rest.permissions import AllowAny -from rest.response import SuccessResponse -from rest.views import APIView - -from .. import settings -from ..models import Fragment, Root -from ..serializers import FragmentSerializer -from .shortcuts import get_node_or_404 - - -def get_fragment_from_root_or_404( - root_pk: uuid.UUID, fragment_pk: uuid.UUID -) -> Fragment: - root = get_node_or_404(Root.nodes, uid=root_pk.hex) - fragment = get_node_or_404(root.fragments, uid=fragment_pk.hex) - - return fragment - - -class RootFragmentListView(APIView): - """List existing fragments of a root or create a new one.""" - - http_method_names = ["get", "post"] - permission_classes = (AllowAny,) - serializer_class = FragmentSerializer - - @neomodel.db.transaction - def get(self, request: Request, pk: uuid.UUID): - """Read a list of root's fragments.""" - - root = get_node_or_404(Root.nodes, uid=pk.hex) - fragments = root.fragments.all() - serializer = self.serializer_class(fragments, many=True) - - return SuccessResponse({"fragments": serializer.data}) - - @neomodel.db.transaction - def post(self, request: Request, pk: uuid.UUID): - """Create a new fragment for this root.""" - - root = get_node_or_404(Root.nodes, uid=pk.hex) - # create a fragment - serializer = self.serializer_class(data=request.data) - serializer.is_valid(raise_exception=True) - fragment = serializer.save() - # connect root to this fragment - root.fragments.connect(fragment) - - return SuccessResponse( - data={"fragment": serializer.data}, - http_status=status.HTTP_201_CREATED, - ) - - -class DefaultRootFragmentListView(RootFragmentListView): - """List existing fragments of the default root or create a new one.""" - - pk = settings.DEFAULT_ROOT_UUID - - def get(self, request: Request): - """Read a list of default root's fragments.""" - - return super().get(request, self.pk) - - def post(self, request: Request): - """Create a new fragment for the default root.""" - - return super().post(request, self.pk) - - -class RootFragmentDetailView(APIView): - """Retrieve, update or delete this root's fragment.""" - - http_method_names = ["get", "put", "delete"] - permission_classes = (AllowAny,) - serializer_class = FragmentSerializer - - @neomodel.db.transaction - def get(self, request: Request, root_pk: uuid.UUID, fragment_pk: uuid.UUID): - """Return root's fragment.""" - - fragment = get_fragment_from_root_or_404(root_pk, fragment_pk) - serializer = self.serializer_class(fragment) - - return SuccessResponse({"fragment": serializer.data}) - - @neomodel.db.transaction - def put(self, request: Request, root_pk: uuid.UUID, fragment_pk: uuid.UUID): - """Update this root's fragment.""" - - old = get_fragment_from_root_or_404(root_pk, fragment_pk) - serializer = self.serializer_class(old, data=request.data) - serializer.is_valid(raise_exception=True) - serializer.save() - - return SuccessResponse({"fragment": serializer.data}) - - @neomodel.db.transaction - def delete(self, request: Request, root_pk: uuid.UUID, fragment_pk: uuid.UUID): - """Delete this root's fragment and its content.""" - - fragment = get_fragment_from_root_or_404(root_pk, fragment_pk) - fragment.delete() - - return SuccessResponse() - - -class DefaultRootFragmentDetailView(RootFragmentDetailView): - """Retrieve, update or delete default root's fragment.""" - - root_pk = settings.DEFAULT_ROOT_UUID - - def get(self, request: Request, pk: uuid.UUID): - fragment_pk = pk - return super().get(request, self.root_pk, fragment_pk) - - def put(self, request: Request, pk: uuid.UUID): - fragment_pk = pk - return super().put(request, self.root_pk, fragment_pk) - - def delete(self, request: Request, pk: uuid.UUID): - fragment_pk = pk - return super().delete(request, self.root_pk, fragment_pk) diff --git a/complex_rest_dtcd_supergraph/views/mixins.py b/complex_rest_dtcd_supergraph/views/mixins.py deleted file mode 100644 index 9997a61..0000000 --- a/complex_rest_dtcd_supergraph/views/mixins.py +++ /dev/null @@ -1,38 +0,0 @@ -import logging - -from .shortcuts import to_content_or_400, replace_or_400 - -logger = logging.getLogger("supergraph") - - -class ContainerManagementMixin: - """Provides `read` and `replace` methods for working with container - and converting back and forth between domain classes and Python primitives. - """ - - converter = None - manager = None - - def read(self, container) -> dict: - """Read container's content as Python primitives in correct format. - - 1. Uses `manager` to query Neo4j database and get `Content` back. - 2. Uses `converter` to convert the `Content` into Python primitives. - """ - - content = self.manager.read(container) - logger.info("Queried content: " + content.info) - data = self.converter.to_data(content) - - return data - - def replace(self, container, data: dict): - """Replace container's content with new one. - - 1. Uses `converter` to convert incoming data to `Content`. - 2. Uses `manager` that maps from `Content` to Neo4j database entities. - """ - - new_content = to_content_or_400(self.converter, data) - logger.info("Converted to content: " + new_content.info) - replace_or_400(self.manager, container, new_content) diff --git a/complex_rest_dtcd_supergraph/views/roots.py b/complex_rest_dtcd_supergraph/views/roots.py deleted file mode 100644 index 70bf9b5..0000000 --- a/complex_rest_dtcd_supergraph/views/roots.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Views for root management operations. -""" - -import uuid - -import neomodel -from rest_framework import status -from rest_framework.request import Request - -from rest.permissions import AllowAny -from rest.response import SuccessResponse -from rest.views import APIView - -from ..models import Root -from ..serializers import RootSerializer -from .shortcuts import get_node_or_404 - - -class RootListView(APIView): - """List existing roots or create a new one.""" - - http_method_names = ["get", "post"] - permission_classes = (AllowAny,) - serializer_class = RootSerializer - - @neomodel.db.transaction - def get(self, request: Request): - """Read a list of existing roots.""" - - roots = list(Root.nodes) - serializer = self.serializer_class(roots, many=True) - - return SuccessResponse({"roots": serializer.data}) - - @neomodel.db.transaction - def post(self, request: Request): - """Create a new root.""" - - serializer = self.serializer_class(data=request.data) - serializer.is_valid(raise_exception=True) - serializer.save() - - return SuccessResponse( - data={"root": serializer.data}, - http_status=status.HTTP_201_CREATED, - ) - - -class RootDetailView(APIView): - """Retrieve, update or delete a root.""" - - http_method_names = ["get", "put", "delete"] - permission_classes = (AllowAny,) - serializer_class = RootSerializer - - @neomodel.db.transaction - def get(self, request: Request, pk: uuid.UUID): - """Return a root.""" - - root = get_node_or_404(Root.nodes, uid=pk.hex) - serializer = self.serializer_class(root) - - return SuccessResponse({"root": serializer.data}) - - @neomodel.db.transaction - def put(self, request: Request, pk: uuid.UUID): - """Update a fragment.""" - - old = get_node_or_404(Root.nodes, uid=pk.hex) - serializer = self.serializer_class(old, data=request.data) - serializer.is_valid(raise_exception=True) - serializer.save() - - return SuccessResponse({"root": serializer.data}) - - @neomodel.db.transaction - def delete(self, request: Request, pk: uuid.UUID): - """Delete a fragment and its content.""" - - root = get_node_or_404(Root.nodes, uid=pk.hex) - root.delete() - - return SuccessResponse() diff --git a/complex_rest_dtcd_supergraph/views/service.py b/complex_rest_dtcd_supergraph/views/service.py deleted file mode 100644 index 44301d1..0000000 --- a/complex_rest_dtcd_supergraph/views/service.py +++ /dev/null @@ -1,20 +0,0 @@ -import neomodel - -from rest.permissions import AllowAny -from rest.response import SuccessResponse -from rest.views import APIView - - -class ResetNeo4jView(APIView): - """A view to reset Neo4j database.""" - - http_method_names = ["post"] - permission_classes = (AllowAny,) - - @neomodel.db.transaction - def post(self, request, *args, **kwargs): - """Delete all nodes and relationships from Neo4j database.""" - - neomodel.clear_neo4j_database(neomodel.db) - - return SuccessResponse() diff --git a/complex_rest_dtcd_supergraph/views/shortcuts.py b/complex_rest_dtcd_supergraph/views/shortcuts.py deleted file mode 100644 index 9dd7f7d..0000000 --- a/complex_rest_dtcd_supergraph/views/shortcuts.py +++ /dev/null @@ -1,64 +0,0 @@ -import logging -from typing import Union - -import neomodel -from rest_framework.exceptions import NotFound - -from ..exceptions import LoadingError, ManagerError - - -logger = logging.getLogger("supergraph") - - -# shortcuts for working with Neomodel -# a la https://docs.djangoproject.com/en/4.0/topics/http/shortcuts/ -def get_node_or_404( - queryset: Union[neomodel.NodeSet, neomodel.RelationshipManager], **kwargs -) -> neomodel.StructuredNode: - """Call `.get()` on a given node set or relationship manager. - - Raises `rest_framework.exceptions.NotFound` if a node is missing. - """ - - try: - return queryset.get(**kwargs) - except neomodel.DoesNotExist: - raise NotFound - - -def func_or_400(func, *args, exception=None, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - logger.error("Error: \n" + str(e)) - raise exception - - -def to_content_or_400(converter, data): - """Try to use the converter to convert the data to content. - - Calls `converter.to_content(data)` and returns the result. Raises - `LoadingError` on exception and logs it. - """ - - # FIXME too broad of an exception - try: - return converter.to_content(data) - except Exception as e: - logger.error("Loading error: \n" + str(e)) - raise LoadingError - - -def replace_or_400(manager, container, new_content): - """Try to use the manager to replace the content of a container with new one. - - Calls `manager.replace(container, content)` and returns the result. - Raises `Manager` on exception and logs it. - """ - - # FIXME too broad of an exception - try: - return manager.replace(container, new_content) - except Exception as e: - logger.error("Manager error: \n" + str(e)) - raise ManagerError diff --git a/docs/scripts/database_init.sh b/docs/scripts/database_init.sh deleted file mode 100755 index 1c833bf..0000000 --- a/docs/scripts/database_init.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Creates the initial Root node and saves its UID -# into a text file "default_root_uid.txt" in plugin's directory. - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR - -# use this plugin's venv -python="venv/bin/python" - -# re-install labels -apps="models" -conf="supergraph.conf" -$python reinstall_labels.py $apps $conf - -# initialize Root -$python create_default_root.py diff --git a/requirements/base.txt b/requirements/base.txt index e530b1a..e69de29 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,2 +0,0 @@ -neo4j-driver==4.3.6 -neomodel==4.0.8 \ No newline at end of file diff --git a/tests/test_converters.py b/tests/test_converters.py deleted file mode 100644 index 3a0285e..0000000 --- a/tests/test_converters.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -import unittest -from pathlib import Path - -from django.test import SimpleTestCase - -from complex_rest_dtcd_supergraph.converters import GraphDataConverter - -from .misc import load_data, sort_payload - -TEST_DIR = Path(__file__).resolve().parent -DATA_DIR = TEST_DIR / "data" - - -class TestGraphDataConverter(SimpleTestCase): - converter = GraphDataConverter() - - def _check_to_content_to_data(self, data): - sort_payload(data) - content = self.converter.to_content(data) - exported = self.converter.to_data(content) - sort_payload(exported) - self.assertEqual(exported, data) - - def test_to_content(self): - data = load_data(DATA_DIR / "basic.json") - content = self.converter.to_content(data) - - self.assertEqual(len(content.vertices), 2) - self.assertEqual(len(content.ports), 2) - self.assertEqual(len(content.edges), 1) - self.assertEqual(len(content.groups), 0) - - def test_to_data(self): - # TODO - pass - - def _check_to_content_to_data_from_json(self, path): - with open(path) as f: - data = json.load(f) - self._check_to_content_to_data(data) - - def test_to_content_to_data_small(self): - self._check_to_content_to_data_from_json(DATA_DIR / "graph-sample-small.json") - - def test_to_content_to_data_n25_e25(self): - self._check_to_content_to_data_from_json(DATA_DIR / "n25_e25.json") - - def test_to_content_to_data_n50_e25(self): - self._check_to_content_to_data_from_json(DATA_DIR / "n50_e25.json") - - def test_to_content_to_data_large(self): - self._check_to_content_to_data_from_json(DATA_DIR / "graph-sample-large.json") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_fields.py b/tests/test_fields.py deleted file mode 100644 index 9561988..0000000 --- a/tests/test_fields.py +++ /dev/null @@ -1,46 +0,0 @@ -import unittest - -from django.test import SimpleTestCase -from rest_framework.validators import ValidationError - -from complex_rest_dtcd_supergraph.fields import EdgeField, VertexField - -from .misc import KEYS - - -class TestVertexField(SimpleTestCase): - def test_invalid(self): - # missing ID key - data = {"spam": 42} - field = VertexField() - - with self.assertRaises(ValidationError): - field.to_internal_value(data) - - -class TestEdgeField(SimpleTestCase): - def test_invalid(self): - data = {"spam": 42} - field = EdgeField() - - with self.assertRaises(ValidationError): - field.to_internal_value(data) - - # missing: end vertex, start/end ports - data[KEYS.source_node] = "n1" - with self.assertRaises(ValidationError): - field.to_internal_value(data) - - # missing: start/end ports - data[KEYS.target_node] = "n2" - with self.assertRaises(ValidationError): - field.to_internal_value(data) - - # missing: end port - data[KEYS.source_port] = "p1" - with self.assertRaises(ValidationError): - field.to_internal_value(data) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_managers.py b/tests/test_managers.py deleted file mode 100644 index c67546f..0000000 --- a/tests/test_managers.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -from pathlib import Path - -from django.test import SimpleTestCase, tag - -# TODO import here causes a strange error: RelationshipClassRedefined -# something with how neomodel builds up the registry & django runs the tests? -# from complex_rest_dtcd_supergraph.managers import Manager - - -TEST_DIR = Path(__file__).resolve().parent - - -@tag("neo4j") -class TestManager(SimpleTestCase): - @classmethod - def setUpClass(cls) -> None: - pass # clear the db - - @classmethod - def tearDownClass(cls) -> None: - pass - - def tearDown(self) -> None: - pass - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py index df0e307..10e6159 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,75 +1 @@ import unittest - -from complex_rest_dtcd_supergraph.utils import ( - homogeneous, - savable_as_property, - valid_property, -) - - -class TestUtils(unittest.TestCase): - def test_valid_property(self): - # int - value = 42 - self.assertTrue(valid_property(value)) - - # invalid: list - value = [1, 2, 3] - self.assertFalse(valid_property(value)) - - # invalid: dict - value = {"age": 42} - self.assertFalse(valid_property(value)) - - def test_is_homogeneous(self): - seq = (1, 2, 3) - self.assertTrue(homogeneous(seq)) - - seq = (1.0, 2.4, 3.1) - self.assertTrue(homogeneous(seq)) - - seq = ("ham", "spam") - self.assertTrue(homogeneous(seq)) - - # empty sequence - seq = [] - self.assertTrue(homogeneous(seq)) - - # one member - seq = (1,) - self.assertTrue(homogeneous(seq)) - - # non-homogeneous - seq = (1, 2.5, "spam") - self.assertFalse(homogeneous(seq)) - - def test_savable_as_property(self): - # valid - # int - value = 42 - self.assertTrue(savable_as_property(value)) - - # empty list - value = [] - self.assertTrue(savable_as_property(value)) - - # list of ints - value = [1, 2, 3] - self.assertTrue(savable_as_property(value)) - - # list of strs - value = ["ham", "spam"] - self.assertTrue(savable_as_property(value)) - - # invalid - # bad value type - value = {"age": 42} - self.assertFalse(savable_as_property(value)) - - # list of non-homogeneous properties - value = ["ham", 1, True] - self.assertFalse(savable_as_property(value)) - - # list of bad types - value = [{"age": 42}, {"age": 17}] - self.assertFalse(savable_as_property(value)) diff --git a/tests/test_views.py b/tests/test_views.py index 94576b2..f375ce4 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -4,7 +4,6 @@ from types import SimpleNamespace import dictdiffer -import neomodel from django.urls import reverse from django.test import Client, tag from rest_framework import status @@ -12,20 +11,13 @@ from .misc import load_data, sort_payload - TEST_DIR = Path(__file__).resolve().parent -DATA_DIR = TEST_DIR / "data" -URL_RESET = reverse("supergraph:reset") # post here resets the db CLIENT = Client() # DEBUG DEBUG_FILEPATH = "debug.txt" -def reset_db(): - neomodel.clear_neo4j_database(neomodel.db) - - class APITestCaseMixin: """Some common attributes and methods for API tests. @@ -37,7 +29,8 @@ class APITestCaseMixin: get=status.HTTP_200_OK, post=status.HTTP_201_CREATED, put=status.HTTP_200_OK, # or 204; 201 if created - delete=status.HTTP_200_OK, # 202 on queue, 204 on no content + delete=status.HTTP_200_OK, # 202 on queue, 204 on noDATA_DIR = TEST_DIR / "data" + URL_RESET=reverse("supergraph:reset") # post here resets the db content ) url = None # the URL endpoint to check @@ -91,494 +84,5 @@ def _test_delete(self): return r1, r2 -class Neo4jTestCaseMixin: - """A mixin for API tests of a Neo4j-based endpoint. - - Adds calls to reset the database on class setup and on teardowns of each test. - """ - - @classmethod - def setUpClass(cls) -> None: - # clean db on start - reset_db() - - @classmethod - def tearDownClass(cls) -> None: - pass - - def tearDown(self) -> None: - # clean after each test - reset_db() - - -@tag("neo4j") -class TestRootListView(Neo4jTestCaseMixin, APITestCaseMixin, APISimpleTestCase): - url = reverse("supergraph:roots") - - def test_post(self): - name = "sales" - response = self.post(data={"name": name}) - obj = response.data["root"] - self.assertIn("id", obj) - self.assertEqual(obj["name"], name) - - def test_get(self): - names = {"hr", "marketing", "sales"} - for name in names: - self.post(data={"name": name}) - response = self.get() - data = response.data - objects = data["roots"] - self.assertEqual({item["name"] for item in objects}, names) - - @classmethod - def create(cls, data: dict) -> dict: - """Send `POST` request to create a root and return it.""" - - response = CLIENT.post(cls.url, data=data, format="json") - return response.data["root"] - - -@tag("neo4j") -class TestRootDetailView(Neo4jTestCaseMixin, APITestCaseMixin, APISimpleTestCase): - root_name = "sales" - - def setUp(self) -> None: - """Create a root object to work with using the given name. - - Creates a root object and saves it to `self.root`. Root ID is - available on `self.pk`, and URL to this object at `self.url`. - """ - - # default root - self.root = TestRootListView.create({"name": self.root_name}) - self.pk = self.root["id"] - self.url = reverse("supergraph:root-detail", args=(self.pk,)) - - def test_get(self): - response = self.get() - obj = response.data["root"] - self.assertEqual(obj["id"], self.pk) - self.assertEqual(obj["name"], self.root_name) - - def test_put(self): - data = {"name": "marketing"} - response = self.put(data=data) - # get detail back with the same pk - response = self.get() - obj = response.data["root"] - self.assertEqual(obj["name"], "marketing") - - def test_delete(self): - self._test_delete() - - -class TestRootFragmentListView(Neo4jTestCaseMixin, APITestCaseMixin, APISimpleTestCase): - root_name = "parent" - - def setUp(self) -> None: - """Create a root object to work with. - - Sets `self.url` to `/roots//fragments`. For the rest of the - attributes see `TestRootDetailView.setUp()`. - """ - - # create default root to work with - TestRootDetailView.setUp(self) - self.url = reverse("supergraph:root-fragments", args=(self.pk,)) - - def test_post(self): - name = "child" - response = self.post(data={"name": name}) - obj = response.data["fragment"] - self.assertIn("id", obj) - self.assertEqual(obj["name"], name) - - def test_get(self): - names = {"hr", "marketing", "sales"} - for name in names: - self.post(data={"name": name}) - response = self.get() - data = response.data - objects = data["fragments"] - self.assertEqual({item["name"] for item in objects}, names) - - -class TestRootFragmentDetailView( - Neo4jTestCaseMixin, APITestCaseMixin, APISimpleTestCase -): - root_name = "parent" - fragment_name = "child" - - def setUp(self) -> None: - """Create a root and a fragment for it using given names. - - The root is at `self.root`. Root ID is - available on `self.root_pk` and the URL is at `self.root_url`. - - The fragment for this root is at - `self.fragment`. Fragment ID is available on `self.fragment_pk`, - and a full URL is at `self.fragment_url`. - - The `self.url` attribute points to `self.fragment_url` since we - work with it in this suite of tests. - """ - - # create the default root - TestRootDetailView.setUp(self) - self.root_pk = self.pk - self.root_url = self.url # URL: roots/ - self.pk = self.url = None # reset - - # create a fragment for this root - endpoint = reverse( - "supergraph:root-fragments", args=(self.root_pk,) - ) # URL: roots//fragments - response = self.client.post( # note explicit request - endpoint, - data={"name": self.fragment_name}, - format="json", - ) - self.fragment = response.data["fragment"] - self.fragment_pk = self.fragment["id"] - - self.fragment_url = reverse( - "supergraph:root-fragment-detail", args=(self.root_pk, self.fragment_pk) - ) - self.url = self.fragment_url - - def test_get(self): - response = self.get() - obj = response.data["fragment"] - self.assertEqual(obj["id"], self.fragment_pk) - self.assertEqual(obj["name"], self.fragment_name) - - def test_put(self): - data = {"name": "marketing"} - response = self.put(data=data) - # get detail back with the same pk - response = self.get() - obj = response.data["fragment"] - self.assertEqual(obj["name"], "marketing") - - def test_delete(self): - self._test_delete() - - -@unittest.skip("deprecated") -@unittest.expectedFailure # DB reset deletes default root -@tag("neo4j") -class TestFragmentListView(Neo4jTestCaseMixin, APISimpleTestCase): - url = reverse("supergraph:default-root-fragments") - - def test_post(self): - response = self.client.post(self.url, data={"name": "sales"}, format="json") - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - fragment = response.data["fragment"] - self.assertIn("id", fragment) - self.assertEqual(fragment["name"], "sales") - - def test_get(self): - names = {"hr", "marketing", "sales"} - for name in names: - self.client.post(self.url, data={"name": name}, format="json") - response = self.client.get(self.url) - data = response.data - fragments = data["fragments"] - self.assertEqual({f["name"] for f in fragments}, names) - - -@unittest.skip("deprecated") -@unittest.expectedFailure # DB reset deletes default root -@tag("neo4j") -class TestFragmentDetailView(Neo4jTestCaseMixin, APISimpleTestCase): - def setUp(self) -> None: - # default fragment - response = self.client.post( - TestFragmentListView.url, - data={"name": "sales"}, - format="json", - ) - self.fragment = response.data["fragment"] - self.pk = self.fragment["id"] - self.url = reverse("supergraph:default-root-fragment-detail", args=(self.pk,)) - - def test_get(self): - response = self.client.get(self.url) - self.assertEqual(response.status_code, status.HTTP_200_OK) - fragment = response.data["fragment"] - self.assertEqual(fragment["id"], self.pk) - self.assertEqual(fragment["name"], "sales") - - def test_put(self): - data = {"name": "marketing"} - response = self.client.put(self.url, data=data, format="json") - self.assertEqual(response.status_code, status.HTTP_200_OK) - # get detail back with the same pk - response = self.client.get(self.url) - self.assertEqual(response.status_code, status.HTTP_200_OK) - fragment = response.data["fragment"] - self.assertEqual(fragment["name"], "marketing") - - def test_delete(self): - response = self.client.delete(self.url) - self.assertEqual(response.status_code, status.HTTP_200_OK) - response = self.client.get(self.url) - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - - -class GraphEndpointTestCaseMixin: - """Common functionality for testing graph management endpoints. - - See docs or `GraphSerializer` for data input/output format. - """ - - def assert_graph_eq(self, new: dict, original: dict): - """Assert that new graph data is equal to original one. - - Calls `.assertEqual(new, original)`. - Graph data must be sorted. - Logs the difference in a debug file. - """ - - try: - self.assertEqual(new, original) - except Exception: - # log the difference - diff = dictdiffer.diff(original, new) # TODO sort it? - diff = list(diff) - msg = pformat(diff, depth=4, compact=True) - msg = "Graphs do not match. Difference:\n" + msg - # TODO better error logging? - with open(DEBUG_FILEPATH, "w") as f: - f.write(msg) - - raise - - def merge(self, data: dict, url): - """Merge new graph data at the given endpoint.""" - - data = {"graph": data} - return self.client.put(url, data=data, format="json") - - def retrieve(self, url) -> dict: - """Retrieve existing graph data from the given endpoint. - - The data is sorted. - """ - - r = self.client.get(url) - data = r.data["graph"] - sort_payload(data) - - return data - - def assert_merge_retrieve_eq(self, data: dict, url): - """Merge given graph data at the given URL, then get the result - back and and assert both of those are equal. - """ - - sort_payload(data) - self.merge(data, url) - fromdb = self.retrieve(url) - self.assert_graph_eq(fromdb, data) - - def assert_merge_retrieve_eq_from_json(self, path, url): - """Load graph data from given path (JSON), merge it, get back - the result and assert both of those are equal. - """ - - data = load_data(path) - self.assert_merge_retrieve_eq(data, url) - - -@tag("neo4j") -class TestRootGraphView( - GraphEndpointTestCaseMixin, - Neo4jTestCaseMixin, - APITestCaseMixin, - APISimpleTestCase, -): - root_name = "sales" - - def setUp(self) -> None: - """Create a root object to work with. - - Sets `self.url` to `roots//graph`. - """ - - # create default root to work with - TestRootDetailView.setUp(self) - self.url = reverse("supergraph:root-graph", args=(self.pk,)) - - def assert_merge_retrieve_eq_from_json(self, path): - return super().assert_merge_retrieve_eq_from_json(path, self.url) - - def test_get_empty(self): - fromdb = self.retrieve(self.url) - self.assertEqual(fromdb, {"nodes": [], "edges": [], "groups": []}) - - def test_basic(self): - path = DATA_DIR / "basic.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_basic_ports(self): - path = DATA_DIR / "basic-ports.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_basic_nested_ports(self): - path = DATA_DIR / "basic-nested-ports.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_sample(self): - path = DATA_DIR / "sample.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_vertex(self): - # meta on node, mix of (non-)savable properties - path = DATA_DIR / "vertex.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_vertex_port(self): - # on ports: meta, mix of (non-)savable properties - path = DATA_DIR / "vertex-port.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_2vertices_1edge(self): - path = DATA_DIR / "2v-1e.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_2vertices_2groups(self): - path = DATA_DIR / "2v-2g.json" - self.assert_merge_retrieve_eq_from_json(path) - - @tag("slow") - def test_n25_e25(self): - path = DATA_DIR / "n25_e25.json" - self.assert_merge_retrieve_eq_from_json(path) - - @tag("slow") - def test_n50_e25(self): - path = DATA_DIR / "n50_e25.json" - self.assert_merge_retrieve_eq_from_json(path) - - def test_replace_vertex_with_another(self): - # merge graph, then merge the same graph again - # prepare original graph - path = DATA_DIR / "vertex.json" - data = load_data(path) - self.merge(data, self.url) - - # merge same data - self.assert_merge_retrieve_eq(data, self.url) - - def test_replace_port_with_another(self): - # merge vertex with port, then replace this port with a new one - # prepare original graph - path = DATA_DIR / "vertex-port.json" - data = load_data(path) - self.merge(data, self.url) - - new = load_data(path) - new["nodes"][0] = { - "primitiveID": "amy", - # merge one, create one - "initPorts": [{"primitiveID": "mobile"}, {"primitiveID": "laptop"}], - } - # make sure a vertex with one port are merged, the other port is created - self.assert_merge_retrieve_eq(new, self.url) - - def test_replace_edge_with_none(self): - # merge 2 vertices with an edge, then remove an edge - # prepare original graph - path = DATA_DIR / "2v-1e.json" - data = load_data(path) - self.merge(data, self.url) - - new = load_data(path) - new["edges"] = [] # same vertices, no edges - # make sure 2 vertices with ports are merged, the edge is removed - self.assert_merge_retrieve_eq(new, self.url) - - def test_replace_edge_with_another(self): - # merge 2 vertices with an edge, then remove an edge - # prepare original graph - path = DATA_DIR / "2v-1e.json" - data = load_data(path) - self.merge(data, self.url) - - new = load_data(path) - new["edges"][0]["meta"] = { - "new-field": {"string": "ham", "age": 167, "online": False} - } - - # make sure 2 vertices with ports are merged, the edge is removed - self.assert_merge_retrieve_eq(new, self.url) - - @tag("slow") - def test_n25_then_n50(self): - # first merge - old = load_data(DATA_DIR / "n25_e25.json") - self.merge(old, self.url) - - # over-write - new_path = DATA_DIR / "n50_e25.json" - self.assert_merge_retrieve_eq_from_json(new_path) - - -@tag("neo4j") -class TestRootFragmentGraphView( - GraphEndpointTestCaseMixin, - Neo4jTestCaseMixin, - APISimpleTestCase, -): - root_name = "parent" - fragment_name = "child" - - def setUp(self) -> None: - """Create graph endpoints for root and its fragment. - - Root graph endpoint URL is at `self.root_url`. - Fragment graph endpoint URL is at `self.fragment_url`. - """ - - TestRootFragmentDetailView.setUp(self) - # re-wire root_url and fragment_url to point at corresp. graph endpoints - # URL: roots//graph - self.root_url = reverse("supergraph:root-graph", args=(self.root_pk,)) - # URL: roots//fragments//graph - self.fragment_url = reverse( - "supergraph:root-fragment-graph", args=(self.root_pk, self.fragment_pk) - ) - self.url = None # we do not use self.url in this suite of tests - - def test_fragment_creates_root_sees(self): - # fragment creates shared graph, root sees it - data = load_data(DATA_DIR / "basic.json") - self.merge(data, self.fragment_url) # as fragment - # root must see the same graph - fromdb_as_root = self.retrieve(self.root_url) - self.assert_graph_eq(fromdb_as_root, data) - - def test_root_merges_fragment_sees(self): - # root changes shared graph, fragment sees it - # initial data on fragment - old = load_data(DATA_DIR / "basic.json") - self.merge(old, self.fragment_url) # as fragment - # root merges shared data - new = load_data(DATA_DIR / "vertex.json") # same node ID - self.merge(new, self.root_url) # as root - # fragment must see changes in shared data - fromdb_as_fragment = self.retrieve(self.fragment_url) - self.assert_graph_eq(fromdb_as_fragment, new) - - def test_root_creates_fragment_does_not_see(self): - # root creates something, fragment does not see it - data = load_data(DATA_DIR / "basic.json") - self.merge(data, self.root_url) # as root - fromdb_as_fragment = self.retrieve(self.fragment_url) - self.assertNotEqual(fromdb_as_fragment, data) - - if __name__ == "__main__": unittest.main() From ace7079528b46a5187c5ea61af8dd8161ae6eeef Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 14:12:55 +0300 Subject: [PATCH 11/25] made a proper url --- complex_rest_dtcd_supergraph/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/complex_rest_dtcd_supergraph/urls.py b/complex_rest_dtcd_supergraph/urls.py index 523a548..bb7e9de 100644 --- a/complex_rest_dtcd_supergraph/urls.py +++ b/complex_rest_dtcd_supergraph/urls.py @@ -1,9 +1,9 @@ -from django.urls import re_path +from django.urls import path from .views import GraphView app_name = "supergraph" urlpatterns = [ - re_path(r'^complex_rest_dtcd_supergraph/v1/graph/?$', GraphView.as_view()), -] \ No newline at end of file + path('graph/', GraphView.as_view()), +] From 97c71af682fcfc062a10e03e08774443086a9d89 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Thu, 7 Dec 2023 14:13:42 +0300 Subject: [PATCH 12/25] cleaned up --- docs/supergraph.conf.example | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/supergraph.conf.example b/docs/supergraph.conf.example index 1bfba0a..cfa5939 100644 --- a/docs/supergraph.conf.example +++ b/docs/supergraph.conf.example @@ -1,17 +1,7 @@ [logging] level = INFO -[neo4j] -protocol = bolt -address = localhost -port = 7687 -user = neo4j -password = password - [graph] base_path = /opt/otp/complex_rest/plugins/supergraph/graphs tmp_path = /opt/otp/complex_rest/plugins/supergraph/tmp id_name_map_path = /opt/otp/complex_rest/plugins/supergraph/id_name_map - -[schema] -default_root_name = ROOT From 2ca775a9f483d09ee1017015096fef400e67e691 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Fri, 8 Dec 2023 14:13:38 +0300 Subject: [PATCH 13/25] minor documentation and type hinting added --- complex_rest_dtcd_supergraph/utils/abc_graphmanager.py | 5 +++++ .../utils/filesystem_graphmanager.py | 5 +++++ complex_rest_dtcd_supergraph/views/graphs.py | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py b/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py index ed23ec2..7f7933c 100644 --- a/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py @@ -5,20 +5,25 @@ class AbstractGraphManager(ABC): @abstractmethod def read(self, graph_id: str) -> None: + """Read graph json file by 'graph_id'""" pass @abstractmethod def read_all(self) -> None: + """Read all graph json files""" pass @abstractmethod def write(self, graph: dict) -> None: + """Create graph json file with `graph` data""" pass @abstractmethod def update(self, graph: dict) -> None: + """Rewrite graph json file with `graph` data""" pass @abstractmethod def remove(self, graph_id: str) -> None: + """Delete graph json file by 'graph_id'""" pass diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index 16bf73d..9488bc5 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -22,6 +22,7 @@ def __init__(self, path, tmp_path, map_path): # or better import it from settin self.default_filename = 'graph.graphml' def read(self, graph_id) -> dict: + """Read graph json file by 'graph_id'""" graph_data = {} try: with open(Path(self.final_path) / graph_id / self.default_filename, 'r') as graph: @@ -35,6 +36,7 @@ def read(self, graph_id) -> dict: raise GraphManagerException(NO_GRAPH, graph_id) def read_all(self) -> list: + """Read all graph json files""" graph_list = [] with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) @@ -43,6 +45,7 @@ def read_all(self) -> list: return graph_list def write(self, graph: dict) -> None: + """Create graph json file with `graph` data""" shutil.copyfile(self.map_path, self.map_backup_path) with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) @@ -62,6 +65,7 @@ def write(self, graph: dict) -> None: os.rename(self.tmp_path, Path(graph_dir / self.default_filename)) # atomic operation def update(self, graph: dict) -> None: + """Rewrite graph json file with `graph` data""" if 'graph_id' not in graph: raise GraphManagerException(NO_ID) if 'name' in graph: @@ -88,6 +92,7 @@ def update(self, graph: dict) -> None: os.rename(self.tmp_path, Path(graph_dir) / self.default_filename) # atomic operation def remove(self, graph_id) -> None: + """Delete graph json file by 'graph_id'""" graph_dir = Path(self.final_path) / graph_id if not os.path.isdir(graph_dir): raise GraphManagerException(NO_GRAPH, graph_id) diff --git a/complex_rest_dtcd_supergraph/views/graphs.py b/complex_rest_dtcd_supergraph/views/graphs.py index 8872fd4..647fceb 100644 --- a/complex_rest_dtcd_supergraph/views/graphs.py +++ b/complex_rest_dtcd_supergraph/views/graphs.py @@ -14,7 +14,7 @@ class GraphView(APIView): http_method_names = ['get', 'post', 'put', 'delete'] graph_manager = FilesystemGraphManager(GRAPH_BASE_PATH, GRAPH_TMP_PATH, GRAPH_ID_NAME_MAP_PATH) - def post(self, request) -> Response: + def post(self, request: Request) -> Response: graphs = request.data for graph in graphs: try: @@ -44,7 +44,7 @@ def put(self, request: Request) -> Response: status.HTTP_200_OK ) - def delete(self, request) -> Response: + def delete(self, request: Request) -> Response: ids = request.data for id in ids: try: From 29a93d61db077b55a49354c872605b3ff2169f9a Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Fri, 8 Dec 2023 16:45:35 +0300 Subject: [PATCH 14/25] added id_name_map folder --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 000caec..99ae507 100644 --- a/.gitignore +++ b/.gitignore @@ -196,3 +196,6 @@ default_root_uid.txt make_build/ venv.tar.gz *.tar.gz + +# id_map folder +id_name_map/ From b09c88562049cdbf6cceb2ae0eb7b14abb2f4b10 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Fri, 8 Dec 2023 16:46:09 +0300 Subject: [PATCH 15/25] major commenting and changes structure of the id_map and json graph files --- .../utils/filesystem_graphmanager.py | 125 ++++++++++++++---- 1 file changed, 99 insertions(+), 26 deletions(-) diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index 9488bc5..29b8235 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -9,9 +9,41 @@ class FilesystemGraphManager(AbstractGraphManager): + """ + This is a manager that reads, writes, rewrites and deletes json files. + + :: final_path : path to a folder where all graph jason files are stored + :: tmp_path : path to a some temporary graph `graphml` file + :: map_path : path to a map (dictionary) `json` file, which stores {graph_id:title} pairs. + + :: map_backup_path : path to a map backup `json` file: temporary storage for a map path + :: map_tmp_path : path to a graph_tmp.json file inside a map folder + :: default_filename : presumably default name of the graph + + the files seem to be stored like this: + Path(final_path / id / graph.graphml) + + and all the graph data stored in that file in a 'content' part, like this + with open(Path(self.final_path) / graph_id / self.default_filename, 'r') as graph: + graph_data['content'] = graph.read() + + id_map json file has this format now: + { + 'some-graph-001-uuid-id' : 'some-graph-001-title', + 'some-graph-002-uuid-id' : 'some-graph-002-title', + } + + each json graph file has this format now: + { + 'graph_id' : 'some-uuid64-data`, <--- here we store its id + 'title' : 'title-of-the-graph`, <--- here we store its title + 'graph': {} <--- here we store all the graph data: nodes, edges, groups + } + + """ def __init__(self, path, tmp_path, map_path): # or better import it from settings here? - self.final_path = path + self.final_path = path # graph base dir self.tmp_path = tmp_path + '/tmp.graphml' self.map_path = map_path + '/graph_map.json' # not empty, at least {} if not os.path.isfile(self.map_path): @@ -19,88 +51,129 @@ def __init__(self, path, tmp_path, map_path): # or better import it from settin map_file.write('{}') self.map_backup_path = map_path + '/graph_backup.json' self.map_tmp_path = map_path + '/graph_tmp.json' - self.default_filename = 'graph.graphml' + self.default_filename = 'graph.json' def read(self, graph_id) -> dict: """Read graph json file by 'graph_id'""" - graph_data = {} + graph_data: dict = {} try: + # read graph.graphml by its name and id and get the 'content' of it with open(Path(self.final_path) / graph_id / self.default_filename, 'r') as graph: - graph_data['content'] = graph.read() - shutil.copyfile(self.map_path, self.map_backup_path) + graph_data['graph'] = graph.read() + + # back up current map file + # shutil.copyfile(self.map_path, + # self.map_backup_path) # TODO why do we need to backup map file if we do not change it? + + # read the map file and get the title of the graph by its id with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) - graph_data['name'] = id_map[graph_id]['name'] + graph_data['title'] = id_map[graph_id] # this is the only place where the NO_GRAPH error may trigger + + # add graph_id to graph data + graph_data['graph_id'] = graph_id + + # return result return graph_data except OSError: raise GraphManagerException(NO_GRAPH, graph_id) - def read_all(self) -> list: + def read_all(self) -> list[dict]: """Read all graph json files""" graph_list = [] with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) for k, v in id_map.items(): - graph_list.append({'graph_id': k, 'name': v['name']}) + graph_list.append({'graph_id': k, 'title': v}) return graph_list def write(self, graph: dict) -> None: """Create graph json file with `graph` data""" + # backup the map file shutil.copyfile(self.map_path, self.map_backup_path) + + # read the map file with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) - name = graph["name"] - for _, v in id_map.items(): - if v['name'] == name: - raise GraphManagerException(NAME_EXISTS, name) + # check if we have this graph already + if graph["title"] in id_map.values(): + raise GraphManagerException(NAME_EXISTS, graph["title"]) + + # create unique id unique_id = str(uuid.uuid4()) - id_map[unique_id] = {'name': graph["name"]} + # save the name to map with new id + id_map[unique_id] = graph["title"] + + # save id_map to map_tmp_path with open(self.map_tmp_path, 'w') as map_tmp_file: json.dump(id_map, map_tmp_file) + # rename map_tmp_path to map_path os.rename(self.map_tmp_path, self.map_path) # atomic operation + + # save graph['content'] to tmp_path file with open(self.tmp_path, 'w') as file: - file.write(graph["content"]) + file.write(graph["graph"]) + # create new graph dir by its id graph_dir = Path(self.final_path) / unique_id os.mkdir(graph_dir) + # move graph data from tmp_path to its new placement os.rename(self.tmp_path, Path(graph_dir / self.default_filename)) # atomic operation def update(self, graph: dict) -> None: """Rewrite graph json file with `graph` data""" if 'graph_id' not in graph: raise GraphManagerException(NO_ID) - if 'name' in graph: + if 'title' in graph: + # backing up the map file | why? shutil.copyfile(self.map_path, self.map_backup_path) + # read the id_map file with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) - name = graph["name"] - for k, v in id_map.items(): - if v['name'] == name and k != graph['graph_id']: - raise GraphManagerException(NAME_EXISTS, name) - try: - id_map[graph['graph_id']]['name'] = name - except KeyError: - raise GraphManagerException(NO_GRAPH, graph['graph_id']) + # check if we have this graph already + if graph["title"] in id_map.values(): + raise GraphManagerException(NAME_EXISTS, graph["title"]) + # try: + # saving the name of the graph to current id_map | why? it is the same + # id_map[graph['graph_id']] = graph["title"] + # except KeyError: + # this error is thrown when there is no such id in the map + # may be it is simpler to just check if current id is in id_map? + # raise GraphManagerException(NO_GRAPH, graph['graph_id']) + + # opening tmp map file for writing and saving current id_map to it with open(self.map_tmp_path, 'w') as map_tmp_file: json.dump(id_map, map_tmp_file) os.rename(self.map_tmp_path, self.map_path) # atomic operation - if 'content' in graph: + + # rewrite the content of the graph + if 'graph' in graph: graph_dir = Path(self.final_path) / graph['graph_id'] if not os.path.isdir(graph_dir): raise GraphManagerException(NO_GRAPH, graph['graph_id']) with open(self.tmp_path, 'w') as file: - file.write(graph["content"]) + file.write(graph["graph"]) os.rename(self.tmp_path, Path(graph_dir) / self.default_filename) # atomic operation def remove(self, graph_id) -> None: """Delete graph json file by 'graph_id'""" graph_dir = Path(self.final_path) / graph_id + # check if we have graph with this id if not os.path.isdir(graph_dir): raise GraphManagerException(NO_GRAPH, graph_id) - shutil.rmtree(Path(self.final_path) / graph_id) # delete directory and it's content + + # delete directory and it's content + shutil.rmtree(Path(self.final_path) / graph_id) + + # back up id_map file shutil.copyfile(self.map_path, self.map_backup_path) + + # read id_map with open(self.map_path, 'r') as map_file: id_map = json.load(map_file) + # delete id from id_map del id_map[graph_id] + + # save the id_map with open(self.map_tmp_path, 'w') as map_tmp_file: json.dump(id_map, map_tmp_file) os.rename(self.map_tmp_path, self.map_path) # atomic operation From 07799216bd4d7f3d0b3ee58a60b68a2bf5b9ea69 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Fri, 8 Dec 2023 17:16:43 +0300 Subject: [PATCH 16/25] added save to file function --- .../utils/filesystem_graphmanager.py | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index 29b8235..07df70a 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -2,6 +2,7 @@ import shutil import uuid import json +import tempfile from ..utils.graphmanager_exception import GraphManagerException, NO_GRAPH, NO_ID, NAME_EXISTS from ..utils.abc_graphmanager import AbstractGraphManager @@ -13,8 +14,8 @@ class FilesystemGraphManager(AbstractGraphManager): This is a manager that reads, writes, rewrites and deletes json files. :: final_path : path to a folder where all graph jason files are stored - :: tmp_path : path to a some temporary graph `graphml` file - :: map_path : path to a map (dictionary) `json` file, which stores {graph_id:title} pairs. + :: tmp_file_path : path to a some temporary graph `graphml` file + :: map_file_path : path to a map (dictionary) `json` file, which stores {graph_id:title} pairs. :: map_backup_path : path to a map backup `json` file: temporary storage for a map path :: map_tmp_path : path to a graph_tmp.json file inside a map folder @@ -42,15 +43,15 @@ class FilesystemGraphManager(AbstractGraphManager): """ - def __init__(self, path, tmp_path, map_path): # or better import it from settings here? + def __init__(self, path, tmp_folder_path, map_folder_path): # or better import it from settings here? self.final_path = path # graph base dir - self.tmp_path = tmp_path + '/tmp.graphml' - self.map_path = map_path + '/graph_map.json' # not empty, at least {} - if not os.path.isfile(self.map_path): - with open(self.map_path, 'w') as map_file: + self.tmp_file_path = tmp_folder_path + '/tmp.graphml' + self.map_file_path = map_folder_path + '/graph_map.json' # not empty, at least {} + if not os.path.isfile(self.map_file_path): + with open(self.map_file_path, 'w') as map_file: map_file.write('{}') - self.map_backup_path = map_path + '/graph_backup.json' - self.map_tmp_path = map_path + '/graph_tmp.json' + self.map_backup_path = map_folder_path + '/graph_backup.json' + self.map_tmp_path = map_folder_path + '/graph_tmp.json' self.default_filename = 'graph.json' def read(self, graph_id) -> dict: @@ -62,11 +63,11 @@ def read(self, graph_id) -> dict: graph_data['graph'] = graph.read() # back up current map file - # shutil.copyfile(self.map_path, + # shutil.copyfile(self.map_file_path, # self.map_backup_path) # TODO why do we need to backup map file if we do not change it? # read the map file and get the title of the graph by its id - with open(self.map_path, 'r') as map_file: + with open(self.map_file_path, 'r') as map_file: id_map = json.load(map_file) graph_data['title'] = id_map[graph_id] # this is the only place where the NO_GRAPH error may trigger @@ -81,7 +82,7 @@ def read(self, graph_id) -> dict: def read_all(self) -> list[dict]: """Read all graph json files""" graph_list = [] - with open(self.map_path, 'r') as map_file: + with open(self.map_file_path, 'r') as map_file: id_map = json.load(map_file) for k, v in id_map.items(): graph_list.append({'graph_id': k, 'title': v}) @@ -90,10 +91,10 @@ def read_all(self) -> list[dict]: def write(self, graph: dict) -> None: """Create graph json file with `graph` data""" # backup the map file - shutil.copyfile(self.map_path, self.map_backup_path) + shutil.copyfile(self.map_file_path, self.map_backup_path) # read the map file - with open(self.map_path, 'r') as map_file: + with open(self.map_file_path, 'r') as map_file: id_map = json.load(map_file) # check if we have this graph already if graph["title"] in id_map.values(): @@ -104,20 +105,20 @@ def write(self, graph: dict) -> None: # save the name to map with new id id_map[unique_id] = graph["title"] - # save id_map to map_tmp_path + # save id_map to map_tmp with open(self.map_tmp_path, 'w') as map_tmp_file: json.dump(id_map, map_tmp_file) - # rename map_tmp_path to map_path - os.rename(self.map_tmp_path, self.map_path) # atomic operation + # rename map_tmp_path to map_folder_path + os.rename(self.map_tmp_path, self.map_file_path) # atomic operation # save graph['content'] to tmp_path file - with open(self.tmp_path, 'w') as file: + with open(self.tmp_file_path, 'w') as file: file.write(graph["graph"]) # create new graph dir by its id graph_dir = Path(self.final_path) / unique_id os.mkdir(graph_dir) # move graph data from tmp_path to its new placement - os.rename(self.tmp_path, Path(graph_dir / self.default_filename)) # atomic operation + os.rename(self.tmp_file_path, Path(graph_dir / self.default_filename)) # atomic operation def update(self, graph: dict) -> None: """Rewrite graph json file with `graph` data""" @@ -125,9 +126,9 @@ def update(self, graph: dict) -> None: raise GraphManagerException(NO_ID) if 'title' in graph: # backing up the map file | why? - shutil.copyfile(self.map_path, self.map_backup_path) + shutil.copyfile(self.map_file_path, self.map_backup_path) # read the id_map file - with open(self.map_path, 'r') as map_file: + with open(self.map_file_path, 'r') as map_file: id_map = json.load(map_file) # check if we have this graph already if graph["title"] in id_map.values(): @@ -143,16 +144,16 @@ def update(self, graph: dict) -> None: # opening tmp map file for writing and saving current id_map to it with open(self.map_tmp_path, 'w') as map_tmp_file: json.dump(id_map, map_tmp_file) - os.rename(self.map_tmp_path, self.map_path) # atomic operation + os.rename(self.map_tmp_path, self.map_file_path) # atomic operation # rewrite the content of the graph if 'graph' in graph: graph_dir = Path(self.final_path) / graph['graph_id'] if not os.path.isdir(graph_dir): raise GraphManagerException(NO_GRAPH, graph['graph_id']) - with open(self.tmp_path, 'w') as file: + with open(self.tmp_file_path, 'w') as file: file.write(graph["graph"]) - os.rename(self.tmp_path, Path(graph_dir) / self.default_filename) # atomic operation + os.rename(self.tmp_file_path, Path(graph_dir) / self.default_filename) # atomic operation def remove(self, graph_id) -> None: """Delete graph json file by 'graph_id'""" @@ -165,10 +166,10 @@ def remove(self, graph_id) -> None: shutil.rmtree(Path(self.final_path) / graph_id) # back up id_map file - shutil.copyfile(self.map_path, self.map_backup_path) + shutil.copyfile(self.map_file_path, self.map_backup_path) # read id_map - with open(self.map_path, 'r') as map_file: + with open(self.map_file_path, 'r') as map_file: id_map = json.load(map_file) # delete id from id_map del id_map[graph_id] @@ -176,4 +177,10 @@ def remove(self, graph_id) -> None: # save the id_map with open(self.map_tmp_path, 'w') as map_tmp_file: json.dump(id_map, map_tmp_file) - os.rename(self.map_tmp_path, self.map_path) # atomic operation + os.rename(self.map_tmp_path, self.map_file_path) # atomic operation + + +def save_data_to_file(data: dict, destination_path: Path) -> None: + with tempfile.NamedTemporaryFile(delete=False) as file: + file.write(data) + os.rename(file.name, destination_path) From 784cae721bd37faf6f514eaf8ab0309f4a4edbc7 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 11 Dec 2023 11:52:00 +0300 Subject: [PATCH 17/25] updated filesystem manager files accroding new endpoint map --- .../utils/abc_graphmanager.py | 2 +- .../utils/filesystem_graphmanager.py | 104 ++++++------------ .../utils/graphmanager_exception.py | 3 + 3 files changed, 40 insertions(+), 69 deletions(-) diff --git a/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py b/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py index 7f7933c..26fa879 100644 --- a/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/abc_graphmanager.py @@ -19,7 +19,7 @@ def write(self, graph: dict) -> None: pass @abstractmethod - def update(self, graph: dict) -> None: + def update(self, graph: dict, graph_id: str) -> None: """Rewrite graph json file with `graph` data""" pass diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index 07df70a..bd5eafc 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -3,8 +3,9 @@ import uuid import json import tempfile +from typing import Union, AnyStr -from ..utils.graphmanager_exception import GraphManagerException, NO_GRAPH, NO_ID, NAME_EXISTS +from ..utils.graphmanager_exception import GraphManagerException, NO_GRAPH, NO_ID, NAME_EXISTS, NO_TITLE from ..utils.abc_graphmanager import AbstractGraphManager from pathlib import Path @@ -30,17 +31,20 @@ class FilesystemGraphManager(AbstractGraphManager): id_map json file has this format now: { - 'some-graph-001-uuid-id' : 'some-graph-001-title', - 'some-graph-002-uuid-id' : 'some-graph-002-title', + "some-graph-001-uuid-id": "some-graph-001-title", + "some-graph-002-uuid-id": "some-graph-002-title" } + graph json file is stored at: + GRAPH_BASE_DIR/some-graph-001-uuid-id/graph.graphml + each json graph file has this format now: { - 'graph_id' : 'some-uuid64-data`, <--- here we store its id - 'title' : 'title-of-the-graph`, <--- here we store its title - 'graph': {} <--- here we store all the graph data: nodes, edges, groups + 'title' : 'title-of-the-graph`, <--- here we store graph's title | why we need to store title in graph json? + 'nodes': {nodes info}, <--- here we store the graph nodes + 'edges': {edges info}, <--- here we store the graph edges + 'groups': {groups info} <--- here we store the graph groups } - """ def __init__(self, path, tmp_folder_path, map_folder_path): # or better import it from settings here? @@ -55,24 +59,13 @@ def __init__(self, path, tmp_folder_path, map_folder_path): # or better import self.default_filename = 'graph.json' def read(self, graph_id) -> dict: - """Read graph json file by 'graph_id'""" + """Read graph json file by 'graph_id' + """ graph_data: dict = {} try: # read graph.graphml by its name and id and get the 'content' of it with open(Path(self.final_path) / graph_id / self.default_filename, 'r') as graph: - graph_data['graph'] = graph.read() - - # back up current map file - # shutil.copyfile(self.map_file_path, - # self.map_backup_path) # TODO why do we need to backup map file if we do not change it? - - # read the map file and get the title of the graph by its id - with open(self.map_file_path, 'r') as map_file: - id_map = json.load(map_file) - graph_data['title'] = id_map[graph_id] # this is the only place where the NO_GRAPH error may trigger - - # add graph_id to graph data - graph_data['graph_id'] = graph_id + graph_data = json.load(graph) # return result return graph_data @@ -80,13 +73,11 @@ def read(self, graph_id) -> dict: raise GraphManagerException(NO_GRAPH, graph_id) def read_all(self) -> list[dict]: - """Read all graph json files""" - graph_list = [] + """Read all graph json files + """ with open(self.map_file_path, 'r') as map_file: - id_map = json.load(map_file) - for k, v in id_map.items(): - graph_list.append({'graph_id': k, 'title': v}) - return graph_list + graph_data = json.load(map_file) + return graph_data def write(self, graph: dict) -> None: """Create graph json file with `graph` data""" @@ -111,49 +102,29 @@ def write(self, graph: dict) -> None: # rename map_tmp_path to map_folder_path os.rename(self.map_tmp_path, self.map_file_path) # atomic operation - # save graph['content'] to tmp_path file - with open(self.tmp_file_path, 'w') as file: - file.write(graph["graph"]) # create new graph dir by its id graph_dir = Path(self.final_path) / unique_id os.mkdir(graph_dir) - # move graph data from tmp_path to its new placement - os.rename(self.tmp_file_path, Path(graph_dir / self.default_filename)) # atomic operation - def update(self, graph: dict) -> None: + # save graph to file + save_data_to_file(graph, Path(graph_dir / self.default_filename)) + + def update(self, graph: dict, graph_id: str) -> None: """Rewrite graph json file with `graph` data""" - if 'graph_id' not in graph: - raise GraphManagerException(NO_ID) - if 'title' in graph: - # backing up the map file | why? - shutil.copyfile(self.map_file_path, self.map_backup_path) - # read the id_map file - with open(self.map_file_path, 'r') as map_file: - id_map = json.load(map_file) - # check if we have this graph already - if graph["title"] in id_map.values(): - raise GraphManagerException(NAME_EXISTS, graph["title"]) - # try: - # saving the name of the graph to current id_map | why? it is the same - # id_map[graph['graph_id']] = graph["title"] - # except KeyError: - # this error is thrown when there is no such id in the map - # may be it is simpler to just check if current id is in id_map? - # raise GraphManagerException(NO_GRAPH, graph['graph_id']) - - # opening tmp map file for writing and saving current id_map to it - with open(self.map_tmp_path, 'w') as map_tmp_file: - json.dump(id_map, map_tmp_file) - os.rename(self.map_tmp_path, self.map_file_path) # atomic operation + if 'title' not in graph: + raise GraphManagerException(NO_TITLE) + + # read the id_map file + with open(self.map_file_path, 'r') as map_file: + id_map = json.load(map_file) + + graph_dir = Path(self.final_path) / graph_id + # check if we don't have this graph or there is no such dir + if graph["title"] not in id_map.values() or not os.path.isdir(graph_dir): + raise GraphManagerException(NO_GRAPH, graph_id) # rewrite the content of the graph - if 'graph' in graph: - graph_dir = Path(self.final_path) / graph['graph_id'] - if not os.path.isdir(graph_dir): - raise GraphManagerException(NO_GRAPH, graph['graph_id']) - with open(self.tmp_file_path, 'w') as file: - file.write(graph["graph"]) - os.rename(self.tmp_file_path, Path(graph_dir) / self.default_filename) # atomic operation + save_data_to_file(graph["graph"], Path(graph_dir) / self.default_filename) def remove(self, graph_id) -> None: """Delete graph json file by 'graph_id'""" @@ -174,13 +145,10 @@ def remove(self, graph_id) -> None: # delete id from id_map del id_map[graph_id] - # save the id_map - with open(self.map_tmp_path, 'w') as map_tmp_file: - json.dump(id_map, map_tmp_file) - os.rename(self.map_tmp_path, self.map_file_path) # atomic operation + save_data_to_file(id_map, self.map_file_path) -def save_data_to_file(data: dict, destination_path: Path) -> None: +def save_data_to_file(data: Union[dict, AnyStr], destination_path: Path) -> None: with tempfile.NamedTemporaryFile(delete=False) as file: file.write(data) os.rename(file.name, destination_path) diff --git a/complex_rest_dtcd_supergraph/utils/graphmanager_exception.py b/complex_rest_dtcd_supergraph/utils/graphmanager_exception.py index eab075f..3393af7 100644 --- a/complex_rest_dtcd_supergraph/utils/graphmanager_exception.py +++ b/complex_rest_dtcd_supergraph/utils/graphmanager_exception.py @@ -1,6 +1,7 @@ NO_ID = 0 NO_GRAPH = 1 NAME_EXISTS = 2 +NO_TITLE = 3 class GraphManagerException(Exception): @@ -13,4 +14,6 @@ def __init__(self, problem, *args): msg = f"No graph found with id -> {args[0]}" elif problem == NAME_EXISTS: msg = f"Name -> {args[0]} already exists" + elif problem == NO_TITLE: + msg = f"No title found in request body" super().__init__(msg) From 4ab9048414e62006e0ae8d2370ac49debade798a Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 11 Dec 2023 16:40:07 +0300 Subject: [PATCH 18/25] fixed all endpoints and urls.py --- .../graph.json | 1 + .../graph.json | 1 + .../graph.json | 1 + complex_rest_dtcd_supergraph/urls.py | 5 +- .../utils/filesystem_graphmanager.py | 35 +++++----- .../views/__init__.py | 1 + complex_rest_dtcd_supergraph/views/graphs.py | 67 ++++++++++--------- 7 files changed, 59 insertions(+), 52 deletions(-) create mode 100644 complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json create mode 100644 complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json create mode 100644 complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json diff --git a/complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json b/complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json new file mode 100644 index 0000000..cfe2703 --- /dev/null +++ b/complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json @@ -0,0 +1 @@ +{"title": "new_test_graph_011", "nodes": [{"properties": {"type": {"expression": "\"\u0414\u0430\u043d\u043d\u044b\u0435\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "outPort1", "type": ["OUT"], "properties": {"status": {"expression": "value", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Data_812_outPort1", "location": {"x": 484.5, "y": 223}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Data", "primitiveID": "Data_812", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": 223, "height": 148, "width": 294}}, {"properties": {"type": {"expression": "\"\u0426\u0435\u043b\u044c\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "input": {"component": "select", "type": "const", "values": ["\"\u0427\u0438\u0441\u0442\u0430\u044f \u043f\u0440\u0438\u0431\u044b\u043b\u044c \u043c\u043b\u043d \u0440\u0443\u0431.\"", "\"\u0413\u0440\u0443\u0437\u043e\u043e\u0431\u043e\u0440\u043e\u0442\"", "\"\u0414\u043e\u0445\u043e\u0434\u043d\u043e\u0441\u0442\u044c \u043d\u0430 \u043e\u0431\u0449\u0438\u0439 \u043f\u0430\u0440\u043a \u0440\u0443\u0431./\u0432\u0430\u0433./\u0441\u0443\u0442.\"", "\"EBITDA\"", "\"\u0427\u0438\u0441\u0442\u044b\u0439 \u0434\u043e\u043b\u0433/EBITDA\"", "\"ROIC\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u043a\u043e\u0440\u043e\u0441\u0442\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438 \u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440\u043e\u0432\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u041a\u043e\u043d\u0442\u0440\u0435\u0439\u043b\u0435\u0440\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u0442\u0440\u0430\u0442\u0435\u0433\u0438\u044f \u0446\u0438\u0444\u0440\u043e\u0432\u043e\u0439 \u0442\u0440\u0430\u043d\u0441\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438\u00bb\""]}, "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "inPort1", "type": ["IN"], "properties": {"status": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Goal_16_inPort1", "location": {"x": 484.5, "y": 84.75}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Goal", "primitiveID": "Goal_16", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": -63.25, "height": 148, "width": 294}}], "edges": [{"bends": [], "sourceNode": "Data_812", "sourcePort": "Data_812_outPort1", "targetNode": "Goal_16", "targetPort": "Goal_16_inPort1", "extensionName": "ExtensionCommonPrimitives", "primitiveName": "SimpleEdge"}], "groups": []} \ No newline at end of file diff --git a/complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json b/complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json new file mode 100644 index 0000000..c169ce2 --- /dev/null +++ b/complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json @@ -0,0 +1 @@ +{"title": "new_test_graph_005", "nodes": [{"properties": {"type": {"expression": "\"\u0414\u0430\u043d\u043d\u044b\u0435\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "outPort1", "type": ["OUT"], "properties": {"status": {"expression": "value", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Data_812_outPort1", "location": {"x": 484.5, "y": 223}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Data", "primitiveID": "Data_812", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": 223, "height": 148, "width": 294}}, {"properties": {"type": {"expression": "\"\u0426\u0435\u043b\u044c\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "input": {"component": "select", "type": "const", "values": ["\"\u0427\u0438\u0441\u0442\u0430\u044f \u043f\u0440\u0438\u0431\u044b\u043b\u044c \u043c\u043b\u043d \u0440\u0443\u0431.\"", "\"\u0413\u0440\u0443\u0437\u043e\u043e\u0431\u043e\u0440\u043e\u0442\"", "\"\u0414\u043e\u0445\u043e\u0434\u043d\u043e\u0441\u0442\u044c \u043d\u0430 \u043e\u0431\u0449\u0438\u0439 \u043f\u0430\u0440\u043a \u0440\u0443\u0431./\u0432\u0430\u0433./\u0441\u0443\u0442.\"", "\"EBITDA\"", "\"\u0427\u0438\u0441\u0442\u044b\u0439 \u0434\u043e\u043b\u0433/EBITDA\"", "\"ROIC\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u043a\u043e\u0440\u043e\u0441\u0442\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438 \u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440\u043e\u0432\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u041a\u043e\u043d\u0442\u0440\u0435\u0439\u043b\u0435\u0440\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u0442\u0440\u0430\u0442\u0435\u0433\u0438\u044f \u0446\u0438\u0444\u0440\u043e\u0432\u043e\u0439 \u0442\u0440\u0430\u043d\u0441\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438\u00bb\""]}, "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "inPort1", "type": ["IN"], "properties": {"status": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Goal_16_inPort1", "location": {"x": 484.5, "y": 84.75}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Goal", "primitiveID": "Goal_16", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": -63.25, "height": 148, "width": 294}}], "edges": [{"bends": [], "sourceNode": "Data_812", "sourcePort": "Data_812_outPort1", "targetNode": "Goal_16", "targetPort": "Goal_16_inPort1", "extensionName": "ExtensionCommonPrimitives", "primitiveName": "SimpleEdge"}], "groups": []} \ No newline at end of file diff --git a/complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json b/complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json new file mode 100644 index 0000000..50e01ac --- /dev/null +++ b/complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json @@ -0,0 +1 @@ +{"nodes": [{"properties": {"type": {"expression": "\"new \u0414\u0430\u043d\u043d\u044b\u0435\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "outPort1", "type": ["OUT"], "properties": {"status": {"expression": "value", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Data_812_outPort1", "location": {"x": 484.5, "y": 223}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Data", "primitiveID": "Data_812", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": 223, "height": 148, "width": 294}}, {"properties": {"type": {"expression": "\"\u0426\u0435\u043b\u044c\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "input": {"component": "select", "type": "const", "values": ["\"\u0427\u0438\u0441\u0442\u0430\u044f \u043f\u0440\u0438\u0431\u044b\u043b\u044c \u043c\u043b\u043d \u0440\u0443\u0431.\"", "\"\u0413\u0440\u0443\u0437\u043e\u043e\u0431\u043e\u0440\u043e\u0442\"", "\"\u0414\u043e\u0445\u043e\u0434\u043d\u043e\u0441\u0442\u044c \u043d\u0430 \u043e\u0431\u0449\u0438\u0439 \u043f\u0430\u0440\u043a \u0440\u0443\u0431./\u0432\u0430\u0433./\u0441\u0443\u0442.\"", "\"EBITDA\"", "\"\u0427\u0438\u0441\u0442\u044b\u0439 \u0434\u043e\u043b\u0433/EBITDA\"", "\"ROIC\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u043a\u043e\u0440\u043e\u0441\u0442\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438 \u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440\u043e\u0432\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u041a\u043e\u043d\u0442\u0440\u0435\u0439\u043b\u0435\u0440\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u0442\u0440\u0430\u0442\u0435\u0433\u0438\u044f \u0446\u0438\u0444\u0440\u043e\u0432\u043e\u0439 \u0442\u0440\u0430\u043d\u0441\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438\u00bb\""]}, "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "inPort1", "type": ["IN"], "properties": {"status": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Goal_16_inPort1", "location": {"x": 484.5, "y": 84.75}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Goal", "primitiveID": "Goal_16", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": -63.25, "height": 148, "width": 294}}], "edges": [{"bends": [], "sourceNode": "Data_812", "sourcePort": "Data_812_outPort1", "targetNode": "Goal_16", "targetPort": "Goal_16_inPort1", "extensionName": "ExtensionCommonPrimitives", "primitiveName": "SimpleEdge"}], "groups": []} \ No newline at end of file diff --git a/complex_rest_dtcd_supergraph/urls.py b/complex_rest_dtcd_supergraph/urls.py index bb7e9de..05bb554 100644 --- a/complex_rest_dtcd_supergraph/urls.py +++ b/complex_rest_dtcd_supergraph/urls.py @@ -1,9 +1,10 @@ from django.urls import path -from .views import GraphView +from .views import GraphView, GraphDetailView app_name = "supergraph" urlpatterns = [ - path('graph/', GraphView.as_view()), + path('graphs/', GraphView.as_view(), name='graphs'), + path('graphs//', GraphDetailView.as_view(), name='graphs_detail') ] diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index bd5eafc..1794f3a 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -59,12 +59,11 @@ def __init__(self, path, tmp_folder_path, map_folder_path): # or better import self.default_filename = 'graph.json' def read(self, graph_id) -> dict: - """Read graph json file by 'graph_id' - """ - graph_data: dict = {} + """Read graph json file by 'graph_id'""" try: # read graph.graphml by its name and id and get the 'content' of it - with open(Path(self.final_path) / graph_id / self.default_filename, 'r') as graph: + file_path = Path(self.final_path) / str(graph_id) / self.default_filename + with open(file_path, 'r') as graph: graph_data = json.load(graph) # return result @@ -77,9 +76,10 @@ def read_all(self) -> list[dict]: """ with open(self.map_file_path, 'r') as map_file: graph_data = json.load(map_file) - return graph_data + result = [{"id": graph_id, "title": title} for graph_id, title in graph_data.items()] + return result - def write(self, graph: dict) -> None: + def write(self, graph: dict) -> dict: """Create graph json file with `graph` data""" # backup the map file shutil.copyfile(self.map_file_path, self.map_backup_path) @@ -108,33 +108,31 @@ def write(self, graph: dict) -> None: # save graph to file save_data_to_file(graph, Path(graph_dir / self.default_filename)) + return {unique_id: graph['title']} def update(self, graph: dict, graph_id: str) -> None: """Rewrite graph json file with `graph` data""" - if 'title' not in graph: - raise GraphManagerException(NO_TITLE) - # read the id_map file with open(self.map_file_path, 'r') as map_file: id_map = json.load(map_file) - graph_dir = Path(self.final_path) / graph_id + graph_dir = Path(self.final_path) / str(graph_id) # check if we don't have this graph or there is no such dir - if graph["title"] not in id_map.values() or not os.path.isdir(graph_dir): + if not os.path.isdir(graph_dir): raise GraphManagerException(NO_GRAPH, graph_id) # rewrite the content of the graph - save_data_to_file(graph["graph"], Path(graph_dir) / self.default_filename) + save_data_to_file(graph, Path(graph_dir) / self.default_filename) - def remove(self, graph_id) -> None: + def remove(self, graph_id: str) -> None: """Delete graph json file by 'graph_id'""" graph_dir = Path(self.final_path) / graph_id # check if we have graph with this id - if not os.path.isdir(graph_dir): - raise GraphManagerException(NO_GRAPH, graph_id) + # if not os.path.isdir(graph_dir): + # raise GraphManagerException(NO_GRAPH, graph_id) - # delete directory and it's content - shutil.rmtree(Path(self.final_path) / graph_id) + # delete directory, and it's content + # shutil.rmtree(Path(self.final_path) / str(graph_id)) # back up id_map file shutil.copyfile(self.map_file_path, self.map_backup_path) @@ -150,5 +148,6 @@ def remove(self, graph_id) -> None: def save_data_to_file(data: Union[dict, AnyStr], destination_path: Path) -> None: with tempfile.NamedTemporaryFile(delete=False) as file: - file.write(data) + file.write(json.dumps(data).encode('utf-8')) + file.flush() os.rename(file.name, destination_path) diff --git a/complex_rest_dtcd_supergraph/views/__init__.py b/complex_rest_dtcd_supergraph/views/__init__.py index 4566980..c4fbf42 100644 --- a/complex_rest_dtcd_supergraph/views/__init__.py +++ b/complex_rest_dtcd_supergraph/views/__init__.py @@ -1,3 +1,4 @@ from .graphs import ( GraphView, + GraphDetailView ) diff --git a/complex_rest_dtcd_supergraph/views/graphs.py b/complex_rest_dtcd_supergraph/views/graphs.py index 647fceb..c6d0166 100644 --- a/complex_rest_dtcd_supergraph/views/graphs.py +++ b/complex_rest_dtcd_supergraph/views/graphs.py @@ -11,60 +11,63 @@ class GraphView(APIView): permission_classes = (AllowAny,) - http_method_names = ['get', 'post', 'put', 'delete'] + http_method_names = ['get', 'post'] graph_manager = FilesystemGraphManager(GRAPH_BASE_PATH, GRAPH_TMP_PATH, GRAPH_ID_NAME_MAP_PATH) def post(self, request: Request) -> Response: graphs = request.data + list_of_ids = [] for graph in graphs: try: - self.graph_manager.write(graph) + list_of_ids.append(self.graph_manager.write(graph)) except Exception as e: return Response( {"status": "ERROR", "msg": str(e)}, status.HTTP_400_BAD_REQUEST ) return Response( - {"status": "SUCCESS"}, + {"status": "SUCCESS", "result": list_of_ids}, status.HTTP_200_OK ) - def put(self, request: Request) -> Response: - graphs = request.data - for graph in graphs: - try: - self.graph_manager.update(graph) - except Exception as e: - return Response( - {"status": "ERROR", "msg": str(e)}, - status.HTTP_400_BAD_REQUEST - ) - return Response( - {"status": "SUCCESS"}, - status.HTTP_200_OK - ) + def get(self, request: Request) -> Response: + return Response(self.graph_manager.read_all(), status.HTTP_200_OK) - def delete(self, request: Request) -> Response: - ids = request.data - for id in ids: - try: - self.graph_manager.remove(id) - except Exception as e: - return Response( - {"status": "ERROR", "msg": str(e)}, - status.HTTP_400_BAD_REQUEST - ) + +class GraphDetailView(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'put', 'delete'] + graph_manager = FilesystemGraphManager(GRAPH_BASE_PATH, GRAPH_TMP_PATH, GRAPH_ID_NAME_MAP_PATH) + + def put(self, request: Request, graph_id: str) -> Response: + try: + self.graph_manager.update(request.data, graph_id) + except Exception as e: + return Response( + {"status": "ERROR", "msg": str(e)}, + status.HTTP_400_BAD_REQUEST + ) return Response( {"status": "SUCCESS"}, status.HTTP_200_OK ) - def get(self, request: Request) -> Response: - qs = dict(request.query_params) - if 'id' not in qs: - return Response(self.graph_manager.read_all(), status.HTTP_200_OK) + def get(self, request: Request, graph_id: str) -> Response: try: - graph_content = self.graph_manager.read(qs['id'][0]) + graph_content = self.graph_manager.read(graph_id) except Exception as e: return Response({"status": "ERROR", "msg": str(e)}, status.HTTP_400_BAD_REQUEST) return Response(graph_content, status.HTTP_200_OK) + + def delete(self, request: Request, graph_id: str) -> Response: + try: + self.graph_manager.remove(graph_id) + except Exception as e: + return Response( + {"status": "ERROR", "msg": str(e)}, + status.HTTP_400_BAD_REQUEST + ) + return Response( + {"status": "SUCCESS"}, + status.HTTP_200_OK + ) From 18c8f49f216d29eb8000873e29fd88ba271a59be Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Wed, 13 Dec 2023 11:37:33 +0300 Subject: [PATCH 19/25] removed unnecesary tests --- tests/test_utils.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/test_utils.py diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index 10e6159..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1 +0,0 @@ -import unittest From f0a7425f2a80940c5d595bc9b94085fa9ebf3538 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 22 Jan 2024 14:29:20 +0300 Subject: [PATCH 20/25] testsa re passing --- .../graph.json | 1 - .../graph.json | 1 - .../graph.json | 1 - .../models/__init__.py | 1 - tests/misc.py | 47 ------ tests/test_views.py | 138 +++++++----------- 6 files changed, 50 insertions(+), 139 deletions(-) delete mode 100644 complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json delete mode 100644 complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json delete mode 100644 complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json delete mode 100644 complex_rest_dtcd_supergraph/models/__init__.py delete mode 100644 tests/misc.py diff --git a/complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json b/complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json deleted file mode 100644 index cfe2703..0000000 --- a/complex_rest_dtcd_supergraph/graphs/142ad932-22d3-48ac-9f23-8d2f0e3e5f89/graph.json +++ /dev/null @@ -1 +0,0 @@ -{"title": "new_test_graph_011", "nodes": [{"properties": {"type": {"expression": "\"\u0414\u0430\u043d\u043d\u044b\u0435\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "outPort1", "type": ["OUT"], "properties": {"status": {"expression": "value", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Data_812_outPort1", "location": {"x": 484.5, "y": 223}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Data", "primitiveID": "Data_812", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": 223, "height": 148, "width": 294}}, {"properties": {"type": {"expression": "\"\u0426\u0435\u043b\u044c\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "input": {"component": "select", "type": "const", "values": ["\"\u0427\u0438\u0441\u0442\u0430\u044f \u043f\u0440\u0438\u0431\u044b\u043b\u044c \u043c\u043b\u043d \u0440\u0443\u0431.\"", "\"\u0413\u0440\u0443\u0437\u043e\u043e\u0431\u043e\u0440\u043e\u0442\"", "\"\u0414\u043e\u0445\u043e\u0434\u043d\u043e\u0441\u0442\u044c \u043d\u0430 \u043e\u0431\u0449\u0438\u0439 \u043f\u0430\u0440\u043a \u0440\u0443\u0431./\u0432\u0430\u0433./\u0441\u0443\u0442.\"", "\"EBITDA\"", "\"\u0427\u0438\u0441\u0442\u044b\u0439 \u0434\u043e\u043b\u0433/EBITDA\"", "\"ROIC\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u043a\u043e\u0440\u043e\u0441\u0442\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438 \u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440\u043e\u0432\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u041a\u043e\u043d\u0442\u0440\u0435\u0439\u043b\u0435\u0440\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u0442\u0440\u0430\u0442\u0435\u0433\u0438\u044f \u0446\u0438\u0444\u0440\u043e\u0432\u043e\u0439 \u0442\u0440\u0430\u043d\u0441\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438\u00bb\""]}, "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "inPort1", "type": ["IN"], "properties": {"status": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Goal_16_inPort1", "location": {"x": 484.5, "y": 84.75}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Goal", "primitiveID": "Goal_16", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": -63.25, "height": 148, "width": 294}}], "edges": [{"bends": [], "sourceNode": "Data_812", "sourcePort": "Data_812_outPort1", "targetNode": "Goal_16", "targetPort": "Goal_16_inPort1", "extensionName": "ExtensionCommonPrimitives", "primitiveName": "SimpleEdge"}], "groups": []} \ No newline at end of file diff --git a/complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json b/complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json deleted file mode 100644 index c169ce2..0000000 --- a/complex_rest_dtcd_supergraph/graphs/28e17023-cfd3-460e-9ddf-5ec2d4051dd0/graph.json +++ /dev/null @@ -1 +0,0 @@ -{"title": "new_test_graph_005", "nodes": [{"properties": {"type": {"expression": "\"\u0414\u0430\u043d\u043d\u044b\u0435\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "outPort1", "type": ["OUT"], "properties": {"status": {"expression": "value", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Data_812_outPort1", "location": {"x": 484.5, "y": 223}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Data", "primitiveID": "Data_812", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": 223, "height": 148, "width": 294}}, {"properties": {"type": {"expression": "\"\u0426\u0435\u043b\u044c\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "input": {"component": "select", "type": "const", "values": ["\"\u0427\u0438\u0441\u0442\u0430\u044f \u043f\u0440\u0438\u0431\u044b\u043b\u044c \u043c\u043b\u043d \u0440\u0443\u0431.\"", "\"\u0413\u0440\u0443\u0437\u043e\u043e\u0431\u043e\u0440\u043e\u0442\"", "\"\u0414\u043e\u0445\u043e\u0434\u043d\u043e\u0441\u0442\u044c \u043d\u0430 \u043e\u0431\u0449\u0438\u0439 \u043f\u0430\u0440\u043a \u0440\u0443\u0431./\u0432\u0430\u0433./\u0441\u0443\u0442.\"", "\"EBITDA\"", "\"\u0427\u0438\u0441\u0442\u044b\u0439 \u0434\u043e\u043b\u0433/EBITDA\"", "\"ROIC\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u043a\u043e\u0440\u043e\u0441\u0442\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438 \u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440\u043e\u0432\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u041a\u043e\u043d\u0442\u0440\u0435\u0439\u043b\u0435\u0440\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u0442\u0440\u0430\u0442\u0435\u0433\u0438\u044f \u0446\u0438\u0444\u0440\u043e\u0432\u043e\u0439 \u0442\u0440\u0430\u043d\u0441\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438\u00bb\""]}, "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "inPort1", "type": ["IN"], "properties": {"status": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Goal_16_inPort1", "location": {"x": 484.5, "y": 84.75}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Goal", "primitiveID": "Goal_16", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": -63.25, "height": 148, "width": 294}}], "edges": [{"bends": [], "sourceNode": "Data_812", "sourcePort": "Data_812_outPort1", "targetNode": "Goal_16", "targetPort": "Goal_16_inPort1", "extensionName": "ExtensionCommonPrimitives", "primitiveName": "SimpleEdge"}], "groups": []} \ No newline at end of file diff --git a/complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json b/complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json deleted file mode 100644 index 50e01ac..0000000 --- a/complex_rest_dtcd_supergraph/graphs/68d9962e-0170-4bcc-9be8-a9bf6b89a612/graph.json +++ /dev/null @@ -1 +0,0 @@ -{"nodes": [{"properties": {"type": {"expression": "\"new \u0414\u0430\u043d\u043d\u044b\u0435\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "outPort1", "type": ["OUT"], "properties": {"status": {"expression": "value", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Data_812_outPort1", "location": {"x": 484.5, "y": 223}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Data", "primitiveID": "Data_812", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": 223, "height": 148, "width": 294}}, {"properties": {"type": {"expression": "\"\u0426\u0435\u043b\u044c\"", "type": "expression", "status": "complete", "value": ""}, "name": {"expression": "", "type": "expression", "input": {"component": "select", "type": "const", "values": ["\"\u0427\u0438\u0441\u0442\u0430\u044f \u043f\u0440\u0438\u0431\u044b\u043b\u044c \u043c\u043b\u043d \u0440\u0443\u0431.\"", "\"\u0413\u0440\u0443\u0437\u043e\u043e\u0431\u043e\u0440\u043e\u0442\"", "\"\u0414\u043e\u0445\u043e\u0434\u043d\u043e\u0441\u0442\u044c \u043d\u0430 \u043e\u0431\u0449\u0438\u0439 \u043f\u0430\u0440\u043a \u0440\u0443\u0431./\u0432\u0430\u0433./\u0441\u0443\u0442.\"", "\"EBITDA\"", "\"\u0427\u0438\u0441\u0442\u044b\u0439 \u0434\u043e\u043b\u0433/EBITDA\"", "\"ROIC\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u043a\u043e\u0440\u043e\u0441\u0442\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438 \u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440\u043e\u0432\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u041a\u043e\u043d\u0442\u0440\u0435\u0439\u043b\u0435\u0440\u043d\u044b\u0435 \u043f\u0435\u0440\u0435\u0432\u043e\u0437\u043a\u0438\u00bb\"", "\"\u041f\u0440\u043e\u0435\u043a\u0442 \u00ab\u0421\u0442\u0440\u0430\u0442\u0435\u0433\u0438\u044f \u0446\u0438\u0444\u0440\u043e\u0432\u043e\u0439 \u0442\u0440\u0430\u043d\u0441\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438\u00bb\""]}, "status": "complete", "value": ""}, "description": {"expression": "", "type": "expression", "status": "complete", "value": ""}, "value": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "initPorts": [{"primitiveName": "inPort1", "type": ["IN"], "properties": {"status": {"expression": "", "type": "expression", "status": "complete", "value": ""}}, "primitiveID": "Goal_16_inPort1", "location": {"x": 484.5, "y": 84.75}}], "extensionName": "ExtensionRiskPrimitives", "primitiveName": "Goal", "primitiveID": "Goal_16", "nodeTitle": "$this.primitiveID$", "layout": {"x": 337.5, "y": -63.25, "height": 148, "width": 294}}], "edges": [{"bends": [], "sourceNode": "Data_812", "sourcePort": "Data_812_outPort1", "targetNode": "Goal_16", "targetPort": "Goal_16_inPort1", "extensionName": "ExtensionCommonPrimitives", "primitiveName": "SimpleEdge"}], "groups": []} \ No newline at end of file diff --git a/complex_rest_dtcd_supergraph/models/__init__.py b/complex_rest_dtcd_supergraph/models/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/complex_rest_dtcd_supergraph/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/misc.py b/tests/misc.py deleted file mode 100644 index 1e612dd..0000000 --- a/tests/misc.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Helper module for tests. -""" - -import json -from operator import itemgetter - -from complex_rest_dtcd_supergraph.settings import KEYS - - -def sort_payload(data: dict) -> None: - """Sort payload dict according to spec in-place. - - See docs or `serializers.py` for more info about the format. - """ - - nodes = data[KEYS.nodes] - - for node in nodes: - if KEYS.init_ports in node: - node[KEYS.init_ports] = sorted( - node[KEYS.init_ports], key=itemgetter(KEYS.yfiles_id) - ) - - data[KEYS.nodes] = sorted(nodes, key=itemgetter(KEYS.yfiles_id)) - data[KEYS.edges] = sorted( - data[KEYS.edges], - key=itemgetter( - KEYS.source_node, - KEYS.source_port, - KEYS.target_node, - KEYS.target_port, - ), - ) - data[KEYS.groups] = sorted( - data.get(KEYS.groups, []), key=itemgetter(KEYS.yfiles_id) - ) - - -def load_data(path) -> dict: - """Load from JSON and return sorted graph data.""" - - with open(path) as f: - data = json.load(f) - sort_payload(data) - - return data diff --git a/tests/test_views.py b/tests/test_views.py index f375ce4..0e5c291 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,88 +1,50 @@ -import unittest -from pathlib import Path -from pprint import pformat -from types import SimpleNamespace - -import dictdiffer -from django.urls import reverse -from django.test import Client, tag -from rest_framework import status -from rest_framework.test import APISimpleTestCase - -from .misc import load_data, sort_payload - -TEST_DIR = Path(__file__).resolve().parent -CLIENT = Client() - -# DEBUG -DEBUG_FILEPATH = "debug.txt" - - -class APITestCaseMixin: - """Some common attributes and methods for API tests. - - The idea: send an HTTP requests to a URL endpoint, get the response - and check the status. - """ - - expected_status = SimpleNamespace( - get=status.HTTP_200_OK, - post=status.HTTP_201_CREATED, - put=status.HTTP_200_OK, # or 204; 201 if created - delete=status.HTTP_200_OK, # 202 on queue, 204 on noDATA_DIR = TEST_DIR / "data" - URL_RESET=reverse("supergraph:reset") # post here resets the db content - ) - url = None # the URL endpoint to check - - def get(self): - """Send `GET` request to `self.url`, validate status and return - the response. - """ - - response = self.client.get(self.url) - self.assertEqual(response.status_code, self.expected_status.get) - return response - - def post(self, data: dict): - """Send `POST` request with given data in JSON format to - `self.url`, validate status and return the response. - """ - - response = self.client.post(self.url, data=data, format="json") - self.assertEqual(response.status_code, self.expected_status.post) - return response - - def put(self, data: dict): - """Send `PUT` request with given data in JSON format to - `self.url`, validate status and return the response. - """ - - response = self.client.put(self.url, data=data, format="json") - self.assertEqual(response.status_code, self.expected_status.put) - return response - - def delete(self): - """Send `DELETE` request to `self.url`, validate status and return - the response. - """ - - response = self.client.delete(self.url) - self.assertEqual(response.status_code, self.expected_status.delete) - return response - - def _test_delete(self): - """Simple implementation for delete endpoint. - - Calls `.delete()`, then gets the resource on the same URL and - checks the status to be 404. - """ - - r1 = self.delete() - r2 = self.client.get(self.url) # note explicit request - self.assertEqual(r2.status_code, status.HTTP_404_NOT_FOUND) - - return r1, r2 - - -if __name__ == "__main__": - unittest.main() +from random import randint + +from django.test import TestCase +from rest_framework.test import APIClient + + +# current tests do run with running complex rest with attached supergraph plugin +# so in order to run these tests user needs to run complex rest Django test suite +# with current tests folder mentioned as a parameter +class GraphViewTestCase(TestCase): + def setUp(self): + self.client = APIClient() + + def test_get(self): + response = self.client.get('/complex_rest_dtcd_supergraph/v1/graphs/') + self.assertEqual(response.status_code, 200) + + def test_post(self): + data = {'graph': 'some graph data'} + response = self.client.post('/complex_rest_dtcd_supergraph/v1/graphs/', data) + self.assertEqual(response.status_code, 400) + random: int = randint(0, 1000) + data = {'graph': 'some graph data', 'title': f'some-new-graph-{random}'} + response = self.client.post('/complex_rest_dtcd_supergraph/v1/graphs/', data) + self.assertEqual(response.status_code, 200) + self.graph_id = response.data['result']['graph_id'] + # clean up + self.client.delete(f'/complex_rest_dtcd_supergraph/v1/graphs/{self.graph_id}/') + + +class GraphDetailViewTestCase(TestCase): + def setUp(self): + self.client = APIClient() + + def test_all(self) -> None: + # create test graph + random: int = randint(0, 1000) + data = {'graph': 'some graph data', 'title': f'some-new-graph-{random}'} + response = self.client.post('/complex_rest_dtcd_supergraph/v1/graphs/', data) + self.graph_id = response.data['result']['graph_id'] + # test get + response = self.client.get(f'/complex_rest_dtcd_supergraph/v1/graphs/{self.graph_id}/') + self.assertEqual(response.status_code, 200) + # test post + data = {'graph': 'updated graph data'} + response = self.client.put(f'/complex_rest_dtcd_supergraph/v1/graphs/{self.graph_id}/', data) + self.assertEqual(response.status_code, 200) + # test delete + response = self.client.delete(f'/complex_rest_dtcd_supergraph/v1/graphs/{self.graph_id}/') + self.assertEqual(response.status_code, 200) From 7498dbd09748927bf4df403cde5ed3d928d24b72 Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 22 Jan 2024 14:30:39 +0300 Subject: [PATCH 21/25] fixed all errors with saving files and ids to map file --- .../utils/filesystem_graphmanager.py | 46 +++++++++++++------ complex_rest_dtcd_supergraph/views/graphs.py | 23 +++++----- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py index 1794f3a..61149e8 100644 --- a/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py +++ b/complex_rest_dtcd_supergraph/utils/filesystem_graphmanager.py @@ -3,6 +3,7 @@ import uuid import json import tempfile +from json import JSONDecodeError from typing import Union, AnyStr from ..utils.graphmanager_exception import GraphManagerException, NO_GRAPH, NO_ID, NAME_EXISTS, NO_TITLE @@ -63,8 +64,11 @@ def read(self, graph_id) -> dict: try: # read graph.graphml by its name and id and get the 'content' of it file_path = Path(self.final_path) / str(graph_id) / self.default_filename + graph_data: dict = {} with open(file_path, 'r') as graph: - graph_data = json.load(graph) + file_content = graph.read() + if file_content: + graph_data = json.loads(file_content) # return result return graph_data @@ -74,22 +78,29 @@ def read(self, graph_id) -> dict: def read_all(self) -> list[dict]: """Read all graph json files """ + graph_data: dict = {} with open(self.map_file_path, 'r') as map_file: - graph_data = json.load(map_file) + file_content = map_file.read() + if file_content: + graph_data = json.loads(file_content) result = [{"id": graph_id, "title": title} for graph_id, title in graph_data.items()] return result - def write(self, graph: dict) -> dict: + def write(self, graph: Union[dict, str]) -> dict: """Create graph json file with `graph` data""" # backup the map file shutil.copyfile(self.map_file_path, self.map_backup_path) + id_map = {} # read the map file with open(self.map_file_path, 'r') as map_file: - id_map = json.load(map_file) - # check if we have this graph already - if graph["title"] in id_map.values(): - raise GraphManagerException(NAME_EXISTS, graph["title"]) + file_content = map_file.read() + if file_content: + id_map = json.loads(file_content) + + # check if we have this graph already + if graph["title"] in id_map.values(): + raise GraphManagerException(NAME_EXISTS, graph["title"]) # create unique id unique_id = str(uuid.uuid4()) @@ -108,13 +119,16 @@ def write(self, graph: dict) -> dict: # save graph to file save_data_to_file(graph, Path(graph_dir / self.default_filename)) - return {unique_id: graph['title']} + return {'graph_id': unique_id, 'title': graph['title']} def update(self, graph: dict, graph_id: str) -> None: """Rewrite graph json file with `graph` data""" # read the id_map file + id_map: dict = {} with open(self.map_file_path, 'r') as map_file: - id_map = json.load(map_file) + file_content = map_file.read() + if file_content: + id_map = json.loads(file_content) graph_dir = Path(self.final_path) / str(graph_id) # check if we don't have this graph or there is no such dir @@ -128,21 +142,24 @@ def remove(self, graph_id: str) -> None: """Delete graph json file by 'graph_id'""" graph_dir = Path(self.final_path) / graph_id # check if we have graph with this id - # if not os.path.isdir(graph_dir): - # raise GraphManagerException(NO_GRAPH, graph_id) + if not os.path.isdir(graph_dir): + raise GraphManagerException(NO_GRAPH, graph_id) # delete directory, and it's content - # shutil.rmtree(Path(self.final_path) / str(graph_id)) + shutil.rmtree(Path(self.final_path) / str(graph_id)) # back up id_map file shutil.copyfile(self.map_file_path, self.map_backup_path) # read id_map + id_map: dict = {} with open(self.map_file_path, 'r') as map_file: - id_map = json.load(map_file) + file_content = map_file.read() + if file_content: + id_map = json.loads(file_content) # delete id from id_map del id_map[graph_id] - + # save map file save_data_to_file(id_map, self.map_file_path) @@ -151,3 +168,4 @@ def save_data_to_file(data: Union[dict, AnyStr], destination_path: Path) -> None file.write(json.dumps(data).encode('utf-8')) file.flush() os.rename(file.name, destination_path) + diff --git a/complex_rest_dtcd_supergraph/views/graphs.py b/complex_rest_dtcd_supergraph/views/graphs.py index c6d0166..38be03e 100644 --- a/complex_rest_dtcd_supergraph/views/graphs.py +++ b/complex_rest_dtcd_supergraph/views/graphs.py @@ -1,3 +1,5 @@ +import json + from rest.views import APIView from rest.response import Response, status from rest.permissions import AllowAny @@ -15,18 +17,17 @@ class GraphView(APIView): graph_manager = FilesystemGraphManager(GRAPH_BASE_PATH, GRAPH_TMP_PATH, GRAPH_ID_NAME_MAP_PATH) def post(self, request: Request) -> Response: - graphs = request.data - list_of_ids = [] - for graph in graphs: - try: - list_of_ids.append(self.graph_manager.write(graph)) - except Exception as e: - return Response( - {"status": "ERROR", "msg": str(e)}, - status.HTTP_400_BAD_REQUEST - ) + graph = request.data + + try: + id_and_title = (self.graph_manager.write(graph)) + except Exception as e: + return Response( + {"status": "ERROR", "msg": str(e)}, + status.HTTP_400_BAD_REQUEST + ) return Response( - {"status": "SUCCESS", "result": list_of_ids}, + {"status": "SUCCESS", "result": id_and_title}, status.HTTP_200_OK ) From 090d14b01eb7bb5126cb1ef8165d3b048c553c7e Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 22 Jan 2024 14:48:09 +0300 Subject: [PATCH 22/25] updated versions and maintainer --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index f2667fe..09b5f3d 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ __author__ = "Aleksei Tsysin" -__copyright__ = "Copyright 2021, ISG Neuro" +__copyright__ = "Copyright 2024, ISG Neuro" __credits__ = [] __license__ = "" -__version__ = "0.3.3" -__api_version__ = "1" -__maintainer__ = "Aleksei Tsysin" -__email__ = "atsysin@isgneuro.com" +__version__ = "0.4.0" +__api_version__ = "2" +__maintainer__ = "Nikita Serditov" +__email__ = "nserditov@isgneuro.com" __status__ = "Dev" From 3575ef9e5106bbb7d0da92e3a1a8e8b49fca9a6e Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 22 Jan 2024 14:48:40 +0300 Subject: [PATCH 23/25] cleaned up & updated README.md --- README.md | 80 ++++--------------------------------------------------- 1 file changed, 5 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 52d10d8..369c5bb 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,10 @@ # Supergraph plugin -[Complex rest](https://github.com/ISGNeuroTeam/complex_rest/tree/develop) plugin for graph management. - -For an introduction check out the [User guide](docs/user-guide.md). - -## Installation - -These instructions will get you a copy of the plugin up and running on your local machine for development and testing purposes. See [deployment](#deployment) for notes on how to deploy the plugin on a live system. +[Complex rest](https://github.com/ISGNeuroTeam/complex_rest/tree/develop) plugin for graph management. For an introduction check out the [User guide](docs/user-guide.md). ### Prerequisites 1. Deploy [complex rest](https://github.com/ISGNeuroTeam/complex_rest). -2. Install [Neo4j](https://neo4j.com/docs/operations-manual/current/installation/) graph database. - 1. Follow installation instructions for your OS [ [Linux](https://neo4j.com/docs/operations-manual/current/installation/linux/) | [Windows](https://neo4j.com/docs/operations-manual/current/installation/windows/) | [Mac](https://neo4j.com/docs/operations-manual/current/installation/osx/) ]. Pay attention to the [required Java version](https://neo4j.com/docs/operations-manual/current/installation/requirements/#deployment-requirements-java); you may need to change system defaults. - 2. [Set initial password](https://neo4j.com/docs/operations-manual/current/configuration/set-initial-password/) to `password` with the following command: - ```sh - neo4j-admin set-initial-password password - ``` - > This must be performed before starting up the database for the first time. - 3. Run the service, make sure it is available on port `7687`: - ```sh - systemctl start neo4j - ``` - 4. (optional) If you installed *Cypher shell*, you can try to connect to Neo4j to make sure everything is ok: - ```sh - cypher-shell -a neo4j://localhost:7687 -u neo4j -p password - ``` ### Deploy from GitHub or Nexus @@ -33,11 +12,10 @@ You can get the latest build from either [GitHub releases](https://github.com/IS 1. Download an archive with the latest build. 2. Unpack the archive into `complex_rest/plugins` directory. -3. Run the [initialization script](#initialization-script) to prepare the database. ### Deploy via Make -If you want to have an access to feature branch version, you can build this plugin locally. +If you want to have access to feature branch version, you can build this plugin locally. 1. Clone the Git repository: ```sh @@ -49,7 +27,6 @@ If you want to have an access to feature branch version, you can build this plug make pack ``` 3. Unpack the archive into `complex_rest/plugins` directory. -4. Run the [initialization script](#initialization-script) to prepare the database. ### Deploy manually @@ -78,66 +55,18 @@ If you are a developer, then follow this section. cd complex_rest/plugins ln -s pathtofolder/repo/complex_rest_dtcd_supergraph ``` -6. [Re-install constraints and indexes](#re-installing-constraints-and-indexes), [initialize the default `Root` node](#initializing-the-default-root). ## Deployment -For deployment we need to get a build archive - see the previous section on how to do that. Then: +For deployment, we need to get a build archive - see the previous section on how to do that. Then: 1. Stop `complex_rest`. 2. Unpack the archive into `complex_rest/plugins` directory. -3. Run the [initialization script](#initialization-script) to prepare the database. -4. **TODO** Backup / reset / migrate the database. -5. Start `complex_rest`. - -## Notes - -### Re-installing constraints and indexes - -Neo4j provides support for applying [indexes](https://neo4j.com/docs/getting-started/current/graphdb-concepts/#graphdb-indexes) and [constraints](https://neo4j.com/docs/getting-started/current/graphdb-concepts/#graphdb-constraints). - -To do this, activate plugin's virtual environment and run: - -```sh -address="bolt://neo4j:password@localhost:7687" -neomodel_remove_labels --db $address -neomodel_install_labels models --db $address -``` - -For password and port use values from `supergraph.conf`. - -### Initializing the default Root - -We need to create the default root node in order to keep backwards compatibility with the API v0.2.0. There is a script just for that! - -Activate plugin's virtual environment, navigate to script's parent directory and run: - -```sh -python create_default_root.py -``` - -It will create a single `Root` node and save its `uid` attribute in the file `default_root_uid.txt` inside source code directory. - -### Initialization script - -There is a helper script `database_init.sh` that prepares the plugin for work when deploying *from build archive*. It combines activation of correct virtual environment, [re-installation of constraints and indexes](#re-installing-constraints-and-indexes) with [default Root creation](#initializing-the-default-root) in one place. - -You can run it from anywhere you like: - -```sh -./database_init.sh -``` +3. Start `complex_rest`. ## TODO - Update [User guide](docs/user-guide.md). -- Resolve `RelationshipClassRedefined` error when trying to test `neomodel` models directly (see `test_managers.py`). -- User-defined properties that were saved before *stay on the node after merge* even if they are missing in new structure and should be deleted (see how `create_or_update` works). We handle it in converter, but this is not nice. -- Some database queries are inefficient (`n+1` problems). - -## Built With - -- [Neo4j](https://neo4j.com/) - Graph data platform. ## Versioning @@ -146,6 +75,7 @@ We use [SemVer](http://semver.org/) for versioning. For the versions available, ## Authors - Aleksei Tsysin (atsysin@isgneuro.com) +- Nikita Serditov (nserditov@isgneuro.com) ## License From ba95c075ac09f2290617a71c27565ac95e2e512e Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 22 Jan 2024 15:48:42 +0300 Subject: [PATCH 24/25] fixed typo --- docs/Format.md | 67 --------------------------------------------- tests/test_views.py | 2 +- 2 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 docs/Format.md diff --git a/docs/Format.md b/docs/Format.md deleted file mode 100644 index e8c2e39..0000000 --- a/docs/Format.md +++ /dev/null @@ -1,67 +0,0 @@ -# Graph data format for communication between Neo4j and Y-files - -The (sub)graph is represented using JSON with the following structure: - -``` -{ - "nodes": [node, node, ...], - "edges": [edge, edge, ...], - "groups" : [group, group, ...] -} -``` - -Arrays may be empty. - -## Nodes - -Top-level `nodes` key corresponds to an *array* of objects, each representing a particular node. - -Each object is a *mapping* that may contain a variable amount of keys and values of arbitrary nesting. Every node object **must** have a `primitiveID` key with an **unique** ID. In addition, a node may have an array of *ports* in its `initPorts` field, and user-defined properties in its `properties` field. - -Example of a valid node object: - -```json -{ - "primitiveID": "n1", - "primitiveName": "name", - "extensionName": "extension", - "nodeTitle": "title", - "properties": { - "custom_field": {"attribute": "value"} - }, - "initPorts": [ - { - "primitiveID": "p1", - "primitiveName": "port_name", - "type": "port_type", - "properties": {"property": "value"} - } - ] -} -``` - -## Edges - -`edges` is an array of objects which represent edges. - -Every edge object **must** have: -- `sourceNode` and `targetNode` keys corresponding to valid node object IDs. -- `sourcePort` and `targetPort` keys corresponding to valid port IDs on start and end nodes. - -Example of a valid edge object - -```json -{ - "sourceNode": "n1", - "targetNode": "n2", - "sourcePort": "p1", - "targetPort": "p3", - "extensionName": "extension", - "meta": {"field": 42} -} -``` - -## Requirements - -- All IDs must be unique. -- Referential integrity must be preserved: referenced entities must exist within the payload. \ No newline at end of file diff --git a/tests/test_views.py b/tests/test_views.py index 0e5c291..dc37a20 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -41,7 +41,7 @@ def test_all(self) -> None: # test get response = self.client.get(f'/complex_rest_dtcd_supergraph/v1/graphs/{self.graph_id}/') self.assertEqual(response.status_code, 200) - # test post + # test put data = {'graph': 'updated graph data'} response = self.client.put(f'/complex_rest_dtcd_supergraph/v1/graphs/{self.graph_id}/', data) self.assertEqual(response.status_code, 200) From 90173ae7ba1036a934554bab78798b4b13b315ed Mon Sep 17 00:00:00 2001 From: Nikita Serditov Date: Mon, 22 Jan 2024 15:53:30 +0300 Subject: [PATCH 25/25] updated user-guide --- docs/user-guide.md | 300 ++++++--------------------------------------- 1 file changed, 40 insertions(+), 260 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index d9a2687..4e592fc 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1,260 +1,40 @@ -# UNDER CONSTRUCTION: User guide - -> The information here is deprecated. - -In this guide you'll find examples of core operations provided by this plugin. - -Check out check out the [Format](Format.md) and `neo-tools` library for details on why and how we represent nested structures in the database and [OpenAPI schema](./openapi.yaml) to get a general idea for the plugin. - -> You can follow examples using Django-provided shell of `complex_rest` project. - -## Vertices and edges - -We get vertices and edges in the form of Python dictionaries (check out the [Format](Format.md) for details). Here is a couple of simple examples. - -```python -vertex = { - "primitiveID": "amy", - "properties": { - "height": {"value": 167}, - "age": {"type": "days"} - } -} -``` - -Front-end guys call these *nodes*, but we use the term *vertex* because Neo4j has its own nodes, and they are different. - -```python -edge = { - "sourceNode": "amy", - "sourcePort": "mobile", - "targetNode": "bob", - "targetPort": "laptop", - "connection": { - "status": 7, - "online": True - } -} -``` - -Edges represent a connection between two nodes and their corresponding ports. - -We get these in the form of graphs, like so: - -```python -graph = { - "nodes": [ - {"primitiveID": "amy"}, - {"primitiveID": "bob"} - ], - "edges": [ - { - "sourceNode": "amy", - "sourcePort": "mobile", - "targetNode": "bob", - "targetPort": "laptop" - } - ] -} -``` - -> We run validation on this data before we proceed further: check consistent IDs, etc. You can see more in `serializers` and `fields` modules. - -## Representation - -We want to represent these as graph structures inside Neo4j. Unfortunately, Neo4j cannot store nested objects on either nodes or relationships, so we *do not* have 1:1 mappings for vertices-nodes and edges-relationships. Instead, we represent vertices & edges as *tree structures* and convert back and forth between them. Check out the [Format](./Format.md) document for more information. - -To support that, we use the `Converter` class: - -```python ->>> from converters import Converter ->>> converter = Converter() -``` - -`Converter` can translate between the `graph` dictionary and structures which can be saved with Neo4j - back and forth: - -```python ->>> subgraph = converter.load(graph) ->>> original = converter.dump(subgraph) ->>> graph == original -True -``` - -We use `py2neo` library to work with Neo4j. `subgraph` is a [`Subgraph`](https://py2neo.org/2021.1/data/index.html#py2neo.data.Subgraph) object - an arbitrary collection of nodes and relationships. - -## Manager - -To start, we need a connection to Neo4j database. `Neo4jGraphManager` provides just that, as well as all database-related operations we are interested in. - -```python ->>> from managers import Neo4jGraphManager ->>> manager = Neo4jGraphManager("bolt://localhost:7687", auth=("neo4j", "neo4j")) ->>> manager.clear() # reset Neo4j database -``` - -## Fragments - -Let's create some **fragments** to work with. These behave just like Django models, except they are [Neo4j OGM models](https://py2neo.org/2021.1/ogm/index.html) from `py2neo`. - -```python ->>> from models import Fragment ->>> marketing = Fragment(name="marketing") ->>> research = Fragment(name="research") ->>> sales = Fragment(name="sales") -``` - -For now these fragments are *unbound*: they exist only in memory, there is no counterpart in Neo4j database. We *bind* them by saving these with the help of a `FragmentManager`. -> `FragmentManager` provides basic CRUD operations for fragments. - -```python ->>> # .fragments is a FragmentManager for this graph ->>> manager.fragments.save(marketing, research, sales) -``` - -Now we have 3 fragments in the database. We use fragments to partition the graph into regions for *security control* and *ease of work*: each fragment may contain a set of vertices and edges between them. - -The graph has a special **root** fragment, which includes all *content* of a graph (vertices, edges, groups, etc). All new and existing entities are included in the root fragment by default. - -Additional notes: - -- Currently, the security control is not implemented. -- Under the hood, we store fragments as **nodes** with a specific label. -- Containment is implemented as a relationship between the fragment node and root nodes of entities (vertices, edges, groups): `(fragment) --> (entity)` -- Currently, we have no idea how to handle connections & interaction between fragments. -- Currently, the root fragment is a *logical construct*; not an instance of `Fragment` model. - -## Graph operations - -The main operation is a **merge with replacement**. We can do this either for a given fragment, or for the whole graph using the *root* fragment. - -### Basic save and read - -Let's add some graph data to the `marketing` fragment: - -```python ->>> data = {'edges': [], 'nodes': [{'primitiveID': 'amy'}, {'primitiveID': 'bob'}]} ->>> subgraph = converter.load(data) ->>> # .content is a ContentManger for this graph ->>> manager.fragments.content.replace(subgraph, marketing) -``` - -We just merged (with replacement) new content into the fragment `marketing`. Now the fragment node has a link to the roots of 2 *vertex trees*: a tree-like subgraph of nodes and relationships representing our initial data. Each tree stores the data for a corresponding node. - -> Remember that we *cannot* save nested data structures as Neo4j properties. We represent these as tree-like entities on the backend. - -We can get this data back just as easy: - -```python ->>> subgraph = manager.fragments.content.read(marketing) ->>> data = converter.dump(subgraph) ->>> data -{'edges': [], 'nodes': [{'primitiveID': 'amy'}, {'primitiveID': 'bob'}]} -``` - -### Merge with replacement - -Now for something interesting - let's try and save the following graph in the same fragment: - -```python ->>> data = {'edges': [], 'nodes': [{'primitiveID': 'amy'}, {'primitiveID': 'dan'}]} ->>> subgraph = converter.load(data) ->>> manager.fragments.content.replace(subgraph, marketing) -``` - -The idea here is that we want to **replace** the content of the `marketing` fragment in a smart way: - -- create new nodes & relationships -- update existing stuff with new data if needed -- delete the old stuff - -We also want to *preserve existing relationships* between updated nodes and other entities in the graph. Here's how we do it: - -1. Merge the *root nodes* of the following entity trees: - 1. Vertex trees. - 2. Edge trees. - 3. Group trees (if any). -2. Remove old nodes & relationships. -3. Re-link fragment with the root nodes of new entities to be created. -4. Merge the rest and fragment-entity links. - -> See `managers.ContentManager._merge` for details. - -In the example above, we create `dan`, delete `bob` and update `amy` vertices. We preserve all connections from `amy` vertex to other intact members of the same graph. - -Now the fragment contains just two vertices: - -```python ->>> subgraph = manager.fragments.content.read(marketing) ->>> data = converter.dump(subgraph) ->>> data -{'edges': [], 'nodes': [{'primitiveID': 'amy'}, {'primitiveID': 'dan'}]} -``` - -### Multiple fragments and the root - -Let's add some more data to another fragment: - -```python ->>> data = { -... 'edges': [{'sourceNode': 'bob', -... 'sourcePort': 'mobile', -... 'targetNode': 'cloe', -... 'targetPort': 'laptop'}], -... 'nodes': [{'primitiveID': 'bob'}, {'primitiveID': 'cloe'}]} ->>> subgraph = converter.load(data) ->>> manager.fragments.content.replace(subgraph, research) -``` - -The `research` fragment now contains 2 vertex trees and 1 *edge tree*, with a *relationship* between roots of entity trees: - -``` -(bob_root) --> (edge_root) --> (cloe_root) -``` - -We can get the *whole* graph (same as the *root* fragment): - -```python ->>> # reads root fragment by default ->>> subgraph = manager.fragments.content.read() ->>> data = converter.dump(subgraph) ->>> data -{'edges': [{'sourceNode': 'bob', - 'sourcePort': 'mobile', - 'targetNode': 'cloe', - 'targetPort': 'laptop'}], - 'nodes': [{'primitiveID': 'amy'}, - {'primitiveID': 'bob'}, - {'primitiveID': 'cloe'}, - {'primitiveID': 'dan'}]} -``` - -Now for something interesting. Let's save the following graph on the `root` fragment: - -```python ->>> data = { -... 'edges': [{'sourceNode': 'bob', -... 'sourcePort': 'IoT device', -... 'targetNode': 'eve', -... 'targetPort': 'server'}], -... 'nodes': [{'primitiveID': 'amy'}, -... {'primitiveID': 'bob'}, -... {'primitiveID': 'eve'}]} ->>> subgraph = converter.load(data) ->>> # replaces root fragment by default ->>> manager.fragments.content.replace(subgraph) -``` - -Here we: - -- create `eve` vertex and `bob-eve` edge -- *update* `amy` and `bob` vertices while preserving relationships to parent fragment -- delete vertices `cloe`, `dan` and `bob-cloe` edge - -The state of fragments' content: -- `marketing` fragment still contains `amy` vertex -- `research` fragment still has `bob` vertex -- `eve` vertex and `bob-eve` edge do not belong to *any* fragment - -Notes: - -- `ContentManger` is responsible for all the logic related to graph updates. \ No newline at end of file +# User guide + +## API: + +### Get list of all graphs: +Method: `GET` +Request: `graphs/` +Result is a list of dicts: {`"id"`: `"graph_id"`, `"title"`: `"graph_title"`} +Example: `[{"id":"25382325-0c26-4ff0-bf26-a3f87fb61502","title":"some-new-graph-012"},{"id":"8d788a5f-310e-40d7-a8d7-060c247bd89e","title":"some-new-graph-013"}, {"id":"73976e44-ea1b-467c-96d2-1e5008726a48","title":"some-new-graph-014"}]` + +### Get exact graph by id +Method: `GET` +Request: `graph/graph_id/` +Example of id: `73976e44-ea1b-467c-96d2-1e5008726a48` +Result is a dict: `{"graph": "graph_data", "title": "graph_title"}` + +### Create new graph +Method: `POST` +Request: `graphs/` +Request body: `{"graph": "some graph data", "title": "some-new-graph-014"}` +`graph` stores all the graph data +`title` stores the title of the graph +Both parameters of the body are **required** +Result is a dict: `{"status" : "status of the request", "result" : "graph and title of the graph"}` +Example: `{"status":"SUCCESS","result":{"graph_id":"73976e44-ea1b-467c-96d2-1e5008726a48","title":"some-new-graph-014"}}` + +### Change graph by id +Method: `PUT` +Request: Request: `graph/graph_id/` +Example of id: `73976e44-ea1b-467c-96d2-1e5008726a48` +Request body: `{"graph": "updated graph_data"}` +Result is a dict: `{"status": "request_status"}` +Example: `{"status": "SUCCESS"}` + +## Delete graph by id +Method: `DELETE` +Request: Request: `graph/graph_id/` +Example of id: `73976e44-ea1b-467c-96d2-1e5008726a48` +Result is a dict: `{"status": "request_status"}` +Example: `{"status": "SUCCESS"}` \ No newline at end of file