diff --git a/.gitignore b/.gitignore index c3e71f5..62cb8ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ file.* *test* -conf/conf.env +aiven_metadata_parser/conf/conf.env __pycache__ certs/* output.* @@ -11,4 +11,5 @@ graph.py kcat.config tmp .vscode -*.log \ No newline at end of file +*.log +conf/* \ No newline at end of file diff --git a/README.md b/README.md index 5acfd6b..bf37195 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The `create_services.sh` uses: Before starting it, please install the required dependences, login with the [Aiven CLI](https://github.com/aiven/aiven-client) and execute: ```bash -./scripts/create_services.sh +./aiven_metadata_parser/scripts/create_services.sh ``` Where `` is the name of your Aiven project. @@ -32,7 +32,7 @@ Where `` is the name of your Aiven project. To delete the services you can call: ```bash -./scripts/delete_services.sh +./aiven_metadata_parser/scripts/delete_services.sh ``` If `` is not passed, the default project will be used. @@ -46,13 +46,13 @@ You need to be on Python 3.7, install the required libraries with: pip install -r requirements.txt ``` -Copy the `conf.env.sample` file to `conf.env` in the `conf` folder and edit the token parameter and the project name from which you want to extract parameters. +In the `aiven_metadata_parser/conf` folder, copy the `conf.env.sample` file to `conf.env` and edit the token parameter and the project name from which you want to extract parameters. If you don't have a project with services already running you can create a sample set of services with the `create_services.sh` file, which requires the `aiven-client` to be installed and the user to be logged in. Once `conf.env` is set, you can start the metadata extraction with: ```bash -python main.py +python aiven_metadata_parser/main.py ``` This will generate: @@ -65,7 +65,7 @@ This will generate: Furthermore if, after executing the `main.py` you also execute the following: ```bash -python app.py +python aiven_metadata_parser/app.py ``` The `app.py` reads the `graph_data.gml` file generated at step 1 and creates a Reactive Web Applications with [Plotly](https://plot.ly/python/) and [Dash](https://plot.ly/dash/) (code taken from [here](https://towardsdatascience.com/python-interactive-network-visualization-using-networkx-plotly-and-dash-e44749161ed7)). @@ -82,7 +82,7 @@ The code is a bare minimum product, doesn't cover all services and options and i Possible issues and solutions ============ -- If you run `python app.py` and see an error saying `No such file or directory: 'neato'`, you will have to install `graphviz` on your machine (the package version from pip doesn't seem to work) - find out how to install it [here](https://graphviz.org/download/). +- If you run `python aiven_metadata_parser/app.py` and see an error saying `No such file or directory: 'neato'`, you will have to install `graphviz` on your machine (the package version from pip doesn't seem to work) - find out how to install it [here](https://graphviz.org/download/). Contributing ============ diff --git a/aiven_metadata_parser/__init__.py b/aiven_metadata_parser/__init__.py new file mode 100644 index 0000000..d2e8e5a --- /dev/null +++ b/aiven_metadata_parser/__init__.py @@ -0,0 +1,18 @@ +from aiven_metadata_parser import ( + add_grafana_dashboard, + app, + backup, + explore_service, + flink, + grafana, + integration, + kafka, + kafka_connect, + mysql, + opensearch, + pg, + pyvis_display, + redis, + sql, + tag, +) diff --git a/src/add_grafana_dashboard.py b/aiven_metadata_parser/add_grafana_dashboard.py similarity index 100% rename from src/add_grafana_dashboard.py rename to aiven_metadata_parser/add_grafana_dashboard.py diff --git a/app.py b/aiven_metadata_parser/app.py similarity index 100% rename from app.py rename to aiven_metadata_parser/app.py diff --git a/src/backup.py b/aiven_metadata_parser/backup.py similarity index 100% rename from src/backup.py rename to aiven_metadata_parser/backup.py diff --git a/conf/conf.env.sample b/aiven_metadata_parser/conf/conf.env.sample similarity index 100% rename from conf/conf.env.sample rename to aiven_metadata_parser/conf/conf.env.sample diff --git a/src/explore_service.py b/aiven_metadata_parser/explore_service.py similarity index 98% rename from src/explore_service.py rename to aiven_metadata_parser/explore_service.py index 4ce7ab6..1687909 100644 --- a/src/explore_service.py +++ b/aiven_metadata_parser/explore_service.py @@ -1,19 +1,14 @@ """Explores a service""" -from src import ( +from aiven_metadata_parser import ( backup, kafka, - kafka_connect, - pg, tag, integration, - grafana, - redis, - mysql, - opensearch, flink, ) - +from aiven_metadata_parser import pg +from aiven_metadata_parser import opensearch, kafka_connect, mysql, grafana, redis SERVICE_MAP = {} EXPLORER_METHODS = {} diff --git a/src/flink.py b/aiven_metadata_parser/flink.py similarity index 100% rename from src/flink.py rename to aiven_metadata_parser/flink.py diff --git a/src/grafana.py b/aiven_metadata_parser/grafana.py similarity index 100% rename from src/grafana.py rename to aiven_metadata_parser/grafana.py diff --git a/src/integration.py b/aiven_metadata_parser/integration.py similarity index 100% rename from src/integration.py rename to aiven_metadata_parser/integration.py diff --git a/src/kafka.py b/aiven_metadata_parser/kafka.py similarity index 99% rename from src/kafka.py rename to aiven_metadata_parser/kafka.py index 5dc523f..3b6439f 100644 --- a/src/kafka.py +++ b/aiven_metadata_parser/kafka.py @@ -1,7 +1,7 @@ """Parsing Apache Kafka® services""" import re -from src import kafka_connect +from aiven_metadata_parser import kafka_connect def explore_kafka(self, service, service_name, project, service_map): diff --git a/src/kafka_connect.py b/aiven_metadata_parser/kafka_connect.py similarity index 100% rename from src/kafka_connect.py rename to aiven_metadata_parser/kafka_connect.py diff --git a/lib/bindings/utils.js b/aiven_metadata_parser/lib/bindings/utils.js similarity index 100% rename from lib/bindings/utils.js rename to aiven_metadata_parser/lib/bindings/utils.js diff --git a/lib/tom-select/tom-select.complete.min.js b/aiven_metadata_parser/lib/tom-select/tom-select.complete.min.js similarity index 100% rename from lib/tom-select/tom-select.complete.min.js rename to aiven_metadata_parser/lib/tom-select/tom-select.complete.min.js diff --git a/lib/tom-select/tom-select.css b/aiven_metadata_parser/lib/tom-select/tom-select.css similarity index 100% rename from lib/tom-select/tom-select.css rename to aiven_metadata_parser/lib/tom-select/tom-select.css diff --git a/lib/vis-9.0.4/vis-network.css b/aiven_metadata_parser/lib/vis-9.0.4/vis-network.css similarity index 100% rename from lib/vis-9.0.4/vis-network.css rename to aiven_metadata_parser/lib/vis-9.0.4/vis-network.css diff --git a/lib/vis-9.0.4/vis-network.min.js b/aiven_metadata_parser/lib/vis-9.0.4/vis-network.min.js similarity index 100% rename from lib/vis-9.0.4/vis-network.min.js rename to aiven_metadata_parser/lib/vis-9.0.4/vis-network.min.js diff --git a/lib/vis-9.1.2/vis-network.css b/aiven_metadata_parser/lib/vis-9.1.2/vis-network.css similarity index 100% rename from lib/vis-9.1.2/vis-network.css rename to aiven_metadata_parser/lib/vis-9.1.2/vis-network.css diff --git a/lib/vis-9.1.2/vis-network.min.js b/aiven_metadata_parser/lib/vis-9.1.2/vis-network.min.js similarity index 100% rename from lib/vis-9.1.2/vis-network.min.js rename to aiven_metadata_parser/lib/vis-9.1.2/vis-network.min.js diff --git a/aiven_metadata_parser/main.py b/aiven_metadata_parser/main.py new file mode 100644 index 0000000..b68d076 --- /dev/null +++ b/aiven_metadata_parser/main.py @@ -0,0 +1,97 @@ +from aiven.client import client +import explore_service +import pyvis_display as pyvis_display +import configparser + + +def main(): + # Reading conf.env configuration file + with open("aiven_metadata_parser/conf/conf.env", "r") as f: + config_string = "[DEFAULT]\n" + f.read() + config = configparser.ConfigParser() + config.read_string(config_string) + + # Creating Aiven client instance + myclient = client.AivenClient(base_url=config["DEFAULT"]["BASE_URL"]) + + # Authenticating and storing the token + # result = myclient.authenticate_user(email=config['DEFAULT']['USERNAME'], password=config['DEFAULT']['PASSWORD']) + myclient.auth_token = config["DEFAULT"]["TOKEN"] + + # Creating empty nodes and edges lists + nodes = [] + edges = [] + + # The service order helps analysis first "standalone" services and then connection services, this might be useful + # to parse first services which can be either source or sink of other services (e.g. Kafka connect might use a PG + # table as a source) + + services_order = { + "opensearch": 1, + "elasticsearch": 1, + "pg": 1, + "redis": 1, + "mysql": 1, + "clickhouse": 1, + "cassandra": 1, + "m3db": 1, + "m3aggregator": 1, + "m3coordinator": 1, + "influxdb": 1, + "kafka": 2, + "kafka_connect": 3, + "kafka_mirrormaker": 3, + "flink": 3, + "grafana": 3, + } + + # Listing the services + services = myclient.get_services(project=config["DEFAULT"]["PROJECT"]) + + # Ordering based on service_order + services.sort(key=lambda x: services_order[x["service_type"]]) + + # Initial loop to find all ip/hostname of existing services + print("Locate IP/hostname of each service") + for i, service in enumerate(services, start=1): + # if service["service_name"]!='test': + print( + f"{i}/{len(services)} {service['service_name']} {service['service_type']}" + ) + # if service["service_type"]=='grafana': + explore_service.populate_service_map( + myclient, + service["service_type"], + service["service_name"], + project=config["DEFAULT"]["PROJECT"], + ) + + # Second loop to find details of each service + print() + print("Find details of each service") + for i, service in enumerate(services, start=1): + print( + f"{i}/{len(services)} Query {service['service_name']} {service['service_type']}" + ) + # if service["service_name"] != 'test': + (newnodes, newedges) = explore_service.explore( + myclient, + service["service_type"], + service["service_name"], + project=config["DEFAULT"]["PROJECT"], + ) + nodes = nodes + newnodes + edges = edges + newedges + + (newnodes, newedges) = explore_service.explore_ext_endpoints( + myclient, project=config["DEFAULT"]["PROJECT"] + ) + nodes = nodes + newnodes + edges = edges + newedges + + # Creating viz with pyviz + pyvis_display.pyviz_graphy(nodes, edges) + + +if __name__ == "__main__": + main() diff --git a/src/mysql.py b/aiven_metadata_parser/mysql.py similarity index 100% rename from src/mysql.py rename to aiven_metadata_parser/mysql.py diff --git a/src/opensearch.py b/aiven_metadata_parser/opensearch.py similarity index 100% rename from src/opensearch.py rename to aiven_metadata_parser/opensearch.py diff --git a/src/pg.py b/aiven_metadata_parser/pg.py similarity index 99% rename from src/pg.py rename to aiven_metadata_parser/pg.py index 0939b45..af181fe 100644 --- a/src/pg.py +++ b/aiven_metadata_parser/pg.py @@ -1,7 +1,7 @@ """Parsing PostgreSQL services""" import psycopg2 -from src import sql +from aiven_metadata_parser import sql def build_conn_string(avnadmin_pwd, service_conn_info): diff --git a/src/pg_store_tbl.sql b/aiven_metadata_parser/pg_store_tbl.sql similarity index 100% rename from src/pg_store_tbl.sql rename to aiven_metadata_parser/pg_store_tbl.sql diff --git a/src/pyvis_display.py b/aiven_metadata_parser/pyvis_display.py similarity index 100% rename from src/pyvis_display.py rename to aiven_metadata_parser/pyvis_display.py diff --git a/src/redis.py b/aiven_metadata_parser/redis.py similarity index 100% rename from src/redis.py rename to aiven_metadata_parser/redis.py diff --git a/scripts/create_mysql_tbl.sql b/aiven_metadata_parser/scripts/create_mysql_tbl.sql similarity index 100% rename from scripts/create_mysql_tbl.sql rename to aiven_metadata_parser/scripts/create_mysql_tbl.sql diff --git a/scripts/create_mysql_usr.sql b/aiven_metadata_parser/scripts/create_mysql_usr.sql similarity index 100% rename from scripts/create_mysql_usr.sql rename to aiven_metadata_parser/scripts/create_mysql_usr.sql diff --git a/scripts/create_pg_tbl.sql b/aiven_metadata_parser/scripts/create_pg_tbl.sql similarity index 100% rename from scripts/create_pg_tbl.sql rename to aiven_metadata_parser/scripts/create_pg_tbl.sql diff --git a/scripts/create_services.sh b/aiven_metadata_parser/scripts/create_services.sh similarity index 97% rename from scripts/create_services.sh rename to aiven_metadata_parser/scripts/create_services.sh index 549a9ee..111c296 100755 --- a/scripts/create_services.sh +++ b/aiven_metadata_parser/scripts/create_services.sh @@ -1,5 +1,5 @@ PROJECT_NAME=$1 -. conf/conf.env +. aiven_metadata_parser/conf/conf.env avn --auth-token $TOKEN project switch $PROJECT_NAME @@ -105,7 +105,7 @@ avn --auth-token $TOKEN service schema create demo-kafka \ avn --auth-token $TOKEN service wait demo-pg avn --auth-token $TOKEN service cli demo-pg << EOF -\i scripts/create_pg_tbl.sql +\i aiven_metadata_parser/scripts/create_pg_tbl.sql EOF PG_PWD=$(avn --auth-token $TOKEN service user-get demo-pg --format '{password}' --username avnadmin) @@ -222,7 +222,7 @@ avn --auth-token $TOKEN service wait demo-mysql MYSQL_HOST=$(avn --auth-token $TOKEN service get demo-mysql --json | jq -r '.service_uri_params.host') MYSQL_PORT=$(avn --auth-token $TOKEN service get demo-mysql --json | jq -r '.service_uri_params.port') MYSQL_PWD=$(avn --auth-token $TOKEN service get demo-mysql --json | jq -r '.service_uri_params.password') -mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < scripts/create_mysql_tbl.sql +mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < aiven_metadata_parser/scripts/create_mysql_tbl.sql KAFKA_FLINK_SI=$(avn --auth-token $TOKEN service integration-list --format '{source_service} {service_integration_id}' demo-flink | grep demo-kafka | awk -F ' ' '{print $2}') @@ -294,11 +294,11 @@ curl --location --request GET "$GRAFANA_URL/api/datasources" \ # Add a grafana dashboard avn --auth-token $TOKEN service wait demo-grafana -python src/add_grafana_dashboard.py $PROJECT_NAME +python aiven_metadata_parser/src/add_grafana_dashboard.py $PROJECT_NAME # Add redis user and ACL avn --auth-token $TOKEN service wait demo-redis avn --auth-token $TOKEN service user-create --project $PROJECT_NAME --username test demo-redis avn --auth-token $TOKEN service user-set-access-control --project $PROJECT_NAME --username test --redis-acl-keys '~app2:*' demo-redis -mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < scripts/create_mysql_usr.sql +mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < aiven_metadata_parser/scripts/create_mysql_usr.sql diff --git a/scripts/delete_services.sh b/aiven_metadata_parser/scripts/delete_services.sh similarity index 93% rename from scripts/delete_services.sh rename to aiven_metadata_parser/scripts/delete_services.sh index c44163a..51cc15c 100755 --- a/scripts/delete_services.sh +++ b/aiven_metadata_parser/scripts/delete_services.sh @@ -1,5 +1,5 @@ PROJECT_NAME=$1 -. conf/conf.env +. aiven_metadata_parser/conf/conf.env avn --auth-token $TOKEN project switch $PROJECT_NAME diff --git a/scripts/pg_queries.md b/aiven_metadata_parser/scripts/pg_queries.md similarity index 100% rename from scripts/pg_queries.md rename to aiven_metadata_parser/scripts/pg_queries.md diff --git a/src/sql.py b/aiven_metadata_parser/sql.py similarity index 100% rename from src/sql.py rename to aiven_metadata_parser/sql.py diff --git a/src/tag.py b/aiven_metadata_parser/tag.py similarity index 100% rename from src/tag.py rename to aiven_metadata_parser/tag.py diff --git a/write_pg.py b/aiven_metadata_parser/write_pg.py similarity index 97% rename from write_pg.py rename to aiven_metadata_parser/write_pg.py index 5706f1a..75baa61 100644 --- a/write_pg.py +++ b/aiven_metadata_parser/write_pg.py @@ -30,7 +30,7 @@ cur.execute("TRUNCATE TABLE metadata_parser_nodes cascade;") conn.commit() else: - cur.execute(open("src/pg_store_tbl.sql", "r").read()) + cur.execute(open("pg_store_tbl.sql", "r").read()) G = read_gml("graph_data.gml") diff --git a/main.py b/main.py deleted file mode 100644 index cc70c36..0000000 --- a/main.py +++ /dev/null @@ -1,92 +0,0 @@ -from aiven.client import client -from src import explore_service -import src.pyvis_display as pyvis_display -import configparser - -# Reading conf.env configuration file -with open("conf/conf.env", "r") as f: - config_string = "[DEFAULT]\n" + f.read() -config = configparser.ConfigParser() -config.read_string(config_string) - -# Creating Aiven client instance -myclient = client.AivenClient(base_url=config["DEFAULT"]["BASE_URL"]) - -# Authenticating and storing the token -# result = myclient.authenticate_user(email=config['DEFAULT']['USERNAME'], password=config['DEFAULT']['PASSWORD']) -myclient.auth_token = config["DEFAULT"]["TOKEN"] - -# Creating empty nodes and edges lists -nodes = [] -edges = [] - -# The service order helps analysis first "standalone" services and then connection services, this might be useful to parse first services which can be either source or sink of other services (e.g. Kafka connect might use a PG table as a source) - -services_order = {} -services_order["opensearch"] = 1 -services_order["elasticsearch"] = 1 -services_order["pg"] = 1 -services_order["redis"] = 1 -services_order["mysql"] = 1 -services_order["clickhouse"] = 1 -services_order["cassandra"] = 1 -services_order["redis"] = 1 -services_order["m3db"] = 1 -services_order["m3aggregator"] = 1 -services_order["m3coordinator"] = 1 - -services_order["influxdb"] = 1 -services_order["kafka"] = 2 -services_order["kafka_connect"] = 3 -services_order["kafka_mirrormaker"] = 3 -services_order["flink"] = 3 -services_order["grafana"] = 3 - - -# Listing the services -services = myclient.get_services(project=config["DEFAULT"]["PROJECT"]) - -# Ordering based on service_order -services.sort(key=lambda x: services_order[x["service_type"]]) - - -# Initial loop to find all ip/hostname of existing services -print("Locate IP/hostname of each service") -for i, service in enumerate(services, start=1): - # if service["service_name"]!='test': - print( - f"{i}/{len(services)} {service['service_name']} {service['service_type']}" - ) - # if service["service_type"]=='grafana': - explore_service.populate_service_map( - myclient, - service["service_type"], - service["service_name"], - project=config["DEFAULT"]["PROJECT"], - ) - -# Second loop to find details of each service -print() -print("Find details of each service") -for i, service in enumerate(services, start=1): - print( - f"{i}/{len(services)} Query {service['service_name']} {service['service_type']}" - ) - # if service["service_name"] != 'test': - (newnodes, newedges) = explore_service.explore( - myclient, - service["service_type"], - service["service_name"], - project=config["DEFAULT"]["PROJECT"], - ) - nodes = nodes + newnodes - edges = edges + newedges - -(newnodes, newedges) = explore_service.explore_ext_endpoints( - myclient, project=config["DEFAULT"]["PROJECT"] -) -nodes = nodes + newnodes -edges = edges + newedges - -# Creating viz with pyviz -pyvis_display.pyviz_graphy(nodes, edges) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..469fc16 --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +from distutils.core import setup + +setup( + name="aiven_metadata_parser", + packages=["aiven_metadata_parser"], + version="1.0.0", + license="apache-2.0", + description=" A python tool scraping Aiven services metadata and building a connected graph.", + author="Open Source @ Aiven", + url="https://github.com/aiven/metadata-parser", + download_url="TBD", + keywords=["aiven", "meta-data", "parser"], + install_requires=[ + "psycopg2-binary", + "requests", + "simplejson", + "aiven.client", + "configparser", + "pyvis==0.3.1", + "networkx==2.8.8", + "dash", + "plotly", + "pymysql", + "pydot", + "colour", + "cryptography", + "sqllineage", + ], + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache license 2.0", + "Programming Language :: Python :: 3.7", + ], +) diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29..0000000