Prepare repo for PyPI packaging

.gitignore

@@ -1,6 +1,6 @@
 file.*
 *test*
-conf/conf.env
+aiven_metadata_parser/conf/conf.env
 __pycache__
 certs/*
 output.*
@@ -11,4 +11,5 @@ graph.py
 kcat.config
 tmp
 .vscode
-*.log
+*.log
+conf/*

README.md

@@ -24,15 +24,15 @@ The `create_services.sh` uses:
 Before starting it, please install the required dependences, login with the [Aiven CLI](https://github.com/aiven/aiven-client) and execute:
 
 ```bash
-./scripts/create_services.sh <PROJECT_NAME>
+./aiven_metadata_parser/scripts/create_services.sh <PROJECT_NAME>
 ```
 
 Where `<PROJECT_NAME>` is the name of your Aiven project.
 
 To delete the services you can call:
 
 ```bash
-./scripts/delete_services.sh <PROJECT_NAME>
+./aiven_metadata_parser/scripts/delete_services.sh <PROJECT_NAME>
 ```
 
 If `<PROJECT_NAME>` is not passed, the default project will be used.
@@ -46,13 +46,13 @@ You need to be on Python 3.7, install the required libraries with:
 pip install -r requirements.txt
 ```
 
-Copy the `conf.env.sample` file to `conf.env` in the `conf` folder and edit the token parameter and the project name from which you want to extract parameters.
+In the `aiven_metadata_parser/conf` folder, copy the `conf.env.sample` file to `conf.env` and edit the token parameter and the project name from which you want to extract parameters.
 If you don't have a project with services already running you can create a sample set of services with the `create_services.sh` file, which requires the `aiven-client` to be installed and the user to be logged in.
 
 Once `conf.env` is set, you can start the metadata extraction with: 
 
 ```bash
-python main.py
+python aiven_metadata_parser/main.py
 ```
 
 This will generate:
@@ -65,7 +65,7 @@ This will generate:
 Furthermore if, after executing the `main.py` you also execute the following:
 
 ```bash
-python app.py
+python aiven_metadata_parser/app.py
 ```
 
 The `app.py` reads the `graph_data.gml` file generated at step 1 and creates a Reactive Web Applications with [Plotly](https://plot.ly/python/) and [Dash](https://plot.ly/dash/) (code taken from [here](https://towardsdatascience.com/python-interactive-network-visualization-using-networkx-plotly-and-dash-e44749161ed7)).
@@ -82,7 +82,7 @@ The code is a bare minimum product, doesn't cover all services and options and i
 Possible issues and solutions
 ============
 
-- If you run `python app.py` and see an error saying `No such file or directory: 'neato'`, you will have to install `graphviz` on your machine (the package version from pip doesn't seem to work) - find out how to install it [here](https://graphviz.org/download/).
+- If you run `python aiven_metadata_parser/app.py` and see an error saying `No such file or directory: 'neato'`, you will have to install `graphviz` on your machine (the package version from pip doesn't seem to work) - find out how to install it [here](https://graphviz.org/download/).
 
 Contributing
 ============

aiven_metadata_parser/__init__.py

@@ -0,0 +1,18 @@
+from aiven_metadata_parser import (
+    add_grafana_dashboard,
+    app,
+    backup,
+    explore_service,
+    flink,
+    grafana,
+    integration,
+    kafka,
+    kafka_connect,
+    mysql,
+    opensearch,
+    pg,
+    pyvis_display,
+    redis,
+    sql,
+    tag,
+)

src/explore_service.py → aiven_metadata_parser/explore_service.py

@@ -1,19 +1,14 @@
 """Explores a service"""
 
-from src import (
+from aiven_metadata_parser import (
     backup,
     kafka,
-    kafka_connect,
-    pg,
     tag,
     integration,
-    grafana,
-    redis,
-    mysql,
-    opensearch,
     flink,
 )
-
+from aiven_metadata_parser import pg
+from aiven_metadata_parser import opensearch, kafka_connect, mysql, grafana, redis
 
 SERVICE_MAP = {}
 EXPLORER_METHODS = {}

src/kafka.py → aiven_metadata_parser/kafka.py

@@ -1,7 +1,7 @@
 """Parsing Apache Kafka® services"""
 
 import re
-from src import kafka_connect
+from aiven_metadata_parser import kafka_connect
 
 
 def explore_kafka(self, service, service_name, project, service_map):

aiven_metadata_parser/main.py

@@ -0,0 +1,97 @@
+from aiven.client import client
+import explore_service
+import pyvis_display as pyvis_display
+import configparser
+
+
+def main():
+    # Reading conf.env configuration file
+    with open("aiven_metadata_parser/conf/conf.env", "r") as f:
+        config_string = "[DEFAULT]\n" + f.read()
+    config = configparser.ConfigParser()
+    config.read_string(config_string)
+
+    # Creating Aiven client instance
+    myclient = client.AivenClient(base_url=config["DEFAULT"]["BASE_URL"])
+
+    # Authenticating and storing the token
+    # result = myclient.authenticate_user(email=config['DEFAULT']['USERNAME'], password=config['DEFAULT']['PASSWORD'])
+    myclient.auth_token = config["DEFAULT"]["TOKEN"]
+
+    # Creating empty nodes and edges lists
+    nodes = []
+    edges = []
+
+    # The service order helps analysis first "standalone" services and then connection services, this might be useful
+    # to parse first services which can be either source or sink of other services (e.g. Kafka connect might use a PG
+    # table as a source)
+
+    services_order = {
+        "opensearch": 1,
+        "elasticsearch": 1,
+        "pg": 1,
+        "redis": 1,
+        "mysql": 1,
+        "clickhouse": 1,
+        "cassandra": 1,
+        "m3db": 1,
+        "m3aggregator": 1,
+        "m3coordinator": 1,
+        "influxdb": 1,
+        "kafka": 2,
+        "kafka_connect": 3,
+        "kafka_mirrormaker": 3,
+        "flink": 3,
+        "grafana": 3,
+    }
+
+    # Listing the services
+    services = myclient.get_services(project=config["DEFAULT"]["PROJECT"])
+
+    # Ordering based on service_order
+    services.sort(key=lambda x: services_order[x["service_type"]])
+
+    # Initial loop to find all ip/hostname of existing services
+    print("Locate IP/hostname of each service")
+    for i, service in enumerate(services, start=1):
+        # if service["service_name"]!='test':
+        print(
+            f"{i}/{len(services)} {service['service_name']} {service['service_type']}"
+        )
+        # if service["service_type"]=='grafana':
+        explore_service.populate_service_map(
+            myclient,
+            service["service_type"],
+            service["service_name"],
+            project=config["DEFAULT"]["PROJECT"],
+        )
+
+    # Second loop to find details of each service
+    print()
+    print("Find details of each service")
+    for i, service in enumerate(services, start=1):
+        print(
+            f"{i}/{len(services)} Query {service['service_name']} {service['service_type']}"
+        )
+        # if service["service_name"] != 'test':
+        (newnodes, newedges) = explore_service.explore(
+            myclient,
+            service["service_type"],
+            service["service_name"],
+            project=config["DEFAULT"]["PROJECT"],
+        )
+        nodes = nodes + newnodes
+        edges = edges + newedges
+
+    (newnodes, newedges) = explore_service.explore_ext_endpoints(
+        myclient, project=config["DEFAULT"]["PROJECT"]
+    )
+    nodes = nodes + newnodes
+    edges = edges + newedges
+
+    # Creating viz with pyviz
+    pyvis_display.pyviz_graphy(nodes, edges)
+
+
+if __name__ == "__main__":
+    main()

src/pg.py → aiven_metadata_parser/pg.py

@@ -1,7 +1,7 @@
 """Parsing PostgreSQL services"""
 
 import psycopg2
-from src import sql
+from aiven_metadata_parser import sql
 
 
 def build_conn_string(avnadmin_pwd, service_conn_info):

scripts/create_services.sh → aiven_metadata_parser/scripts/create_services.sh

@@ -1,5 +1,5 @@
 PROJECT_NAME=$1
-. conf/conf.env
+. aiven_metadata_parser/conf/conf.env
 
 
 avn --auth-token $TOKEN project switch $PROJECT_NAME
@@ -105,7 +105,7 @@ avn --auth-token $TOKEN service schema create demo-kafka \
 avn --auth-token $TOKEN service wait demo-pg
 
 avn --auth-token $TOKEN service cli demo-pg << EOF
-\i scripts/create_pg_tbl.sql
+\i aiven_metadata_parser/scripts/create_pg_tbl.sql
 EOF
 
 PG_PWD=$(avn --auth-token $TOKEN service user-get demo-pg --format '{password}' --username avnadmin)
@@ -222,7 +222,7 @@ avn --auth-token $TOKEN service wait demo-mysql
 MYSQL_HOST=$(avn --auth-token $TOKEN service get demo-mysql --json | jq -r '.service_uri_params.host')
 MYSQL_PORT=$(avn --auth-token $TOKEN service get demo-mysql --json | jq -r '.service_uri_params.port')
 MYSQL_PWD=$(avn --auth-token $TOKEN service get demo-mysql --json | jq -r '.service_uri_params.password')
-mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < scripts/create_mysql_tbl.sql
+mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < aiven_metadata_parser/scripts/create_mysql_tbl.sql
 
 
 KAFKA_FLINK_SI=$(avn --auth-token $TOKEN service integration-list --format '{source_service} {service_integration_id}' demo-flink | grep demo-kafka | awk -F ' ' '{print $2}')
@@ -294,11 +294,11 @@ curl --location --request GET "$GRAFANA_URL/api/datasources" \
 
 # Add a grafana dashboard
 avn --auth-token $TOKEN service wait demo-grafana
-python src/add_grafana_dashboard.py $PROJECT_NAME
+python aiven_metadata_parser/src/add_grafana_dashboard.py $PROJECT_NAME
 
 # Add redis user and ACL
 avn --auth-token $TOKEN service wait demo-redis
 avn --auth-token $TOKEN service user-create --project $PROJECT_NAME --username test demo-redis
 avn --auth-token $TOKEN service user-set-access-control --project $PROJECT_NAME --username test --redis-acl-keys '~app2:*' demo-redis
 
-mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < scripts/create_mysql_usr.sql
+mysql -u avnadmin -P $MYSQL_PORT -h $MYSQL_HOST -D defaultdb -p$MYSQL_PWD < aiven_metadata_parser/scripts/create_mysql_usr.sql

scripts/delete_services.sh → aiven_metadata_parser/scripts/delete_services.sh

@@ -1,5 +1,5 @@
 PROJECT_NAME=$1
-. conf/conf.env
+. aiven_metadata_parser/conf/conf.env
 
 
 avn --auth-token $TOKEN project switch $PROJECT_NAME

write_pg.py → aiven_metadata_parser/write_pg.py

@@ -30,7 +30,7 @@
     cur.execute("TRUNCATE TABLE metadata_parser_nodes cascade;")
     conn.commit()
 else:
-    cur.execute(open("src/pg_store_tbl.sql", "r").read())
+    cur.execute(open("pg_store_tbl.sql", "r").read())
 
 G = read_gml("graph_data.gml")
 

setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md