Domain-based restructure (#137)

## What is the goal of this PR?

We categorize the examples by domain and provide a summary in the top-level readme. In addition, we now benchmark each example in typedb factory in order to keep track of performance changes in typedb.

## What are the changes implemented in this PR?

- Top-level README: a link to each example with a summary, and a bit of a restructure;
- closes #123;
- Domain-based package structure, with one package per domain;
- closes #120;
- Benchmarks: for most examples just runs the tests 20 times;
- closes #124.

Author: dmitrii-ubskii

.gitignore

@@ -39,7 +39,6 @@ bin/
 
 # NPM
 node_modules/
-yarn.lock
 
 # Python
 __pycache__

.grabl/automation.yml

@@ -33,40 +33,124 @@ build:
        bazel run //:typedb-extractor -- dist/typedb-all-linux
        sudo systemd-run ./dist/typedb-all-linux/typedb server
        sleep 60
-       bazel test //catalogue_of_life:tests --test_output=errors
+       bazel test //biology/catalogue_of_life:tests --test_output=errors
     test-gitapp:
       image: vaticle-ubuntu-21.04
       command: |
         bazel run //:typedb-extractor -- dist/typedb-all-linux
         sudo systemd-run ./dist/typedb-all-linux/typedb server
         sleep 60
-        bazel test //github/tests:test-migrator-explorer --test_output=errors
+        bazel test //software/github/tests:test-migrator-explorer --test_output=errors
     test-phone-calls-java:
       image: vaticle-ubuntu-21.04
       command: |
         bazel run //:typedb-extractor -- dist/typedb-all-linux
         sudo systemd-run ./dist/typedb-all-linux/typedb server
         sleep 60
-        bazel test //phone_calls/java:test --test_output=errors
+        bazel test //telecom/phone_calls/java:test --test_output=errors
     test-phone-calls-nodejs:
       image: vaticle-ubuntu-21.04
       command: |
         bazel run //:typedb-extractor -- dist/typedb-all-linux
         sudo systemd-run ./dist/typedb-all-linux/typedb server
         sleep 60
-        bazel test //phone_calls/nodejs:test --test_output=errors
+        bazel test //telecom/phone_calls/nodejs:test --test_output=errors
     test-phone-calls-python:
       image: vaticle-ubuntu-21.04
       command: |
         pyenv global system
         bazel run //:typedb-extractor -- dist/typedb-all-linux
         sudo systemd-run ./dist/typedb-all-linux/typedb server
         sleep 60
-        bazel test //phone_calls/python:test --test_output=errors
+        bazel test //telecom/phone_calls/python:test --test_output=errors
     test-xcom:
       image: vaticle-ubuntu-21.04
       command: |
         bazel run //:typedb-extractor -- dist/typedb-all-linux
         sudo systemd-run ./dist/typedb-all-linux/typedb server
         sleep 60
-        bazel test //xcom:test --test_output=errors
+        bazel test //gaming/xcom:test --test_output=errors
+  performance:
+    test-performance-typedb-core-server:
+      machine: 16-core-32-gb
+      image: vaticle-ubuntu-21.04
+      type: background
+      timeout: "30m"
+      command: |
+        cat > typedb.service <<- EOM
+        [Unit]
+        Description=TypeDB
+
+        [Service]
+        Type=simple
+        Environment=SERVER_JAVAOPTS=-Xmx16G
+        ExecStart=/home/grabl/$GRABL_REPO/dist/typedb-all-linux/typedb server
+        Restart=on-failure
+        RestartSec=10
+        KillMode=process
+
+        [Install]
+        WantedBy=multi-user.target
+        EOM
+
+        sudo mv typedb.service /etc/systemd/system/
+
+        bazel run //:typedb-extractor -- dist/typedb-all-linux
+        cd ./dist/typedb-all-linux/
+
+        sudo systemctl daemon-reload
+        sudo systemctl start typedb
+        export GRABL_EXPORT_PERFORMANCE_TYPEDB_CORE_URI="${HOSTNAME}:1729"
+      monitor: |
+        sleep 10s
+        tail -f -n +1 /home/grabl/$GRABL_REPO/dist/typedb-all-linux/server/logs/typedb.log
+    benchmark-catalogue-of-life-loader:
+      machine: 16-core-32-gb
+      image: vaticle-ubuntu-21.04
+      dependencies: [test-performance-typedb-core-server]
+      timeout: "30m"
+      command: |
+        bazel run //biology/catalogue_of_life:loader -- --typedb $GRABL_EXPORT_PERFORMANCE_TYPEDB_CORE_URI
+    benchmark-catalogue-of-life-tests:
+      image: vaticle-ubuntu-21.04
+      command: |
+        bazel run //:typedb-extractor -- dist/typedb-all-linux
+        sudo systemd-run ./dist/typedb-all-linux/typedb server
+        sleep 60
+        bazel test //biology/catalogue_of_life:tests --test_output=errors --runs_per_test=20 --jobs=1
+    benchmark-gitapp-tests:
+      image: vaticle-ubuntu-21.04
+      command: |
+        bazel run //:typedb-extractor -- dist/typedb-all-linux
+        sudo systemd-run ./dist/typedb-all-linux/typedb server
+        sleep 60
+        bazel test //software/github/tests:test-migrator-explorer --test_output=errors --runs_per_test=20 --jobs=1
+    benchmark-phone-calls-java-test:
+      image: vaticle-ubuntu-21.04
+      command: |
+        bazel run //:typedb-extractor -- dist/typedb-all-linux
+        sudo systemd-run ./dist/typedb-all-linux/typedb server
+        sleep 60
+        bazel test //telecom/phone_calls/java:test --test_output=errors --runs_per_test=20 --jobs=1
+#    benchmark-phone-calls-nodejs-test:
+#      image: vaticle-ubuntu-21.04
+#      command: |
+#        bazel run //:typedb-extractor -- dist/typedb-all-linux
+#        sudo systemd-run ./dist/typedb-all-linux/typedb server
+#        sleep 60
+#        bazel test //telecom/phone_calls/nodejs:test --test_output=errors --runs_per_test=20 --jobs=1
+    benchmark-phone-calls-python-test:
+      image: vaticle-ubuntu-21.04
+      command: |
+        pyenv global system
+        bazel run //:typedb-extractor -- dist/typedb-all-linux
+        sudo systemd-run ./dist/typedb-all-linux/typedb server
+        sleep 60
+        bazel test //telecom/phone_calls/python:test --test_output=errors --runs_per_test=20 --jobs=1
+    benchmark-xcom-test:
+      image: vaticle-ubuntu-21.04
+      command: |
+        bazel run //:typedb-extractor -- dist/typedb-all-linux
+        sudo systemd-run ./dist/typedb-all-linux/typedb server
+        sleep 60
+        bazel test //gaming/xcom:test --test_output=errors --runs_per_test=20 --jobs=1

README.md

@@ -2,43 +2,69 @@
 
 [![Grabl](https://grabl.io/api/status/vaticle/typedb-examples/badge.svg)](https://grabl.io/vaticle/typedb-examples)
 
-This repository includes examples that showcase usage of TypeDB Clients in reading from and writing to a TypeDB knowledge graph.
+This repository includes examples that showcase usage of TypeDB Clients in reading from and writing to a TypeDB
+database.
 
-## Explore Client Examples
+## [Biology: Catalogue of Life](biology/catalogue_of_life)
+
+[Catalogue of Life](https://www.catalogueoflife.org/) is a database of over 4.5 million currently known taxa in biology,
+compiled from over a hundred different sources. The example showcases simple data preparation and a sample configuration
+file for loading a large taxonomic dataset using [TypeDB Loader.](https://github.com/typedb-osi/typedb-loader)
+
+## [Gaming: XCOM Project](gaming/xcom)
+
+The XCOM 2 example contains a database of interdependent research tasks in the game XCOM 2, featuring automatic
+inference of available research based on completed tasks and available items. See [the schema](gaming/xcom/schema.tql)
+for the examples of reasoner rules inferring attributes.
+
+## [Software: GitHub](software/github)
+
+The GitHub example showcases migration of heavily interconnected data from a live repository on GitHub or from a Vaticle
+GitHub snapshot, and provides a visual interface to explore some sample queries.
+
+## [Telecom: Phone Calls](telecom/phone_calls)
 
 TypeDB officially supports clients for Java, Node.js and Python. Learn more about [TypeDB Clients](http://docs.vaticle.com/docs/client-api/overview).
 
-### Client Java Examples
-- [Phone Calls: Migrate and perform expressive queries using Client Java](phone_calls/java/)
-- [XCOM Project: Migrate and perform logical queries on tree-structured game data using Client Java](xcom/)
+The Phone Calls example showcases basic migration of a database of customers of a fictional telecom company and calls
+they make in various formats (CSV, JSON, and XML) and expressive TypeQL queries using
+the [Java,](telecom/phone_calls/java) [Node.js,](telecom/phone_calls/nodejs) and [Python](telecom/phone_calls/python)
+clients.
+
+### Data migration
+
+- Java: [CSV](telecom/phone_calls/java/CSVMigration.java) | [JSON](telecom/phone_calls/java/JSONMigration.java)
+  | [XML](telecom/phone_calls/java/XMLMigration.java)
+- Node.js: [CSV](telecom/phone_calls/nodejs/migrateCsv.js) | [JSON](telecom/phone_calls/nodejs/migrateJson.js)
+  | [XML](telecom/phone_calls/nodejs/migrateXml.js)
+- Python: [CSV](telecom/phone_calls/python/migrate_csv.py) | [JSON](telecom/phone_calls/python/migrate_json.py)
+  | [XML](telecom/phone_calls/python/migrate_xml.py)
+
+### Query examples
 
-### Client Node.js Examples
-- [Phone Calls: Migrate and perform expressive queries using Client Node.js](phone_calls/nodejs/)
+- [Java](telecom/phone_calls/java/Queries.java)
+- [Node.js](telecom/phone_calls/nodejs/queries.js)
+- [Python](telecom/phone_calls/python/queries.py)
 
-### Client Python Examples
-- [Phone Calls: Migrate and perform expressive queries using Client Python](phone_calls/python/)
+## Visualise data using TypeDB Studio
 
-## Explore Use Cases
+After loading the data of any of the examples, you can
+use [TypeDB Studio](https://github.com/vaticle/typedb-studio/releases) to explore
+the graph structure of the database.
 
-### Migrate data to TypeDB:
+## TypeDB Bio
 
-#### serialised:
-- Java: [CSV](phone_calls/java/CSVMigration.java) | [JSON](phone_calls/java/JSONMigration.java) | [XML](phone_calls/java/XMLMigration.java)
-- Node.js: [CSV](phone_calls/nodejs/migrateCsv.js) | [JSON](phone_calls/nodejs/migrateJson.js) | [XML](phone_calls/nodejs/migrateXml.js)
-- Python: [CSV](phone_calls/python/migrate_csv.py) | [JSON](phone_calls/python/migrate_json.py) | [XML](phone_calls/python/migrate_xml.py)
+[TypeDB Bio](https://github.com/vaticle/typedb-bio) is a collection of knowledge graphs of biomedical data.
 
-### Perform expressive queries:
-- [Java](phone_calls/java/Queries.java)
-- [Node.js](phone_calls/nodejs/queries.js)
-- [Python](phone_calls/python/queries.py)
-    
-## Explore TypeDB Bio
-[BioGrakn](https://github.com/vaticle/typedb-bio) is a collection of knowledge graphs of biomedical data.
+TypeDB Bio provides an intuitive way to query interconnected and heterogeneous biomedical data in one single place. The
+schema that models the underlying knowledge graph alongside the descriptive query language, TypeQL, makes writing
+complex queries an extremely straightforward and intuitive process. Furthermore, the automated reasoning capability of
+TypeDB, allows TypeDB Bio to become an intelligent database of biomedical data that infers implicit knowledge based on
+the explicitly stored data. TypeDB Bio can understand biological facts, infer based on new findings and enforce research
+constraints, all at query (run) time.
 
-TypeDB Bio provides an intuitive way to query interconnected and heterogeneous biomedical data in one single place. The schema that models the underlying knowledge graph alongside the descriptive query language, TypeQL, makes writing complex queries an extremely straightforward and intuitive process. Furthermore, the automated reasoning capability of TypeDB, allows TypeDB Bio to become an intelligent database of biomedical data that infers implicit knowledge based on the explicitly stored data. TypeDB Bio can understand biological facts, infer based on new findings and enforce research constraints, all at query (run) time.
+## TypeDB CTI
 
-## Explore Schemas
-- [Catalogue of Life](catalogue_of_life/schema.tql)
-- [GitHub](github/schemas)
-- [Phone Calls](phone_calls/schema.tql)
-- [XCOM](xcom/schema.tql)
+[TypeDB CTI](https://github.com/typedb-osi/typedb-cti) is an open source threat intelligence platform for organisations
+to store and manage their cyber threat intelligence (CTI) knowledge. It enables threat intel professionals to bring
+together their disparate CTI information into one database and find new insights about cyber threats.

WORKSPACE

@@ -142,7 +142,7 @@ pip_install(
 # Load python example dependencies
 pip_install(
     name = "phone_calls_pip",
-    requirements = "//phone_calls/python:requirements.txt"
+    requirements = "//telecom/phone_calls/python:requirements.txt"
 )
 
 # Load java example dependencies
@@ -151,8 +151,8 @@ load("//dependencies/maven:artifacts.bzl", vaticle_typedb_examples_maven_artifac
 # Load nodejs example dependencies
 yarn_install(
     name = "npm",
-    package_json = "//phone_calls/nodejs:package.json",
-    yarn_lock = "//phone_calls/nodejs:yarn.lock"
+    package_json = "//telecom/phone_calls/nodejs:package.json",
+    yarn_lock = "//telecom/phone_calls/nodejs:yarn.lock"
 )
 
 ############################

catalogue_of_life/BUILD renamed to biology/catalogue_of_life/BUILD

@@ -21,7 +21,7 @@
 
 java_binary(
     name = "loader",
-    main_class = "com.vaticle.typedb.example.catalogueOfLife.Loader",
+    main_class = "com.vaticle.typedb.example.biology.catalogueOfLife.Loader",
     srcs = ["Loader.java"],
     deps = [
         "@maven//:com_univocity_univocity_parsers",
@@ -36,7 +36,7 @@ java_binary(
 
 java_test(
     name = "tests",
-    test_class = "com.vaticle.typedb.example.catalogueOfLife.test.CoLTest",
+    test_class = "com.vaticle.typedb.example.biology.catalogueOfLife.test.CoLTest",
     srcs = [
         "test/CoLTest.java"
     ],

catalogue_of_life/Loader.java renamed to biology/catalogue_of_life/Loader.java

@@ -19,7 +19,7 @@
  * under the License.
  */
 
-package com.vaticle.typedb.example.catalogueOfLife;
+package com.vaticle.typedb.example.biology.catalogueOfLife;
 
 import com.univocity.parsers.common.record.Record;
 import com.univocity.parsers.tsv.TsvParser;
@@ -231,7 +231,7 @@ public static void loadData(Path dataDirectory, CLIOptions options) throws IOExc
     }
 
     public static void extractSchema(Path dataDirectory) throws IOException {
-        InputStream schema = Loader.class.getResourceAsStream("/catalogue_of_life/schema.tql");
+        InputStream schema = Loader.class.getResourceAsStream("/biology/catalogue_of_life/schema.tql");
         if (schema == null) throw new FileNotFoundException("schema.tql not found within the jar!");
         Files.copy(schema, dataDirectory.resolve("schema.tql"), StandardCopyOption.REPLACE_EXISTING);
     }
@@ -240,7 +240,7 @@ public static Path populateLoaderConfigTemplate(Path dataDirectory) throws IOExc
         Path config = dataDirectory.resolve("loader-config.json");
 
         try (FileWriter configWriter = new FileWriter(config.toFile())) {
-            InputStream template = Loader.class.getResourceAsStream("/catalogue_of_life/loader-config.json.template");
+            InputStream template = Loader.class.getResourceAsStream("/biology/catalogue_of_life/loader-config.json.template");
             if (template == null) throw new FileNotFoundException("loader-config.json.template not found within the jar!");
 
             BufferedReader reader = new BufferedReader(new InputStreamReader(template));

catalogue_of_life/README.md renamed to biology/catalogue_of_life/README.md

@@ -19,14 +19,14 @@
  * under the License.
  */
 
-package com.vaticle.typedb.example.catalogueOfLife.test;
+package com.vaticle.typedb.example.biology.catalogueOfLife.test;
 
 import com.vaticle.typedb.client.TypeDB;
 import com.vaticle.typedb.client.api.TypeDBClient;
 import com.vaticle.typedb.client.api.TypeDBOptions;
 import com.vaticle.typedb.client.api.TypeDBSession;
 import com.vaticle.typedb.client.api.TypeDBTransaction;
-import com.vaticle.typedb.example.catalogueOfLife.Loader;
+import com.vaticle.typedb.example.biology.catalogueOfLife.Loader;
 import com.vaticle.typeql.lang.TypeQL;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -50,7 +50,7 @@ public class CoLTest {
 
     @BeforeClass
     public static void loadTestData() throws IOException {
-        Path dataDirectory = Path.of("catalogue_of_life/test/data");
+        Path dataDirectory = Path.of("biology/catalogue_of_life/test/data");
         Loader.prepareData(dataDirectory);
         Loader.loadData(dataDirectory, Loader.parseCLIOptions(new String[]{}).get());
 

catalogue_of_life/loader-config.json.template renamed to biology/catalogue_of_life/loader-config.json.template

@@ -45,8 +45,8 @@ java_library(
 
 java_test(
     name = "test",
-    classpath_resources = ["//xcom/config:logback-xml"],
-    test_class = "com.vaticle.typedb.example.xcom.XCOMTest",
+    classpath_resources = ["//gaming/xcom/config:logback-xml"],
+    test_class = "com.vaticle.typedb.example.gaming.xcom.XCOMTest",
     srcs = [
         "XCOMTest.java"
     ],
@@ -68,8 +68,8 @@ java_test(
 
 java_binary(
     name = "migration",
-    classpath_resources = ["//xcom/config:logback-xml"],
-    main_class = "com.vaticle.typedb.example.xcom.Migration",
+    classpath_resources = ["//gaming/xcom/config:logback-xml"],
+    main_class = "com.vaticle.typedb.example.gaming.xcom.Migration",
     srcs = ["Migration.java"],
     deps = [
         "@vaticle_typedb_client_java//:client-java",
@@ -84,8 +84,8 @@ java_binary(
 
 java_binary(
     name = "queries",
-    classpath_resources = ["//xcom/config:logback-xml"],
-    main_class = "com.vaticle.typedb.example.xcom.Queries",
+    classpath_resources = ["//gaming/xcom/config:logback-xml"],
+    main_class = "com.vaticle.typedb.example.gaming.xcom.Queries",
     srcs = ["Queries.java", "TransactionMode.java", "ResearchTask.java", "InventoryItem.java", "Result.java"],
     deps = [
         "@vaticle_typedb_client_java//:client-java",

catalogue_of_life/schema.tql renamed to biology/catalogue_of_life/schema.tql

@@ -19,7 +19,7 @@
  * under the License.
  */
 
-package com.vaticle.typedb.example.xcom;
+package com.vaticle.typedb.example.gaming.xcom;
 
 public class InventoryItem {
     public InventoryItem(String name, long quantity) {