Distribution (Assembly and Deployment) Tests (typedb#5036)

## What is the goal of this PR?
Added test assembly and test deployments. These tests are designed to verify whether Grakn distributions which comes in various formats (Zip, Tar, Deb, RPM and Docker Image) works on the following target operating systems: Mac, Windows, Ubuntu, CentOS and Docker.

## What are the changes implemented in this PR?
1. `test-assembly-windows-zip` (currently disabled due to [this particular error](https://circleci.com/gh/lolski/grakn/3869?utm_campaign=vcs-integration-link&utm_medium=referral&utm_source=github-build-link) from `build-tools`: `error: invalid command 'bdist_wheel'`
2. `test-assembly-mac-zip`
3. `test-assembly-linux-targz`
4. `test-assembly-linux-apt` and `test-deployment-linux-apt`
5. `test-assembly-linux-rpm` and `test-deployment-linux-rpm`
6. `test-assembly-docker`

## Remaining Issues 
1. test-deployment-linux-rpm - The yum install command needs to specify the exact Grakn version typedb#5037
2. test-deployment-linux-rpm - fix infinite symlink expansion issue in CentOS and enable Bazel test typedb#5038
3. test-assembly-windows-zip - enable the test after fixing the python issue regarding missing bdist_wheel typedb#5039
4. test-assembly-windows-zip - need to verify that Grakn can be started from within a directory of which the name contains whitespace typedb#5040
5. test-deployment-linux-apt - figure out why repo.grakn.ai only keeps the latest deb package typedb#5041
6. test-assembly-docker - needs to add timeout typedb#5042
7. Move test-end-to-end and test-integration to //test package for consistency typedb#5043

.bazelignore

@@ -0,0 +1 @@
+.circleci

.circleci/config.yml

@@ -16,9 +16,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
+
 version: 2.1
 commands:
-  install-bazel:
+  install-bazel-linux:
     steps:
       - run:
           name: Install bazel and RBE credential
@@ -31,6 +32,19 @@ commands:
             # Install RBE credential
             bazel run @graknlabs_build_tools//ci:install-bazel-rbe-credential
 
+  install-bazel-mac:
+    steps:
+      - run:
+          name: Install bazel and RBE credential
+          command: |
+            # Install bazel
+            curl -OL https://github.com/bazelbuild/bazel/releases/download/0.20.0/bazel-0.20.0-installer-darwin-x86_64.sh
+            chmod +x bazel-0.20.0-installer-darwin-x86_64.sh
+            sudo ./bazel-0.20.0-installer-darwin-x86_64.sh
+
+            # Install RBE credential
+            bazel run @graknlabs_build_tools//ci:install-bazel-rbe-credential
+
   install-rpm:
     steps:
       - run: sudo apt update && sudo apt install rpm -y
@@ -48,7 +62,7 @@ jobs:
     working_directory: ~/grakn
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - install-rpm
       - run: bazel run @graknlabs_build_tools//checkstyle:test-coverage
       - run-bazel-rbe:
@@ -59,7 +73,7 @@ jobs:
     working_directory: ~/grakn
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - run-bazel-rbe:
           command: bazel test //common/... --test_output=errors
 
@@ -68,7 +82,7 @@ jobs:
     working_directory: ~/grakn
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - install-rpm
       - run-bazel-rbe:
           command: bazel test //console/... --test_output=errors
@@ -78,7 +92,7 @@ jobs:
     working_directory: ~/grakn
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - install-rpm
       - run-bazel-rbe:
           command: bazel test //server/... --test_output=errors
@@ -88,7 +102,7 @@ jobs:
     working_directory: ~/grakn
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - run-bazel-rbe:
           command: bazel test //test-integration/server/... --test_output=errors
       - run-bazel-rbe:
@@ -101,7 +115,7 @@ jobs:
     working_directory: ~/grakn
     steps:
     - checkout
-    - install-bazel
+    - install-bazel-linux
     - run: bazel run @graknlabs_build_tools//ci:install-bazel-rbe-credential
     - run-bazel-rbe:
         command: bazel test //test-integration/graql/reasoner/... --test_output=errors
@@ -111,7 +125,7 @@ jobs:
     working_directory: ~/grakn
     steps:
     - checkout
-    - install-bazel
+    - install-bazel-linux
     - run: bazel run @graknlabs_build_tools//ci:install-bazel-rbe-credential
     - run-bazel-rbe:
         command: bazel test //test-integration/graql/analytics/... --test_output=errors
@@ -121,15 +135,126 @@ jobs:
     working_directory: ~/grakn
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - run-bazel-rbe:
           command: bazel test //test-end-to-end:test-end-to-end --test_output=streamed --spawn_strategy=standalone
 
+  test-assembly-mac-zip:
+    macos:
+      xcode: "9.0"
+    working_directory: ~/grakn
+    steps:
+      - checkout
+      - install-bazel-mac
+      - run: bazel build //:assemble-mac-zip
+      - run: unzip bazel-genfiles/grakn-core-all-mac.zip -d bazel-genfiles/
+      - run: nohup bazel-genfiles/grakn-core-all-mac/grakn server start
+      - run: bazel test //test/common:grakn-application-test --test_output=streamed --spawn_strategy=standalone --cache_test_results=no
+      - run: bazel-genfiles/grakn-core-all-mac/grakn server stop
+
+# TODO: add after fixing the python issue regarding missing bdist_wheel
+#  test-assembly-windows-zip:
+#    machine: true
+#    working_directory: ~/grakn
+#    steps:
+#      - checkout
+#      - run:
+#          command: test/assembly/windows/windows-zip.py
+#          no_output_timeout: 20m
+
+  test-assembly-linux-targz:
+    machine: true
+    working_directory: ~/grakn
+    steps:
+      - checkout
+      - install-bazel-linux
+      - run: bazel build //:assemble-linux-targz
+      - run: tar -xf bazel-genfiles/grakn-core-all-linux.tar.gz -C bazel-genfiles
+      - run: nohup bazel-genfiles/grakn-core-all-linux/grakn server start
+      - run: bazel test //test/common:grakn-application-test --test_output=streamed --spawn_strategy=standalone --cache_test_results=no
+      - run: bazel-genfiles/grakn-core-all-linux/grakn server stop
+
+  test-assembly-linux-apt:
+    machine: true
+    working_directory: ~/grakn
+    steps:
+      - checkout
+      - install-bazel-linux
+      - run: echo $(date +%s)-$(cat VERSION)-$CIRCLE_SHA1 > VERSION
+      - run: echo VERSION=$(cat VERSION)
+      - run: cat VERSION
+      - run: bazel build //bin:assemble-linux-apt
+      - run: bazel build //server:assemble-linux-apt
+      - run: bazel build //console:assemble-linux-apt
+      - run: sudo dpkg -i bazel-bin/bin/grakn-core-bin__all.deb
+      - run: sudo dpkg -i bazel-bin/server/grakn-core-server__all.deb
+      - run: sudo dpkg -i bazel-bin/console/grakn-core-console__all.deb
+      - run: sudo chown -R circleci:circleci /opt/grakn/ # TODO: how do we avoid having to chown?
+      - run: nohup grakn server start
+      - run: bazel test //test/common:grakn-application-test --test_output=streamed --spawn_strategy=standalone --cache_test_results=no
+      - run: grakn server stop
+      - run: bazel run //bin:deploy-apt -- test $REPO_GRAKN_USERNAME $REPO_GRAKN_PASSWORD
+      - run: bazel run //console:deploy-apt -- test $REPO_GRAKN_USERNAME $REPO_GRAKN_PASSWORD
+      - run: bazel run //server:deploy-apt -- test $REPO_GRAKN_USERNAME $REPO_GRAKN_PASSWORD
+      - persist_to_workspace:
+          root: ~/grakn
+          paths:
+            - VERSION
+
+  test-deployment-apt:
+    machine: true
+    working_directory: ~/grakn
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ~/circleci-workspace
+      - install-bazel-linux
+      - run: echo "deb [ arch=all ] https://repo.grakn.ai/repository/test-deb/ trusty main" | sudo tee -a /etc/apt/sources.list.d/grakn-core.list
+      - run: sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 8F3DA4B5E9AEF44C
+      - run: sudo apt update
+      - run: mv ~/circleci-workspace/VERSION VERSION
+      - run: cat VERSION
+      - run: sudo apt install grakn-core-server=$(cat VERSION) grakn-core-console=$(cat VERSION)
+      - run: sudo chown -R circleci:circleci /opt/grakn/ # TODO: how do we avoid having to chown?
+      - run: nohup grakn server start
+      - run: bazel test //test/common:grakn-application-test --test_output=streamed --spawn_strategy=standalone --cache_test_results=no
+      - run: grakn server stop
+
+  test-assembly-linux-rpm:
+    machine: true
+    working_directory: ~/grakn
+    steps:
+      - checkout
+      - install-bazel-linux
+      - run: sudo apt install rpm
+      - run: echo $(cat VERSION)-$CIRCLE_SHA1 > VERSION
+      - run: sed -i -e 's/-/_/g' VERSION
+      - run: cat VERSION
+      - run: bazel run //bin:deploy-rpm -- test $REPO_GRAKN_USERNAME $REPO_GRAKN_PASSWORD
+      - run: bazel run //server:deploy-rpm -- test $REPO_GRAKN_USERNAME $REPO_GRAKN_PASSWORD
+      - run: bazel run //console:deploy-rpm -- test $REPO_GRAKN_USERNAME $REPO_GRAKN_PASSWORD
+
+
+  test-deployment-rpm:
+    machine: true
+    working_directory: ~/grakn
+    steps:
+      - checkout
+      - run: test/deployment/rpm.py
+
+  test-assembly-docker:
+    machine: true
+    working_directory: ~/grakn
+    steps:
+      - checkout
+      - install-bazel-linux
+      - run: test/assembly/docker.py
+
   sync-dependencies:
     machine: true
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - run: |
           bazel run @graknlabs_build_tools//ci:sync-dependencies -- \
           --source grakn@$CIRCLE_SHA1 \
@@ -141,15 +266,15 @@ jobs:
     machine: true
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - run: bazel run @graknlabs_build_tools//ci:release-approval
 
   release-github-draft:
     machine: true
     working_directory: ~/grakn
     steps:
       - checkout
-      - install-bazel
+      - install-bazel-linux
       - run: bazel run //:deploy-github-zip -- ${GRABL_CREDENTIAL}
 
   release-cleanup:
@@ -193,6 +318,42 @@ workflows:
           filters:
             branches:
               ignore: grakn-core-release-branch
+      - test-assembly-mac-zip:
+          filters:
+            branches:
+              only: master
+#      - test-assembly-windows-zip:
+#          filters:
+#            branches:
+#              only: master
+      - test-assembly-linux-targz:
+          filters:
+            branches:
+              only: master
+      - test-assembly-linux-apt:
+          filters:
+            branches:
+              only: master
+      - test-deployment-apt:
+          filters:
+            branches:
+              only: master
+          requires:
+            - test-assembly-linux-apt
+      - test-assembly-linux-rpm:
+          filters:
+            branches:
+              only: master
+      - test-deployment-rpm:
+          filters:
+            branches:
+              only: master
+          requires:
+            - test-assembly-linux-rpm
+      - test-assembly-docker:
+          filters:
+            branches:
+              only: master
       - sync-dependencies:
           filters:
             branches:
@@ -224,4 +385,4 @@ workflows:
             branches:
               only: grakn-core-release-branch
           requires:
-            - release-github-draft
+            - release-github-draft

bin/BUILD

@@ -43,7 +43,7 @@ assemble_targz(
 )
 
 assemble_apt(
-    name = "assemble-apt",
+    name = "assemble-linux-apt",
     package_name = "grakn-core-bin",
     maintainer = "Grakn Labs <[email protected]>",
     description = "Grakn Core (binaries)",
@@ -67,12 +67,12 @@ assemble_apt(
 
 deploy_apt(
     name = "deploy-apt",
-    target = ":assemble-apt",
+    target = ":assemble-linux-apt",
     deployment_properties = "@graknlabs_build_tools//:deployment.properties",
 )
 
 assemble_rpm(
-    name = "assemble-rpm",
+    name = "assemble-linux-rpm",
     package_name = "grakn-core-bin",
     installation_dir = "/opt/grakn/core/",
     version_file = "//:VERSION",
@@ -92,6 +92,6 @@ assemble_rpm(
 
 deploy_rpm(
     name = "deploy-rpm",
-    target = ":assemble-rpm",
+    target = ":assemble-linux-rpm",
     deployment_properties = "@graknlabs_build_tools//:deployment.properties",
 )

bin/grakn-docker.sh

@@ -27,7 +27,7 @@ function cleanup() {
 
 trap cleanup SIGINT SIGTERM
 
-pushd ./grakn-core-all-linux &>/dev/null
+pushd grakn-core-all-linux &>/dev/null
 ./grakn server start
 tail -f logs/grakn.log &