[FLORA-794] Replace the usage of datalog with Haskell for category normalisation

CONTRIBUTING.md

@@ -18,7 +18,6 @@ The following Haskell command-line tools will have to be installed:
 
 (Some of the above packages have incompatible dependencies, so don't try to install them all at once with `cabal install`)
 
-* [Soufflé datalog engine v2.2](https://github.com/souffle-lang/souffle/releases/tag/2.2): The datalog engine for package classification
 * `libsodium-1.0.18`: The system library that powers most of the cryptography happening in flora
 * `yarn`: The tool that handles the JavaScript code bases
 * `esbuild`: The tool that handles asset bundling

Dockerfile

@@ -1,5 +1,4 @@
-# this is a pinned ubuntu:22.04 (newer versions have incomptible
-# library versions for souffle)
+# this is a pinned ubuntu:22.04
 FROM ubuntu@sha256:67211c14fa74f070d27cc59d69a7fa9aeff8e28ea118ef3babc295a0428a6d21
 
 ARG GID=1000
@@ -19,7 +18,7 @@ ARG POSTGRESQL_MIGRATION_VERSION=0.2.1.8
 USER "root"
 ARG USER="local"
 RUN groupadd -g "$GID" -o "$USER" \
-    && useradd -r -u "$UID" -g "$GID" -m -s /bin/zsh "$USER"
+  && useradd -r -u "$UID" -g "$GID" -m -s /bin/zsh "$USER"
 
 # We create the folder explicitly so that we can give nonprivileged user the appropriate access
 RUN mkdir /flora-server
@@ -30,7 +29,20 @@ RUN chown -R $USER:$USER /home/$USER/.cabal
 WORKDIR /flora-server
 
 RUN apt update && \
-    apt install -y build-essential curl libffi-dev libffi8 libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5 git libsodium-dev pkg-config
+  apt install -y  \
+    build-essential \
+    curl \
+    libffi-dev \
+    libffi8 \
+    libgmp-dev \
+    libgmp10 \
+    libncurses-dev \
+    libncurses5 \
+    libtinfo5 \
+    git \
+    libsodium-dev \
+    pkg-config \
+    zlib1g-dev
 
 # install dependencies (pg_config, postgresql-client, yarn)
 ENV BOOTSTRAP_HASKELL_NONINTERACTIVE="YES"
@@ -48,18 +60,12 @@ RUN git config --global --add safe.directory "*"
 RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh
 
 RUN ghcup install hls $HLS_VERSION \
-    && ghcup install ghc $GHC_VERSION \
-    && ghcup set ghc $GHC_VERSION \
-    && ghcup install cabal $CABAL_VERSION
+  && ghcup install ghc $GHC_VERSION \
+  && ghcup set ghc $GHC_VERSION \
+  && ghcup install cabal $CABAL_VERSION
 
 USER ${USER}
 
-# install soufflé
-USER "root"
-RUN wget --content-disposition https://github.com/souffle-lang/souffle/releases/download/2.2/x86_64-ubuntu-2004-souffle-2.2-Linux.deb
-RUN apt install -f -y ./x86_64-ubuntu-2004-souffle-2.2-Linux.deb
-USER ${USER}
-
 RUN echo $PATH
 
 # install Haskell tooling (note that for cabal, it's probably better
@@ -83,10 +89,8 @@ COPY --chown=${USER} scripts/.zshrc /home/$USER/.zshrc
 COPY --chown=${USER} cabal.project flora.cabal cabal.project.freeze ./
 RUN cabal build --only-dependencies -j
 
-# compile Souffle source files
+# copy makefile
 COPY --chown=${USER} Makefile ./
-COPY --chown=${USER} cbits ./cbits
-RUN make souffle
 
 # copy and build the assets
 COPY --chown=${USER} assets ./assets

Makefile

@@ -4,10 +4,10 @@ init: ## Set up git hooks properly - needs calling once when cloning the repo
 start: ## Start flora-server
 	@cabal run exe:flora-server
 
-build: soufflé ## Build the server
+build: ## Build the server
 	@cabal build
 
-build-release: soufflé ## Build the server for production
+build-release: ## Build the server for production
 	@cabal freeze --project-file cabal.project.release
 	@cabal build --project-file cabal.project.release
 
@@ -91,21 +91,21 @@ db-test-provision-packages: ## Load development data in the database
 import-from-hackage: ## Imports every cabal file from the ./index-01 directory
 	@cabal run -- flora-cli import-packages ./01-index
 
-repl: soufflé ## Start a cabal REPL
+repl: ## Start a cabal REPL
 	@cabal repl lib:flora
 
 ghci: repl ## Start a cabal REPL (alias for `make repl`)
 
-watch: soufflé ## Load the main library and reload on file change
+watch: ## Load the main library and reload on file change
 	@ghcid --target flora-server -l
 
-test: build soufflé ## Run the test suite
+test:  ## Run the test suite
 	./scripts/run-tests.sh
 
-watch-test: soufflé ## Load the tests in ghcid and reload them on file change
+watch-test: ## Load the tests in ghcid and reload them on file change
 	./scripts/run-tests.sh --watch
 
-watch-server: soufflé ## Start flora-server in ghcid
+watch-server: ## Start flora-server in ghcid
 	@ghcid --target=flora-server --restart="src" --test 'FloraWeb.Server.runFlora'
 
 lint-hs: ## Run the code linter (HLint)
@@ -145,9 +145,6 @@ docker-enter: ## Enter the docker environment
 start-tmux: ## Start a Tmux session with hot code reloading
 	./scripts/start-tmux.sh
 
-soufflé: ## Generate C++ files from the Soufflé Datalog definitions
-	cd cbits ; souffle -g categorise.{cpp,dl}
-
 tags: ## Generate ctags for the project with `ghc-tags`
 	@ghc-tags -c
 

cabal.project

@@ -15,7 +15,6 @@ allow-newer: hashable:filepath
            , qrcode-juicypixels:bytestring
            , qrcode-juicypixels:text
            , servant-lucid:text
-           , souffle-haskell:text
            , tasty-test-reporter:ansi-terminal
            , tasty-test-reporter:tasty
            , type-errors-pretty:base
@@ -88,11 +87,6 @@ source-repository-package
     location: https://github.com/saurabhnanda/odd-jobs
     tag: a159d7a17913725a0cb2a2251fa0b812acd3d160
 
-source-repository-package
-    type: git
-    location: https://github.com/luc-tielen/souffle-haskell
-    tag: 268a11283ca9293b5eacabf7a0b79d9368232478
-
 source-repository-package
   type: git
   location: https://github.com/goodlyrottenapple/tasty-test-reporter

cabal.project.freeze

@@ -395,7 +395,6 @@ constraints: any.Cabal ==3.10.3.0,
              any.some ==1.0.6,
              some +newtype-unsafe,
              any.sop-core ==0.5.0.2,
-             any.souffle-haskell ==4.0.0,
              any.split ==0.2.5,
              any.splitmix ==0.1.1,
              splitmix -optimised-mixer,

changelog.d/794

@@ -0,0 +1,8 @@
+synopsis: Replace the usage of datalog with Haskell for category normalisation
+
+prs: #822
+issues: #794
+
+description: {
+  Replaces Souffle as much as possible for category normalisation
+}