Deploy with SQLite

.dockerignore

@@ -1,5 +1,6 @@
 *.beam
 *.ez
+*.sqlite*
 .env
 .github
 .gitignore

Dockerfile

@@ -1,20 +1,21 @@
 FROM ghcr.io/gleam-lang/gleam:v0.31.0-rc1-erlang-alpine
 
+# Add LiteFS binary, to replicate the SQLite database.
+COPY --from=flyio/litefs:0.5 /usr/local/bin/litefs /usr/local/bin/litefs
+
 # Add project code
 COPY . /build/
 
 # Compile the Gleam application
 RUN cd /build \
+  && apk add fuse3 ca-certificates sqlite gcc build-base \
   && gleam export erlang-shipment \
   && mv build/erlang-shipment /app \
   && rm -r /build \
-  && apk del gcc build-base \
-  && addgroup -S packages \
-  && adduser -S packages -G packages \
-  && chown -R packages /app
+  && apk del gcc build-base
+
+COPY litefs.yml /etc/litefs.yml
 
 # Run the application
-USER packages
 WORKDIR /app
-ENTRYPOINT ["/app/entrypoint.sh"]
-CMD ["run", "server"]
+ENTRYPOINT litefs mount

README.md

@@ -2,34 +2,42 @@
 
 📦 Search for Gleam packages on [Hex](https://hex.pm).
 
-## Local development
+A Gleam application served with the [Mist](https://github.com/rawhat/mist) web
+server, using a SQLite database.
 
-Install Gleam and PostgreSQL. The application will respect the `PGUSER` and
-`PGPASSWORD` environment variables, defaulting to the user `postgres` with no
-password if they are not set.
+The application is deployed on [Fly](https://fly.io) where
+[LiteFS](https://github.com/superfly/litefs) is used to replicate the
+SQLite database across all instances of the application.
 
-A read-only API key for the Hex API should be supplied via the `HEX_API_KEY`
-environment variable. You can generate one via [the Hex dashboard](https://hex.pm/dashboard/keys).
+## Environment variables
 
-```shell
-# Create the PostgreSQL databases
-createdb gleam_packages
-createdb gleam_packages_test
+The application is configured with a series of environment variables.
 
-# Run the tests
-gleam test
+- `HEX_API_KEY` - **Required**. A read-only API key for the Hex API. You can
+  generate one via [the Hex dashboard](https://hex.pm/dashboard/keys).
+- `DATABASE_PATH` - A path where the SQLite database will be stored. Defaults
+  to `./database.sqlite`. In production this should be set to
+  `$LITEFS_MOUNT_PATH/database.sqlite`.
+- `LITEFS_PRIMARY_FILE` - If this environment variable is set then the
+  application will only attempt to pull information from Hex and insert into the
+  database if there is no file present at this path. When deployed to Fly this
+  file is created by LiteFS for the node that has been elected leader, and the
+  path will be `$LITEFS_MOUNT_PATH/.primary`.
+
+## Local development
 
-# Run the server
-gleam run server
+Install Gleam! See `./Dockerfile` for which version is used in production.
+
+```shell
+gleam test        # Run the tests
+gleam run server  # Run the server
 ```
 
 The SQL query functions are generated from the `sql` directory. To regenerate
 them run `gleam run -m codegen`.
 
 ## Deployment
 
-The application and the PostgreSQL database are hosted on [Fly](https://fly.io).
-
 ```shell
 # Deploy the application
 fly deploy

fly.toml

@@ -1,4 +1,3 @@
-# fly.toml app configuration file generated for gleam-packages on 2023-05-29T14:48:30+01:00
 #
 # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
 #
@@ -8,15 +7,22 @@ primary_region = "lhr"
 kill_signal = "SIGINT"
 kill_timeout = "5s"
 
+[deploy]
+# strategy = "canary"
+strategy = "rolling"
+
 [experimental]
   auto_rollback = true
 
 [env]
   PRIMARY_REGION = "lhr"
+  DATABASE_PATH = "/litefs/packages.sqlite"
+  LITEFS_PRIMARY_FILE = "/litefs/.primary"
+  WOBBLE = "buggle"
 
 [[services]]
   protocol = "tcp"
-  internal_port = 3000
+  internal_port = 3001
   processes = ["app"]
 
   [[services.ports]]
@@ -37,3 +43,7 @@ kill_timeout = "5s"
     timeout = "2s"
     grace_period = "1s"
     restart_limit = 0
+
+[mounts]
+  source = "packages_litefs"
+  destination = "/var/lib/litefs"

gleam.toml

@@ -27,6 +27,7 @@ nakai = "~> 0.4"
 gleam_otp = "~> 0.5"
 gleam_hexpm = "~> 0.1"
 sqlight = "~> 0.8"
+simplifile = "~> 0.1"
 
 [dev-dependencies]
 gleeunit = "~> 0.7"

litefs.yml

@@ -0,0 +1,49 @@
+# The fuse section describes settings for the FUSE file system. This file system
+# is used as a thin layer between the SQLite client in your application and the
+# storage on disk. It intercepts disk writes to determine transaction boundaries
+# so that those transactions can be saved and shipped to replicas.
+fuse:
+  dir: "/litefs"
+
+# The data section describes settings for the internal LiteFS storage. We'll 
+# mount a volume to the data directory so it can be persisted across restarts.
+# However, this data should not be accessed directly by the user application.
+data:
+  dir: "/var/lib/litefs"
+
+# This flag ensure that LiteFS continues to run if there is an issue on starup.
+# It makes it easy to ssh in and debug any issues you might be having rather
+# than continually restarting on initialization failure.
+exit-on-error: false
+
+# This section defines settings for the option HTTP proxy.
+# This proxy can handle primary forwarding & replica consistency
+# for applications that use a single SQLite database.
+proxy:
+  addr: ":3001"
+  target: "localhost:3000"
+  db: "packages.sqlite"
+  passthrough: 
+    - "*.css"
+    - "*.js"
+
+# This section defines a list of commands to run after LiteFS has connected
+# and sync'd with the cluster. You can run multiple commands but LiteFS expects
+# the last command to be long-running (e.g. an application server). When the
+# last command exits, LiteFS is shut down.
+exec:
+  - cmd: "/app/entrypoint.sh run server"
+
+# The lease section specifies how the cluster will be managed. We're using the
+# "consul" lease type so that our application can dynamically change the primary.
+#
+# These environment variables will be available in your Fly.io application.
+lease:
+  type: "consul"
+  advertise-url: "http://${HOSTNAME}.vm.${FLY_APP_NAME}.internal:20202"
+  candidate: ${FLY_REGION == PRIMARY_REGION}
+  promote: true
+
+  consul:
+    url: "${FLY_CONSUL_URL}"
+    key: "litefs/${FLY_APP_NAME}"

manifest.toml

@@ -2,27 +2,28 @@
 # You typically do not need to edit this file
 
 packages = [
-  { name = "birl", version = "0.16.1", build_tools = ["gleam"], requirements = ["ranger", "gleam_stdlib"], otp_app = "birl", source = "hex", outer_checksum = "90051B6CBDC1D74E4E007EE0202C52D755BF50D0147F9D02C2159D5C05AD33EC" },
+  { name = "birl", version = "0.16.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "ranger"], otp_app = "birl", source = "hex", outer_checksum = "90051B6CBDC1D74E4E007EE0202C52D755BF50D0147F9D02C2159D5C05AD33EC" },
   { name = "certifi", version = "2.12.0", build_tools = ["rebar3"], requirements = [], otp_app = "certifi", source = "hex", outer_checksum = "EE68D85DF22E554040CDB4BE100F33873AC6051387BAF6A8F6CE82272340FF1C" },
   { name = "esqlite", version = "0.8.6", build_tools = ["rebar3"], requirements = [], otp_app = "esqlite", source = "hex", outer_checksum = "607E45F4DA42601D8F530979417F57A4CD629AB49085891849302057E68EA188" },
   { name = "gleam_erlang", version = "0.22.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "367D8B41A7A86809928ED1E7E55BFD0D46D7C4CF473440190F324AFA347109B4" },
-  { name = "gleam_hackney", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "hackney"], otp_app = "gleam_hackney", source = "hex", outer_checksum = "CA69AD9061C4A8775A7BD445DE33ECEFD87379AF8E5B028F3DD0216BECA5DD0B" },
-  { name = "gleam_hexpm", version = "0.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "birl"], otp_app = "gleam_hexpm", source = "hex", outer_checksum = "0F1080C3DCB8E69D844CB6127118C82D5EED1C3906F8F9F1DFF089FC8AC286C7" },
+  { name = "gleam_hackney", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "gleam_http", "hackney"], otp_app = "gleam_hackney", source = "hex", outer_checksum = "CA69AD9061C4A8775A7BD445DE33ECEFD87379AF8E5B028F3DD0216BECA5DD0B" },
+  { name = "gleam_hexpm", version = "0.1.0", build_tools = ["gleam"], requirements = ["birl", "gleam_stdlib"], otp_app = "gleam_hexpm", source = "hex", outer_checksum = "0F1080C3DCB8E69D844CB6127118C82D5EED1C3906F8F9F1DFF089FC8AC286C7" },
   { name = "gleam_http", version = "3.5.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "FAE9AE3EB1CA90C2194615D20FFFD1E28B630E84DACA670B28D959B37BCBB02C" },
-  { name = "gleam_json", version = "0.6.0", build_tools = ["gleam"], requirements = ["thoas", "gleam_stdlib"], otp_app = "gleam_json", source = "hex", outer_checksum = "C6CC5BEECA525117E97D0905013AB3F8836537455645DDDD10FE31A511B195EF" },
-  { name = "gleam_otp", version = "0.7.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "gleam_erlang"], otp_app = "gleam_otp", source = "hex", outer_checksum = "ED7381E90636E18F5697FD7956EECCA635A3B65538DC2BE2D91A38E61DCE8903" },
+  { name = "gleam_json", version = "0.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "thoas"], otp_app = "gleam_json", source = "hex", outer_checksum = "C6CC5BEECA525117E97D0905013AB3F8836537455645DDDD10FE31A511B195EF" },
+  { name = "gleam_otp", version = "0.7.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "ED7381E90636E18F5697FD7956EECCA635A3B65538DC2BE2D91A38E61DCE8903" },
   { name = "gleam_stdlib", version = "0.30.2", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "8D8BF3790AA31176B1E1C0B517DD74C86DA8235CF3389EA02043EE4FD82AE3DC" },
   { name = "gleeunit", version = "0.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "1397E5C4AC4108769EE979939AC39BF7870659C5AFB714630DEEEE16B8272AD5" },
-  { name = "glisten", version = "0.8.2", build_tools = ["gleam"], requirements = ["gleam_otp", "gleam_erlang", "gleam_stdlib"], otp_app = "glisten", source = "hex", outer_checksum = "364E9B3D4F78308D2EEE5D73E0FB16C686E08516943EFDA501B17177B382907C" },
-  { name = "hackney", version = "1.19.1", build_tools = ["rebar3"], requirements = ["ssl_verify_fun", "unicode_util_compat", "certifi", "idna", "mimerl", "parse_trans", "metrics"], otp_app = "hackney", source = "hex", outer_checksum = "8AA08234BDEFC269995C63C2282CF3CD0E36FEBE3A6BFAB11B610572FDD1CAD0" },
+  { name = "glisten", version = "0.8.2", build_tools = ["gleam"], requirements = ["gleam_otp", "gleam_stdlib", "gleam_erlang"], otp_app = "glisten", source = "hex", outer_checksum = "364E9B3D4F78308D2EEE5D73E0FB16C686E08516943EFDA501B17177B382907C" },
+  { name = "hackney", version = "1.19.1", build_tools = ["rebar3"], requirements = ["metrics", "certifi", "mimerl", "parse_trans", "ssl_verify_fun", "unicode_util_compat", "idna"], otp_app = "hackney", source = "hex", outer_checksum = "8AA08234BDEFC269995C63C2282CF3CD0E36FEBE3A6BFAB11B610572FDD1CAD0" },
   { name = "idna", version = "6.1.1", build_tools = ["rebar3"], requirements = ["unicode_util_compat"], otp_app = "idna", source = "hex", outer_checksum = "92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA" },
   { name = "metrics", version = "1.0.1", build_tools = ["rebar3"], requirements = [], otp_app = "metrics", source = "hex", outer_checksum = "69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16" },
   { name = "mimerl", version = "1.2.0", build_tools = ["rebar3"], requirements = [], otp_app = "mimerl", source = "hex", outer_checksum = "F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323" },
-  { name = "mist", version = "0.13.2", build_tools = ["gleam"], requirements = ["gleam_otp", "gleam_http", "gleam_stdlib", "glisten", "gleam_erlang"], otp_app = "mist", source = "hex", outer_checksum = "51C385C58A78A2013A30F92705814560AD9A2B8EC3ECBA94C620F22D3ACB50BC" },
+  { name = "mist", version = "0.13.2", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "gleam_otp", "glisten", "gleam_erlang"], otp_app = "mist", source = "hex", outer_checksum = "51C385C58A78A2013A30F92705814560AD9A2B8EC3ECBA94C620F22D3ACB50BC" },
   { name = "nakai", version = "0.8.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "nakai", source = "hex", outer_checksum = "E4BA604229121481776CB4DD42C6E5E7FB1A9BF303D7B1DCDFECFE5FA0B34382" },
   { name = "parse_trans", version = "3.4.1", build_tools = ["rebar3"], requirements = [], otp_app = "parse_trans", source = "hex", outer_checksum = "620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A" },
   { name = "ranger", version = "0.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "ranger", source = "hex", outer_checksum = "A3D54DD2BFCC654F1AA7A0D245F6F59A5D9FF6B2D68449C849BFE321E945EA7A" },
-  { name = "sqlight", version = "0.8.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "esqlite"], otp_app = "sqlight", source = "hex", outer_checksum = "63001BC125F481A15D459AE883DAACB3B3CB828814DE3D885AE70CCC324C2BC2" },
+  { name = "simplifile", version = "0.1.14", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "10EA0207796F20488A3A166C50A189C9385333F3C9FAC187729DE7B9CE4ADDBC" },
+  { name = "sqlight", version = "0.8.0", build_tools = ["gleam"], requirements = ["esqlite", "gleam_stdlib"], otp_app = "sqlight", source = "hex", outer_checksum = "63001BC125F481A15D459AE883DAACB3B3CB828814DE3D885AE70CCC324C2BC2" },
   { name = "ssl_verify_fun", version = "1.1.7", build_tools = ["mix", "rebar3", "make"], requirements = [], otp_app = "ssl_verify_fun", source = "hex", outer_checksum = "FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8" },
   { name = "thoas", version = "0.4.1", build_tools = ["rebar3"], requirements = [], otp_app = "thoas", source = "hex", outer_checksum = "4918D50026C073C4AB1388437132C77A6F6F7C8AC43C60C13758CC0ADCE2134E" },
   { name = "unicode_util_compat", version = "0.7.0", build_tools = ["rebar3"], requirements = [], otp_app = "unicode_util_compat", source = "hex", outer_checksum = "25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521" },
@@ -40,4 +41,5 @@ gleam_stdlib = { version = "~> 0.23" }
 gleeunit = { version = "~> 0.7" }
 mist = { version = "~> 0.10" }
 nakai = { version = "~> 0.4" }
+simplifile = { version = "~> 0.1" }
 sqlight = { version = "~> 0.8" }

src/packages.gleam

@@ -72,7 +72,6 @@ fn server() {
     |> mist.new
     |> mist.port(3000)
     |> mist.start_http
-  io.println("Started listening on http://localhost:3000 ✨")
 
   // Start syncing new releases periodically
   let assert Ok(_) =

src/packages/index.gleam

@@ -9,12 +9,25 @@ import gleam/option.{None, Option, Some}
 import gleam/result.{try}
 import packages/error.{Error}
 import packages/generated/sql
+import simplifile
 import sqlight
+import gleam/erlang/os
 
 pub opaque type Connection {
   Connection(inner: sqlight.Connection)
 }
 
+/// The application is considered to have write permissions if the
+/// `DATABASE_LOCK_PATH` environment variable is not set, or if it is set and a
+/// file exists at that path.
+///
+pub fn has_write_permission() -> Bool {
+  case os.get_env("LITEFS_PRIMARY_FILE") {
+    Ok(path) -> !simplifile.is_file(path)
+    Error(_) -> True
+  }
+}
+
 const schema = "
 pragma foreign_keys = on;
 pragma journal_mode = wal;

src/packages/syncing.gleam

@@ -38,19 +38,28 @@ pub fn sync_new_gleam_releases(
   hex_api_key: String,
   db: index.Connection,
 ) -> Result(Nil, Error) {
-  io.println("Syncing new releases from Hex")
-  use limit <- try(index.get_most_recent_hex_timestamp(db))
-  use latest <- try(sync_packages(State(
-    page: 1,
-    limit: limit,
-    newest: limit,
-    hex_api_key: hex_api_key,
-    last_logged: time.now(),
-    db: db,
-  )))
-  let latest = index.upsert_most_recent_hex_timestamp(db, latest)
-  io.println("\nUp to date!")
-  latest
+  case index.has_write_permission() {
+    True -> {
+      io.println("Syncing new releases from Hex")
+      use limit <- try(index.get_most_recent_hex_timestamp(db))
+      use latest <- try(sync_packages(State(
+        page: 1,
+        limit: limit,
+        newest: limit,
+        hex_api_key: hex_api_key,
+        last_logged: time.now(),
+        db: db,
+      )))
+      let latest = index.upsert_most_recent_hex_timestamp(db, latest)
+      io.println("\nUp to date!")
+      latest
+    }
+
+    False -> {
+      io.println("Node lacks write permission to DB, not syncing")
+      Ok(Nil)
+    }
+  }
 }
 
 fn sync_packages(state: State) -> Result(DateTime, Error) {