Emitting Otel traces when the database syncs (#3)

This is a very simple span, but it gives us insight into if/when syncs are occurring.

Author: clintonb

.dockerignore

@@ -1,2 +1,4 @@
 *
-!docker-build-resources/
+!docker-build-resources/etc
+!docker-build-resources/opentelemetry-shell/library
+!docker-build-resources/sync.sh

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "docker-build-resources/opentelemetry-shell"]
+	path = docker-build-resources/opentelemetry-shell
+	url = https://github.com/voriteam/opentelemetry-shell.git

Dockerfile

@@ -26,6 +26,7 @@ ENV LITEFS_DB_DIRECTORY=/litefs
 ENV LITEFS_INTERNAL_DATA_DIRECTORY=/var/lib/litefs
 
 WORKDIR /
+COPY docker-build-resources/opentelemetry-shell /opentelemetry-shell
 COPY docker-build-resources/sync.sh /sync.sh
 
 ENTRYPOINT litefs mount

README.md

@@ -58,5 +58,6 @@ docker buildx build --platform linux/amd64,linux/arm64 -t us-docker.pkg.dev/vori
 ```
 
 ### Homebrew
-The POS machines run a service installed via Homebrew. Create a release on GitHub, and follow the instructions at 
-https://github.com/voriteam/homebrew-voripos to update the tap with the latest version.
+1. Update `VORIPOS_DOMAIN_SYNC_VERSION`.
+2. Create a release on GitHub.
+3. Follow the instructions at https://github.com/voriteam/homebrew-voripos to update the tap with the latest version.

docker-build-resources/etc/litefs.yml

@@ -19,4 +19,4 @@ lease:
   # and false on the replicas.
   candidate: $IS_PRIMARY
 
-exec: "sh sync.sh"
+exec: "bash sync.sh"

docker-build-resources/opentelemetry-shell

@@ -4,6 +4,12 @@ set -e
 set +v
 set +x
 
+# OpenTelemetry setup
+export OTEL_EXPORTER_OTEL_ENDPOINT="http://host.docker.internal:4318"
+export OTEL_SH_LIB_PATH="opentelemetry-shell/library"
+service_version=$OTEL_SERVICE_VERSION
+. opentelemetry-shell/library/otel_traces.sh
+
 Normal='\033[0m'
 Underline='\033[4m'
 
@@ -40,7 +46,21 @@ sync_database() {
     if test -s $sourcePath; then
       # NOTE: We execute the restore in the container since it is more performant than restoring on a host volume.
       echo -e "${Yellow}Copying $sourcePath to $destPath...${Normal}"
-      time cp $sourcePath $destPath
+
+      # Create an Otel span
+      generatedAt=$(sqlite3 $sourcePath "SELECT generated_at FROM metadata;")
+      schemaVersion=$(sqlite3 $sourcePath "SELECT schema_version FROM metadata;")
+      local span_name="Copy domain data to host"
+      local custom_resource_attributes=(
+        "domain_data_generated_at:$generatedAt"
+        "domain_data_schema_version:$schemaVersion"
+      )
+      local linkedTraceId=$(sqlite3 $sourcePath "SELECT linked_trace_id FROM metadata;")
+      local linkedTraceState=$(sqlite3 $sourcePath "SELECT linked_trace_state FROM metadata;")
+      local linkedSpanId=$(sqlite3 $sourcePath "SELECT linked_span_id FROM metadata;")
+      echo "[Linked Span Data] linkedTraceId: $linkedTraceId, linkedTraceState: $linkedTraceState, linkedSpanId: $linkedSpanId"
+      local linked_span=("$linkedTraceId" "$linkedSpanId" "$linkedTraceState")
+      otel_trace_start_parent_span cp $sourcePath $destPath
 
       echo -e "${Green}Successfully copied ${name} to ${destPath}$Normal"
       sqlite3 $destPath ".mode table" "SELECT * FROM metadata;"

docker-build-resources/sync.sh

@@ -14,6 +14,10 @@ services:
       LITEFS_CLOUD_TOKEN: ${LITEFS_CLOUD_TOKEN}
       FILE_RETENTION_COUNT: ${FILE_RETENTION_COUNT}
       DOMAIN_DB_NAME: Domain.sqlite3
+      OTEL_EXPORTER_OTEL_ENDPOINT: "http://host.docker.internal:4318"
+      OTEL_SERVICE_NAME: voripos-domain-sync
+      OTEL_SERVICE_VERSION: ${VORIPOS_DOMAIN_SYNC_VERSION}
+      OTEL_LOG_LEVEL: debug
     volumes:
       # Bind mounts help decrease data transfer latency. We no longer perform SQLite restores but,
       # when we tested them with bind mounts, the bind mount dropped restore time from 4 minutes down to 20 seconds.

docker-compose.yml

@@ -1,11 +1,18 @@
 #!/bin/bash
 
+set -e
+set +v
+set +x
+
 # This is needed to locate the Docker executable
 export PATH="/usr/local/bin:$PATH"
 
+export VORIPOS_DOMAIN_SYNC_VERSION=0.4.0
+
 # NOTE: Bash must be given Full Disk Access in order to read the user defaults.
 #   See https://www.kith.org/jed/2022/02/15/launchctl-scheduling-shell-scripts-on-macos-and-full-disk-access/.
-export LITEFS_CLOUD_TOKEN=$(defaults read com.vori.VoriPOS provisioned_litefsCloudToken)
+LITEFS_CLOUD_TOKEN=$(defaults read com.vori.VoriPOS provisioned_litefsCloudToken)
+export LITEFS_CLOUD_TOKEN
 export VORIPOS_DATA_DIR="$HOME/Library/Containers/com.vori.VoriPOS/Data/Library/Application Support/Domain"
 
 # Keep the most-recent 100 files to ensure we (a) don't fill the disk while