Feat improve local devex

chart/templates/backend.yaml

@@ -55,6 +55,12 @@ spec:
         - name: api
           image: {{ .Values.backend_image }}
           imagePullPolicy: {{ .Values.backend_pull_policy }}
+          {{- if .Values.skaffold_dev }}
+          command:
+            {{- range .Values.backend_api_command_override }}
+            - {{ . | quote }}
+            {{- end }}
+          {{- else }}
           command:
               - gunicorn
               - btrixcloud.main:app_root
@@ -66,6 +72,7 @@ spec:
               - "{{ .Values.backend_workers | default 1 }}"
               - --worker-class
               - uvicorn.workers.UvicornWorker
+          {{- end }}
 
           envFrom:
             - configMapRef:
@@ -140,6 +147,12 @@ spec:
         - name: op
           image: {{ .Values.backend_image }}
           imagePullPolicy: {{ .Values.backend_pull_policy }}
+          {{- if .Values.skaffold_dev }}
+          command:
+            {{- range .Values.backend_op_command_override }}
+            - {{ . | quote }}
+            {{- end }}
+          {{- else }}
           command:
               - gunicorn
               - btrixcloud.main_op:app_root
@@ -151,7 +164,7 @@ spec:
               - "{{ .Values.backend_workers | default 1 }}"
               - --worker-class
               - uvicorn.workers.UvicornWorker
-
+          {{- end }}
           envFrom:
             - configMapRef:
                 name: backend-env-config
@@ -185,6 +198,7 @@ spec:
               cpu: {{ .Values.backend_cpu }}
               memory: {{ .Values.backend_memory }}
 
+          {{- if not (.Values.skaffold_dev) }}
           startupProbe:
             httpGet:
               path: /healthz
@@ -193,6 +207,7 @@ spec:
             periodSeconds: 5
             failureThreshold: 5
             successThreshold: 1
+          {{- end }}
 
           readinessProbe:
             httpGet:
@@ -203,6 +218,7 @@ spec:
             failureThreshold: 5
             successThreshold: 1
 
+          {{- if not (.Values.skaffold_dev) }}
           livenessProbe:
             httpGet:
               path: /healthz
@@ -211,7 +227,7 @@ spec:
             periodSeconds: 30
             failureThreshold: 15
             successThreshold: 1
-
+          {{- end }}
 
 
 ---

docs/develop/backend-hot-reload-and-debugger.md

@@ -0,0 +1,84 @@
+# Deploy the Backend with Hot Reloading and Interactive Debugging
+
+This guide explains how to deploy Browsertrix with [skaffold](https://skaffold.dev/) 
+so the backend hot reloads and allows interactive debugging.
+
+This may save time since you don't need to rebuild the backend container every time you change code
+and can use a debugger to step through code.
+
+## Requirements
+
+Follow the documentation to [install skaffold](https://skaffold.dev/docs/install/), i.e. if you are on
+Mac OS run:
+
+```sh
+brew install skaffold
+```
+
+To install helm and set up a local Kubernetes cluster, see the section on [local dev set up](local-dev-setup.md).
+
+## Quickstart
+
+From the command line, run:
+
+```sh
+skaffold dev
+```
+
+This will deploy Browsertrix into the cluster and port forward the API with hot reloading. 
+
+Navigate to `localhost:8000/api/redoc` or `localhost:8000/api/docs` to see the documentation 
+for the api container.
+
+Navigate to `http://localhost:8756/redoc` or `http://localhost:8756/docs` to see the documentation
+for the op container.
+
+Changing any code in `backend/btrixcloud` will trigger a reload in both the op and api containers.
+
+### Debugger
+
+Interactive debugging uses [debugpy](https://github.com/microsoft/debugpy), which 
+works on VSCode but not PyCharm.
+
+Use these debug configurations in VSCode:
+
+```JSON
+{
+    "name": "Attach to Browsertrix Backend API",
+    "type": "debugpy",
+    "request": "attach",
+    "connect": {
+        "host": "127.0.0.1",
+        "port": 5678
+    },
+    "pathMappings": [
+        {
+        "localRoot": "${workspaceFolder}/backend/btrixcloud/",
+        "remoteRoot": "/app/btrixcloud/"
+        }
+    ],
+    "justMyCode": false
+},
+{
+    "name": "Attach to Browsertrix Backend OP",
+    "type": "debugpy",
+    "request": "attach",
+    "connect": {
+        "host": "127.0.0.1",
+        "port": 5679
+    },
+    "pathMappings": [
+        {
+        "localRoot": "${workspaceFolder}/backend/btrixcloud/",
+        "remoteRoot": "/app/btrixcloud/"
+        }
+    ],
+    "justMyCode": false
+}
+```
+
+This will attach to the Kubernetes pod running Browsertrix and persist between
+hot reloads. Change your code, wait for the application to reload, 
+and still hit breakpoints in the same debugging session.
+
+

docs/develop/frontend-dev.md

@@ -72,6 +72,12 @@ If connecting to a local deployment cluster, set `API_BASE_URL` to:
 API_BASE_URL=http://localhost:30870
 ```
 
+If the API is not running at `localhost:30870`, port forward it from your local cluster:
+
+```sh
+kubectl port-forward svc/browsertrix-cloud-backend 30870:8000
+```
+
 ??? info "Port when using Minikube (on macOS)"
 
     When using Minikube on macOS, the port will not be 30870. Instead, Minikube opens a tunnel to a random port,
@@ -85,7 +91,7 @@ Start the frontend development server:
 yarn start
 ```
 
-This will open `localhost:9870` in a new tab in your default browser.
+This will open `localhost:9870` in a new tab in your default browser. It will take a few minutes to finish.
 
 Saving changes to files in `src` will automatically reload your browser window with the latest UI updates.
 

docs/develop/local-dev-setup.md

@@ -116,7 +116,7 @@ helm upgrade --install -f ./chart/values.yaml \
 
     ??? info "MicroK8S"
 
-        If using microk8s, the commend will be:
+        If using microk8s, the command will be:
 
         ```sh
         microk8s helm3 upgrade --install -f ./chart/values.yaml -f ./chart/local.yaml btrix ./chart/
@@ -133,3 +133,7 @@ Changes to settings in `./chart/local.yaml` can be deployed with `helm upgrade .
 ## Deploying Frontend Only
 
 If you are just making changes to the frontend, you can also [deploy the frontend separately](frontend-dev.md) using a dev server for quicker iteration.
+
+## Deploying Backend with Hot Reloading
+
+If you want to iterate faster on the backend, read [deploy the backend with hot reloading and interactive debugging](backend-hot-reload-and-debugger.md).

mkdocs.yml

@@ -63,6 +63,7 @@ nav:
       - develop/local-dev-setup.md
       - develop/frontend-dev.md
       - develop/docs.md
+      - develop/backend-hot-reload-and-debugger.md
   - User Guide:
       - user-guide/index.md
       - user-guide/signup.md

skaffold.yaml

@@ -0,0 +1,80 @@
+apiVersion: skaffold/v4beta11
+kind: Config
+metadata:
+  name: browsertrix
+build:
+  artifacts:
+    # only put the backend in here since it's quite easy now
+    # to run frontend locally and get hot reloading with yarn
+    - image: docker.io/webrecorder/browsertrix-backend
+      context: backend
+      docker:
+        dockerfile: Dockerfile
+      sync:
+        manual:
+        - src: 'btrixcloud/**/*'
+          dest: .
+portForward:
+# so you can hit the API on `localhost:8000`
+- resourceType: service
+  resourceName: browsertrix-cloud-backend
+  namespace: default
+  port: 8000
+  localPort: 8000
+# for the debugger
+- resourceType: deployment
+  resourceName: browsertrix-cloud-backend
+  namespace: default
+  port: 5678
+  localPort: 5678
+# so you can hit the op container on `localhost:8756`
+- resourceType: service
+  resourceName: browsertrix-cloud-backend
+  namespace: default
+  # can't references {{ Values. }} hence hardcoding
+  port: 8756
+  localPort: 8756
+# for the debugger
+- resourceType: deployment
+  resourceName: browsertrix-cloud-backend
+  namespace: default
+  port: 5679
+  localPort: 5679
+deploy:
+  helm:
+    releases:
+      - name: btrix
+        chartPath: chart
+        valuesFiles:
+          - chart/values.yaml
+          # local must come after values since it is expected to override
+          - chart/local.yaml
+        # See https://skaffold.dev/docs/deployers/helm/
+        # must do this to make skaffold use local images with helm
+        setValues:
+          # we hardcoded this earlier so make sure it has same value
+          # across the helm chart
+          opPort: 8756
+          # we need to override some settings for skaffold dev
+          skaffold_dev: true
+          # hot reloading doesn't work with default gunicorn command
+          # so need to override to use uvicorn
+          # plus need to start the process with debugpy to debug
+          backend_api_command_override:
+            - sh
+            - -c
+            -  >
+                pip install --no-input debugpy
+                &&
+                python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678
+                -m uvicorn btrixcloud.main:app_root
+                --reload --host 0.0.0.0 --port 8000
+          backend_op_command_override:
+            - sh
+            - -c
+            -  >
+                pip install --no-input debugpy
+                &&
+                python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5679
+                -m uvicorn btrixcloud.main_op:app_root
+                --reload --host 0.0.0.0 --port 8756