feat(modules): add dagster and mage exposure modules (#253)

add recon modules for self-hosted data-orchestration webservers that ship no built-in authentication: dagster /server_info discloses the webserver and core versions, and mage /api/status discloses the scheduler status and server repository path; both reach an editor or graphql api on the same instance that can execute arbitrary code.

Author: TBX3D

internal/modules/data_orchestration_exposure_test.go

@@ -0,0 +1,99 @@
+package modules_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/vmfunc/sif/internal/modules"
+)
+
+func runDataOrchModule(t *testing.T, file string, status int, body string) *modules.Result {
+	t.Helper()
+	def, err := modules.ParseYAMLModule(file)
+	if err != nil {
+		t.Fatalf("parse %s: %v", file, err)
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	defer srv.Close()
+
+	res, err := modules.ExecuteHTTPModule(context.Background(), srv.URL, def, modules.Options{
+		Timeout: 5 * time.Second,
+		Threads: 2,
+	})
+	if err != nil {
+		t.Fatalf("execute %s: %v", file, err)
+	}
+	return res
+}
+
+func dataOrchExtract(res *modules.Result, key string) string {
+	for _, f := range res.Findings {
+		if v := f.Extracted[key]; v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+func TestDataOrchestrationExposureModules(t *testing.T) {
+	const dagster = "../../modules/recon/dagster-webserver-exposure.yaml"
+	const mage = "../../modules/recon/mage-status-exposure.yaml"
+
+	t.Run("a dagster server_info is flagged with its version", func(t *testing.T) {
+		body := `{"dagster_webserver_version":"1.7.0","dagster_version":"1.7.0","dagster_graphql_version":"1.7.0"}`
+		res := runDataOrchModule(t, dagster, 200, body)
+		if len(res.Findings) == 0 {
+			t.Fatal("expected a dagster finding")
+		}
+		if v := dataOrchExtract(res, "dagster_version"); v != "1.7.0" {
+			t.Errorf("dagster_version=%q, want 1.7.0", v)
+		}
+	})
+
+	t.Run("a bare core-version body is not flagged as dagster", func(t *testing.T) {
+		if res := runDataOrchModule(t, dagster, 200, `{"dagster_version":"1.7.0"}`); len(res.Findings) > 0 {
+			t.Errorf("a body without dagster_webserver_version should not match, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("a mage status is flagged with its repo path", func(t *testing.T) {
+		body := `{"statuses":[{"is_instance_manager":false,"repo_path":"/home/src/default_repo",` +
+			`"repo_path_relative":"default_repo","scheduler_status":"running","project_type":"standalone",` +
+			`"project_uuid":"abc-123"}]}`
+		res := runDataOrchModule(t, mage, 200, body)
+		if len(res.Findings) == 0 {
+			t.Fatal("expected a mage finding")
+		}
+		if v := dataOrchExtract(res, "mage_repo_path"); v != "/home/src/default_repo" {
+			t.Errorf("mage_repo_path=%q, want /home/src/default_repo", v)
+		}
+	})
+
+	t.Run("a statuses collection without scheduler fields is not flagged as mage", func(t *testing.T) {
+		if res := runDataOrchModule(t, mage, 200, `{"statuses":[{"id":1,"name":"ok"}]}`); len(res.Findings) > 0 {
+			t.Errorf("a generic statuses array should not match mage, got %d findings", len(res.Findings))
+		}
+	})
+
+	t.Run("a plain 200 body is not a leak", func(t *testing.T) {
+		for _, file := range []string{dagster, mage} {
+			if res := runDataOrchModule(t, file, 200, "ok"); len(res.Findings) > 0 {
+				t.Errorf("%s: a plain 200 body should not match, got %d findings", file, len(res.Findings))
+			}
+		}
+	})
+
+	t.Run("a 404 is not a leak", func(t *testing.T) {
+		for _, file := range []string{dagster, mage} {
+			if res := runDataOrchModule(t, file, 404, "not found"); len(res.Findings) > 0 {
+				t.Errorf("%s: a 404 should not match, got %d findings", file, len(res.Findings))
+			}
+		}
+	})
+}

modules/recon/dagster-webserver-exposure.yaml

@@ -0,0 +1,36 @@
+# Dagster Webserver Exposure Detection Module
+
+id: dagster-webserver-exposure
+info:
+  name: Dagster Webserver Exposure
+  author: sif
+  severity: medium
+  description: Detects a Dagster webserver that discloses its version and exposes a run-launching graphql api without authentication
+  tags: [dagster, dagit, data-orchestration, pipeline, mlops, exposure, unauth, recon]
+
+type: http
+
+http:
+  method: GET
+  paths:
+    - "{{BaseURL}}/server_info"
+
+  matchers:
+    - type: word
+      part: body
+      words:
+        - "\"dagster_webserver_version\""
+        - "\"dagster_version\""
+      condition: and
+
+    - type: status
+      status:
+        - 200
+
+  extractors:
+    - type: regex
+      name: dagster_version
+      part: body
+      regex:
+        - '"dagster_version"\s*:\s*"([^"]+)"'
+      group: 1

modules/recon/mage-status-exposure.yaml

@@ -0,0 +1,37 @@
+# Mage Data Pipeline Status Exposure Detection Module
+
+id: mage-status-exposure
+info:
+  name: Mage Data Pipeline Status Exposure
+  author: sif
+  severity: medium
+  description: Detects a Mage data-pipeline server with auth disabled that leaks the scheduler status and the server-side repository path
+  tags: [mage, mage-ai, data-orchestration, pipeline, mlops, exposure, unauth, recon]
+
+type: http
+
+http:
+  method: GET
+  paths:
+    - "{{BaseURL}}/api/status"
+
+  matchers:
+    - type: word
+      part: body
+      words:
+        - "\"statuses\""
+        - "\"scheduler_status\""
+        - "\"repo_path\""
+      condition: and
+
+    - type: status
+      status:
+        - 200
+
+  extractors:
+    - type: regex
+      name: mage_repo_path
+      part: body
+      regex:
+        - '"repo_path"\s*:\s*"([^"]+)"'
+      group: 1