Merge pull request #246 from yokowu/feat-scanner

feat: 实现扫描器

Author: yokowu

.github/workflows/backend-ci-cd.yml

@@ -160,6 +160,26 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Build and push scanner image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          file: ./backend/build/Dockerfile.scanner
+          push: true
+          platforms: ${{ matrix.platform }}
+          tags: |
+            ${{ env.REGISTRY }}/scanner:${{ steps.get_version.outputs.VERSION }}-${{ matrix.arch }}
+            ${{ env.REGISTRY }}/scanner:latest-${{ matrix.arch }}
+          build-args: |
+            GOCACHE=/tmp/go-build
+            GOMODCACHE=/tmp/go-mod
+            REPO_COMMIT=${{ github.sha }}
+            VERSION=${{ steps.get_version.outputs.VERSION }}
+            BUILD_TIME=${{ steps.get_build_time.outputs.BUILD_TIME }}
+            GIT_COMMIT=${{ steps.get_git_commit.outputs.GIT_COMMIT }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
   create-manifest:
     needs: build
     runs-on: ubuntu-latest
@@ -176,14 +196,26 @@ jobs:
         run: |
           VERSION=${{ needs.build.outputs.version }}
           
-          # Create and push version manifest
+          # Create and push backend version manifest
           docker manifest create ${{ env.REGISTRY }}/backend:${VERSION} \
             ${{ env.REGISTRY }}/backend:${VERSION}-x86_64 \
             ${{ env.REGISTRY }}/backend:${VERSION}-aarch64
           docker manifest push ${{ env.REGISTRY }}/backend:${VERSION}
           
-          # Create and push latest manifest
+          # Create and push backend latest manifest
           docker manifest create ${{ env.REGISTRY }}/backend:latest \
             ${{ env.REGISTRY }}/backend:latest-x86_64 \
             ${{ env.REGISTRY }}/backend:latest-aarch64
-          docker manifest push ${{ env.REGISTRY }}/backend:latest
+          docker manifest push ${{ env.REGISTRY }}/backend:latest
+          
+          # Create and push scanner version manifest
+          docker manifest create ${{ env.REGISTRY }}/scanner:${VERSION} \
+            ${{ env.REGISTRY }}/scanner:${VERSION}-x86_64 \
+            ${{ env.REGISTRY }}/scanner:${VERSION}-aarch64
+          docker manifest push ${{ env.REGISTRY }}/scanner:${VERSION}
+          
+          # Create and push scanner latest manifest
+          docker manifest create ${{ env.REGISTRY }}/scanner:latest \
+            ${{ env.REGISTRY }}/scanner:latest-x86_64 \
+            ${{ env.REGISTRY }}/scanner:latest-aarch64
+          docker manifest push ${{ env.REGISTRY }}/scanner:latest

backend/Makefile

@@ -23,6 +23,20 @@ image:
 	  --output ${OUTPUT} \
 	  .
 
+image-scanner:
+	docker buildx build \
+	  -f build/Dockerfile.scanner \
+	  --build-arg GOCACHE=${GOCACHE} \
+	  --build-arg GOMODCACHE=${GOMODCACHE} \
+	  --build-arg REPO_COMMIT=$(shell git rev-parse HEAD) \
+	  --build-arg VERSION=${VERSION} \
+	  --build-arg BUILD_TIME=${BUILD_TIME} \
+	  --build-arg GIT_COMMIT=${GIT_COMMIT} \
+	  --platform ${PLATFORM} \
+	  --tag ${REGISTRY}/scanner:${TAG} \
+	  --output ${OUTPUT} \
+	  .
+
 wire:
 	wire pro/cmd/server/wire.go pro/cmd/server/main.go
 

backend/build/Dockerfile

@@ -21,18 +21,12 @@ go build \
 -o /out/main \
 pro/cmd/server/main.go pro/cmd/server/wire_gen.go 
 
-FROM debian:bullseye-20250721-slim as binary
-
-RUN apt-get update && \
-    apt-get install -y ca-certificates && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    update-ca-certificates
+FROM alpine:3.22.1 as binary
 
 WORKDIR /app
 
 ADD migration ./migration
-ADD assets ./assets
+ADD assets/vsix ./assets/vsix
 
 COPY --from=builder /out/main /app/main
 

backend/build/Dockerfile.scanner

@@ -0,0 +1,32 @@
+FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder
+
+WORKDIR /src
+ENV CGO_ENABLED=0
+
+COPY go.* .
+ARG GOMODCACHE
+RUN --mount=type=cache,target=${GOMODCACHE} \
+    go mod download
+
+ARG TARGETOS TARGETARCH GOCACHE
+ARG VERSION
+ARG BUILD_TIME
+ARG GIT_COMMIT
+RUN --mount=type=bind,target=. \
+--mount=type=cache,target=${GOMODCACHE} \
+--mount=type=cache,target=${GOCACHE} \
+GOOS=$TARGETOS GOARCH=$TARGETARCH \
+go build \
+-ldflags "-w -s -X 'github.com/chaitin/MonkeyCode/backend/pkg/version.Version=${VERSION}' -X 'github.com/chaitin/MonkeyCode/backend/pkg/version.BuildTime=${BUILD_TIME}' -X 'github.com/chaitin/MonkeyCode/backend/pkg/version.GitCommit=${GIT_COMMIT}'" \
+-o /out/main \
+cmd/scanner/main.go cmd/scanner/wire_gen.go 
+
+FROM debian:bullseye-20250721-slim as binary
+
+WORKDIR /app
+
+ADD assets/sgp ./assets/sgp
+
+COPY --from=builder /out/main /app/main
+
+CMD [ "./main" ]

backend/cmd/scanner/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"context"
+
+	"github.com/google/wire"
+
+	"github.com/chaitin/MonkeyCode/backend/config"
+	v1 "github.com/chaitin/MonkeyCode/backend/internal/scanner/handler/http/v1"
+	"github.com/chaitin/MonkeyCode/backend/pkg"
+	"github.com/chaitin/MonkeyCode/backend/pkg/logger"
+	"github.com/chaitin/MonkeyCode/backend/pkg/service"
+	"github.com/chaitin/MonkeyCode/backend/pkg/version"
+)
+
+func main() {
+	s, err := newServer()
+	if err != nil {
+		panic(err)
+	}
+
+	s.version.Print()
+	s.logger.With("config", s.config).Debug("config")
+	s.web.PrintRoutes()
+
+	svc := service.NewService(service.WithPprof())
+	svc.Add(s)
+	if err := svc.Run(); err != nil {
+		panic(err)
+	}
+}
+
+// Name implements service.Servicer.
+func (s *Server) Name() string {
+	return "Scanner Server"
+}
+
+// Start implements service.Servicer.
+func (s *Server) Start() error {
+	return s.web.Run(s.config.Server.Addr)
+}
+
+// Stop implements service.Servicer.
+func (s *Server) Stop() error {
+	return s.web.Echo().Shutdown(context.Background())
+}
+
+var AppSet = wire.NewSet(
+	wire.FieldsOf(new(*config.Config), "Logger"),
+	config.Init,
+	pkg.NewWeb,
+	logger.NewLogger,
+	version.NewVersionInfo,
+	v1.NewScannerHandler,
+)

backend/cmd/scanner/wire.go

@@ -0,0 +1,32 @@
+//go:build wireinject
+// +build wireinject
+
+package main
+
+import (
+	"log/slog"
+
+	"github.com/google/wire"
+
+	"github.com/GoYoko/web"
+
+	"github.com/chaitin/MonkeyCode/backend/config"
+	v1 "github.com/chaitin/MonkeyCode/backend/internal/scanner/handler/http/v1"
+	"github.com/chaitin/MonkeyCode/backend/pkg/version"
+)
+
+type Server struct {
+	config  *config.Config
+	web     *web.Web
+	logger  *slog.Logger
+	version *version.VersionInfo
+	scanner *v1.ScannerHandler
+}
+
+func newServer() (*Server, error) {
+	wire.Build(
+		wire.Struct(new(Server), "*"),
+		AppSet,
+	)
+	return &Server{}, nil
+}

backend/cmd/scanner/wire_gen.go

@@ -70,6 +70,13 @@ func (s *SecurityScanningBrief) From(e *db.SecurityScanning) *SecurityScanningBr
 	return s
 }
 
+type ScanReq struct {
+	TaskID    string                          `json:"task_id"`
+	UserID    string                          `json:"user_id"`
+	Workspace string                          `json:"workspace"` // 项目目录
+	Language  consts.SecurityScanningLanguage `json:"language"`  // 扫描语言
+}
+
 type CreateSecurityScanningReq struct {
 	UserID    string                          `json:"user_id"`
 	Workspace string                          `json:"workspace"` // 项目目录

backend/domain/security.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"net/http"
 	"os"
 	"path"
-	"strings"
 	"time"
 
 	"github.com/redis/go-redis/v9"
@@ -38,7 +38,13 @@ func NewProxyUsecase(
 	cfg *config.Config,
 	redis *redis.Client,
 ) domain.ProxyUsecase {
-	client := request.NewClient("http", "monkeycode-scanner:8888", 15*time.Second)
+	client := request.NewClient("http", "monkeycode-scanner:8888", 30*time.Minute, request.WithTransport(&http.Transport{
+		MaxIdleConns:        100,
+		MaxIdleConnsPerHost: 100,
+		MaxConnsPerHost:     100,
+		IdleConnTimeout:     90 * time.Second,
+		ForceAttemptHTTP2:   true,
+	}))
 	client.SetDebug(cfg.Debug)
 	p := &ProxyUsecase{
 		repo:         repo,
@@ -128,7 +134,7 @@ func (p *ProxyUsecase) TaskHandle(ctx context.Context, task *queuerunner.Task[do
 		p.logger.With("id", id).With("error", err).ErrorContext(ctx, "failed to get security scanning")
 		return err
 	}
-	prefix := fmt.Sprintf("/app/codes/%s", id)
+	prefix := fmt.Sprintf("/app/static/codes/%s", id)
 	rootPath := path.Join(prefix, scanning.Edges.WorkspaceEdge.RootPath)
 	defer os.RemoveAll(prefix)
 
@@ -151,8 +157,13 @@ func (p *ProxyUsecase) TaskHandle(ctx context.Context, task *queuerunner.Task[do
 		return err
 	}
 
-	rule := strings.ToLower(string(scanning.Language))
-	result, err := scan.Scan(task.ID, rootPath, rule)
+	result, err := request.Post[scan.Result](p.client, "/api/v1/scan", domain.ScanReq{
+		TaskID:    task.ID,
+		UserID:    task.Data.UserID,
+		Workspace: rootPath,
+		Language:  task.Data.Language,
+	})
+
 	if err != nil {
 		if err = p.securityRepo.Update(ctx, id, consts.SecurityScanningStatusFailed, &scan.Result{
 			Output: err.Error(),

backend/internal/proxy/usecase/proxy.go

@@ -0,0 +1,38 @@
+package v1
+
+import (
+	"fmt"
+	"log/slog"
+	"net/http"
+	"strings"
+
+	"github.com/GoYoko/web"
+
+	"github.com/chaitin/MonkeyCode/backend/domain"
+	"github.com/chaitin/MonkeyCode/backend/pkg/scan"
+)
+
+type ScannerHandler struct {
+	logger *slog.Logger
+}
+
+func NewScannerHandler(w *web.Web, logger *slog.Logger) *ScannerHandler {
+	s := &ScannerHandler{
+		logger: logger,
+	}
+
+	w.POST("/api/v1/scan", web.BindHandler(s.Scan))
+
+	return s
+}
+
+func (s *ScannerHandler) Scan(ctx *web.Context, req domain.ScanReq) error {
+	rule := strings.ToLower(string(req.Language))
+	result, err := scan.Scan(req.TaskID, req.Workspace, rule)
+	if err != nil {
+		s.logger.With("id", req.TaskID).With("error", err).ErrorContext(ctx.Request().Context(), "failed to scan")
+		return fmt.Errorf("failed to scan: %w", err)
+	}
+	s.logger.With("id", req.TaskID).InfoContext(ctx.Request().Context(), "task done")
+	return ctx.JSON(http.StatusOK, result)
+}