diff --git a/.dockerignore b/.dockerignore index c3c141b452..747f625d9e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,5 +4,3 @@ !/go.mod !/go.sum !/zadig-portal -!/third_party - diff --git a/GOVERNANCE.md b/GOVERNANCE.md index a05d520a14..3bf965707d 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -56,3 +56,5 @@ The council will make a decision for the nomination. The candidate has to have d - [piao100101](https://github.com/piao100101) - [nanzm](https://github.com/nanzm) - [solomon-cc](https://github.com/solomon-cc) + + diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go new file mode 100644 index 0000000000..8e7a30db49 --- /dev/null +++ b/cmd/hubagent/main.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "log" + + "github.com/koderover/zadig/pkg/microservice/hubagent/server" +) + +func main() { + if err := server.Serve(); err != nil { + log.Fatal(err) + } +} diff --git a/pkg/tool/gitlab/client.go b/cmd/hubserver/main.go similarity index 61% rename from pkg/tool/gitlab/client.go rename to cmd/hubserver/main.go index b3c9049e17..9b03aebc73 100644 --- a/pkg/tool/gitlab/client.go +++ b/cmd/hubserver/main.go @@ -14,27 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gitlab +package main import ( - "fmt" + "context" + "log" + "os/signal" + "syscall" - "github.com/xanzy/go-gitlab" + "github.com/koderover/zadig/pkg/microservice/hubserver/server" ) -type Client struct { - *gitlab.Client -} - -func NewGitlabClient(address, accessToken string) (*Client, error) { - cli, err := gitlab.NewOAuthClient(accessToken, gitlab.WithBaseURL(address)) - if err != nil { - return nil, fmt.Errorf("set base url failed, err:%v", err) +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + go func() { + select { + case <-ctx.Done(): + stop() + } + }() + + if err := server.Serve(ctx); err != nil { + log.Fatal(err) } - - client := &Client{ - Client: cli, - } - - return client, nil } diff --git a/docker/hub-agent.Dockerfile.template b/docker/hub-agent.Dockerfile.template new file mode 100644 index 0000000000..dca0b25939 --- /dev/null +++ b/docker/hub-agent.Dockerfile.template @@ -0,0 +1,11 @@ +#golang-deps.Dockerfile.inc + +RUN go build -v -o /hubagent ./cmd/hubagent/main.go + +#alpine-base.Dockerfile.inc + +WORKDIR /app + +COPY --from=build /hubagent . + +ENTRYPOINT ["/app/hubagent"] diff --git a/docker/hub-server.Dockerfile.template b/docker/hub-server.Dockerfile.template new file mode 100644 index 0000000000..bcdb2344a6 --- /dev/null +++ b/docker/hub-server.Dockerfile.template @@ -0,0 +1,11 @@ +#golang-deps.Dockerfile.inc + +RUN go build -v -o /hubserver ./cmd/hubserver/main.go + +#alpine-base.Dockerfile.inc + +WORKDIR /app + +COPY --from=build /hubserver . + +ENTRYPOINT ["/app/hubserver"] diff --git a/examples/website/yaml/website-example.yaml b/examples/website/yaml/website-example.yaml index 7528cdcf41..86f93ec5eb 100644 --- a/examples/website/yaml/website-example.yaml +++ b/examples/website/yaml/website-example.yaml @@ -11,7 +11,6 @@ spec: - port: 80 selector: app: website-example - tier: frontend --- apiVersion: apps/v1 kind: Deployment diff --git a/go.mod b/go.mod index eeb0af84ae..f08876d885 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect github.com/bugsnag/bugsnag-go v2.1.0+incompatible // indirect github.com/bugsnag/panicwrap v1.3.1 // indirect + github.com/cenkalti/backoff/v3 v3.0.0 github.com/coocood/freecache v1.1.0 github.com/docker/distribution v2.7.1+incompatible github.com/docker/docker v1.4.2-0.20200204220554-5f6d6f3f2203 @@ -24,14 +25,14 @@ require ( github.com/go-openapi/spec v0.19.5 // indirect github.com/go-resty/resty/v2 v2.6.0 github.com/gofrs/uuid v4.0.0+incompatible // indirect - github.com/google/go-github/v35 v35.1.0 + github.com/google/go-github/v35 v35.3.0 github.com/google/uuid v1.2.0 github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.2 github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f - github.com/hashicorp/go-multierror v1.0.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/jasonlvhit/gocron v0.0.0-20171226191223-3c914c8681c3 github.com/jinzhu/now v1.1.2 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect @@ -55,7 +56,7 @@ require ( github.com/swaggo/gin-swagger v1.3.0 github.com/swaggo/swag v1.5.1 github.com/ugorji/go v1.2.0 // indirect - github.com/xanzy/go-gitlab v0.44.0 + github.com/xanzy/go-gitlab v0.50.0 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.7 // indirect diff --git a/go.sum b/go.sum index e5299003af..a14acc3f5f 100644 --- a/go.sum +++ b/go.sum @@ -113,7 +113,10 @@ github.com/bugsnag/bugsnag-go v2.1.0+incompatible/go.mod h1:2oa8nejYd4cQ/b0hMIop github.com/bugsnag/panicwrap v1.3.1 h1:pmuhHlhbUV4OOrGDvoiMjHSZzwRcL+I9cIzYKiW4lII= github.com/bugsnag/panicwrap v1.3.1/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -382,12 +385,13 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts= github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= -github.com/google/go-github/v35 v35.1.0 h1:KkwZnKWQ/0YryvXjZlCN/3EGRJNp6VCZPKo+RG9mG28= -github.com/google/go-github/v35 v35.1.0/go.mod h1:s0515YVTI+IMrDoy9Y4pHt9ShGpzHvHO8rZ7L7acgvs= +github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho= +github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -451,10 +455,11 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-retryablehttp v0.6.4 h1:BbgctKO892xEyOXnGiaAwIoSq1QZ/SS4AhjoAh9DnfY= -github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= +github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -614,6 +619,7 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -830,8 +836,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243 h1:R43TdZy32XXSXjJn7M/HhALJ9imq6ztLnChfYJpVDnM= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs= -github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= +github.com/xanzy/go-gitlab v0.50.0 h1:t7IoYTrnLSbdEZN7d8X/5zcr+ZM4TZQ2mXa8MqWlAZQ= +github.com/xanzy/go-gitlab v0.50.0/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= @@ -947,7 +953,6 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/image.Makefile b/image.Makefile index 81b44d5554..21fb9778ab 100644 --- a/image.Makefile +++ b/image.Makefile @@ -12,15 +12,7 @@ IMAGE_TAG:=$(shell ./docker/image-tag) VCS_REF:=$(shell git rev-parse HEAD) TAG ?= ${DATE}-${IMAGE_TAG} -BASE_TARGETS = aslan warpdrive cron podexec jenkins-plugin predator-plugin - - -TARGETS = -ifndef ENTERPRISE_TARGETS -TARGETS = $(BASE_TARGETS) -else -TARGETS = $(ENTERPRISE_TARGETS) -endif +TARGETS = aslan warpdrive cron podexec jenkins-plugin predator-plugin hub-server hub-agent ALL_IMAGES=$(TARGETS:=.image) ALL_PUSHES=$(TARGETS:=.push) diff --git a/pkg/config/config.go b/pkg/config/config.go index ee63b5022a..64572323d3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -60,3 +60,15 @@ func LogFile() string { func RequestLogFile() string { return LogPath() + RequestLogName() } + +func AslanURL() string { + return viper.GetString(setting.ENVAslanURL) +} + +func PoetryAPIServer() string { + return viper.GetString(setting.ENVPoetryAPIServer) +} + +func PoetryAPIRootKey() string { + return viper.GetString(setting.ENVPoetryAPIRootKey) +} diff --git a/pkg/microservice/aslan/config/config.go b/pkg/microservice/aslan/config/config.go index 6993a116a9..cca6d1d804 100644 --- a/pkg/microservice/aslan/config/config.go +++ b/pkg/microservice/aslan/config/config.go @@ -18,6 +18,7 @@ package config import ( "errors" + "fmt" "strconv" "strings" @@ -125,7 +126,7 @@ func S3StoragePath() string { } func EnableGitCheck() bool { - return viper.GetString(setting.EnableGitCheck) == "true" + return true } func S3StorageAK() string { @@ -253,3 +254,7 @@ func SonarRootToken() string { func SonarInternalAddr() string { return viper.GetString(setting.EnvSonarInternalAddr) } + +func WebHookURL() string { + return fmt.Sprintf("%s/api/aslan/webhook", AslanURL()) +} diff --git a/pkg/microservice/aslan/core/build/handler/build.go b/pkg/microservice/aslan/core/build/handler/build.go index be2b988da5..a70c3e626f 100644 --- a/pkg/microservice/aslan/core/build/handler/build.go +++ b/pkg/microservice/aslan/core/build/handler/build.go @@ -25,7 +25,7 @@ import ( buildservice "github.com/koderover/zadig/pkg/microservice/aslan/core/build/service" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/build/handler/router.go b/pkg/microservice/aslan/core/build/handler/router.go index 1b50162ce1..831d69ed21 100644 --- a/pkg/microservice/aslan/core/build/handler/router.go +++ b/pkg/microservice/aslan/core/build/handler/router.go @@ -19,23 +19,23 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) build := router.Group("build") { build.GET("/:name/:version", FindBuildModule) build.GET("", ListBuildModules) - build.POST("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.BuildManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateBuildModule) - build.PUT("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.BuildManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateBuildModule) - build.DELETE("", middleware.IsHavePermission([]string{permission.BuildDeleteUUID}, permission.QueryType), middleware.UpdateOperationLogStatus, DeleteBuildModule) - build.POST("/targets", middleware.IsHavePermission([]string{permission.BuildManageUUID}, permission.QueryType), middleware.UpdateOperationLogStatus, UpdateBuildTargets) + build.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.BuildManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateBuildModule) + build.PUT("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.BuildManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateBuildModule) + build.DELETE("", gin2.IsHavePermission([]string{permission.BuildDeleteUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeleteBuildModule) + build.POST("/targets", gin2.IsHavePermission([]string{permission.BuildManageUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, UpdateBuildTargets) } target := router.Group("targets") diff --git a/pkg/microservice/aslan/core/build/handler/target.go b/pkg/microservice/aslan/core/build/handler/target.go index 373d4f6cfc..71f5e16008 100644 --- a/pkg/microservice/aslan/core/build/handler/target.go +++ b/pkg/microservice/aslan/core/build/handler/target.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" buildservice "github.com/koderover/zadig/pkg/microservice/aslan/core/build/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func ListDeployTarget(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/build/service/build.go b/pkg/microservice/aslan/core/build/service/build.go index 1879145c29..a5b41afd32 100644 --- a/pkg/microservice/aslan/core/build/service/build.go +++ b/pkg/microservice/aslan/core/build/service/build.go @@ -318,6 +318,9 @@ func UpdateBuildTargets(name, productName string, targets []*commonmodels.Servic return e.ErrUpdateBuildParam.AddErr(err) } + //处理云主机服务组件逻辑 + handleServiceTargets(name, productName, targets) + err := commonrepo.NewBuildColl().UpdateTargets(name, productName, targets) if err != nil { log.Errorf("[Build.UpdateServices] %s error: %v", name, err) diff --git a/pkg/microservice/aslan/core/build/service/target.go b/pkg/microservice/aslan/core/build/service/target.go index 1b3ba11677..e88296c542 100644 --- a/pkg/microservice/aslan/core/build/service/target.go +++ b/pkg/microservice/aslan/core/build/service/target.go @@ -86,6 +86,19 @@ func ListDeployTarget(productName string, log *zap.SugaredLogger) ([]*commonmode } } } + case setting.PMDeployType: + target := fmt.Sprintf("%s-%s-%s", serviceTmpl.ProductName, serviceTmpl.ServiceName, serviceTmpl.ServiceName) + if _, ok := targetMap[target]; !ok { + targetMap[target] = true + ServiceObject := &commonmodels.ServiceModuleTarget{ + ProductName: serviceTmpl.ProductName, + ServiceName: serviceTmpl.ServiceName, + ServiceModule: serviceTmpl.ServiceName, + } + if !buildTargets.Has(target) { + serviceObjects = append(serviceObjects, ServiceObject) + } + } } } return serviceObjects, nil @@ -121,6 +134,12 @@ func ListContainers(productName string, log *zap.SugaredLogger) ([]*commonmodels ServiceModule: container.Name, }) } + } else if service.Type == setting.PMDeployType { + containerList = append(containerList, &commonmodels.ServiceModuleTarget{ + ProductName: service.ProductName, + ServiceName: service.ServiceName, + ServiceModule: service.ServiceName, + }) } } return containerList, nil diff --git a/pkg/microservice/aslan/core/code/handler/codehost.go b/pkg/microservice/aslan/core/code/handler/codehost.go index f1f6b2edf3..edddb991bf 100644 --- a/pkg/microservice/aslan/core/code/handler/codehost.go +++ b/pkg/microservice/aslan/core/code/handler/codehost.go @@ -22,11 +22,11 @@ import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/core/code/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -57,7 +57,7 @@ func CodeHostGetNamespaceList(c *gin.Context) { return } chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodehostListNamespaces(chID, keyword, ctx.Logger) + ctx.Resp, ctx.Err = service.CodeHostListNamespaces(chID, keyword, ctx.Logger) } func CodeHostGetProjectsList(c *gin.Context) { @@ -87,7 +87,7 @@ func CodeHostGetProjectsList(c *gin.Context) { } chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodehostListProjects( + ctx.Resp, ctx.Err = service.CodeHostListProjects( chID, strings.Replace(namespace, "%2F", "/", -1), namespaceType, @@ -117,7 +117,7 @@ func CodeHostGetBranchList(c *gin.Context) { } chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodehostListBranches( + ctx.Resp, ctx.Err = service.CodeHostListBranches( chID, projectName, strings.Replace(namespace, "%2F", "/", -1), @@ -146,7 +146,7 @@ func CodeHostGetTagList(c *gin.Context) { } chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodehostListTags(chID, projectName, strings.Replace(namespace, "%2F", "/", -1), ctx.Logger) + ctx.Resp, ctx.Err = service.CodeHostListTags(chID, projectName, strings.Replace(namespace, "%2F", "/", -1), ctx.Logger) } func CodeHostGetPRList(c *gin.Context) { @@ -173,7 +173,7 @@ func CodeHostGetPRList(c *gin.Context) { targetBr := c.Query("targetBranch") chID, _ := strconv.Atoi(codehostID) - ctx.Resp, ctx.Err = service.CodehostListPRs(chID, projectName, strings.Replace(namespace, "%2F", "/", -1), targetBr, ctx.Logger) + ctx.Resp, ctx.Err = service.CodeHostListPRs(chID, projectName, strings.Replace(namespace, "%2F", "/", -1), targetBr, ctx.Logger) } func ListRepoInfos(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/code/handler/router.go b/pkg/microservice/aslan/core/code/handler/router.go index 5bbb1ecc2b..9e85a52d5c 100644 --- a/pkg/microservice/aslan/core/code/handler/router.go +++ b/pkg/microservice/aslan/core/code/handler/router.go @@ -19,14 +19,14 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) codehost := router.Group("codehost") { @@ -44,7 +44,7 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- workspace := router.Group("workspace") { - workspace.DELETE("", GetProductNameByWorkspacePipeline, middleware.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CleanWorkspace) + workspace.DELETE("", GetProductNameByWorkspacePipeline, gin2.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CleanWorkspace) workspace.GET("/file", GetWorkspaceFile) workspace.GET("/git/:codehostId/:repoName/:branchName/:remoteName", GetGitRepoInfo) workspace.GET("/publicRepo", GetPublicGitRepoInfo) diff --git a/pkg/microservice/aslan/core/code/handler/workspace.go b/pkg/microservice/aslan/core/code/handler/workspace.go index 1eb1d400e6..8ec11f43d0 100644 --- a/pkg/microservice/aslan/core/code/handler/workspace.go +++ b/pkg/microservice/aslan/core/code/handler/workspace.go @@ -25,7 +25,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/code/service" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/code/service/branch.go b/pkg/microservice/aslan/core/code/service/branch.go index fa0b00615f..98124d5675 100644 --- a/pkg/microservice/aslan/core/code/service/branch.go +++ b/pkg/microservice/aslan/core/code/service/branch.go @@ -18,73 +18,52 @@ package service import ( "context" - "fmt" - "github.com/google/go-github/v35/github" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/gitlab" + "github.com/koderover/zadig/pkg/tool/git/gitlab" ) -func CodehostListBranches(codehostID int, projectName, namespace string, log *zap.SugaredLogger) ([]*gitlab.Branch, error) { - projectID := fmt.Sprintf("%s/%s", namespace, projectName) - +func CodeHostListBranches(codeHostID int, projectName, namespace string, log *zap.SugaredLogger) ([]*Branch, error) { opt := &codehost.Option{ - CodeHostID: codehostID, + CodeHostID: codeHostID, } - codehost, err := codehost.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { return nil, e.ErrCodehostListBranches.AddDesc("git client is nil") } - if codehost.Type == codeHostGitlab { - client, err := gitlab.NewGitlabClient(codehost.Address, codehost.AccessToken) + if ch.Type == codeHostGitlab { + client, err := gitlab.NewClient(ch.Address, ch.AccessToken) if err != nil { log.Errorf("get gitlab client failed, err:%v", err) return nil, e.ErrCodehostListBranches.AddDesc(err.Error()) } - brList, err := client.ListBranches(projectID) + brList, err := client.ListBranches(namespace, projectName, nil) if err != nil { return nil, err } - return brList, nil - } else if codehost.Type == gerrit.CodehostTypeGerrit { - cli := gerrit.NewClient(codehost.Address, codehost.AccessToken) - result := make([]*gitlab.Branch, 0) + return ToBranches(brList), nil + } else if ch.Type == gerrit.CodehostTypeGerrit { + cli := gerrit.NewClient(ch.Address, ch.AccessToken) branches, err := cli.ListBranches(projectName) - if err == nil { - for _, branch := range branches { - result = append(result, &gitlab.Branch{ - Name: branch, - Protected: false, - Merged: false, - }) - } + if err != nil { + return nil, err } - - return result, err + return ToBranches(branches), nil } else { // github - gitClient := git.NewClient(codehost.AccessToken, config.ProxyHTTPSAddr()) - opt := &github.BranchListOptions{ListOptions: github.ListOptions{Page: page, PerPage: perPage}} - branches, _, err := gitClient.Repositories.ListBranches(context.Background(), namespace, projectName, opt) + gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) + branches, err := gh.ListBranches(context.TODO(), namespace, projectName, nil) if err != nil { return nil, err } - gitBranches := make([]*gitlab.Branch, 0) - for _, branch := range branches { - gitBranch := new(gitlab.Branch) - gitBranch.Name = *branch.Name - gitBranch.Protected = *branch.Protected - - gitBranches = append(gitBranches, gitBranch) - } - return gitBranches, nil + return ToBranches(branches), nil } } diff --git a/pkg/microservice/aslan/core/code/service/merge_request.go b/pkg/microservice/aslan/core/code/service/merge_request.go index da5f4501f2..3cf83ba238 100644 --- a/pkg/microservice/aslan/core/code/service/merge_request.go +++ b/pkg/microservice/aslan/core/code/service/merge_request.go @@ -18,74 +18,50 @@ package service import ( "context" - "fmt" - "github.com/google/go-github/v35/github" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/gitlab" + "github.com/koderover/zadig/pkg/tool/git/gitlab" ) -func CodehostListPRs(codehostID int, projectName, namespace, targetBr string, log *zap.SugaredLogger) ([]*gitlab.MergeRequest, error) { - projectID := fmt.Sprintf("%s/%s", namespace, projectName) +func CodeHostListPRs(codeHostID int, projectName, namespace, targetBr string, log *zap.SugaredLogger) ([]*PullRequest, error) { opt := &codehost.Option{ - CodeHostID: codehostID, + CodeHostID: codeHostID, } - codehost, err := codehost.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { return nil, e.ErrCodehostListPrs.AddDesc("git client is nil") } - if codehost.Type == codeHostGitlab { - client, err := gitlab.NewGitlabClient(codehost.Address, codehost.AccessToken) + if ch.Type == codeHostGitlab { + client, err := gitlab.NewClient(ch.Address, ch.AccessToken) if err != nil { log.Error(err) return nil, e.ErrCodehostListPrs.AddDesc(err.Error()) } - prs, err := client.ListOpened(projectID, targetBr) + prs, err := client.ListOpenedProjectMergeRequests(namespace, projectName, targetBr, nil) if err != nil { log.Error(err) return nil, e.ErrCodehostListPrs.AddDesc(err.Error()) } - if len(prs) == 0 { - prs = []*gitlab.MergeRequest{} - } - return prs, nil - } else if codehost.Type == gerrit.CodehostTypeGerrit { - return make([]*gitlab.MergeRequest, 0), nil + return ToPullRequests(prs), nil + + } else if ch.Type == gerrit.CodehostTypeGerrit { + return nil, nil } else { // github - gitClient := git.NewClient(codehost.AccessToken, config.ProxyHTTPSAddr()) - opt := &github.PullRequestListOptions{ - ListOptions: github.ListOptions{Page: page, PerPage: perPage}, - } - pullRequests, _, err := gitClient.PullRequests.List(context.Background(), namespace, projectName, opt) + gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) + pullRequests, err := gh.ListPullRequests(context.TODO(), namespace, projectName, nil) if err != nil { return nil, err } - mergeRequests := make([]*gitlab.MergeRequest, 0) - for _, pullRequest := range pullRequests { - mergeRequest := new(gitlab.MergeRequest) - mergeRequest.MRID = *pullRequest.Number - mergeRequest.CreatedAt = pullRequest.CreatedAt.Unix() - mergeRequest.UpdatedAt = pullRequest.UpdatedAt.Unix() - mergeRequest.State = *pullRequest.State - mergeRequest.User = *pullRequest.User.Login - mergeRequest.Number = *pullRequest.Number - mergeRequest.AuthorUsername = *pullRequest.User.Login - mergeRequest.Title = *pullRequest.Title - mergeRequest.SourceBranch = *pullRequest.Head.Ref - mergeRequest.TargetBranch = *pullRequest.Base.Ref - - mergeRequests = append(mergeRequests, mergeRequest) - } - return mergeRequests, nil + return ToPullRequests(pullRequests), nil } } diff --git a/pkg/microservice/aslan/core/code/service/namespace.go b/pkg/microservice/aslan/core/code/service/namespace.go index e14e127dd0..d0e9cbb88f 100644 --- a/pkg/microservice/aslan/core/code/service/namespace.go +++ b/pkg/microservice/aslan/core/code/service/namespace.go @@ -19,15 +19,14 @@ package service import ( "context" - "github.com/google/go-github/v35/github" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/gitlab" + "github.com/koderover/zadig/pkg/tool/git/gitlab" ) const ( @@ -39,65 +38,50 @@ const ( perPage = 100 ) -func CodehostListNamespaces(codehostID int, keyword string, log *zap.SugaredLogger) ([]*gitlab.Namespace, error) { +func CodeHostListNamespaces(codeHostID int, keyword string, log *zap.SugaredLogger) ([]*Namespace, error) { opt := &codehost.Option{ - CodeHostID: codehostID, + CodeHostID: codeHostID, } - codehost, err := codehost.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { return nil, e.ErrCodehostListNamespaces.AddDesc("git client is nil") } - if codehost.Type == codeHostGitlab { - client, err := gitlab.NewGitlabClient(codehost.Address, codehost.AccessToken) + if ch.Type == codeHostGitlab { + client, err := gitlab.NewClient(ch.Address, ch.AccessToken) if err != nil { log.Error(err) return nil, e.ErrCodehostListNamespaces.AddDesc(err.Error()) } - nsList, err := client.ListNamespaces(keyword) + nsList, err := client.ListNamespaces(keyword, nil) if err != nil { return nil, err } - return nsList, nil - } else if codehost.Type == gerrit.CodehostTypeGerrit { - return []*gitlab.Namespace{{ + return ToNamespaces(nsList), nil + } else if ch.Type == gerrit.CodehostTypeGerrit { + return []*Namespace{{ Name: gerrit.DefaultNamespace, Path: gerrit.DefaultNamespace, Kind: OrgKind, }}, nil } else { // github - gitClient := git.NewClient(codehost.AccessToken, config.ProxyHTTPSAddr()) - namespaces := make([]*gitlab.Namespace, 0) + gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) + namespaces := make([]*Namespace, 0) - // authenticated user - user, _, err := gitClient.Users.Get(context.Background(), "") + user, err := gh.GetAuthenticatedUser(context.TODO()) if err != nil { return nil, err } - namespace := &gitlab.Namespace{ - Name: user.GetLogin(), - Path: user.GetLogin(), - Kind: UserKind, - } - namespaces = append(namespaces, namespace) + namespaces = append(namespaces, ToNamespaces(user)...) - // organizations for the authenticated user - opt := &github.ListOptions{Page: page, PerPage: perPage} - organizations, _, err := gitClient.Organizations.List(context.Background(), "", opt) + organizations, err := gh.ListOrganizationsForAuthenticatedUser(context.TODO(), nil) if err != nil { return nil, err } - for _, organization := range organizations { - namespace := &gitlab.Namespace{ - Name: organization.GetLogin(), - Path: organization.GetLogin(), - Kind: OrgKind, - } + namespaces = append(namespaces, ToNamespaces(organizations)...) - namespaces = append(namespaces, namespace) - } return namespaces, nil } } diff --git a/pkg/microservice/aslan/core/code/service/project.go b/pkg/microservice/aslan/core/code/service/project.go index 6f2ec928b7..b8cb1e5878 100644 --- a/pkg/microservice/aslan/core/code/service/project.go +++ b/pkg/microservice/aslan/core/code/service/project.go @@ -19,96 +19,72 @@ package service import ( "context" - "github.com/google/go-github/v35/github" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/gitlab" + "github.com/koderover/zadig/pkg/tool/git/gitlab" ) -func CodehostListProjects(codehostID int, namespace, namespaceType, keyword string, log *zap.SugaredLogger) ([]*gitlab.Project, error) { +func CodeHostListProjects(codeHostID int, namespace, namespaceType, keyword string, log *zap.SugaredLogger) ([]*Project, error) { opt := &codehost.Option{ - CodeHostID: codehostID, + CodeHostID: codeHostID, } - codehost, err := codehost.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc("git client is nil") } - if codehost.Type == codeHostGitlab { - client, err := gitlab.NewGitlabClient(codehost.Address, codehost.AccessToken) + if ch.Type == codeHostGitlab { + client, err := gitlab.NewClient(ch.Address, ch.AccessToken) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) } if namespaceType == GroupKind { - projects, err := client.ListGroupProjects(namespace, keyword) + projects, err := client.ListGroupProjects(namespace, keyword, nil) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) } - return projects, nil + return ToProjects(projects), nil } // user - projects, err := client.ListUserProjects(namespace, keyword) + projects, err := client.ListUserProjects(namespace, keyword, nil) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) } - return projects, nil + return ToProjects(projects), nil - } else if codehost.Type == gerrit.CodehostTypeGerrit { - cli := gerrit.NewClient(codehost.Address, codehost.AccessToken) + } else if ch.Type == gerrit.CodehostTypeGerrit { + cli := gerrit.NewClient(ch.Address, ch.AccessToken) projects, err := cli.ListProjectsByKey(keyword) - result := make([]*gitlab.Project, 0) - if err == nil { - id := 0 - for _, p := range projects { - result = append(result, &gitlab.Project{ - ID: id, // fake id - Name: gerrit.Unescape(p.ID), // id could have %2F - Description: p.Description, - DefaultBranch: "master", - Namespace: gerrit.DefaultNamespace, - }) - id++ - } + if err != nil { + log.Error(err) + return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) } - return result, err + return ToProjects(projects), nil } else { // github - gitClient := git.NewClient(codehost.AccessToken, config.ProxyHTTPSAddr()) - opt := &github.RepositoryListOptions{ - ListOptions: github.ListOptions{Page: page, PerPage: perPage}, - } - - projects := make([]*gitlab.Project, 0) - repos, _, err := gitClient.Repositories.List(context.Background(), "", opt) + gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) + repos, err := gh.ListRepositoriesForAuthenticatedUser(context.TODO(), nil) if err != nil { return nil, err } - - for _, repo := range repos { - if repo.Owner == nil || *repo.Owner.Login != namespace { + var projects []*Project + for _, p := range ToProjects(repos) { + if p.Namespace != namespace { continue } - projects = append(projects, getProject(repo)) + projects = append(projects, p) } + return projects, nil } } - -func getProject(repo *github.Repository) *gitlab.Project { - project := new(gitlab.Project) - project.ID = int(*repo.ID) - project.Name = *repo.Name - project.DefaultBranch = *repo.DefaultBranch - project.Namespace = *repo.Owner.Login - return project -} diff --git a/pkg/microservice/aslan/core/code/service/repo.go b/pkg/microservice/aslan/core/code/service/repo.go index b6c7603cce..8d01371397 100644 --- a/pkg/microservice/aslan/core/code/service/repo.go +++ b/pkg/microservice/aslan/core/code/service/repo.go @@ -22,8 +22,6 @@ import ( "github.com/hashicorp/go-multierror" "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/tool/gitlab" ) type RepoInfoList struct { @@ -31,16 +29,15 @@ type RepoInfoList struct { } type GitRepoInfo struct { - Owner string `json:"repo_owner"` - Repo string `json:"repo"` - CodehostID int `json:"codehost_id"` - Source string `json:"source"` - DefaultBranch string `json:"default_branch"` - ErrorMsg string `json:"error_msg"` // repo信息是否拉取成功 - Branches []*gitlab.Branch `json:"branches"` - Tags []*gitlab.Tag `json:"tags"` - //Releases []*GitRelease `json:"releases"` - PRs []*gitlab.MergeRequest `json:"prs"` + Owner string `json:"repo_owner"` + Repo string `json:"repo"` + CodehostID int `json:"codehost_id"` + Source string `json:"source"` + DefaultBranch string `json:"default_branch"` + ErrorMsg string `json:"error_msg"` // repo信息是否拉取成功 + Branches []*Branch `json:"branches"` + Tags []*Tag `json:"tags"` + PRs []*PullRequest `json:"prs"` } // ListRepoInfos ... @@ -57,11 +54,11 @@ func ListRepoInfos(infos []*GitRepoInfo, param string, log *zap.SugaredLogger) ( defer func() { wg.Done() }() - info.PRs, err = CodehostListPRs(info.CodehostID, info.Repo, strings.Replace(info.Owner, "%2F", "/", -1), "", log) + info.PRs, err = CodeHostListPRs(info.CodehostID, info.Repo, strings.Replace(info.Owner, "%2F", "/", -1), "", log) if err != nil { errList = multierror.Append(errList, err) info.ErrorMsg = err.Error() - info.PRs = []*gitlab.MergeRequest{} + info.PRs = []*PullRequest{} return } }(info) @@ -72,11 +69,11 @@ func ListRepoInfos(infos []*GitRepoInfo, param string, log *zap.SugaredLogger) ( defer func() { wg.Done() }() - info.Branches, err = CodehostListBranches(info.CodehostID, info.Repo, strings.Replace(info.Owner, "%2F", "/", -1), log) + info.Branches, err = CodeHostListBranches(info.CodehostID, info.Repo, strings.Replace(info.Owner, "%2F", "/", -1), log) if err != nil { errList = multierror.Append(errList, err) info.ErrorMsg = err.Error() - info.Branches = []*gitlab.Branch{} + info.Branches = []*Branch{} return } @@ -89,11 +86,11 @@ func ListRepoInfos(infos []*GitRepoInfo, param string, log *zap.SugaredLogger) ( defer func() { wg.Done() }() - info.Tags, err = CodehostListTags(info.CodehostID, info.Repo, strings.Replace(info.Owner, "%2F", "/", -1), log) + info.Tags, err = CodeHostListTags(info.CodehostID, info.Repo, strings.Replace(info.Owner, "%2F", "/", -1), log) if err != nil { errList = multierror.Append(errList, err) info.ErrorMsg = err.Error() - info.Tags = []*gitlab.Tag{} + info.Tags = []*Tag{} return } }(info) diff --git a/pkg/microservice/aslan/core/code/service/tag.go b/pkg/microservice/aslan/core/code/service/tag.go index fa70784d54..bbc7e97629 100644 --- a/pkg/microservice/aslan/core/code/service/tag.go +++ b/pkg/microservice/aslan/core/code/service/tag.go @@ -18,75 +18,55 @@ package service import ( "context" - "fmt" - "github.com/google/go-github/v35/github" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" - "github.com/koderover/zadig/pkg/tool/gitlab" + "github.com/koderover/zadig/pkg/tool/git/gitlab" ) -func CodehostListTags(codehostID int, projectName string, namespace string, log *zap.SugaredLogger) ([]*gitlab.Tag, error) { - projectID := fmt.Sprintf("%s/%s", namespace, projectName) +func CodeHostListTags(codeHostID int, projectName string, namespace string, log *zap.SugaredLogger) ([]*Tag, error) { opt := &codehost.Option{ - CodeHostID: codehostID, + CodeHostID: codeHostID, } - codehost, err := codehost.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { log.Error(err) return nil, e.ErrCodehostListTags.AddDesc("git client is nil") } - if codehost.Type == codeHostGitlab { - client, err := gitlab.NewGitlabClient(codehost.Address, codehost.AccessToken) + if ch.Type == codeHostGitlab { + client, err := gitlab.NewClient(ch.Address, ch.AccessToken) if err != nil { log.Error(err) return nil, e.ErrCodehostListTags.AddDesc(err.Error()) } - brList, err := client.ListTags(projectID) + brList, err := client.ListTags(namespace, projectName, nil) if err != nil { return nil, err } - return brList, nil - } else if codehost.Type == gerrit.CodehostTypeGerrit { - client := gerrit.NewClient(codehost.Address, codehost.AccessToken) - result := make([]*gitlab.Tag, 0) + return ToTags(brList), nil + } else if ch.Type == gerrit.CodehostTypeGerrit { + client := gerrit.NewClient(ch.Address, ch.AccessToken) tags, err := client.ListTags(projectName) if err != nil { - return result, nil - } - - for _, tag := range tags { - result = append(result, &gitlab.Tag{ - Name: tag.Ref, - Message: tag.Message, - }) + return nil, err } - return result, nil + return ToTags(tags), nil } else { // github - gitClient := git.NewClient(codehost.AccessToken, config.ProxyHTTPSAddr()) - opt := &github.ListOptions{Page: page, PerPage: perPage} - tags, _, err := gitClient.Repositories.ListTags(context.Background(), namespace, projectName, opt) + gh := git.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()) + tags, err := gh.ListTags(context.TODO(), namespace, projectName, nil) if err != nil { return nil, err } - gitlabTags := make([]*gitlab.Tag, 0) - for _, tag := range tags { - gitTag := new(gitlab.Tag) - gitTag.Name = *tag.Name - gitTag.ZipballURL = *tag.ZipballURL - gitTag.TarballURL = *tag.TarballURL - gitlabTags = append(gitlabTags, gitTag) - } - return gitlabTags, nil + return ToTags(tags), nil } } diff --git a/pkg/microservice/aslan/core/code/service/types.go b/pkg/microservice/aslan/core/code/service/types.go new file mode 100644 index 0000000000..ba7c2ff92b --- /dev/null +++ b/pkg/microservice/aslan/core/code/service/types.go @@ -0,0 +1,243 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "github.com/andygrunwald/go-gerrit" + "github.com/google/go-github/v35/github" + "github.com/xanzy/go-gitlab" + + gerrittool "github.com/koderover/zadig/pkg/tool/gerrit" +) + +type Branch struct { + Name string `json:"name"` + Protected bool `json:"protected"` + Merged bool `json:"merged"` +} + +type PullRequest struct { + ID int `json:"id"` + TargetBranch string `json:"targetBranch"` + SourceBranch string `json:"sourceBranch"` + ProjectID int `json:"projectId"` + Title string `json:"title"` + State string `json:"state"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` + AuthorUsername string `json:"authorUsername"` + Number int `json:"number"` + User string `json:"user"` + Base string `json:"base,omitempty"` +} + +type Namespace struct { + Name string `json:"name"` + Path string `json:"path"` + Kind string `json:"kind"` +} + +type Project struct { + ID int `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"defaultBranch"` + Namespace string `json:"namespace"` +} + +type Tag struct { + Name string `json:"name"` + ZipballURL string `json:"zipball_url"` + TarballURL string `json:"tarball_url"` + Message string `json:"message"` +} + +func ToBranches(obj interface{}) []*Branch { + var res []*Branch + + switch os := obj.(type) { + case []*github.Branch: + for _, o := range os { + res = append(res, &Branch{ + Name: o.GetName(), + Protected: o.GetProtected(), + }) + } + case []*gitlab.Branch: + for _, o := range os { + res = append(res, &Branch{ + Name: o.Name, + Protected: o.Protected, + Merged: o.Merged, + }) + } + case []string: + for _, o := range os { + res = append(res, &Branch{ + Name: o, + }) + } + } + + return res +} + +func ToPullRequests(obj interface{}) []*PullRequest { + var res []*PullRequest + + switch os := obj.(type) { + case []*github.PullRequest: + for _, o := range os { + res = append(res, &PullRequest{ + ID: o.GetNumber(), + CreatedAt: o.GetCreatedAt().Unix(), + UpdatedAt: o.GetUpdatedAt().Unix(), + State: o.GetState(), + User: o.GetUser().GetLogin(), + Number: o.GetNumber(), + AuthorUsername: o.GetUser().GetLogin(), + Title: o.GetTitle(), + SourceBranch: o.GetHead().GetRef(), + TargetBranch: o.GetBase().GetRef(), + }) + } + case []*gitlab.MergeRequest: + for _, o := range os { + res = append(res, &PullRequest{ + ID: o.IID, + TargetBranch: o.TargetBranch, + SourceBranch: o.SourceBranch, + ProjectID: o.ProjectID, + Title: o.Title, + State: o.State, + CreatedAt: o.CreatedAt.Unix(), + UpdatedAt: o.UpdatedAt.Unix(), + AuthorUsername: o.Author.Username, + }) + } + } + + return res +} + +func ToNamespaces(obj interface{}) []*Namespace { + var res []*Namespace + + switch os := obj.(type) { + case *github.User: + res = append(res, &Namespace{ + Name: os.GetLogin(), + Path: os.GetLogin(), + Kind: UserKind, + }) + case []*github.User: + for _, o := range os { + res = append(res, &Namespace{ + Name: o.GetLogin(), + Path: o.GetLogin(), + Kind: UserKind, + }) + } + case []*github.Organization: + for _, o := range os { + res = append(res, &Namespace{ + Name: o.GetLogin(), + Path: o.GetLogin(), + Kind: OrgKind, + }) + } + case []*gitlab.Namespace: + for _, o := range os { + res = append(res, &Namespace{ + Name: o.Path, + Path: o.FullPath, + Kind: o.Kind, + }) + } + } + + return res +} + +func ToProjects(obj interface{}) []*Project { + var res []*Project + + switch os := obj.(type) { + case []*github.Repository: + for _, o := range os { + res = append(res, &Project{ + ID: int(o.GetID()), + Name: o.GetName(), + DefaultBranch: o.GetDefaultBranch(), + Namespace: o.GetOwner().GetLogin(), + }) + } + case []*gitlab.Project: + for _, o := range os { + res = append(res, &Project{ + ID: o.ID, + Name: o.Path, + Namespace: o.Namespace.FullPath, + Description: o.Description, + DefaultBranch: o.DefaultBranch, + }) + } + case []*gerrit.ProjectInfo: + for ind, o := range os { + res = append(res, &Project{ + ID: ind, // fake id + Name: gerrittool.Unescape(o.ID), // id could have %2F + Description: o.Description, + DefaultBranch: "master", + Namespace: gerrittool.DefaultNamespace, + }) + } + } + + return res +} + +func ToTags(obj interface{}) []*Tag { + var res []*Tag + + switch os := obj.(type) { + case []*github.RepositoryTag: + for _, o := range os { + res = append(res, &Tag{ + Name: o.GetName(), + ZipballURL: o.GetZipballURL(), + TarballURL: o.GetTarballURL(), + }) + } + case []*gitlab.Tag: + for _, o := range os { + res = append(res, &Tag{ + Name: o.Name, + Message: o.Message, + }) + } + case []*gerrit.TagInfo: + for _, o := range os { + res = append(res, &Tag{ + Name: o.Ref, + Message: o.Message, + }) + } + } + + return res +} diff --git a/pkg/microservice/aslan/core/code/service/workspace.go b/pkg/microservice/aslan/core/code/service/workspace.go index 822214620d..7698b43ed1 100644 --- a/pkg/microservice/aslan/core/code/service/workspace.go +++ b/pkg/microservice/aslan/core/code/service/workspace.go @@ -32,8 +32,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/common/handler/tool.go b/pkg/microservice/aslan/core/common/handler/tool.go index bf852d89e5..6134b5b3a9 100644 --- a/pkg/microservice/aslan/core/common/handler/tool.go +++ b/pkg/microservice/aslan/core/common/handler/tool.go @@ -22,7 +22,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/config" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func GetToolDownloadURL(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/common/repository/models/build.go b/pkg/microservice/aslan/core/common/repository/models/build.go index d3e072aee2..6e83e041fd 100644 --- a/pkg/microservice/aslan/core/common/repository/models/build.go +++ b/pkg/microservice/aslan/core/common/repository/models/build.go @@ -33,18 +33,19 @@ type Build struct { // 在任一编译配置模板中只能出现一次 // 对于k8s部署是传入容器名称 // 对于物理机部署是服务名称 - Targets []*ServiceModuleTarget `bson:"targets" json:"targets"` - Description string `bson:"desc,omitempty" json:"desc"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - UpdateBy string `bson:"update_by" json:"update_by"` - Repos []*types.Repository `bson:"repos,omitempty" json:"repos"` - PreBuild *PreBuild `bson:"pre_build" json:"pre_build"` - JenkinsBuild *JenkinsBuild `bson:"jenkins_build,omitempty" json:"jenkins_build,omitempty"` - Scripts string `bson:"scripts" json:"scripts"` - PostBuild *PostBuild `bson:"post_build,omitempty" json:"post_build"` - Caches []string `bson:"caches" json:"caches"` - ProductName string `bson:"product_name" json:"product_name"` - SSHs []string `bson:"sshs,omitempty" json:"sshs,omitempty"` + Targets []*ServiceModuleTarget `bson:"targets" json:"targets"` + Description string `bson:"desc,omitempty" json:"desc"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + Repos []*types.Repository `bson:"repos,omitempty" json:"repos"` + PreBuild *PreBuild `bson:"pre_build" json:"pre_build"` + JenkinsBuild *JenkinsBuild `bson:"jenkins_build,omitempty" json:"jenkins_build,omitempty"` + Scripts string `bson:"scripts" json:"scripts"` + PostBuild *PostBuild `bson:"post_build,omitempty" json:"post_build"` + Caches []string `bson:"caches" json:"caches"` + ProductName string `bson:"product_name" json:"product_name"` + SSHs []string `bson:"sshs,omitempty" json:"sshs,omitempty"` + PMDeployScripts string `bson:"pm_deploy_scripts" json:"pm_deploy_scripts"` } // PreBuild prepares an environment for a job diff --git a/pkg/microservice/aslan/core/common/repository/models/build_stat.go b/pkg/microservice/aslan/core/common/repository/models/build_stat.go deleted file mode 100644 index f3b1979462..0000000000 --- a/pkg/microservice/aslan/core/common/repository/models/build_stat.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package models - -type BuildStat struct { - ProductName string `bson:"product_name" json:"productName"` - TotalSuccess int `bson:"total_success" json:"totalSuccess"` - TotalFailure int `bson:"total_failure" json:"totalFailure"` - TotalTimeout int `bson:"total_timeout" json:"totalTimeout"` - TotalDuration int64 `bson:"total_duration" json:"totalDuration"` - TotalBuildCount int `bson:"total_build_count" json:"totalBuildCount"` - MaxDuration int64 `bson:"max_duration" json:"maxDuration"` - MaxDurationPipeline *PipelineInfo `bson:"max_duration_pipeline" json:"maxDurationPipeline"` - Date string `bson:"date" json:"date"` - CreateTime int64 `bson:"create_time" json:"createTime"` - UpdateTime int64 `bson:"update_time" json:"updateTime"` -} - -// PipelineInfo -type PipelineInfo struct { - TaskID int64 `bson:"task_id" json:"taskId"` - PipelineName string `bson:"pipeline_name" json:"pipelineName"` - Type string `bson:"type" json:"type"` - MaxDuration int64 `bson:"max_duration" json:"maxDuration"` -} - -func (BuildStat) TableName() string { - return "build_stat" -} diff --git a/pkg/microservice/aslan/core/common/repository/models/deploy_stat.go b/pkg/microservice/aslan/core/common/repository/models/deploy_stat.go deleted file mode 100644 index d687c6170f..0000000000 --- a/pkg/microservice/aslan/core/common/repository/models/deploy_stat.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package models - -type DeployStat struct { - ProductName string `bson:"product_name" json:"productName"` - TotalTaskSuccess int `bson:"total_task_success" json:"totalTaskSuccess"` - TotalTaskFailure int `bson:"total_task_failure" json:"totalTaskFailure"` - TotalDeploySuccess int `bson:"total_deploy_success" json:"totalDeploySuccess"` - TotalDeployFailure int `bson:"total_deploy_failure" json:"totalDeployFailure"` - MaxDeployServiceNum int `bson:"max_deploy_service_num" json:"maxDeployServiceNum"` - MaxDeployServiceFailureNum int `bson:"max_deploy_service_failure_num" json:"maxDeployServiceFailureNum"` - MaxDeployServiceName string `bson:"max_deploy_service_name" json:"maxDeployServiceName"` - MaxDeployFailureServiceNum int `bson:"max_deploy_failure_service_num" json:"maxDeployFailureServiceNum"` - MaxDeployFailureServiceName string `bson:"max_deploy_failure_service_name" json:"maxDeployFailureServiceName"` - Date string `bson:"date" json:"date"` - CreateTime int64 `bson:"create_time" json:"createTime"` - UpdateTime int64 `bson:"update_time" json:"updateTime"` -} - -func (DeployStat) TableName() string { - return "deploy_stat" -} diff --git a/pkg/microservice/aslan/core/common/repository/models/operation_log.go b/pkg/microservice/aslan/core/common/repository/models/operation_log.go deleted file mode 100644 index 5a9cc65090..0000000000 --- a/pkg/microservice/aslan/core/common/repository/models/operation_log.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type OperationLog struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - Username string `bson:"username" json:"username"` - ProductName string `bson:"product_name" json:"product_name"` - Method string `bson:"method" json:"method"` - PermissionUUID string `bson:"permission_uuid" json:"permission_uuid"` - Function string `bson:"function" json:"function"` - Name string `bson:"name" json:"name"` - RequestBody string `bson:"request_body" json:"request_body"` - Status int `bson:"status" json:"status"` - CreatedAt int64 `bson:"created_at" json:"created_at"` -} - -func (OperationLog) TableName() string { - return "operation_log" -} diff --git a/pkg/microservice/aslan/core/common/repository/models/queue.go b/pkg/microservice/aslan/core/common/repository/models/queue.go index 239af175c7..d6279a8b30 100644 --- a/pkg/microservice/aslan/core/common/repository/models/queue.go +++ b/pkg/microservice/aslan/core/common/repository/models/queue.go @@ -138,6 +138,7 @@ type ConfigPayload struct { // ResetCache means ignore workspace cache ResetCache bool `json:"reset_cache"` JenkinsBuildConfig JenkinsBuildConfig `json:"jenkins_build_config"` + PrivateKeys []*PrivateKey `json:"private_keys"` } type AslanConfig struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/service.go b/pkg/microservice/aslan/core/common/repository/models/service.go index 45d6c2bcd3..cc82c0006f 100644 --- a/pkg/microservice/aslan/core/common/repository/models/service.go +++ b/pkg/microservice/aslan/core/common/repository/models/service.go @@ -21,39 +21,40 @@ package models // 1. Kubernetes service, and yaml+config is held in aslan: type == "k8s"; source == "spock"; yaml != "" // 2. Kubernetes service, and yaml+config is held in gitlab: type == "k8s"; source == "gitlab"; src_path != "" type Service struct { - ServiceName string `bson:"service_name" json:"service_name"` - Type string `bson:"type" json:"type"` - Team string `bson:"team,omitempty" json:"team,omitempty"` - ProductName string `bson:"product_name" json:"product_name"` - Revision int64 `bson:"revision" json:"revision"` - Source string `bson:"source,omitempty" json:"source,omitempty"` - GUIConfig *GUIConfig `bson:"gui_config,omitempty" json:"gui_config,omitempty"` - Yaml string `bson:"yaml,omitempty" json:"yaml"` - SrcPath string `bson:"src_path,omitempty" json:"src_path,omitempty"` - Commit *Commit `bson:"commit,omitempty" json:"commit,omitempty"` - KubeYamls []string `bson:"-" json:"-"` - Hash string `bson:"hash256,omitempty" json:"hash256,omitempty"` - CreateTime int64 `bson:"create_time" json:"create_time"` - CreateBy string `bson:"create_by" json:"create_by"` - Containers []*Container `bson:"containers,omitempty" json:"containers,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - Visibility string `bson:"visibility,omitempty" json:"visibility,omitempty"` - Status string `bson:"status,omitempty" json:"status,omitempty"` - GerritRepoName string `bson:"gerrit_repo_name,omitempty" json:"gerrit_repo_name,omitempty"` - GerritBranchName string `bson:"gerrit_branch_name,omitempty" json:"gerrit_branch_name,omitempty"` - GerritRemoteName string `bson:"gerrit_remote_name,omitempty" json:"gerrit_remote_name,omitempty"` - GerritPath string `bson:"gerrit_path,omitempty" json:"gerrit_path,omitempty"` - GerritCodeHostID int `bson:"gerrit_codeHost_id,omitempty" json:"gerrit_codeHost_id,omitempty"` - BuildName string `bson:"build_name,omitempty" json:"build_name,omitempty"` - HelmChart *HelmChart `bson:"helm_chart,omitempty" json:"helm_chart,omitempty"` - EnvConfigs []*EnvConfig `bson:"env_configs,omitempty" json:"env_configs,omitempty"` - EnvStatuses []*EnvStatus `bson:"env_statuses,omitempty" json:"env_statuses,omitempty"` - CodehostID int `bson:"codehost_id" json:"codehost_id"` - RepoOwner string `bson:"repo_owner" json:"repo_owner"` - RepoName string `bson:"repo_name" json:"repo_name"` - BranchName string `bson:"branch_name" json:"branch_name"` - LoadPath string `bson:"load_path" json:"load_path"` - LoadFromDir bool `bson:"is_dir" json:"is_dir"` + ServiceName string `bson:"service_name" json:"service_name"` + Type string `bson:"type" json:"type"` + Team string `bson:"team,omitempty" json:"team,omitempty"` + ProductName string `bson:"product_name" json:"product_name"` + Revision int64 `bson:"revision" json:"revision"` + Source string `bson:"source,omitempty" json:"source,omitempty"` + GUIConfig *GUIConfig `bson:"gui_config,omitempty" json:"gui_config,omitempty"` + Yaml string `bson:"yaml,omitempty" json:"yaml"` + SrcPath string `bson:"src_path,omitempty" json:"src_path,omitempty"` + Commit *Commit `bson:"commit,omitempty" json:"commit,omitempty"` + KubeYamls []string `bson:"-" json:"-"` + Hash string `bson:"hash256,omitempty" json:"hash256,omitempty"` + CreateTime int64 `bson:"create_time" json:"create_time"` + CreateBy string `bson:"create_by" json:"create_by"` + Containers []*Container `bson:"containers,omitempty" json:"containers,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + Visibility string `bson:"visibility,omitempty" json:"visibility,omitempty"` + Status string `bson:"status,omitempty" json:"status,omitempty"` + GerritRepoName string `bson:"gerrit_repo_name,omitempty" json:"gerrit_repo_name,omitempty"` + GerritBranchName string `bson:"gerrit_branch_name,omitempty" json:"gerrit_branch_name,omitempty"` + GerritRemoteName string `bson:"gerrit_remote_name,omitempty" json:"gerrit_remote_name,omitempty"` + GerritPath string `bson:"gerrit_path,omitempty" json:"gerrit_path,omitempty"` + GerritCodeHostID int `bson:"gerrit_codeHost_id,omitempty" json:"gerrit_codeHost_id,omitempty"` + BuildName string `bson:"build_name,omitempty" json:"build_name,omitempty"` + HelmChart *HelmChart `bson:"helm_chart,omitempty" json:"helm_chart,omitempty"` + EnvConfigs []*EnvConfig `bson:"env_configs,omitempty" json:"env_configs,omitempty"` + EnvStatuses []*EnvStatus `bson:"env_statuses,omitempty" json:"env_statuses,omitempty"` + CodehostID int `bson:"codehost_id" json:"codehost_id"` + RepoOwner string `bson:"repo_owner" json:"repo_owner"` + RepoName string `bson:"repo_name" json:"repo_name"` + BranchName string `bson:"branch_name" json:"branch_name"` + LoadPath string `bson:"load_path" json:"load_path"` + LoadFromDir bool `bson:"is_dir" json:"is_dir"` + HealthChecks []*PmHealthCheck `bson:"health_checks,omitempty" json:"health_checks,omitempty"` } type GUIConfig struct { @@ -187,6 +188,18 @@ type EnvConfig struct { HostIDs []string `bson:"host_ids,omitempty" json:"host_ids"` } +type PmHealthCheck struct { + Protocol string `bson:"protocol,omitempty" json:"protocol,omitempty"` + Port int `bson:"port,omitempty" json:"port,omitempty"` + Path string `bson:"path,omitempty" json:"path,omitempty"` + TimeOut int64 `bson:"time_out,omitempty" json:"time_out,omitempty"` + Interval uint64 `bson:"interval,omitempty" json:"interval,omitempty"` + HealthyThreshold int `bson:"healthy_threshold,omitempty" json:"healthy_threshold,omitempty"` + UnhealthyThreshold int `bson:"unhealthy_threshold,omitempty" json:"unhealthy_threshold,omitempty"` + CurrentHealthyNum int `bson:"current_healthy_num,omitempty" json:"current_healthy_num,omitempty"` + CurrentUnhealthyNum int `bson:"current_unhealthy_num,omitempty" json:"current_unhealthy_num,omitempty"` +} + func (Service) TableName() string { return "template_service" } diff --git a/pkg/microservice/aslan/core/common/repository/models/task/build.go b/pkg/microservice/aslan/core/common/repository/models/task/build.go index 944fd8ce77..39f59e5e5a 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/build.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/build.go @@ -128,6 +128,7 @@ type JobCtx struct { // TestJobCtx TestThreshold int `bson:"test_threshold" json:"test_threshold"` TestResultPath string `bson:"test_result_path,omitempty" json:"test_result_path,omitempty"` + TestReportPath string `bson:"test_report_path" json:"test_report_path"` TestJobName string `bson:"test_job_name,omitempty" json:"test_job_name,omitempty"` // DockerBuildCtx DockerBuildCtx *DockerBuildCtx `bson:"docker_build_ctx,omitempty" json:"docker_build_ctx,omitempty"` @@ -142,8 +143,9 @@ type JobCtx struct { //StorageUri string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` // ClassicBuild used by qbox build - ClassicBuild bool `bson:"classic_build" json:"classic_build"` - PostScripts string `bson:"post_scripts,omitempty" json:"post_scripts"` + ClassicBuild bool `bson:"classic_build" json:"classic_build"` + PostScripts string `bson:"post_scripts,omitempty" json:"post_scripts"` + PMDeployScripts string `bson:"pm_deploy_scripts,omitempty" json:"pm_deploy_scripts"` } type BuildStep struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/testing.go b/pkg/microservice/aslan/core/common/repository/models/testing.go index cdd9468592..7ea1b46e84 100644 --- a/pkg/microservice/aslan/core/common/repository/models/testing.go +++ b/pkg/microservice/aslan/core/common/repository/models/testing.go @@ -24,30 +24,33 @@ import ( ) type Testing struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - Name string `bson:"name" json:"name"` - ProductName string `bson:"product_name" json:"product_name"` - Desc string `bson:"desc" json:"desc"` - Timeout int `bson:"timeout" json:"timeout"` - Team string `bson:"team" json:"team"` - Repos []*types.Repository `bson:"repos,omitempty" json:"repos"` - PreTest *PreTest `bson:"pre_test" json:"pre_test"` - Scripts string `bson:"scripts" json:"scripts"` - UpdateTime int64 `bson:"update_time" json:"update_time"` - UpdateBy string `bson:"update_by" json:"update_by"` - TestResultPath string `bson:"test_result_path" json:"test_result_path"` - Threshold int `bson:"threshold" json:"threshold"` - TestType string `bson:"test_type" json:"test_type"` - Caches []string `bson:"caches" json:"caches"` - ArtifactPaths []string `bson:"artifact_paths,omitempty" json:"artifact_paths,omitempty"` - TestCaseNum int `bson:"-" json:"test_case_num,omitempty"` - ExecuteNum int `bson:"-" json:"execute_num,omitempty"` - PassRate float64 `bson:"-" json:"pass_rate,omitempty"` - AvgDuration float64 `bson:"-" json:"avg_duration,omitempty"` - Workflows []*Workflow `bson:"-" json:"workflows,omitempty"` - Schedules *ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` - HookCtl *TestingHookCtrl `bson:"hook_ctl" json:"hook_ctl"` - ScheduleEnabled bool `bson:"schedule_enabled" json:"-"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Name string `bson:"name" json:"name"` + ProductName string `bson:"product_name" json:"product_name"` + Desc string `bson:"desc" json:"desc"` + Timeout int `bson:"timeout" json:"timeout"` + Team string `bson:"team" json:"team"` + Repos []*types.Repository `bson:"repos,omitempty" json:"repos"` + PreTest *PreTest `bson:"pre_test" json:"pre_test"` + Scripts string `bson:"scripts" json:"scripts"` + UpdateTime int64 `bson:"update_time" json:"update_time"` + UpdateBy string `bson:"update_by" json:"update_by"` + // Junit 测试报告 + TestResultPath string `bson:"test_result_path" json:"test_result_path"` + // html 测试报告 + TestReportPath string `bson:"test_report_path" json:"test_report_path"` + Threshold int `bson:"threshold" json:"threshold"` + TestType string `bson:"test_type" json:"test_type"` + Caches []string `bson:"caches" json:"caches"` + ArtifactPaths []string `bson:"artifact_paths,omitempty" json:"artifact_paths,omitempty"` + TestCaseNum int `bson:"-" json:"test_case_num,omitempty"` + ExecuteNum int `bson:"-" json:"execute_num,omitempty"` + PassRate float64 `bson:"-" json:"pass_rate,omitempty"` + AvgDuration float64 `bson:"-" json:"avg_duration,omitempty"` + Workflows []*Workflow `bson:"-" json:"workflows,omitempty"` + Schedules *ScheduleCtrl `bson:"schedules,omitempty" json:"schedules,omitempty"` + HookCtl *TestingHookCtrl `bson:"hook_ctl" json:"hook_ctl"` + ScheduleEnabled bool `bson:"schedule_enabled" json:"-"` } type TestingHookCtrl struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/webhook.go b/pkg/microservice/aslan/core/common/repository/models/webhook.go new file mode 100644 index 0000000000..e32c8cbdcc --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/webhook.go @@ -0,0 +1,34 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type WebHook struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Owner string `bson:"owner" json:"owner"` + Repo string `bson:"repo" json:"repo"` + Address string `bson:"address" json:"address"` + // References is a record to store all the workflows/services who are using this webhook + References []string `bson:"references" json:"references"` +} + +func (WebHook) TableName() string { + return "webhook" +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/build_stat.go b/pkg/microservice/aslan/core/common/repository/mongodb/build_stat.go deleted file mode 100644 index e016135ff6..0000000000 --- a/pkg/microservice/aslan/core/common/repository/mongodb/build_stat.go +++ /dev/null @@ -1,183 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mongodb - -import ( - "context" - "errors" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - mongotool "github.com/koderover/zadig/pkg/tool/mongo" -) - -// BuildPipeResp ... -type BuildPipeResp struct { - ID BuildItem `bson:"_id" json:"_id"` - TotalSuccess int `bson:"total_success" json:"total_success"` - TotalFailure int `bson:"total_failure" json:"total_failure"` -} - -// BuildItem ... -type BuildItem struct { - ProductName string `bson:"product_name" json:"product_name"` - TotalSuccess int `bson:"total_success" json:"total_success"` - TotalFailure int `bson:"total_failure" json:"total_failure"` -} - -type BuildStatOption struct { - StartDate int64 - EndDate int64 - Limit int - Skip int - IsAsc bool - ProductNames []string -} - -type BuildStatColl struct { - *mongo.Collection - - coll string -} - -func NewBuildStatColl() *BuildStatColl { - name := models.BuildStat{}.TableName() - return &BuildStatColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} -} - -func (c *BuildStatColl) GetCollectionName() string { - return c.coll -} - -func (c *BuildStatColl) EnsureIndex(ctx context.Context) error { - mod := mongo.IndexModel{ - Keys: bson.D{ - bson.E{Key: "product_name", Value: 1}, - bson.E{Key: "date", Value: 1}, - }, - Options: options.Index().SetUnique(true), - } - - _, err := c.Indexes().CreateOne(ctx, mod) - - return err -} - -func (c *BuildStatColl) Create(args *models.BuildStat) error { - if args == nil { - return errors.New("nil buildStat args") - } - - _, err := c.InsertOne(context.TODO(), args) - return err -} - -func (c *BuildStatColl) Update(args *models.BuildStat) error { - if args == nil { - return errors.New("nil buildStat args") - } - - query := bson.M{"date": args.Date, "product_name": args.ProductName} - update := bson.M{"$set": args} - _, err := c.UpdateOne(context.TODO(), query, update) - return err -} - -func (c *BuildStatColl) FindCount() (int, error) { - count, err := c.EstimatedDocumentCount(context.TODO()) - if err != nil { - return 0, err - } - return int(count), nil -} - -func (c *BuildStatColl) ListBuildStat(option *BuildStatOption) ([]*models.BuildStat, error) { - resp := make([]*models.BuildStat, 0) - query := bson.M{} - - if len(option.ProductNames) > 0 { - query["product_name"] = bson.M{"$in": option.ProductNames} - } - - if option.StartDate > 0 { - query["create_time"] = bson.M{"$gte": option.StartDate, "$lte": option.EndDate} - } - - opt := &options.FindOptions{} - - if option.Limit > 0 { - opt.SetSort(bson.D{{"max_duration", -1}}).SetSkip(int64(option.Skip)).SetLimit(int64(option.Limit)) - } else if option.IsAsc { - opt.SetSort(bson.D{{"create_time", 1}}) - } else { - opt.SetSort(bson.D{{"create_time", -1}}) - } - - cursor, err := c.Collection.Find(context.TODO(), query, opt) - if err != nil { - return nil, err - } - - err = cursor.All(context.TODO(), &resp) - if err != nil { - return nil, err - } - return resp, nil -} - -func (c *BuildStatColl) GetBuildTotalAndSuccess() ([]*BuildItem, error) { - var result []*BuildPipeResp - var pipeline []bson.M - var resp []*BuildItem - - pipeline = append(pipeline, - bson.M{ - "$group": bson.M{ - "_id": bson.M{ - "product_name": "$product_name", - }, - "total_success": bson.M{ - "$sum": "$total_success", - }, - "total_failure": bson.M{ - "$sum": "$total_failure", - }, - }, - }) - - cursor, err := c.Aggregate(context.TODO(), pipeline) - if err != nil { - return nil, err - } - if err := cursor.All(context.TODO(), &result); err != nil { - return nil, err - } - for _, res := range result { - buildItem := &BuildItem{ - ProductName: res.ID.ProductName, - TotalSuccess: res.TotalSuccess, - TotalFailure: res.TotalFailure, - } - resp = append(resp, buildItem) - } - - return resp, nil -} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/deploy_stat.go b/pkg/microservice/aslan/core/common/repository/mongodb/deploy_stat.go deleted file mode 100644 index d8e3f98c9f..0000000000 --- a/pkg/microservice/aslan/core/common/repository/mongodb/deploy_stat.go +++ /dev/null @@ -1,348 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mongodb - -import ( - "context" - "errors" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - mongotool "github.com/koderover/zadig/pkg/tool/mongo" -) - -type DeployStatOption struct { - StartDate int64 - EndDate int64 - Limit int - Skip int - IsAsc bool - IsMaxDeploy bool - ProductNames []string -} - -type DeployTotalPipeResp struct { - ID DeployTotalItem `bson:"_id" json:"_id"` - TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` - TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` -} - -type DeployTotalItem struct { - ProductName string `bson:"product_name" json:"product_name"` - TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` - TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` -} - -type DeployDailyPipeResp struct { - ID DeployDailyItem `bson:"_id" json:"_id"` - TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` - TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` -} - -type DeployDailyItem struct { - Date string `bson:"date" json:"date"` - TotalSuccess int `bson:"total_deploy_success" json:"total_deploy_success"` - TotalFailure int `bson:"total_deploy_failure" json:"total_deploy_failure"` -} - -type DeployPipeResp struct { - ID DeployPipeInfo `bson:"_id" json:"_id"` - MaxDeployServiceNum int `bson:"max_deploy_service_num" json:"maxDeployServiceNum"` -} - -type DeployPipeInfo struct { - MaxDeployServiceName string `bson:"max_deploy_service_name" json:"maxDeployServiceName"` -} - -type DeployFailurePipeResp struct { - ID DeployFailurePipeInfo `bson:"_id" json:"_id"` - MaxDeployFailureServiceNum int `bson:"max_deploy_failure_service_num" json:"maxDeployFailureServiceNum"` -} - -type DeployFailurePipeInfo struct { - ProductName string `bson:"product_name" json:"productName"` - MaxDeployFailureServiceName string `bson:"max_deploy_failure_service_name" json:"maxDeployFailureServiceName"` -} - -type DeployStatColl struct { - *mongo.Collection - - coll string -} - -func NewDeployStatColl() *DeployStatColl { - name := models.DeployStat{}.TableName() - return &DeployStatColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} -} - -func (c *DeployStatColl) GetCollectionName() string { - return c.coll -} - -func (c *DeployStatColl) EnsureIndex(ctx context.Context) error { - mod := mongo.IndexModel{ - Keys: bson.D{ - bson.E{Key: "product_name", Value: 1}, - bson.E{Key: "date", Value: 1}, - }, - Options: options.Index().SetUnique(true), - } - - _, err := c.Indexes().CreateOne(ctx, mod) - - return err -} - -func (c *DeployStatColl) Create(args *models.DeployStat) error { - if args == nil { - return errors.New("nil deployStat args") - } - - _, err := c.InsertOne(context.TODO(), args) - return err -} - -type DeployStatGetOption struct { - MaxDeployServiceNum int - ServiceName string -} - -func (c *DeployStatColl) Get(args *DeployStatGetOption) (*models.DeployStat, error) { - ret := new(models.DeployStat) - query := bson.M{"max_deploy_service_num": args.MaxDeployServiceNum, "max_deploy_service_name": args.ServiceName} - err := c.FindOne(context.TODO(), query).Decode(ret) - if err != nil { - return nil, err - } - return ret, nil -} - -func (c *DeployStatColl) Update(args *models.DeployStat) error { - if args == nil { - return errors.New("nil deployStat args") - } - - query := bson.M{ - "date": args.Date, - "product_name": args.ProductName, - } - update := bson.M{"$set": args} - _, err := c.UpdateOne(context.TODO(), query, update) - return err -} - -func (c *DeployStatColl) FindCount() (int, error) { - count, err := c.CountDocuments(context.TODO(), bson.M{}) - return int(count), err -} - -func (c *DeployStatColl) GetDeployTotalAndSuccess() ([]*DeployTotalItem, error) { - var result []*DeployTotalPipeResp - var resp []*DeployTotalItem - - pipeline := []bson.M{{ - "$group": bson.M{ - "_id": bson.M{ - "product_name": "$product_name", - }, - "total_deploy_success": bson.M{ - "$sum": "$total_deploy_success", - }, - "total_deploy_failure": bson.M{ - "$sum": "$total_deploy_failure", - }, - }, - }} - - cursor, err := c.Aggregate(context.TODO(), pipeline) - if err != nil { - return nil, err - } - if err := cursor.All(context.TODO(), &result); err != nil { - return nil, err - } - for _, res := range result { - deployItem := &DeployTotalItem{ - ProductName: res.ID.ProductName, - TotalSuccess: res.TotalSuccess, - TotalFailure: res.TotalFailure, - } - resp = append(resp, deployItem) - } - - return resp, nil -} - -func (c *DeployStatColl) ListDeployStat(option *DeployStatOption) ([]*models.DeployStat, error) { - ret := make([]*models.DeployStat, 0) - query := bson.M{} - if option.Limit > 0 { - var deployStatQuery []bson.M - if len(option.ProductNames) > 0 { - query["product_name"] = bson.M{"$in": option.ProductNames} - } - if option.StartDate > 0 { - query["create_time"] = bson.M{ - "$gte": option.StartDate, - "$lte": option.EndDate, - } - } - deployStatQuery = append(deployStatQuery, bson.M{"$match": query}) - - if option.IsMaxDeploy { - var results []DeployPipeResp - deployStatQuery = append(deployStatQuery, - bson.M{"$group": bson.M{ - "_id": bson.M{"max_deploy_service_name": "$max_deploy_service_name"}, - "max_deploy_service_num": bson.M{"$max": "$max_deploy_service_num"}}, - }) - deployStatQuery = append(deployStatQuery, bson.M{"$sort": bson.M{"max_deploy_service_num": -1}}) - deployStatQuery = append(deployStatQuery, bson.M{"$limit": 5}) - deployStatQuery = append(deployStatQuery, bson.M{"$skip": 0}) - - cursor, err := c.Aggregate(context.TODO(), deployStatQuery) - if err != nil { - return nil, err - } - if err := cursor.All(context.TODO(), &results); err != nil { - return nil, err - } - for _, result := range results { - deployItem := &models.DeployStat{ - MaxDeployServiceName: result.ID.MaxDeployServiceName, - MaxDeployServiceNum: result.MaxDeployServiceNum, - } - ret = append(ret, deployItem) - } - } else { - var results []DeployFailurePipeResp - - deployStatQuery = append(deployStatQuery, - bson.M{"$group": bson.M{ - "_id": bson.M{ - "max_deploy_failure_service_name": "$max_deploy_failure_service_name", - "product_name": "$product_name", - }, - "max_deploy_failure_service_num": bson.M{ - "$max": "$max_deploy_failure_service_num", - }, - }}) - deployStatQuery = append(deployStatQuery, bson.M{"$sort": bson.M{"max_deploy_failure_service_num": -1}}) - deployStatQuery = append(deployStatQuery, bson.M{"$limit": 5}) - deployStatQuery = append(deployStatQuery, bson.M{"$skip": 0}) - cursor, err := c.Aggregate(context.TODO(), deployStatQuery) - if err != nil { - return nil, err - } - if err := cursor.All(context.TODO(), &results); err != nil { - return nil, err - } - for _, result := range results { - deployItem := &models.DeployStat{ - ProductName: result.ID.ProductName, - MaxDeployFailureServiceName: result.ID.MaxDeployFailureServiceName, - MaxDeployFailureServiceNum: result.MaxDeployFailureServiceNum, - } - ret = append(ret, deployItem) - } - } - return ret, nil - } - - if len(option.ProductNames) > 0 { - query["product_name"] = bson.M{"$in": option.ProductNames} - } - - if option.StartDate > 0 { - query["create_time"] = bson.M{"$gte": option.StartDate, "$lte": option.EndDate} - } - - opt := &options.FindOptions{} - if option.IsAsc { - opt.SetSort(bson.D{{"create_time", 1}}) - } else { - opt.SetSort(bson.D{{"create_time", -1}}) - } - - cursor, err := c.Collection.Find(context.TODO(), query, opt) - if err != nil { - return nil, err - } - err = cursor.All(context.TODO(), &ret) - if err != nil { - return nil, err - } - return ret, nil -} - -func (c *DeployStatColl) GetDeployDailyTotal(args *DeployStatOption) ([]*DeployDailyItem, error) { - var result []*DeployDailyPipeResp - var resp []*DeployDailyItem - - pipeline := []bson.M{{ - "$sort": bson.M{ - "create_time": 1, - }, - }} - - if args.StartDate > 0 || args.EndDate > 0 { - timeRange := bson.M{} - if args.StartDate > 0 { - timeRange["$gte"] = args.StartDate - } - if args.EndDate > 0 { - timeRange["$lte"] = args.EndDate - } - pipeline = append(pipeline, bson.M{"$match": bson.M{"create_time": timeRange}}) - } - - pipeline = append(pipeline, bson.M{ - "$group": bson.M{ - "_id": bson.M{ - "date": "$date", - }, - "total_deploy_success": bson.M{ - "$sum": "$total_deploy_success", - }, - "total_deploy_failure": bson.M{ - "$sum": "$total_deploy_failure", - }, - }, - }) - - cursor, err := c.Aggregate(context.TODO(), pipeline) - if err != nil { - return nil, err - } - if err := cursor.All(context.TODO(), &result); err != nil { - return nil, err - } - for _, res := range result { - deployDailyItem := &DeployDailyItem{ - Date: res.ID.Date, - TotalSuccess: res.TotalSuccess, - TotalFailure: res.TotalFailure, - } - resp = append(resp, deployDailyItem) - } - - return resp, nil -} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/operation_log.go b/pkg/microservice/aslan/core/common/repository/mongodb/operation_log.go deleted file mode 100644 index e3ff0380e7..0000000000 --- a/pkg/microservice/aslan/core/common/repository/mongodb/operation_log.go +++ /dev/null @@ -1,135 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mongodb - -import ( - "context" - "errors" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/koderover/zadig/pkg/microservice/aslan/config" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - mongotool "github.com/koderover/zadig/pkg/tool/mongo" -) - -type OperationLogArgs struct { - Username string `json:"username"` - ProductName string `json:"product_name"` - Function string `json:"function"` - Status int `json:"status"` - PerPage int `json:"per_page"` - Page int `json:"page"` -} - -type OperationLogColl struct { - *mongo.Collection - - coll string -} - -func NewOperationLogColl() *OperationLogColl { - name := models.OperationLog{}.TableName() - return &OperationLogColl{ - Collection: mongotool.Database(config.MongoDatabase()).Collection(name), - coll: name, - } -} - -func (c *OperationLogColl) GetCollectionName() string { - return c.coll -} - -func (c *OperationLogColl) EnsureIndex(_ context.Context) error { - return nil -} - -func (c *OperationLogColl) Insert(args *models.OperationLog) error { - if args == nil { - return errors.New("nil operation_log args") - } - - res, err := c.InsertOne(context.TODO(), args) - if err != nil || res == nil { - return err - } - - if oid, ok := res.InsertedID.(primitive.ObjectID); ok { - args.ID = oid - } - - return nil -} - -func (c *OperationLogColl) Update(id string, status int) error { - if id == "" { - return errors.New("nil operation_log args") - } - - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - return err - } - - change := bson.M{"$set": bson.M{ - "status": status, - }} - _, err = c.UpdateByID(context.TODO(), oid, change, options.Update().SetUpsert(true)) - - return err -} - -func (c *OperationLogColl) Find(args *OperationLogArgs) ([]*models.OperationLog, int, error) { - var res []*models.OperationLog - query := bson.M{} - if args.ProductName != "" { - query["product_name"] = bson.M{"$regex": args.ProductName} - } - if args.Username != "" { - query["username"] = bson.M{"$regex": args.Username} - } - if args.Function != "" { - query["function"] = bson.M{"$regex": args.Function} - } - if args.Status != 0 { - query["status"] = args.Status - } - - opts := options.Find() - opts.SetSort(bson.D{{"created_at", -1}}) - if args.Page > 0 && args.PerPage > 0 { - opts.SetSkip(int64(args.PerPage * (args.Page - 1))).SetLimit(int64(args.PerPage)) - } - cursor, err := c.Collection.Find(context.TODO(), query, opts) - if err != nil { - return nil, 0, err - } - err = cursor.All(context.TODO(), &res) - if err != nil { - return nil, 0, err - } - - count, err := c.CountDocuments(context.TODO(), query) - if err != nil { - return nil, 0, err - } - - return res, int(count), err -} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/private_key.go b/pkg/microservice/aslan/core/common/repository/mongodb/private_key.go new file mode 100644 index 0000000000..238c6e5da7 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/private_key.go @@ -0,0 +1,148 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type PrivateKeyArgs struct { + Name string +} + +type PrivateKeyColl struct { + *mongo.Collection + + coll string +} + +func NewPrivateKeyColl() *PrivateKeyColl { + name := models.PrivateKey{}.TableName() + return &PrivateKeyColl{ + Collection: mongotool.Database(config.MongoDatabase()).Collection(name), + coll: name, + } +} + +func (c *PrivateKeyColl) GetCollectionName() string { + return c.coll +} + +func (c *PrivateKeyColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.M{"label": 1}, + Options: options.Index().SetUnique(false), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + return err +} + +func (c *PrivateKeyColl) Find(id string) (*models.PrivateKey, error) { + privateKey := new(models.PrivateKey) + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + + query := bson.M{"_id": oid} + err = c.FindOne(context.TODO(), query).Decode(privateKey) + return privateKey, err +} + +func (c *PrivateKeyColl) List(args *PrivateKeyArgs) ([]*models.PrivateKey, error) { + query := bson.M{} + if args.Name != "" { + query["name"] = args.Name + } + resp := make([]*models.PrivateKey, 0) + ctx := context.Background() + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, err +} + +func (c *PrivateKeyColl) Create(args *models.PrivateKey) error { + if args == nil { + return errors.New("nil PrivateKey info") + } + + args.CreateTime = time.Now().Unix() + args.UpdateTime = time.Now().Unix() + + _, err := c.InsertOne(context.TODO(), args) + + return err +} + +func (c *PrivateKeyColl) Update(id string, args *models.PrivateKey) error { + if args == nil { + return errors.New("nil PrivateKey info") + } + + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + + query := bson.M{"_id": oid} + change := bson.M{"$set": bson.M{ + "name": args.Name, + "user_name": args.UserName, + "ip": args.IP, + "label": args.Label, + "is_prod": args.IsProd, + "private_key": args.PrivateKey, + "update_by": args.UpdateBy, + "update_time": time.Now().Unix(), + }} + + _, err = c.UpdateOne(context.TODO(), query, change) + return err +} + +func (c *PrivateKeyColl) Delete(id string) error { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + + query := bson.M{"_id": oid} + + _, err = c.DeleteOne(context.TODO(), query) + return err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/strategy.go b/pkg/microservice/aslan/core/common/repository/mongodb/strategy.go new file mode 100644 index 0000000000..25f6f8bb3d --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/strategy.go @@ -0,0 +1,69 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +const ( + fieldTarget = "target" +) + +type StrategyColl struct { + *mongo.Collection + + coll string +} + +func NewStrategyColl() *StrategyColl { + name := models.CapacityStrategy{}.TableName() + return &StrategyColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *StrategyColl) GetCollectionName() string { + return c.coll +} + +func (c *StrategyColl) EnsureIndex(ctx context.Context) error { + return nil +} + +func (c *StrategyColl) Upsert(strategy *models.CapacityStrategy) error { + query := bson.M{fieldTarget: strategy.Target} + change := bson.M{"$set": strategy} + + _, err := c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) + return err +} + +// GetByTarget returns either one selected CapacityStrategy or error +func (c *StrategyColl) GetByTarget(target models.CapacityTarget) (*models.CapacityStrategy, error) { + query := bson.M{fieldTarget: target} + result := &models.CapacityStrategy{} + + err := c.FindOne(context.TODO(), query).Decode(result) + return result, err +} diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/webhook.go b/pkg/microservice/aslan/core/common/repository/mongodb/webhook.go new file mode 100644 index 0000000000..5625b697e5 --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/mongodb/webhook.go @@ -0,0 +1,120 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type WebHookColl struct { + *mongo.Collection + + coll string +} + +func NewWebHookColl() *WebHookColl { + name := models.WebHook{}.TableName() + return &WebHookColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} +} + +func (c *WebHookColl) GetCollectionName() string { + return c.coll +} + +func (c *WebHookColl) EnsureIndex(ctx context.Context) error { + mod := mongo.IndexModel{ + Keys: bson.D{ + bson.E{Key: "owner", Value: 1}, + bson.E{Key: "repo", Value: 1}, + bson.E{Key: "address", Value: 1}, + }, + Options: options.Index().SetUnique(true), + } + + _, err := c.Indexes().CreateOne(ctx, mod) + + return err +} + +// AddReferenceOrCreate updates the existing record by adding a new reference if there is a matching record, +// otherwise, it will create a new record and return true. +// It is an atomic operation and is safe cross goroutines (because of the unique index). +func (c *WebHookColl) AddReferenceOrCreate(owner, repo, address, ref string) (bool, error) { + // the query must meet the unique index + query := bson.M{"owner": owner, "repo": repo, "address": address} + change := bson.M{ + "$set": bson.M{ + "owner": owner, + "repo": repo, + "address": address, + }, + "$addToSet": bson.M{ + "references": ref, + }} + opts := options.FindOneAndUpdate().SetUpsert(true) + err := c.FindOneAndUpdate(context.TODO(), query, change, opts).Err() + if err != nil { + if err == mongo.ErrNoDocuments { + return true, nil + } + + return false, err + } + + return false, nil +} + +// RemoveReference removes a reference in the webhook record +func (c *WebHookColl) RemoveReference(owner, repo, address, ref string) (*models.WebHook, error) { + // the query must meet the unique index + query := bson.M{"owner": owner, "repo": repo, "address": address} + change := bson.M{ + "$pull": bson.M{ + "references": ref, + }} + + updated := &models.WebHook{} + err := c.FindOneAndUpdate(context.TODO(), query, change, options.FindOneAndUpdate().SetReturnDocument(options.After)).Decode(updated) + + // for compatibility + if err != nil && err != mongo.ErrNoDocuments { + return updated, err + } + + return updated, nil +} + +// Delete deletes the webhook record +func (c *WebHookColl) Delete(owner, repo, address string) error { + // the query must meet the unique index + query := bson.M{"owner": owner, "repo": repo, "address": address} + + _, err := c.DeleteOne(context.TODO(), query) + if err != nil && err != mongo.ErrNoDocuments { + return err + } + + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/base/convert.go b/pkg/microservice/aslan/core/common/service/base/convert.go new file mode 100644 index 0000000000..b691354808 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/base/convert.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package base + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" +) + +type Preview struct { + TaskType config.TaskType `json:"type"` + Enabled bool `json:"enabled"` +} + +func ToPreview(sb map[string]interface{}) (*Preview, error) { + var pre *Preview + if err := task.IToi(sb, &pre); err != nil { + return nil, fmt.Errorf("convert interface to SubTaskPreview error: %v", err) + } + return pre, nil +} + +func ToBuildTask(sb map[string]interface{}) (*task.Build, error) { + var t *task.Build + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %v", err) + } + return t, nil +} + +func ToTestingTask(sb map[string]interface{}) (*task.Testing, error) { + var t *task.Testing + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to Testing error: %v", err) + } + return t, nil +} + +func ToArtifactTask(sb map[string]interface{}) (*task.Artifact, error) { + var t *task.Artifact + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to ArtifactTask error: %v", err) + } + return t, nil +} + +func ToDockerBuildTask(sb map[string]interface{}) (*task.DockerBuild, error) { + var t *task.DockerBuild + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to DockerBuildTask error: %v", err) + } + return t, nil +} + +func ToDeployTask(sb map[string]interface{}) (*task.Deploy, error) { + var t *task.Deploy + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to DeployTask error: %v", err) + } + return t, nil +} + +func ToDistributeToS3Task(sb map[string]interface{}) (*task.DistributeToS3, error) { + var t *task.DistributeToS3 + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to DistributeToS3Task error: %v", err) + } + return t, nil +} + +func ToReleaseImageTask(sb map[string]interface{}) (*task.ReleaseImage, error) { + var t *task.ReleaseImage + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %v", err) + } + return t, nil +} + +func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { + var t *task.Jira + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to JiraTask error: %v", err) + } + return t, nil +} + +func ToSecurityTask(sb map[string]interface{}) (*task.Security, error) { + var t *task.Security + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to securityTask error: %v", err) + } + return t, nil +} + +func ToJenkinsBuildTask(sb map[string]interface{}) (*task.JenkinsBuild, error) { + var jenkinsBuild *task.JenkinsBuild + if err := task.IToi(sb, &jenkinsBuild); err != nil { + return nil, fmt.Errorf("convert interface to JenkinsBuildTask error: %v", err) + } + return jenkinsBuild, nil +} diff --git a/pkg/microservice/aslan/core/common/service/collie/client.go b/pkg/microservice/aslan/core/common/service/collie/client.go new file mode 100644 index 0000000000..a2b94c6a0a --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/collie/client.go @@ -0,0 +1,43 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collie + +import ( + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string + token string +} + +func New(host, token string) *Client { + c := httpclient.New( + httpclient.SetAuthScheme(setting.RootAPIKey), + httpclient.SetAuthToken(token), + httpclient.SetHostURL(host), + ) + + return &Client{ + Client: c, + host: host, + token: token, + } +} diff --git a/pkg/microservice/aslan/core/common/service/collie/pipeline.go b/pkg/microservice/aslan/core/common/service/collie/pipeline.go new file mode 100644 index 0000000000..07435d1bd6 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/collie/pipeline.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collie + +import ( + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type CiPipelineResource struct { + Version string `json:"version"` + Kind string `json:"kind"` + Metadata PipelineMetadata `json:"metadata"` +} + +type PipelineMetadata struct { + Name string `json:"name"` + Project string `json:"project"` + ProjectID string `json:"projectId"` + Revision int `json:"revision"` + FilePath string `json:"filePath"` + Source string `json:"source"` + OriginYamlString string `json:"originYamlString"` + ID string `json:"id"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func (c *Client) ListCIPipelines(productName string, log *zap.SugaredLogger) ([]*CiPipelineResource, error) { + url := "/api/collie/api/pipelines" + + ciPipelines := make([]*CiPipelineResource, 0) + _, err := c.Get(url, httpclient.SetResult(&ciPipelines), httpclient.SetQueryParam("project", productName)) + if err != nil { + log.Errorf("ListCIPipelines from collie failed, productName:%s, err:%+v", productName, err) + return nil, err + } + + return ciPipelines, nil +} + +func (c *Client) DeleteCIPipelines(productName string, log *zap.SugaredLogger) error { + url := "/api/collie/api/pipelines/" + productName + + _, err := c.Delete(url) + if err != nil { + log.Errorf("call collie delete pipeline err:%+v", err) + return err + } + + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/command/git_cmd.go b/pkg/microservice/aslan/core/common/service/command/git_cmd.go index e4479ac1f6..060e53b0c3 100644 --- a/pkg/microservice/aslan/core/common/service/command/git_cmd.go +++ b/pkg/microservice/aslan/core/common/service/command/git_cmd.go @@ -30,8 +30,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" "github.com/koderover/zadig/pkg/tool/log" ) diff --git a/pkg/microservice/aslan/core/common/service/config_payload.go b/pkg/microservice/aslan/core/common/service/config_payload.go index 57faaabacd..b624d8562e 100644 --- a/pkg/microservice/aslan/core/common/service/config_payload.go +++ b/pkg/microservice/aslan/core/common/service/config_payload.go @@ -20,8 +20,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" ) func GetConfigPayload(codeHostID int) *models.ConfigPayload { @@ -88,5 +88,10 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { payload.Proxy = *proxies[0] } + privateKeys, _ := mongodb.NewPrivateKeyColl().List(&mongodb.PrivateKeyArgs{}) + if len(privateKeys) != 0 { + payload.PrivateKeys = privateKeys + } + return payload } diff --git a/pkg/microservice/aslan/core/common/service/delivery.go b/pkg/microservice/aslan/core/common/service/delivery.go index 6add23a1ce..9efda65d4f 100644 --- a/pkg/microservice/aslan/core/common/service/delivery.go +++ b/pkg/microservice/aslan/core/common/service/delivery.go @@ -27,6 +27,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" @@ -148,7 +149,7 @@ func GetSubTaskContent(deliveryVersion *commonmodels.DeliveryVersion, pipelineTa deliveryBuild := new(commonmodels.DeliveryBuild) deliveryBuild.ReleaseID = deliveryVersion.ID deliveryBuild.ServiceName = serviceName - buildInfo, err := ToBuildTask(subTask) + buildInfo, err := base.ToBuildTask(subTask) if err != nil { log.Errorf("get buildInfo ToBuildTask failed ! err:%v", err) continue @@ -174,7 +175,7 @@ func GetSubTaskContent(deliveryVersion *commonmodels.DeliveryVersion, pipelineTa jiraSubBuildTaskMap := jiraSubStage.SubTasks var jira *taskmodels.Jira for _, jiraSubTask := range jiraSubBuildTaskMap { - jira, err = ToJiraTask(jiraSubTask) + jira, err = base.ToJiraTask(jiraSubTask) if err != nil { log.Errorf("ToJiraTask failed ! err:%v", err) break @@ -199,7 +200,7 @@ func GetSubTaskContent(deliveryVersion *commonmodels.DeliveryVersion, pipelineTa } subDeployTaskMap := subStage.SubTasks for _, subTask := range subDeployTaskMap { - deployInfo, err := ToDeployTask(subTask) + deployInfo, err := base.ToDeployTask(subTask) if err != nil { log.Errorf("get deployInfo failed, err:%v", err) continue @@ -257,7 +258,7 @@ func GetSubTaskContent(deliveryVersion *commonmodels.DeliveryVersion, pipelineTa deliveryTest.ReleaseID = deliveryVersion.ID testReports := make([]commonmodels.TestReportObject, 0) for _, subTask := range subTestTaskMap { - testInfo, err := ToTestingTask(subTask) + testInfo, err := base.ToTestingTask(subTask) if err != nil { log.Errorf("get testInfo ToTestingTask failed ! err:%v", err) continue @@ -307,7 +308,7 @@ func GetSubTaskContent(deliveryVersion *commonmodels.DeliveryVersion, pipelineTa deliveryDistribute.ReleaseID = deliveryVersion.ID deliveryDistribute.ServiceName = serviceName deliveryDistribute.DistributeType = config.Image - releaseImageInfo, err := ToReleaseImageTask(subTask) + releaseImageInfo, err := base.ToReleaseImageTask(subTask) if err != nil { log.Errorf("get releaseImage ToReleaseImageTask failed ! err:%v", err) continue @@ -335,7 +336,7 @@ func GetSubTaskContent(deliveryVersion *commonmodels.DeliveryVersion, pipelineTa deliveryDistributeFile.ReleaseID = deliveryVersion.ID deliveryDistributeFile.ServiceName = serviceName deliveryDistributeFile.DistributeType = config.File - releaseFileInfo, err := ToDistributeToS3Task(subTask) + releaseFileInfo, err := base.ToDistributeToS3Task(subTask) if err != nil { log.Errorf("get releasefile ToDistributeToS3Task failed ! err:%v", err) continue diff --git a/pkg/microservice/aslan/core/common/service/environment.go b/pkg/microservice/aslan/core/common/service/environment.go index be845ac26d..1059645162 100644 --- a/pkg/microservice/aslan/core/common/service/environment.go +++ b/pkg/microservice/aslan/core/common/service/environment.go @@ -25,13 +25,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/kube/resource" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" ) diff --git a/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go b/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go index 844c74fa46..5706817753 100644 --- a/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go +++ b/pkg/microservice/aslan/core/common/service/gerrit/gerrit.go @@ -26,7 +26,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" + "github.com/koderover/zadig/pkg/shared/codehost" "github.com/koderover/zadig/pkg/tool/gerrit" ) diff --git a/pkg/microservice/aslan/core/common/service/git/hook.go b/pkg/microservice/aslan/core/common/service/git/hook.go new file mode 100644 index 0000000000..dcff7666cf --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/git/hook.go @@ -0,0 +1,42 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "sync" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/tool/log" +) + +var once sync.Once +var secret string + +func GetHookSecret() string { + once.Do(func() { + poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) + org, err := poetryClient.GetOrganization(poetry.DefaultOrganization) + if err != nil { + log.Errorf("failed to find default organization: %v", err) + secret = "--impossible-token--" + } + secret = org.Token + }) + + return secret +} diff --git a/pkg/microservice/aslan/core/common/service/github/client.go b/pkg/microservice/aslan/core/common/service/github/client.go index 62fb9f6302..fc33e1db22 100644 --- a/pkg/microservice/aslan/core/common/service/github/client.go +++ b/pkg/microservice/aslan/core/common/service/github/client.go @@ -19,7 +19,7 @@ package github import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/tool/github" + "github.com/koderover/zadig/pkg/tool/git/github" ) type Client struct { diff --git a/pkg/microservice/aslan/core/common/service/github/webhook.go b/pkg/microservice/aslan/core/common/service/github/webhook.go new file mode 100644 index 0000000000..e84ebf1613 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/github/webhook.go @@ -0,0 +1,53 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "context" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" + "github.com/koderover/zadig/pkg/tool/git" + "github.com/koderover/zadig/pkg/tool/log" +) + +func (c *Client) CreateWebHook(owner, repo string) error { + _, err := c.CreateHook(context.TODO(), owner, repo, &git.Hook{ + URL: config.WebHookURL(), + Secret: gitservice.GetHookSecret(), + Events: []string{git.PushEvent, git.PullRequestEvent, git.BranchOrTagCreateEvent, git.CheckRunEvent}, + }) + + return err +} + +func (c *Client) DeleteWebHook(owner, repo string) error { + whs, err := c.ListHooks(context.TODO(), owner, repo, nil) + if err != nil { + log.Errorf("Failed to list hooks from %s/%s, err: %s", owner, repo, err) + return err + } + + for _, wh := range whs { + // we assume that there is only one webhook matching this url + if wh.Config != nil && wh.Config["url"] == config.WebHookURL() { + return c.DeleteHook(context.TODO(), owner, repo, *wh.ID) + } + } + + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/helmclient/spec.go b/pkg/microservice/aslan/core/common/service/gitlab/client.go similarity index 69% rename from pkg/microservice/aslan/core/common/service/helmclient/spec.go rename to pkg/microservice/aslan/core/common/service/gitlab/client.go index 3aebd48436..4d5051e087 100644 --- a/pkg/microservice/aslan/core/common/service/helmclient/spec.go +++ b/pkg/microservice/aslan/core/common/service/gitlab/client.go @@ -14,20 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helmclient +package gitlab import ( - "sigs.k8s.io/yaml" + "github.com/koderover/zadig/pkg/tool/git/gitlab" ) -// GetValuesMap returns the mapped out values of a chart -func (spec *ChartSpec) GetValuesMap() (map[string]interface{}, error) { - var values map[string]interface{} +type Client struct { + *gitlab.Client +} - err := yaml.Unmarshal([]byte(spec.ValuesYaml), &values) +func NewClient(address, accessToken string) (*Client, error) { + c, err := gitlab.NewClient(address, accessToken) if err != nil { return nil, err } - return values, nil + return &Client{Client: c}, nil } diff --git a/pkg/microservice/aslan/core/common/service/gitlab/webhook.go b/pkg/microservice/aslan/core/common/service/gitlab/webhook.go new file mode 100644 index 0000000000..ff80e8ddf5 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/gitlab/webhook.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "github.com/koderover/zadig/pkg/microservice/aslan/config" + gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" + "github.com/koderover/zadig/pkg/tool/git" + "github.com/koderover/zadig/pkg/tool/log" +) + +func (c *Client) CreateWebHook(owner, repo string) error { + _, err := c.AddProjectHook(owner, repo, &git.Hook{ + URL: config.WebHookURL(), + Secret: gitservice.GetHookSecret(), + Events: []string{git.PushEvent, git.PullRequestEvent, git.BranchOrTagCreateEvent}, + }) + + return err +} + +func (c *Client) DeleteWebHook(owner, repo string) error { + whs, err := c.ListProjectHooks(owner, repo, nil) + if err != nil { + log.Errorf("Failed to list hooks from %s/%s, err: %s", owner, repo, err) + return err + } + + for _, wh := range whs { + // we assume that there is only one webhook matching this url + if wh.URL == config.WebHookURL() { + return c.DeleteProjectHook(owner, repo, wh.ID) + } + } + + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/helmclient/client.go b/pkg/microservice/aslan/core/common/service/helmclient/client.go deleted file mode 100644 index 601f63cc84..0000000000 --- a/pkg/microservice/aslan/core/common/service/helmclient/client.go +++ /dev/null @@ -1,631 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helmclient - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "os" - "strings" - - "github.com/spf13/pflag" - "go.uber.org/zap" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/downloader" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/repo" - "helm.sh/helm/v3/pkg/storage/driver" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/client-go/rest" - - "github.com/koderover/zadig/pkg/tool/log" -) - -var storage = repo.File{} - -const ( - DefaultCachePath = "/tmp/.helmcache" - DefaultRepositoryConfigPath = "/tmp/.helmrepo" -) - -// NewClientFromKubeConf returns a new Helm client constructed with the provided kubeconfig options -func NewClientFromKubeConf(options *KubeConfClientOptions) (Client, error) { - settings := cli.New() - if options.KubeConfig == nil { - return nil, fmt.Errorf("kubeconfig missing") - } - - clientGetter := NewRESTClientGetter(options.Namespace, options.KubeConfig, nil) - err := setEnvSettings(options.Options, settings) - if err != nil { - return nil, err - } - - if options.KubeContext != "" { - settings.KubeContext = options.KubeContext - } - - return newClient(options.Options, clientGetter, settings) -} - -// NewClientFromRestConf returns a new Helm client constructed with the provided REST config options -func NewClientFromRestConf(restConfig *rest.Config, namespace string) (Client, error) { - options := &RestConfClientOptions{ - Options: &Options{ - RepositoryCache: DefaultCachePath, - RepositoryConfig: DefaultRepositoryConfigPath, - Debug: true, - Linting: true, - Namespace: namespace, - }, - RestConfig: restConfig, - } - - settings := cli.New() - clientGetter := NewRESTClientGetter(options.Namespace, nil, options.RestConfig) - - err := setEnvSettings(options.Options, settings) - if err != nil { - return nil, err - } - - return newClient(options.Options, clientGetter, settings) -} - -// newClient returns a new Helm client via the provided options and REST config -func newClient(options *Options, clientGetter genericclioptions.RESTClientGetter, settings *cli.EnvSettings) (Client, error) { - err := setEnvSettings(options, settings) - if err != nil { - return nil, err - } - - actionConfig := new(action.Configuration) - err = actionConfig.Init( - clientGetter, - settings.Namespace(), - os.Getenv("HELM_DRIVER"), - func(format string, v ...interface{}) { - log.Infof(format, v) - }, - ) - if err != nil { - return nil, err - } - - return &HelmClient{ - Settings: settings, - Providers: getter.All(settings), - storage: &storage, - ActionConfig: actionConfig, - linting: options.Linting, - }, nil -} - -// setEnvSettings sets the client's environment settings based on the provided client configuration -func setEnvSettings(options *Options, settings *cli.EnvSettings) error { - if options == nil { - options = &Options{ - RepositoryConfig: DefaultRepositoryConfigPath, - RepositoryCache: DefaultCachePath, - Linting: true, - } - } - - // set the namespace with this ugly workaround because cli.EnvSettings.namespace is private - // thank you helm! - if options.Namespace != "" { - pflags := pflag.NewFlagSet("", pflag.ContinueOnError) - settings.AddFlags(pflags) - err := pflags.Parse([]string{"-n", options.Namespace}) - if err != nil { - return err - } - } - - if options.RepositoryConfig == "" { - options.RepositoryConfig = DefaultRepositoryConfigPath - } - - if options.RepositoryCache == "" { - options.RepositoryCache = DefaultCachePath - } - - settings.RepositoryCache = options.RepositoryCache - settings.RepositoryConfig = DefaultRepositoryConfigPath - settings.Debug = options.Debug - - return nil -} - -// AddOrUpdateChartRepo adds or updates the provided helm chart repository -func (c *HelmClient) AddOrUpdateChartRepo(entry repo.Entry, isUpdate bool, log *zap.SugaredLogger) error { - chartRepo, err := repo.NewChartRepository(&entry, c.Providers) - if err != nil { - log.Errorf("chartRepo NewChartRepository err :%+v", err) - return err - } - - chartRepo.CachePath = c.Settings.RepositoryCache - - _, err = chartRepo.DownloadIndexFile() - if err != nil { - log.Errorf(fmt.Sprintf("chartRepo DownloadIndexFile err :%+v", err)) - return err - } - - if c.storage.Has(entry.Name) && !isUpdate { - log.Warnf("WARNING: repository name %q already exists", entry.Name) - return nil - } - - c.storage.Update(&entry) - err = c.storage.WriteFile(c.Settings.RepositoryConfig, 0o644) - if err != nil { - log.Errorf("chartRepo WriteFile err :%+v", err) - return err - } - - return nil -} - -// UpdateChartRepos updates the list of chart repositories stored in the client's cache -func (c *HelmClient) UpdateChartRepos(log *zap.SugaredLogger) error { - for _, entry := range c.storage.Repositories { - chartRepo, err := repo.NewChartRepository(entry, c.Providers) - if err != nil { - return err - } - - chartRepo.CachePath = c.Settings.RepositoryCache - _, err = chartRepo.DownloadIndexFile() - if err != nil { - return err - } - - c.storage.Update(entry) - } - - return c.storage.WriteFile(c.Settings.RepositoryConfig, 0o644) -} - -// InstallOrUpgradeChart triggers the installation of the provided chart. -// If the chart is already installed, trigger an upgrade instead -func (c *HelmClient) InstallOrUpgradeChart(ctx context.Context, spec *ChartSpec, chartOption *ChartOption, log *zap.SugaredLogger) error { - installed, err := c.chartIsInstalled(spec.ReleaseName) - if err != nil { - return err - } - if installed { - return c.upgrade(ctx, spec, chartOption) - } - return c.install(spec, chartOption, log) -} - -// DeleteChartFromCache deletes the provided chart from the client's cache -func (c *HelmClient) DeleteChartFromCache(spec *ChartSpec, chartOption *ChartOption) error { - return c.deleteChartFromCache(spec, chartOption) -} - -// UninstallRelease uninstalls the provided release -func (c *HelmClient) UninstallRelease(spec *ChartSpec) error { - return c.uninstallRelease(spec) -} - -// install lints and installs the provided chart -func (c *HelmClient) install(spec *ChartSpec, chartOption *ChartOption, log *zap.SugaredLogger) error { - client := action.NewInstall(c.ActionConfig) - mergeInstallOptions(spec, client) - - if client.Version == "" { - client.Version = ">0.0.0-0" - } - - helmChart, chartPath, err := c.getChart(spec.ChartName, &client.ChartPathOptions, chartOption) - if err != nil { - return err - } - - if helmChart.Metadata.Type != "" && helmChart.Metadata.Type != "application" { - return fmt.Errorf( - "chart %q has an unsupported type and is not installable: %q", - helmChart.Metadata.Name, - helmChart.Metadata.Type, - ) - } - - if req := helmChart.Metadata.Dependencies; req != nil { - if err := action.CheckDependencies(helmChart, req); err != nil { - if client.DependencyUpdate { - man := &downloader.Manager{ - ChartPath: chartPath, - Keyring: client.ChartPathOptions.Keyring, - SkipUpdate: false, - Getters: c.Providers, - RepositoryConfig: c.Settings.RepositoryConfig, - RepositoryCache: c.Settings.RepositoryCache, - } - if err := man.Update(); err != nil { - return err - } - } else { - return err - } - } - } - - values, err := spec.GetValuesMap() - if err != nil { - return err - } - - if c.linting { - err = c.lint(chartPath, values) - if err != nil { - return err - } - } - - _, err = client.Run(helmChart, values) - if err != nil { - log.Errorf("release installed: %s/%s-%s err: %+v", spec.ChartName, spec.Namespace, spec.Version, err) - return err - } - - return nil -} - -// upgrade upgrades a chart and CRDs -func (c *HelmClient) upgrade(ctx context.Context, spec *ChartSpec, chartOption *ChartOption) error { - client := action.NewUpgrade(c.ActionConfig) - mergeUpgradeOptions(spec, client) - - if client.Version == "" { - client.Version = ">0.0.0-0" - } - - helmChart, chartPath, err := c.getChart(spec.ChartName, &client.ChartPathOptions, chartOption) - if err != nil { - return err - } - - if req := helmChart.Metadata.Dependencies; req != nil { - if err := action.CheckDependencies(helmChart, req); err != nil { - return err - } - } - - values, err := spec.GetValuesMap() - if err != nil { - return err - } - - if c.linting { - err = c.lint(chartPath, values) - if err != nil { - return err - } - } - - if !spec.SkipCRDs && spec.UpgradeCRDs { - err = c.upgradeCRDs(ctx, helmChart) - if err != nil { - return err - } - } - - _, err = client.Run(spec.ReleaseName, helmChart, values) - if err != nil { - log.Errorf("release upgrade: %s/%s-%s err: %+v", spec.ChartName, spec.Namespace, spec.Version, err) - return err - } - return nil -} - -// deleteChartFromCache deletes the provided chart from the client's cache -func (c *HelmClient) deleteChartFromCache(spec *ChartSpec, chartOption *ChartOption) error { - client := action.NewChartRemove(c.ActionConfig) - - helmChart, _, err := c.getChart(spec.ChartName, &action.ChartPathOptions{}, chartOption) - if err != nil { - log.Errorf("deleteChartFromCache getChart err:%+v", err) - return err - } - - var deleteOutputBuffer bytes.Buffer - err = client.Run(&deleteOutputBuffer, helmChart.Name()) - if err != nil { - log.Errorf("deleteChartFromCache chart removed: %s/%s-%s err: %+v", helmChart.Name(), spec.ReleaseName, helmChart.AppVersion(), err) - return err - } - - return nil -} - -// uninstallRelease uninstalls the provided release -func (c *HelmClient) uninstallRelease(spec *ChartSpec) error { - client := action.NewUninstall(c.ActionConfig) - mergeUninstallReleaseOptions(spec, client) - - resp, err := client.Run(spec.ReleaseName) - if err != nil { - log.Errorf("release removed, response: %v, err: %+v", resp, err) - return err - } - - return nil -} - -// lint lints a chart's values -func (c *HelmClient) lint(chartPath string, values map[string]interface{}) error { - client := action.NewLint() - - result := client.Run([]string{chartPath}, values) - - for _, err := range result.Errors { - log.Errorf("Error: %s", err) - } - - if len(result.Errors) > 0 { - return fmt.Errorf("linting for chartpath %q failed", chartPath) - } - - return nil -} - -// chart template -func (c *HelmClient) TemplateChart(spec *ChartSpec, chartOption *ChartOption, log *zap.SugaredLogger) ([]byte, error) { - client := action.NewInstall(c.ActionConfig) - mergeInstallOptions(spec, client) - - client.DryRun = true - client.ReleaseName = spec.ReleaseName - client.Replace = true // Skip the name check - client.ClientOnly = true - client.APIVersions = []string{} - client.IncludeCRDs = true - - if client.Version == "" { - client.Version = ">0.0.0-0" - } - - helmChart, chartPath, err := c.getChart(spec.ChartName, &client.ChartPathOptions, chartOption) - if err != nil { - return nil, err - } - - if helmChart.Metadata.Type != "" && helmChart.Metadata.Type != "application" { - return nil, fmt.Errorf( - "chart %q has an unsupported type and is not installable: %q", - helmChart.Metadata.Name, - helmChart.Metadata.Type, - ) - } - - if req := helmChart.Metadata.Dependencies; req != nil { - if err := action.CheckDependencies(helmChart, req); err != nil { - if client.DependencyUpdate { - man := &downloader.Manager{ - ChartPath: chartPath, - Keyring: client.ChartPathOptions.Keyring, - SkipUpdate: false, - Getters: c.Providers, - RepositoryConfig: c.Settings.RepositoryConfig, - RepositoryCache: c.Settings.RepositoryCache, - } - if err := man.Update(); err != nil { - return nil, err - } - } else { - return nil, err - } - } - } - - values, err := spec.GetValuesMap() - if err != nil { - return nil, err - } - - out := new(bytes.Buffer) - rel, err := client.Run(helmChart, values) - if err != nil { - log.Errorf("TemplateChart run err: %+v", err) - } - - if rel != nil { - var manifests bytes.Buffer - fmt.Fprint(&manifests, strings.TrimSpace(rel.Manifest)) - if !client.DisableHooks { - for _, m := range rel.Hooks { - fmt.Fprintf(&manifests, "---\n# Source: %s\n%s\n", m.Path, m.Manifest) - } - } - - fmt.Fprint(out, manifests.String()) - } - - return out.Bytes(), err -} - -// 检测chart信息 -func (c *HelmClient) LintChart(spec *ChartSpec, chartOption *ChartOption, log *zap.SugaredLogger) error { - _, chartPath, err := c.getChart(spec.ChartName, &action.ChartPathOptions{}, chartOption) - if err != nil { - return err - } - - values, err := spec.GetValuesMap() - if err != nil { - return err - } - - return c.lint(chartPath, values) -} - -// upgradeCRDs -func (c *HelmClient) upgradeCRDs(ctx context.Context, chartInstance *chart.Chart) error { - cfg, err := c.Settings.RESTClientGetter().ToRESTConfig() - if err != nil { - return err - } - k8sClient, err := clientset.NewForConfig(cfg) - if err != nil { - return err - } - - for _, crd := range chartInstance.CRDObjects() { - jsonCRD, err := yaml.ToJSON(crd.File.Data) - if err != nil { - return err - } - - var meta metav1.TypeMeta - err = json.Unmarshal(jsonCRD, &meta) - if err != nil { - return err - } - - switch meta.APIVersion { - case "apiextensions.k8s.io/v1": - var crdObj v1.CustomResourceDefinition - err = json.Unmarshal(jsonCRD, &crdObj) - if err != nil { - return err - } - existingCRDObj, err := k8sClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdObj.Name, metav1.GetOptions{}) - if err != nil { - return err - } - crdObj.ResourceVersion = existingCRDObj.ResourceVersion - _, err = k8sClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, &crdObj, metav1.UpdateOptions{}) - if err != nil { - return err - } - - case "apiextensions.k8s.io/v1beta1": - var crdObj v1beta1.CustomResourceDefinition - err = json.Unmarshal(jsonCRD, &crdObj) - if err != nil { - return err - } - existingCRDObj, err := k8sClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(ctx, crdObj.Name, metav1.GetOptions{}) - if err != nil { - return err - } - crdObj.ResourceVersion = existingCRDObj.ResourceVersion - _, err = k8sClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(ctx, &crdObj, metav1.UpdateOptions{}) - if err != nil { - return err - } - - default: - return fmt.Errorf("failed to update crd %q: unsupported api-version %q", crd.Name, meta.APIVersion) - } - } - - return nil -} - -// getChart returns a chart matching the provided chart name and options -func (c *HelmClient) getChart(chartName string, chartPathOptions *action.ChartPathOptions, chartOption *ChartOption) (*chart.Chart, string, error) { - var ( - chartPath = "" - err error - ) - if chartOption == nil { - chartPath, err = chartPathOptions.LocateChart(chartName, c.Settings) - } else { - chartPath = chartOption.ChartPath - } - if err != nil { - return nil, "", err - } - - helmChart, err := loader.Load(chartPath) - if err != nil { - return nil, "", err - } - - if helmChart.Metadata.Deprecated { - log.Infof("WARNING: This chart (%q) is deprecated", helmChart.Metadata.Name) - } - - return helmChart, chartPath, err -} - -// chartIsInstalled checks whether a chart is already installed or not by the provided release name -func (c *HelmClient) chartIsInstalled(release string) (bool, error) { - histClient := action.NewHistory(c.ActionConfig) - histClient.Max = 1 - if _, err := histClient.Run(release); err == driver.ErrReleaseNotFound { - return false, nil - } else if err != nil { - return false, err - } - - return true, nil -} - -// mergeInstallOptions merges values of the provided chart to helm install options used by the client -func mergeInstallOptions(chartSpec *ChartSpec, installOptions *action.Install) { - installOptions.DisableHooks = chartSpec.DisableHooks - installOptions.Replace = chartSpec.Replace - installOptions.Wait = chartSpec.Wait - installOptions.DependencyUpdate = chartSpec.DependencyUpdate - installOptions.Timeout = chartSpec.Timeout - installOptions.Namespace = chartSpec.Namespace - installOptions.ReleaseName = chartSpec.ReleaseName - installOptions.Version = chartSpec.Version - installOptions.GenerateName = chartSpec.GenerateName - installOptions.NameTemplate = chartSpec.NameTemplate - installOptions.Atomic = chartSpec.Atomic - installOptions.SkipCRDs = chartSpec.SkipCRDs - installOptions.SubNotes = chartSpec.SubNotes -} - -// mergeUpgradeOptions merges values of the provided chart to helm upgrade options used by the client -func mergeUpgradeOptions(chartSpec *ChartSpec, upgradeOptions *action.Upgrade) { - upgradeOptions.Version = chartSpec.Version - upgradeOptions.Namespace = chartSpec.Namespace - upgradeOptions.Timeout = chartSpec.Timeout - upgradeOptions.Wait = chartSpec.Wait - upgradeOptions.DisableHooks = chartSpec.DisableHooks - upgradeOptions.Force = chartSpec.Force - upgradeOptions.ResetValues = chartSpec.ResetValues - upgradeOptions.ReuseValues = chartSpec.ReuseValues - upgradeOptions.Recreate = chartSpec.Recreate - upgradeOptions.MaxHistory = chartSpec.MaxHistory - upgradeOptions.Atomic = chartSpec.Atomic - upgradeOptions.CleanupOnFail = chartSpec.CleanupOnFail - upgradeOptions.SubNotes = chartSpec.SubNotes -} - -// mergeUninstallReleaseOptions merges values of the provided chart to helm uninstall options used by the client -func mergeUninstallReleaseOptions(chartSpec *ChartSpec, uninstallReleaseOptions *action.Uninstall) { - uninstallReleaseOptions.DisableHooks = chartSpec.DisableHooks - uninstallReleaseOptions.Timeout = chartSpec.Timeout -} diff --git a/pkg/microservice/aslan/core/common/service/helmclient/client_getter.go b/pkg/microservice/aslan/core/common/service/helmclient/client_getter.go deleted file mode 100644 index 810c1914bf..0000000000 --- a/pkg/microservice/aslan/core/common/service/helmclient/client_getter.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helmclient - -import ( - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/client-go/discovery" - "k8s.io/client-go/discovery/cached/memory" - "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" - "k8s.io/client-go/tools/clientcmd" -) - -// NewRESTClientGetter -func NewRESTClientGetter(namespace string, kubeConfig []byte, restConfig *rest.Config) *RESTClientGetter { - return &RESTClientGetter{ - namespace: namespace, - kubeConfig: kubeConfig, - restConfig: restConfig, - } -} - -// ToRESTConfig returns a REST config build from a given kubeconfig -func (c *RESTClientGetter) ToRESTConfig() (*rest.Config, error) { - if c.restConfig != nil { - return c.restConfig, nil - } - - return clientcmd.RESTConfigFromKubeConfig(c.kubeConfig) -} - -func (c *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { - config, err := c.ToRESTConfig() - if err != nil { - return nil, err - } - - // The more groups you have, the more discovery requests you need to make. - // given 25 groups (our groups + a few custom conf) with one-ish version each, discovery needs to make 50 requests - // double it just so we don't end up here again for a while. This config is only used for discovery. - config.Burst = 100 - - discoveryClient, _ := discovery.NewDiscoveryClientForConfig(config) - return memory.NewMemCacheClient(discoveryClient), nil -} - -func (c *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) { - discoveryClient, err := c.ToDiscoveryClient() - if err != nil { - return nil, err - } - - mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) - expander := restmapper.NewShortcutExpander(mapper, discoveryClient) - return expander, nil -} - -func (c *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig { - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - // use the standard defaults for this client command - // DEPRECATED: remove and replace with something more accurate - loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig - - overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmd.ClusterDefaults} - overrides.Context.Namespace = c.namespace - - return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides) -} diff --git a/pkg/microservice/aslan/core/common/service/helmclient/interface.go b/pkg/microservice/aslan/core/common/service/helmclient/interface.go deleted file mode 100644 index a6bb878d8b..0000000000 --- a/pkg/microservice/aslan/core/common/service/helmclient/interface.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helmclient - -import ( - "context" - - "go.uber.org/zap" - "helm.sh/helm/v3/pkg/repo" -) - -type Client interface { - AddOrUpdateChartRepo(entry repo.Entry, isUpdate bool, log *zap.SugaredLogger) error - UpdateChartRepos(log *zap.SugaredLogger) error - InstallOrUpgradeChart(ctx context.Context, spec *ChartSpec, chartOption *ChartOption, log *zap.SugaredLogger) error - DeleteChartFromCache(spec *ChartSpec, chartOption *ChartOption) error - UninstallRelease(spec *ChartSpec) error - TemplateChart(spec *ChartSpec, chartOption *ChartOption, log *zap.SugaredLogger) ([]byte, error) - LintChart(spec *ChartSpec, chartOption *ChartOption, log *zap.SugaredLogger) error -} diff --git a/pkg/microservice/aslan/core/common/service/helmclient/types.go b/pkg/microservice/aslan/core/common/service/helmclient/types.go deleted file mode 100644 index 8c85001a1f..0000000000 --- a/pkg/microservice/aslan/core/common/service/helmclient/types.go +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helmclient - -import ( - "time" - - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/repo" - "k8s.io/client-go/rest" -) - -// KubeConfClientOptions defines the options used for constructing a client via kubeconfig -type KubeConfClientOptions struct { - *Options - KubeContext string - KubeConfig []byte -} - -// RestConfClientOptions defines the options used for constructing a client via REST config -type RestConfClientOptions struct { - *Options - RestConfig *rest.Config -} - -// Options defines the options of a client -type Options struct { - Namespace string - RepositoryConfig string - RepositoryCache string - Debug bool - Linting bool -} - -// RESTClientGetter defines the values of a helm REST client -type RESTClientGetter struct { - namespace string - kubeConfig []byte - restConfig *rest.Config -} - -// Client defines the values of a helm client -type HelmClient struct { - Settings *cli.EnvSettings - Providers getter.Providers - storage *repo.File - ActionConfig *action.Configuration - linting bool -} - -// ChartSpec defines the values of a helm chart -type ChartSpec struct { - ReleaseName string `json:"release"` - ChartName string `json:"chart"` - Namespace string `json:"namespace"` - - // use string instead of map[string]interface{} - // https://github.com/kubernetes-sigs/kubebuilder/issues/528#issuecomment-466449483 - // and https://github.com/kubernetes-sigs/controller-tools/pull/317 - // +optional - ValuesYaml string `json:"valuesYaml,omitempty"` - - // +optional - Version string `json:"version,omitempty"` - - // +optional - DisableHooks bool `json:"disableHooks,omitempty"` - - // +optional - Replace bool `json:"replace,omitempty"` - - // +optional - Wait bool `json:"wait,omitempty"` - - // +optional - DependencyUpdate bool `json:"dependencyUpdate,omitempty"` - - // +optional - Timeout time.Duration `json:"timeout,omitempty"` - - // +optional - GenerateName bool `json:"generateName,omitempty"` - - // +optional - NameTemplate string `json:"NameTemplate,omitempty"` - - // +optional - Atomic bool `json:"atomic,omitempty"` - - // +optional - SkipCRDs bool `json:"skipCRDs,omitempty"` - - // +optional - UpgradeCRDs bool `json:"upgradeCRDs,omitempty"` - - // +optional - SubNotes bool `json:"subNotes,omitempty"` - - // +optional - Force bool `json:"force,omitempty"` - - // +optional - ResetValues bool `json:"resetValues,omitempty"` - - // +optional - ReuseValues bool `json:"reuseValues,omitempty"` - - // +optional - Recreate bool `json:"recreate,omitempty"` - - // +optional - MaxHistory int `json:"maxHistory,omitempty"` - - // +optional - CleanupOnFail bool `json:"cleanupOnFail,omitempty"` -} - -type ChartOption struct { - ChartPath string `json:"chart_path"` -} diff --git a/pkg/microservice/aslan/core/common/service/kube/status.go b/pkg/microservice/aslan/core/common/service/kube/status.go index 4b2e6588f7..cf2460ad5b 100644 --- a/pkg/microservice/aslan/core/common/service/kube/status.go +++ b/pkg/microservice/aslan/core/common/service/kube/status.go @@ -23,8 +23,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" "github.com/koderover/zadig/pkg/tool/kube/getter" ) diff --git a/pkg/microservice/aslan/core/common/service/notify/client.go b/pkg/microservice/aslan/core/common/service/notify/client.go index 398be69c38..d08af85229 100644 --- a/pkg/microservice/aslan/core/common/service/notify/client.go +++ b/pkg/microservice/aslan/core/common/service/notify/client.go @@ -19,7 +19,6 @@ package notify import ( "encoding/json" "fmt" - "time" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" @@ -57,12 +56,6 @@ func (c *client) CreateNotify(sender string, nf *models.Notify) error { return fmt.Errorf("marshal content error: %v", err) } switch nf.Type { - case config.Announcement: - var content *models.AnnouncementCtx - if err = json.Unmarshal(b, &content); err != nil { - return fmt.Errorf("[%s] convert announcement error: %v", sender, err) - } - nf.Content = content case config.PipelineStatus: var content *models.PipelineStatusCtx if err = json.Unmarshal(b, &content); err != nil { @@ -74,6 +67,7 @@ func (c *client) CreateNotify(sender string, nf *models.Notify) error { if err = json.Unmarshal(b, &content); err != nil { return fmt.Errorf("[%s] convert message error: %v", sender, err) } + nf.Content = content default: return fmt.Errorf("notify type not found") @@ -85,37 +79,6 @@ func (c *client) CreateNotify(sender string, nf *models.Notify) error { return nil } -func (c *client) PullNotifyAnnouncement(user string) ([]*models.Notify, error) { - resp := make([]*models.Notify, 0) - systemNotifyList, err := c.notifyColl.List("*") - if err != nil { - return resp, fmt.Errorf("[*] pull notify error: %v", err) - } - //取出所有当前生效的系统通告 - for _, systemNotify := range systemNotifyList { - b, err := json.Marshal(systemNotify.Content) - if err != nil { - return resp, fmt.Errorf("marshal content error: %v", err) - } - var content *models.AnnouncementCtx - if err = json.Unmarshal(b, &content); err != nil { - return resp, fmt.Errorf("[%s] convert announcement error: %v", user, err) - } - if content.StartTime < time.Now().Unix() && time.Now().Unix() < content.EndTime { - resp = append(resp, systemNotify) - } - } - return resp, nil -} - -func (c *client) PullAllAnnouncement(user string) ([]*models.Notify, error) { - systemNotifyList, err := c.notifyColl.List("*") - if err != nil { - return nil, fmt.Errorf("[*] pull notify error: %v", err) - } - return systemNotifyList, nil -} - func (c *client) PullNotify(user string) ([]*models.Notify, error) { resp := make([]*models.Notify, 0) notifyList, err := c.notifyColl.List(user) @@ -172,28 +135,6 @@ func (c *client) Read(user string, notifyIDs []string) error { return nil } -func (c *client) Update(user string, notifyID string, ctx *models.Notify) error { - b, err := json.Marshal(ctx.Content) - if err != nil { - return fmt.Errorf("marshal content error: %v", err) - } - switch ctx.Type { - case config.Announcement: - var content *models.AnnouncementCtx - if err = json.Unmarshal(b, &content); err != nil { - return fmt.Errorf("[%s] convert announcement error: %v", user, err) - } - ctx.Content = content - default: - return fmt.Errorf("notify type not found") - } - err = c.notifyColl.Update(notifyID, ctx) - if err != nil { - return fmt.Errorf("[%s] update Notify error: %v", user, err) - } - return nil -} - func (c *client) DeleteNotifies(user string, notifyIDs []string) error { for _, id := range notifyIDs { err := c.notifyColl.DeleteByID(id) diff --git a/pkg/microservice/aslan/core/common/service/operation_log.go b/pkg/microservice/aslan/core/common/service/operation_log.go deleted file mode 100644 index a80b8fe858..0000000000 --- a/pkg/microservice/aslan/core/common/service/operation_log.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "go.uber.org/zap" - - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - e "github.com/koderover/zadig/pkg/tool/errors" -) - -func InsertOperation(args *models.OperationLog, log *zap.SugaredLogger) error { - err := mongodb.NewOperationLogColl().Insert(args) - if err != nil { - log.Errorf("insert operation log error: %v", err) - return e.ErrCreateOperationLog - } - return nil -} - -func UpdateOperation(id string, status int, log *zap.SugaredLogger) error { - err := mongodb.NewOperationLogColl().Update(id, status) - if err != nil { - log.Errorf("update operation log error: %v", err) - return e.ErrUpdateOperationLog - } - return nil -} diff --git a/pkg/microservice/aslan/core/common/service/product.go b/pkg/microservice/aslan/core/common/service/product.go index cf919f6670..8ea3087231 100644 --- a/pkg/microservice/aslan/core/common/service/product.go +++ b/pkg/microservice/aslan/core/common/service/product.go @@ -25,13 +25,14 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/kube/updater" ) @@ -52,6 +53,11 @@ func DeleteProduct(username, envName, productName, requestID string, log *zap.Su return e.ErrDeleteEnv.AddErr(err) } + restConfig, err := kube.GetRESTConfig(productInfo.ClusterID) + if err != nil { + return e.ErrDeleteEnv.AddErr(err) + } + // 设置产品状态 err = mongodb.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusDeleting) if err != nil { @@ -65,6 +71,68 @@ func DeleteProduct(username, envName, productName, requestID string, log *zap.Su poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) switch productInfo.Source { + case setting.SourceFromHelm: + err = mongodb.NewProductColl().Delete(envName, productName) + if err != nil { + log.Errorf("Product.Delete error: %v", err) + } + + _, err = poetryClient.DeleteEnvRolePermission(productName, envName, log) + if err != nil { + log.Errorf("DeleteEnvRole error: %v", err) + } + + go func() { + var err error + defer func() { + if err != nil { + // 发送删除产品失败消息给用户 + title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 失败!", productName, envName) + SendErrorMessage(username, title, requestID, err, log) + _ = mongodb.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUnknown) + } else { + title := fmt.Sprintf("删除项目:[%s] 环境:[%s] 成功!", productName, envName) + content := fmt.Sprintf("namespace:%s", productInfo.Namespace) + SendMessage(username, title, content, requestID, log) + } + }() + + //卸载helm release资源 + if helmClient, err := helmclient.NewClientFromRestConf(restConfig, productInfo.Namespace); err == nil { + for _, services := range productInfo.Services { + for _, service := range services { + if err = helmClient.UninstallRelease(&helmclient.ChartSpec{ + ReleaseName: fmt.Sprintf("%s-%s", productInfo.Namespace, service.ServiceName), + Namespace: productInfo.Namespace, + Wait: true, + Force: true, + Timeout: timeout * time.Second * 10, + }); err != nil { + log.Errorf("UninstallRelease err:%v", err) + } + } + } + } else { + log.Errorf("获取helmClient err:%v", err) + } + + //删除namespace + s := labels.Set{setting.EnvCreatedBy: setting.EnvCreator}.AsSelector() + if err1 := updater.DeleteMatchingNamespace(productInfo.Namespace, s, kubeClient); err1 != nil { + err = e.ErrDeleteEnv.AddDesc(e.DeleteNamespaceErrMsg + ": " + err1.Error()) + return + } + }() + case setting.SourceFromExternal: + err = mongodb.NewProductColl().Delete(envName, productName) + if err != nil { + log.Errorf("Product.Delete error: %v", err) + } + + _, err = poetryClient.DeleteEnvRolePermission(productName, envName, log) + if err != nil { + log.Errorf("DeleteEnvRole error: %v", err) + } default: go func() { var err error diff --git a/pkg/microservice/aslan/core/common/service/registry.go b/pkg/microservice/aslan/core/common/service/registry.go index ccfe065921..5177609a11 100644 --- a/pkg/microservice/aslan/core/common/service/registry.go +++ b/pkg/microservice/aslan/core/common/service/registry.go @@ -48,6 +48,15 @@ func FindDefaultRegistry(log *zap.SugaredLogger) (*models.RegistryNamespace, err return resp, nil } +func GetDefaultRegistryNamespace(log *zap.SugaredLogger) (*models.RegistryNamespace, error) { + resp, err := mongodb.NewRegistryNamespaceColl().Find(&mongodb.FindRegOps{IsDefault: true}) + if err != nil { + log.Errorf("get default registry error: %v", err) + return resp, fmt.Errorf("get default registry error: %v", err) + } + return resp, nil +} + func ListRegistryNamespaces(log *zap.SugaredLogger) ([]*models.RegistryNamespace, error) { resp, err := mongodb.NewRegistryNamespaceColl().FindAll(&mongodb.FindRegOps{}) if err != nil { diff --git a/pkg/microservice/aslan/core/common/service/registry/service_test.go b/pkg/microservice/aslan/core/common/service/registry/service_test.go index 39580eeae3..70b9bcf4c7 100644 --- a/pkg/microservice/aslan/core/common/service/registry/service_test.go +++ b/pkg/microservice/aslan/core/common/service/registry/service_test.go @@ -26,6 +26,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/tool/log" + _ "github.com/koderover/zadig/pkg/util/testing" ) func Test_v2RegistryService_ListRepoImages(t *testing.T) { diff --git a/pkg/microservice/aslan/core/common/service/render.go b/pkg/microservice/aslan/core/common/service/render.go index df9011d58b..25af3a99f9 100644 --- a/pkg/microservice/aslan/core/common/service/render.go +++ b/pkg/microservice/aslan/core/common/service/render.go @@ -180,6 +180,31 @@ func CreateRenderSet(args *commonmodels.RenderSet, log *zap.SugaredLogger) error return nil } +// CreateHelmRenderSet 添加renderSet +func CreateHelmRenderSet(args *commonmodels.RenderSet, log *zap.SugaredLogger) error { + opt := &commonrepo.RenderSetFindOption{Name: args.Name} + rs, err := commonrepo.NewRenderSetColl().Find(opt) + if rs != nil && err == nil { + // 已经存在渲染配置集 + // 判断是否有修改 + if rs.HelmRenderDiff(args) { + args.IsDefault = rs.IsDefault + } else { + return nil + } + } + if err := ensureHelmRenderSetArgs(args); err != nil { + log.Error(err) + return e.ErrCreateRenderSet.AddDesc(err.Error()) + } + if err := commonrepo.NewRenderSetColl().Create(args); err != nil { + errMsg := fmt.Sprintf("[RenderSet.Create] %s error: %v", args.Name, err) + log.Error(errMsg) + return e.ErrCreateRenderSet.AddDesc(errMsg) + } + return nil +} + func UpdateRenderSet(args *commonmodels.RenderSet, log *zap.SugaredLogger) error { err := commonrepo.NewRenderSetColl().Update(args) if err != nil { @@ -533,3 +558,22 @@ func ensureRenderSetArgs(args *commonmodels.RenderSet) error { args.Revision = rev return nil } + +// ensureHelmRenderSetArgs ... +func ensureHelmRenderSetArgs(args *commonmodels.RenderSet) error { + if args == nil { + return errors.New("nil RenderSet") + } + + if len(args.Name) == 0 { + return errors.New("empty render set name") + } + // 设置新的版本号 + rev, err := commonrepo.NewCounterColl().GetNextSeq("renderset:" + args.Name) + if err != nil { + return fmt.Errorf("get next render set revision error: %v", err) + } + + args.Revision = rev + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/scmnotify/client.go b/pkg/microservice/aslan/core/common/service/scmnotify/client.go index c4b90a2369..f7299ce333 100644 --- a/pkg/microservice/aslan/core/common/service/scmnotify/client.go +++ b/pkg/microservice/aslan/core/common/service/scmnotify/client.go @@ -27,9 +27,9 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" + "github.com/koderover/zadig/pkg/shared/codehost" "github.com/koderover/zadig/pkg/tool/gerrit" - gitlabtool "github.com/koderover/zadig/pkg/tool/gitlab" + gitlabtool "github.com/koderover/zadig/pkg/tool/git/gitlab" "github.com/koderover/zadig/pkg/tool/log" ) @@ -61,7 +61,7 @@ func (c *Client) Comment(notify *models.Notification) error { } if strings.ToLower(codeHostDetail.Type) == "gitlab" { var note *gitlab.Note - cli, err := gitlabtool.NewGitlabClient(codeHostDetail.Address, codeHostDetail.AccessToken) + cli, err := gitlabtool.NewClient(codeHostDetail.Address, codeHostDetail.AccessToken) if err != nil { c.logger.Errorf("create gitlab client failed err: %v", err) return fmt.Errorf("create gitlab client failed err: %v", err) diff --git a/pkg/microservice/aslan/core/common/service/service.go b/pkg/microservice/aslan/core/common/service/service.go index 8120335c18..7326b9f00f 100644 --- a/pkg/microservice/aslan/core/common/service/service.go +++ b/pkg/microservice/aslan/core/common/service/service.go @@ -25,12 +25,12 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/yaml" - "github.com/koderover/zadig/pkg/internal/poetry" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -49,15 +49,16 @@ type ServiceTmplBuildObject struct { } type ServiceTmplObject struct { - ProductName string `json:"product_name"` - ServiceName string `json:"service_name"` - Visibility string `json:"visibility"` - Revision int64 `json:"revision"` - Type string `json:"type"` - Username string `json:"username"` - EnvConfigs []*commonmodels.EnvConfig `json:"env_configs"` - EnvStatuses []*commonmodels.EnvStatus `json:"env_statuses,omitempty"` - From string `json:"from,omitempty"` + ProductName string `json:"product_name"` + ServiceName string `json:"service_name"` + Visibility string `json:"visibility"` + Revision int64 `json:"revision"` + Type string `json:"type"` + Username string `json:"username"` + EnvConfigs []*commonmodels.EnvConfig `json:"env_configs"` + EnvStatuses []*commonmodels.EnvStatus `json:"env_statuses,omitempty"` + From string `json:"from,omitempty"` + HealthChecks []*commonmodels.PmHealthCheck `json:"health_checks"` } type ServiceProductMap struct { @@ -341,3 +342,39 @@ func GetServiceTemplate(serviceName, serviceType, productName, excludeStatus str return resp, nil } + +func UpdatePmServiceTemplate(username string, args *ServiceTmplBuildObject, log *zap.SugaredLogger) error { + //该请求来自环境中的服务更新时,from=createEnv + if args.ServiceTmplObject.From == "" { + if err := UpdateBuild(username, args.Build, log); err != nil { + return err + } + } + + //先比较healthcheck是否有变动 + preService, err := GetServiceTemplate(args.ServiceTmplObject.ServiceName, setting.PMDeployType, args.ServiceTmplObject.ProductName, setting.ProductStatusDeleting, args.ServiceTmplObject.Revision, log) + if err != nil { + return err + } + + //更新服务 + serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, preService.ServiceName, setting.PMDeployType) + rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) + if err != nil { + return err + } + preService.HealthChecks = args.ServiceTmplObject.HealthChecks + preService.EnvConfigs = args.ServiceTmplObject.EnvConfigs + preService.Revision = rev + preService.CreateBy = username + preService.BuildName = args.Build.Name + + if err := commonrepo.NewServiceColl().Delete(preService.ServiceName, setting.PMDeployType, "", setting.ProductStatusDeleting, preService.Revision); err != nil { + return err + } + + if err := commonrepo.NewServiceColl().Create(preService); err != nil { + return err + } + return nil +} diff --git a/pkg/microservice/aslan/core/common/service/task.go b/pkg/microservice/aslan/core/common/service/task.go index 05dd2a4571..49675c3f09 100644 --- a/pkg/microservice/aslan/core/common/service/task.go +++ b/pkg/microservice/aslan/core/common/service/task.go @@ -162,97 +162,3 @@ func covertTaskToQueue(t *task.Task) *models.Queue { CreateTime: t.CreateTime, } } - -type Preview struct { - TaskType config.TaskType `json:"type"` - Enabled bool `json:"enabled"` -} - -func ToPreview(sb map[string]interface{}) (*Preview, error) { - var pre *Preview - if err := task.IToi(sb, &pre); err != nil { - return nil, fmt.Errorf("convert interface to SubTaskPreview error: %v", err) - } - return pre, nil -} - -func ToBuildTask(sb map[string]interface{}) (*task.Build, error) { - var t *task.Build - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %v", err) - } - return t, nil -} - -func ToArtifactTask(sb map[string]interface{}) (*task.Artifact, error) { - var t *task.Artifact - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to ArtifactTask error: %v", err) - } - return t, nil -} - -func ToDockerBuildTask(sb map[string]interface{}) (*task.DockerBuild, error) { - var t *task.DockerBuild - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DockerBuildTask error: %v", err) - } - return t, nil -} - -func ToDeployTask(sb map[string]interface{}) (*task.Deploy, error) { - var t *task.Deploy - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DeployTask error: %v", err) - } - return t, nil -} - -func ToTestingTask(sb map[string]interface{}) (*task.Testing, error) { - var t *task.Testing - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to Testing error: %v", err) - } - return t, nil -} - -func ToDistributeToS3Task(sb map[string]interface{}) (*task.DistributeToS3, error) { - var t *task.DistributeToS3 - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DistributeToS3Task error: %v", err) - } - return t, nil -} - -func ToReleaseImageTask(sb map[string]interface{}) (*task.ReleaseImage, error) { - var t *task.ReleaseImage - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %v", err) - } - return t, nil -} - -func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { - var t *task.Jira - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to JiraTask error: %v", err) - } - return t, nil -} - -func ToSecurityTask(sb map[string]interface{}) (*task.Security, error) { - var t *task.Security - if err := task.IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to securityTask error: %v", err) - } - return t, nil -} - -// ToJenkinsTask ... -func ToJenkinsBuildTask(sb map[string]interface{}) (*task.JenkinsBuild, error) { - var jenkinsBuild *task.JenkinsBuild - if err := task.IToi(sb, &jenkinsBuild); err != nil { - return nil, fmt.Errorf("convert interface to JenkinsBuildTask error: %v", err) - } - return jenkinsBuild, nil -} diff --git a/pkg/microservice/aslan/core/common/service/template_product.go b/pkg/microservice/aslan/core/common/service/template_product.go index 7ad384a930..04ce0eb7c5 100644 --- a/pkg/microservice/aslan/core/common/service/template_product.go +++ b/pkg/microservice/aslan/core/common/service/template_product.go @@ -23,13 +23,14 @@ import ( "go.uber.org/zap" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" "github.com/koderover/zadig/pkg/microservice/aslan/internal/cache" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -49,6 +50,28 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P return nil, e.ErrGetProduct.AddDesc(err.Error()) } + totalFreeStyles := make([]*collie.CiPipelineResource, 0) + features, err := GetFeatures(log) + if err != nil { + log.Errorf("GetProductTemplate GetFeatures err : %v", err) + } + + if strings.Contains(features, string(config.FreestyleType)) { + // CI场景onboarding流程处于第二步时,需要返回ci工作流id,用于前端跳转 + collieAPIAddress := config.CollieAPIAddress() + cl := collie.New(collieAPIAddress, config.PoetryAPIRootKey()) + if resp.ProductFeature != nil && resp.ProductFeature.DevelopHabit == "yaml" && resp.OnboardingStatus == setting.OnboardingStatusSecond && collieAPIAddress != "" { + ciPipelines, err := cl.ListCIPipelines(productName, log) + if err != nil { + log.Errorf("GetProductTemplate error: %v", err) + return nil, e.ErrGetProduct.AddDesc(err.Error()) + } + if len(ciPipelines) != 0 { + resp.CiPipelineID = ciPipelines[0].Metadata.ID + } + } + } + err = FillProductTemplateVars([]*template.Product{resp}, log) if err != nil { return nil, fmt.Errorf("FillProductTemplateVars err : %v", err) @@ -84,6 +107,15 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P return resp, fmt.Errorf("Pipeline.List err : %v", err) } + if strings.Contains(features, string(config.FreestyleType)) { + collieAPIAddress := config.CollieAPIAddress() + cl := collie.New(collieAPIAddress, config.PoetryAPIRootKey()) + totalFreeStyles, err = cl.ListCIPipelines(productName, log) + if err != nil { + log.Errorf("GetProductTemplate freestyle.List err : %v", err) + } + } + totalEnvTemplateServiceNum := 0 for _, services := range resp.Services { totalEnvTemplateServiceNum += len(services) @@ -114,6 +146,7 @@ func GetProductTemplate(productName string, log *zap.SugaredLogger) (*template.P resp.LatestEnvUpdateBy = totalEnvs[0].UpdateBy } resp.TotalWorkflowNum = len(totalWorkflows) + len(totalPiplines) + resp.TotalWorkflowNum += len(totalFreeStyles) if len(totalWorkflows) > 0 { resp.LatestWorkflowUpdateTime = totalWorkflows[0].UpdateTime resp.LatestWorkflowUpdateBy = totalWorkflows[0].UpdateBy diff --git a/pkg/microservice/aslan/core/common/service/utils.go b/pkg/microservice/aslan/core/common/service/utils.go index afd2b726c6..a2cc8d0af3 100644 --- a/pkg/microservice/aslan/core/common/service/utils.go +++ b/pkg/microservice/aslan/core/common/service/utils.go @@ -24,11 +24,11 @@ import ( "go.uber.org/zap" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/notify" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/types/permission" ) diff --git a/pkg/microservice/aslan/core/common/service/webhook/client.go b/pkg/microservice/aslan/core/common/service/webhook/client.go new file mode 100644 index 0000000000..bbe5e1f476 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/webhook/client.go @@ -0,0 +1,111 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "fmt" + "time" +) + +const ( + WorkflowPrefix = "workflow-" + PipelinePrefix = "pipeline-" + ColliePrefix = "collie-" + ServicePrefix = "service-" + + taskTimeoutSecond = 10 +) + +type client struct { + enabled bool +} + +func NewClient() *client { + return &client{ + enabled: false, + } +} + +type task struct { + owner, repo, address, token, ref string + from string + add bool + err error + doneCh chan struct{} +} + +func (c *client) AddWebHook(owner, repo, address, token, ref, from string) error { + if !c.enabled { + return nil + } + + t := &task{ + owner: owner, + repo: repo, + address: address, + token: token, + ref: ref, + from: from, + add: true, + doneCh: make(chan struct{}), + } + + select { + case webhookController().queue <- t: + default: + return fmt.Errorf("queue is full, please retry it later") + } + + select { + case <-t.doneCh: + case <-time.After(taskTimeoutSecond * time.Second): + t.err = fmt.Errorf("timed out waiting for the task") + } + + return t.err +} + +func (c *client) RemoveWebHook(owner, repo, address, token, ref, from string) error { + if !c.enabled { + return nil + } + + t := &task{ + owner: owner, + repo: repo, + address: address, + token: token, + ref: ref, + from: from, + add: false, + doneCh: make(chan struct{}), + } + + select { + case webhookController().queue <- t: + default: + return fmt.Errorf("queue is full, please retry it later") + } + + select { + case <-t.doneCh: + case <-time.After(taskTimeoutSecond * time.Second): + t.err = fmt.Errorf("timed out waiting for the task") + } + + return t.err +} diff --git a/pkg/microservice/aslan/core/common/service/webhook/controller.go b/pkg/microservice/aslan/core/common/service/webhook/controller.go new file mode 100644 index 0000000000..8c61c3ecae --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/webhook/controller.go @@ -0,0 +1,205 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "fmt" + "sync" + "time" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitlab" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" +) + +type hookCreateDeleter interface { + CreateWebHook(owner, repo string) error + DeleteWebHook(owner, repo string) error +} + +type controller struct { + queue chan *task + + logger *zap.Logger +} + +var once sync.Once +var c *controller + +func webhookController() *controller { + once.Do(func() { + c = &controller{ + queue: make(chan *task, 100), + logger: log.Logger(), + } + }) + + return c +} + +func NewWebhookController() *controller { + return webhookController() +} + +// Run starts the controller and blocks until receiving signal from stopCh. +func (c *controller) Run(workers int, stopCh <-chan struct{}) { + defer close(c.queue) + + if workers > 1 { + c.logger.Panic("webhook controller only accept one worker") + } + + c.logger.Info("Starting webhook controller") + defer c.logger.Info("Shutting down webhook controller") + + //for i := 0; i < workers; i++ { + // go wait.Until(c.runWorker, time.Second, stopCh) + //} + go wait.Until(c.runWorker, time.Second, stopCh) + + <-stopCh +} + +func (c *controller) runWorker() { + for c.processNextWorkItem() { + } +} + +// processNextWorkItem deals with one key off the queue. It returns false when it's time to quit. +func (c *controller) processNextWorkItem() bool { + t, ok := <-c.queue + if !ok { + return false + } + + logger := c.logger.With( + zap.String("owner", t.owner), + zap.String("repo", t.repo), + zap.String("address", t.address), + zap.String("ref", t.ref), + ) + + if t.err != nil { + logger.Warn(fmt.Sprintf("Task is canceled with reason: %s", t.err)) + return true + } + + if t.add { + addWebhook(t, logger) + } else { + removeWebhook(t, logger) + } + + return true +} + +func removeWebhook(t *task, logger *zap.Logger) { + coll := mongodb.NewWebHookColl() + var cl hookCreateDeleter + var err error + + switch t.from { + case setting.SourceFromGithub: + cl = github.NewClient(t.token, config.ProxyHTTPSAddr()) + case setting.SourceFromGitlab: + cl, err = gitlab.NewClient(t.address, t.token) + if err != nil { + t.err = err + t.doneCh <- struct{}{} + return + } + default: + t.err = fmt.Errorf("invaild source: %s", t.from) + t.doneCh <- struct{}{} + return + } + + logger.Info("Removing webhook") + updated, err := coll.RemoveReference(t.owner, t.repo, t.address, t.ref) + if err != nil { + t.err = err + t.doneCh <- struct{}{} + return + } + + if len(updated.References) == 0 { + logger.Info("Deleting webhook") + err = cl.DeleteWebHook(t.owner, t.repo) + if err != nil { + logger.Error("Failed to delete webhook", zap.Error(err)) + t.err = err + t.doneCh <- struct{}{} + return + } + + err = coll.Delete(t.owner, t.repo, t.address) + if err != nil { + logger.Error("Failed to delete webhook record in db", zap.Error(err)) + t.err = err + } + } + + t.doneCh <- struct{}{} +} + +func addWebhook(t *task, logger *zap.Logger) { + coll := mongodb.NewWebHookColl() + var cl hookCreateDeleter + var err error + + switch t.from { + case setting.SourceFromGithub: + cl = github.NewClient(t.token, config.ProxyHTTPSAddr()) + case setting.SourceFromGitlab: + cl, err = gitlab.NewClient(t.address, t.token) + if err != nil { + t.err = err + t.doneCh <- struct{}{} + return + } + default: + t.err = fmt.Errorf("invaild source: %s", t.from) + t.doneCh <- struct{}{} + return + } + + logger.Info("Adding webhook") + created, err := coll.AddReferenceOrCreate(t.owner, t.repo, t.address, t.ref) + if err != nil || !created { + t.err = err + t.doneCh <- struct{}{} + return + } + + logger.Info("Creating webhook") + err = cl.CreateWebHook(t.owner, t.repo) + if err != nil { + t.err = err + logger.Error("Failed to create webhook", zap.Error(err)) + if err = coll.Delete(t.owner, t.repo, t.address); err != nil { + logger.Error("Failed to delete webhook record in db", zap.Error(err)) + } + } + + t.doneCh <- struct{}{} +} diff --git a/pkg/microservice/aslan/core/common/service/wechat/service.go b/pkg/microservice/aslan/core/common/service/wechat/service.go index 5c31c8a822..dc318b4104 100644 --- a/pkg/microservice/aslan/core/common/service/wechat/service.go +++ b/pkg/microservice/aslan/core/common/service/wechat/service.go @@ -28,6 +28,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/httpclient" "github.com/koderover/zadig/pkg/tool/log" ) @@ -275,6 +277,20 @@ func (w *Service) createNotifyBody(weChatNotification *wechatNotification) (cont "- 创建人:{{.Task.TaskCreator}} \n" + "- 总运行时长:{{ .TotalTime}} 秒 \n" + testNames := getHTMLTestReport(weChatNotification.Task) + if len(testNames) != 0 { + tmplSource += "- 测试报告:\n" + } + + for _, testName := range testNames { + url := fmt.Sprintf("{{.BaseURI}}/api/aslan/testing/report?pipelineName={{.Task.PipelineName}}&pipelineType={{.Task.Type}}&taskID={{.Task.TaskID}}&testName=%s\n", testName) + if weChatNotification.WebHookType == feiShuType { + tmplSource += url + continue + } + tmplSource += fmt.Sprintf("[%s](%s)\n", url, url) + } + if weChatNotification.WebHookType == dingDingType { if len(weChatNotification.AtMobiles) > 0 && !weChatNotification.IsAtAll { tmplSource = fmt.Sprintf("%s - 相关人员:@%s \n", tmplSource, strings.Join(weChatNotification.AtMobiles, "@")) @@ -307,3 +323,30 @@ func (w *Service) createNotifyBody(weChatNotification *wechatNotification) (cont return buffer.String(), nil } + +func getHTMLTestReport(task *task.Task) []string { + if task.Type != config.WorkflowType { + return nil + } + + testNames := make([]string, 0) + for _, stage := range task.Stages { + if stage.TaskType != config.TaskTestingV2 { + continue + } + + for testName, subTask := range stage.SubTasks { + testInfo, err := base.ToTestingTask(subTask) + if err != nil { + log.Errorf("parse testInfo failed, err:%s", err) + continue + } + + if testInfo.JobCtx.TestType == setting.FunctionTest && testInfo.JobCtx.TestReportPath != "" { + testNames = append(testNames, testName) + } + } + } + + return testNames +} diff --git a/pkg/microservice/aslan/core/cron/handler/cron.go b/pkg/microservice/aslan/core/cron/handler/cron.go index eaf2ed89e6..abe686bec6 100644 --- a/pkg/microservice/aslan/core/cron/handler/cron.go +++ b/pkg/microservice/aslan/core/cron/handler/cron.go @@ -21,7 +21,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" cronservice "github.com/koderover/zadig/pkg/microservice/aslan/core/cron/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func CleanJobCronJob(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/cron/handler/router.go b/pkg/microservice/aslan/core/cron/handler/router.go index 21232e85a9..9d06de008e 100644 --- a/pkg/microservice/aslan/core/cron/handler/router.go +++ b/pkg/microservice/aslan/core/cron/handler/router.go @@ -19,13 +19,13 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) // --------------------------------------------------------------------------------------- // 定时任务管理接口 diff --git a/pkg/microservice/aslan/core/delivery/handler/artifact.go b/pkg/microservice/aslan/core/delivery/handler/artifact.go index f29a778ebc..75b11324bf 100644 --- a/pkg/microservice/aslan/core/delivery/handler/artifact.go +++ b/pkg/microservice/aslan/core/delivery/handler/artifact.go @@ -25,7 +25,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" deliveryservice "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/delivery/handler/product.go b/pkg/microservice/aslan/core/delivery/handler/product.go index 0c142891df..59eb70b2ce 100644 --- a/pkg/microservice/aslan/core/delivery/handler/product.go +++ b/pkg/microservice/aslan/core/delivery/handler/product.go @@ -22,7 +22,7 @@ import ( "github.com/gin-gonic/gin" deliveryservice "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -38,3 +38,16 @@ func ListDeliveryProduct(c *gin.Context) { } ctx.Resp, ctx.Err = deliveryservice.FindDeliveryProduct(orgID, ctx.Logger) } + +func GetProductByDeliveryInfo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + releaseID := c.Param("releaseId") + if releaseID == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("releaseId can't be empty!") + return + } + + ctx.Resp, ctx.Err = deliveryservice.GetProductByDeliveryInfo(ctx.Username, releaseID, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/delivery/handler/router.go b/pkg/microservice/aslan/core/delivery/handler/router.go index 042ff75211..9bd3cfe463 100644 --- a/pkg/microservice/aslan/core/delivery/handler/router.go +++ b/pkg/microservice/aslan/core/delivery/handler/router.go @@ -19,14 +19,14 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) deliveryArtifact := router.Group("artifacts") { @@ -40,14 +40,15 @@ func (*Router) Inject(router *gin.RouterGroup) { deliveryProduct := router.Group("products") { deliveryProduct.GET("", ListDeliveryProduct) + deliveryProduct.GET("/:releaseId", GetProductByDeliveryInfo) } deliveryRelease := router.Group("releases") { - deliveryRelease.POST("", middleware.IsHavePermission([]string{permission.WorkflowDeliveryUUID}, permission.QueryType), middleware.UpdateOperationLogStatus, AddDeliveryVersion) + deliveryRelease.POST("", gin2.IsHavePermission([]string{permission.WorkflowDeliveryUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, AddDeliveryVersion) deliveryRelease.GET("/:id", GetDeliveryVersion) deliveryRelease.GET("", ListDeliveryVersion) - deliveryRelease.DELETE("/:id", GetProductNameByDelivery, middleware.IsHavePermission([]string{permission.ReleaseDeleteUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, DeleteDeliveryVersion) + deliveryRelease.DELETE("/:id", GetProductNameByDelivery, gin2.IsHavePermission([]string{permission.ReleaseDeleteUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, DeleteDeliveryVersion) } deliveryPackage := router.Group("packages") diff --git a/pkg/microservice/aslan/core/delivery/handler/security.go b/pkg/microservice/aslan/core/delivery/handler/security.go index 58a7897083..8a19d15789 100644 --- a/pkg/microservice/aslan/core/delivery/handler/security.go +++ b/pkg/microservice/aslan/core/delivery/handler/security.go @@ -25,7 +25,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" deliveryservice "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/delivery/handler/version.go b/pkg/microservice/aslan/core/delivery/handler/version.go index 6aa8f35b55..c854f69d7e 100644 --- a/pkg/microservice/aslan/core/delivery/handler/version.go +++ b/pkg/microservice/aslan/core/delivery/handler/version.go @@ -30,9 +30,10 @@ import ( taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" deliveryservice "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" @@ -248,7 +249,7 @@ func ListDeliveryVersion(c *gin.Context) { if subStage.TaskType == config.TaskSecurity { subSecurityTaskMap := subStage.SubTasks for _, subTask := range subSecurityTaskMap { - securityInfo, _ := commonservice.ToSecurityTask(subTask) + securityInfo, _ := base.ToSecurityTask(subTask) deliverySecurityStats := new(DeliverySecurityStats) deliverySecurityStats.ImageName = securityInfo.ImageName diff --git a/pkg/microservice/aslan/core/delivery/service/product.go b/pkg/microservice/aslan/core/delivery/service/product.go index b5a28c92c4..9b2fdf8b7f 100644 --- a/pkg/microservice/aslan/core/delivery/service/product.go +++ b/pkg/microservice/aslan/core/delivery/service/product.go @@ -19,6 +19,7 @@ package service import ( "go.uber.org/zap" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -32,3 +33,14 @@ func FindDeliveryProduct(orgID int, log *zap.SugaredLogger) ([]string, error) { return productNames, err } + +func GetProductByDeliveryInfo(username, releaseID string, log *zap.SugaredLogger) (*commonmodels.Product, error) { + version := new(commonrepo.DeliveryVersionArgs) + version.ID = releaseID + deliveryVersion, err := GetDeliveryVersion(version, log) + if err != nil { + log.Errorf("[User:%s][releaseID:%s] GetDeliveryVersion error: %v", username, releaseID, err) + return nil, e.ErrGetEnv + } + return deliveryVersion.ProductEnvInfo, nil +} diff --git a/pkg/microservice/aslan/core/enterprise/handler/notification.go b/pkg/microservice/aslan/core/enterprise/handler/notification.go deleted file mode 100644 index 17877629be..0000000000 --- a/pkg/microservice/aslan/core/enterprise/handler/notification.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package handler - -import ( - "github.com/gin-gonic/gin" - - enterpriseservice "github.com/koderover/zadig/pkg/microservice/aslan/core/enterprise/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" - e "github.com/koderover/zadig/pkg/tool/errors" -) - -type deleteNotificationsArgs struct { - IDs []string `json:"ids"` -} - -func DeleteNotifies(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() - - args := new(deleteNotificationsArgs) - - if err := c.BindJSON(args); err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("invalid deletenotificationsargs") - return - } - ctx.Err = enterpriseservice.DeleteNotifies(ctx.Username, args.IDs, ctx.Logger) -} - -func PullNotify(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() - - ctx.Resp, ctx.Err = enterpriseservice.PullNotify(ctx.Username, ctx.Logger) -} - -type readNotificationsArgs struct { - IDs []string `json:"ids"` -} - -func ReadNotify(c *gin.Context) { - ctx := internalhandler.NewContext(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(readNotificationsArgs) - - err := c.BindJSON(args) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc("invalid readnotificationsargs") - return - } - ctx.Err = enterpriseservice.ReadNotify(ctx.Username, args.IDs, ctx.Logger) -} diff --git a/pkg/microservice/aslan/core/enterprise/service/notification.go b/pkg/microservice/aslan/core/enterprise/service/notification.go deleted file mode 100644 index 3402d91284..0000000000 --- a/pkg/microservice/aslan/core/enterprise/service/notification.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "go.uber.org/zap" - - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/notify" - e "github.com/koderover/zadig/pkg/tool/errors" -) - -func DeleteNotifies(user string, notifyIDs []string, log *zap.SugaredLogger) error { - if err := notify.NewNotifyClient().DeleteNotifies(user, notifyIDs); err != nil { - log.Errorf("NotifyCli.DeleteNotifies error: %v", err) - return e.ErrDeleteNotifies - } - return nil -} - -func PullNotify(user string, log *zap.SugaredLogger) ([]*commonmodels.Notify, error) { - resp, err := notify.NewNotifyClient().PullNotify(user) - if err != nil { - log.Errorf("NotifyCli.PullNotify error: %v", err) - return resp, e.ErrPullNotify - } - return resp, nil -} - -func ReadNotify(user string, notifyIDs []string, log *zap.SugaredLogger) error { - if err := notify.NewNotifyClient().Read(user, notifyIDs); err != nil { - log.Errorf("NotifyCli.Read error: %v", err) - return e.ErrReadNotify - } - return nil -} diff --git a/pkg/microservice/aslan/core/environment/handler/configmap.go b/pkg/microservice/aslan/core/environment/handler/configmap.go index 6800cd5ee1..fbbe61a416 100644 --- a/pkg/microservice/aslan/core/environment/handler/configmap.go +++ b/pkg/microservice/aslan/core/environment/handler/configmap.go @@ -25,7 +25,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/environment/handler/diff.go b/pkg/microservice/aslan/core/environment/handler/diff.go index 7efa6d9e3e..f3dd4809f4 100644 --- a/pkg/microservice/aslan/core/environment/handler/diff.go +++ b/pkg/microservice/aslan/core/environment/handler/diff.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) //func ConfigDiff(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/environment/handler/environment.go b/pkg/microservice/aslan/core/environment/handler/environment.go index f6a3976394..30b988bd3c 100644 --- a/pkg/microservice/aslan/core/environment/handler/environment.go +++ b/pkg/microservice/aslan/core/environment/handler/environment.go @@ -29,13 +29,13 @@ import ( "github.com/gin-gonic/gin" "k8s.io/apimachinery/pkg/util/wait" - "github.com/koderover/zadig/pkg/internal/kube/resource" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + "github.com/koderover/zadig/pkg/shared/kube/resource" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" @@ -184,6 +184,74 @@ func UpdateProductRecycleDay(c *gin.Context) { ctx.Err = service.UpdateProductRecycleDay(envName, productName, recycleDay) } +func UpdateHelmProduct(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Query("envName") + updateType := c.Query("updateType") + productName := c.Param("productName") + + ctx.Err = service.UpdateHelmProduct(productName, envName, updateType, ctx.Username, ctx.RequestID, ctx.Logger) + if ctx.Err != nil { + ctx.Logger.Errorf("failed to update product %s %s: %v", envName, productName, ctx.Err) + } +} + +func UpdateHelmProductVariable(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Query("envName") + productName := c.Param("productName") + + args := new(ChartInfoArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateHelmProductVariable c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdateHelmProductVariable json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新", "helm集成环境变量", "", permission.TestEnvManageUUID, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Logger.Error(err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Err = service.UpdateHelmProductVariable(productName, envName, ctx.Username, ctx.RequestID, args.ChartInfos, ctx.Logger) + if ctx.Err != nil { + ctx.Logger.Errorf("failed to update product Variable %s %s: %v", envName, productName, ctx.Err) + } +} + +func UpdateMultiHelmProduct(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(UpdateEnvs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateMultiHelmProduct c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdateMultiHelmProduct json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新helm环境", "集成环境", strings.Join(args.EnvNames, ","), permission.TestEnvManageUUID, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Logger.Error(err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + ctx.Resp = service.UpdateMultiHelmProduct(args.EnvNames, args.UpdateType, c.Param("productName"), ctx.User.ID, ctx.User.IsSuperUser, ctx.RequestID, ctx.Logger) +} + func GetProduct(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -225,6 +293,19 @@ func ListRenderCharts(c *gin.Context) { } } +func GetHelmChartVersions(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + envName := c.Query("envName") + productName := c.Param("productName") + + ctx.Resp, ctx.Err = service.GetHelmChartVersions(productName, envName, ctx.Logger) + if ctx.Err != nil { + ctx.Logger.Errorf("failed to get helmVersions %s %s: %v", envName, productName, ctx.Err) + } +} + func DeleteProduct(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -234,6 +315,40 @@ func DeleteProduct(c *gin.Context) { ctx.Err = commonservice.DeleteProduct(ctx.Username, envName, c.Param("productName"), ctx.RequestID, ctx.Logger) } +func EnvShare(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + productName := c.Param("productName") + + args := new(service.ProductParams) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreateProduct c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreateProduct json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, productName, "更新", "集成环境-环境授权", args.EnvName, permission.TestEnvShareUUID, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if productName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("productName can't be empty!") + return + } + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + + if args.EnvName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("envName can't be empty!") + return + } + + ctx.Err = service.UpdateProductPublic(productName, args, ctx.Logger) +} + func ListGroups(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() diff --git a/pkg/microservice/aslan/core/environment/handler/export.go b/pkg/microservice/aslan/core/environment/handler/export.go index 8f0908849c..6f0a9917c5 100644 --- a/pkg/microservice/aslan/core/environment/handler/export.go +++ b/pkg/microservice/aslan/core/environment/handler/export.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func ExportYaml(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/environment/handler/image.go b/pkg/microservice/aslan/core/environment/handler/image.go index d23b01cd3e..196008f803 100644 --- a/pkg/microservice/aslan/core/environment/handler/image.go +++ b/pkg/microservice/aslan/core/environment/handler/image.go @@ -25,8 +25,8 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/environment/handler/kube.go b/pkg/microservice/aslan/core/environment/handler/kube.go index c7ba13eec3..2e3d589d06 100644 --- a/pkg/microservice/aslan/core/environment/handler/kube.go +++ b/pkg/microservice/aslan/core/environment/handler/kube.go @@ -22,8 +22,8 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/types/permission" ) diff --git a/pkg/microservice/aslan/core/environment/handler/product.go b/pkg/microservice/aslan/core/environment/handler/product.go index 20706aeedd..7ba604f02f 100644 --- a/pkg/microservice/aslan/core/environment/handler/product.go +++ b/pkg/microservice/aslan/core/environment/handler/product.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func CleanProductCronJob(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/environment/handler/revision.go b/pkg/microservice/aslan/core/environment/handler/revision.go index 7c3bc8688c..a299d157da 100644 --- a/pkg/microservice/aslan/core/environment/handler/revision.go +++ b/pkg/microservice/aslan/core/environment/handler/revision.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func ListProductsRevision(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/environment/handler/router.go b/pkg/microservice/aslan/core/environment/handler/router.go index 990e0c6c8e..694b8643a3 100644 --- a/pkg/microservice/aslan/core/environment/handler/router.go +++ b/pkg/microservice/aslan/core/environment/handler/router.go @@ -19,14 +19,14 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) // --------------------------------------------------------------------------------------- // Kube配置管理接口 ConfigMap @@ -34,8 +34,8 @@ func (*Router) Inject(router *gin.RouterGroup) { configmaps := router.Group("configmaps") { configmaps.GET("", ListConfigMaps) - configmaps.PUT("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateConfigMap) - configmaps.POST("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, RollBackConfigMap) + configmaps.PUT("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateConfigMap) + configmaps.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RollBackConfigMap) } // --------------------------------------------------------------------------------------- @@ -60,7 +60,7 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- export := router.Group("export") { - export.GET("/service", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.QueryType), ExportYaml) + export.GET("/service", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.QueryType), ExportYaml) // export.GET("/pipelines/:name", ExportBuildYaml) } @@ -70,8 +70,8 @@ func (*Router) Inject(router *gin.RouterGroup) { image := router.Group("image") { // image.POST("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateContainerImage) - image.POST("/deployment", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateDeploymentContainerImage) - image.POST("/statefulset", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateStatefulSetContainerImage) + image.POST("/deployment", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateDeploymentContainerImage) + image.POST("/statefulset", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateStatefulSetContainerImage) } // 查询环境创建时的服务和变量信息 @@ -87,7 +87,7 @@ func (*Router) Inject(router *gin.RouterGroup) { kube.GET("/events", ListKubeEvents) kube.POST("/pods", ListServicePods) - kube.DELETE("/pods/:podName", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.QueryType), middleware.UpdateOperationLogStatus, DeletePod) + kube.DELETE("/pods/:podName", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeletePod) kube.GET("/pods/:podName/events", ListPodEvents) } @@ -97,26 +97,31 @@ func (*Router) Inject(router *gin.RouterGroup) { environments := router.Group("environments") { environments.GET("", ListProducts) - environments.POST("/:productName/auto", middleware.IsHavePermission([]string{permission.TestEnvCreateUUID}, permission.ParamType), AutoCreateProduct) - environments.PUT("/:productName/autoUpdate", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, AutoUpdateProduct) - environments.POST("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.TestEnvCreateUUID, permission.ProdEnvCreateUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateProduct) - environments.POST("/:productName", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, UpdateProduct) - environments.PUT("/:productName/envRecycle", middleware.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, UpdateProductRecycleDay) + environments.POST("/:productName/auto", gin2.IsHavePermission([]string{permission.TestEnvCreateUUID}, permission.ParamType), AutoCreateProduct) + environments.PUT("/:productName/autoUpdate", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, AutoUpdateProduct) + environments.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.TestEnvCreateUUID, permission.ProdEnvCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateProduct) + environments.POST("/:productName", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProduct) + environments.PUT("/:productName/envRecycle", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProductRecycleDay) + environments.PUT("/:productName/helmEnv", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateHelmProduct) + environments.PUT("/:productName/helmEnvVariable", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateHelmProductVariable) + environments.GET("/:productName/helmChartVersions", GetHelmChartVersions) + environments.PUT("/:productName", gin2.IsHavePermission([]string{permission.TestEnvShareUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, EnvShare) + environments.PUT("/:productName/updateMultiEnv", gin2.IsHavePermission([]string{permission.TestEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateMultiHelmProduct) environments.GET("/:productName", GetProduct) environments.GET("/:productName/productInfo", GetProductInfo) environments.GET("/:productName/ingressInfo", GetProductIngress) environments.GET("/:productName/helmRenderCharts", ListRenderCharts) - environments.DELETE("/:productName", middleware.IsHavePermission([]string{permission.TestEnvDeleteUUID, permission.ProdEnvDeleteUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, DeleteProduct) + environments.DELETE("/:productName", gin2.IsHavePermission([]string{permission.TestEnvDeleteUUID, permission.ProdEnvDeleteUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, DeleteProduct) environments.GET("/:productName/groups", ListGroups) environments.GET("/:productName/groups/:source", ListGroupsBySource) environments.GET("/:productName/services/:serviceName", GetService) - environments.PUT("/:productName/services/:serviceName/:serviceType", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, UpdateService) - environments.POST("/:productName/services/:serviceName/restart", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, RestartService) - environments.POST("/:productName/services/:serviceName/restartNew", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, RestartNewService) - environments.POST("/:productName/services/:serviceName/scale/:number", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, ScaleService) - environments.POST("/:productName/services/:serviceName/scaleNew/:number", middleware.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, ScaleNewService) + environments.PUT("/:productName/services/:serviceName/:serviceType", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateService) + environments.POST("/:productName/services/:serviceName/restart", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, RestartService) + environments.POST("/:productName/services/:serviceName/restartNew", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID, permission.TestUpdateEnvUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, RestartNewService) + environments.POST("/:productName/services/:serviceName/scale/:number", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, ScaleService) + environments.POST("/:productName/services/:serviceName/scaleNew/:number", gin2.IsHavePermission([]string{permission.TestEnvManageUUID, permission.ProdEnvManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, ScaleNewService) environments.GET("/:productName/services/:serviceName/containers/:container/namespaces/:namespace", GetServiceContainer) } diff --git a/pkg/microservice/aslan/core/environment/handler/service.go b/pkg/microservice/aslan/core/environment/handler/service.go index 4b7b7a2ac3..0a13c5cfff 100644 --- a/pkg/microservice/aslan/core/environment/handler/service.go +++ b/pkg/microservice/aslan/core/environment/handler/service.go @@ -23,8 +23,8 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/types/permission" ) diff --git a/pkg/microservice/aslan/core/environment/service/configmap.go b/pkg/microservice/aslan/core/environment/service/configmap.go index d1164860a1..045d23381c 100644 --- a/pkg/microservice/aslan/core/environment/service/configmap.go +++ b/pkg/microservice/aslan/core/environment/service/configmap.go @@ -29,11 +29,11 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/updater" diff --git a/pkg/microservice/aslan/core/environment/service/env.go b/pkg/microservice/aslan/core/environment/service/env.go index c369f46982..477694aa94 100644 --- a/pkg/microservice/aslan/core/environment/service/env.go +++ b/pkg/microservice/aslan/core/environment/service/env.go @@ -38,6 +38,10 @@ func envHandleFunc(projectType string, log *zap.SugaredLogger) envHandle { return &K8sService{ log: log, } + case setting.PMDeployType: + return &PMService{ + log: log, + } } return nil } diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index 6d59393d44..9752f9c3a7 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/go-multierror" "go.uber.org/zap" "helm.sh/helm/v3/pkg/releaseutil" + "helm.sh/helm/v3/pkg/strvals" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -41,9 +42,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" @@ -51,14 +51,17 @@ import ( templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gerrit" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/helmclient" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/serializer" "github.com/koderover/zadig/pkg/tool/kube/updater" "github.com/koderover/zadig/pkg/types/permission" + "github.com/koderover/zadig/pkg/util/converter" ) const ( @@ -99,6 +102,32 @@ type ProductParams struct { PermissionUUIDs []string `json:"permissionUUIDs"` } +func UpdateProductPublic(productName string, args *ProductParams, log *zap.SugaredLogger) error { + err := commonrepo.NewProductColl().UpdateIsPublic(args.EnvName, productName, args.IsPublic) + if err != nil { + log.Errorf("UpdateProductPublic error: %v", err) + return fmt.Errorf("UpdateProductPublic error: %v", err) + } + + poetryCtl := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) + if !args.IsPublic { //把公开设置成不公开 + _, err := poetryCtl.AddEnvRolePermission(productName, args.EnvName, args.PermissionUUIDs, args.RoleID, log) + if err != nil { + log.Errorf("UpdateProductPublic AddEnvRole error: %v", err) + return fmt.Errorf("UpdateProductPublic AddEnvRole error: %v", err) + } + return nil + } + //把不公开设成公开 删除原来环境绑定的角色 + _, err = poetryCtl.DeleteEnvRolePermission(productName, args.EnvName, log) + if err != nil { + log.Errorf("UpdateProductPublic DeleteEnvRole error: %v", err) + return fmt.Errorf("UpdateProductPublic DeleteEnvRole error: %v", err) + } + + return nil +} + func ListProducts(productNameParam, envType string, userName string, userID int, superUser bool, log *zap.SugaredLogger) ([]*ProductResp, error) { var ( err error @@ -276,7 +305,17 @@ func ListProducts(productNameParam, envType string, userName string, userID int, func FillProductVars(products []*commonmodels.Product, log *zap.SugaredLogger) error { for _, product := range products { - renderSet, err := commonservice.GetRenderSet(product.Namespace, 0, log) + if product.Source == setting.SourceFromExternal || product.Source == setting.SourceFromHelm { + continue + } + renderName := product.Namespace + var revision int64 + // if the environment is backtracking, render.name will be different with product.Namespace + if product.Render != nil && product.Render.Name != renderName { + renderName = product.Render.Name + revision = product.Render.Revision + } + renderSet, err := commonservice.GetRenderSet(renderName, revision, log) if err != nil { log.Errorf("Failed to find render set, productName: %s, namespace: %s, err: %s", product.ProductName, product.Namespace, err) return e.ErrGetRenderSet.AddDesc(err.Error()) @@ -792,6 +831,68 @@ func CreateProduct(user, requestID string, args *commonmodels.Product, log *zap. log.Errorf("[%s][%s] create product record error: %v", args.EnvName, args.ProductName, err) return e.ErrCreateEnv.AddDesc(err.Error()) } + case setting.SourceFromHelm: + kubeClient, err := kube.GetKubeClient(args.ClusterID) + if err != nil { + log.Errorf("[%s][%s] GetKubeClient error: %v", args.EnvName, args.ProductName, err) + return e.ErrCreateEnv.AddErr(err) + } + + //判断namespace是否存在 + namespace := args.GetNamespace() + args.Namespace = namespace + _, found, err := getter.GetNamespace(namespace, kubeClient) + if err != nil { + log.Errorf("GetNamespace error: %v", err) + return e.ErrCreateEnv.AddDesc(err.Error()) + } + if found { + log.Warnf("%s[%s]%s", "namespace", namespace, "已经存在,请换个环境名称尝试!") + return e.ErrCreateEnv.AddDesc(fmt.Sprintf("%s[%s]%s", "namespace", namespace, "已经存在,请换个环境名称尝试!")) + } + + restConfig, err := kube.GetRESTConfig(args.ClusterID) + if err != nil { + log.Errorf("GetRESTConfig error: %v", err) + return e.ErrCreateEnv.AddDesc(err.Error()) + } + helmClient, err := helmclient.NewClientFromRestConf(restConfig, namespace) + if err != nil { + log.Errorf("[%s][%s] NewClientFromRestConf error: %v", args.EnvName, args.ProductName, err) + return e.ErrCreateEnv.AddErr(err) + } + + if err := preCreateProduct(args.EnvName, args, kubeClient, log); err != nil { + log.Errorf("CreateProduct preCreateProduct error: %v", err) + return e.ErrCreateEnv.AddDesc(err.Error()) + } + eventStart := time.Now().Unix() + if args.Render == nil { + args.Render = &commonmodels.RenderInfo{ProductTmpl: args.ProductName} + } + + renderSet, err := FindHelmRenderSet(args.ProductName, args.Render.Name, log) + if err != nil { + log.Errorf("[%s][P:%s] find product renderset error: %v", args.EnvName, args.ProductName, err) + return e.ErrCreateEnv.AddDesc(err.Error()) + } + // 设置产品render revsion + args.Render.Revision = renderSet.Revision + // 记录服务当前对应render版本 + setServiceRender(args) + + args.Status = setting.ProductStatusCreating + args.RecycleDay = config.DefaultRecycleDay() + if args.IsForkedProduct { + args.RecycleDay = 7 + } + err = commonrepo.NewProductColl().Create(args) + if err != nil { + log.Errorf("[%s][%s] create product record error: %v", args.EnvName, args.ProductName, err) + return e.ErrCreateEnv.AddDesc(err.Error()) + } + + go installOrUpdateHelmChart(user, args.EnvName, requestID, args, eventStart, helmClient, log) default: kubeClient, err := kube.GetKubeClient(args.ClusterID) if err != nil { @@ -862,6 +963,193 @@ func UpdateProductRecycleDay(envName, productName string, recycleDay int) error return commonrepo.NewProductColl().UpdateProductRecycleDay(envName, productName, recycleDay) } +func UpdateHelmProduct(productName, envName, updateType, username, requestID string, log *zap.SugaredLogger) error { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + productResp, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("GetProduct envName:%s, productName:%s, err:%+v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(err.Error()) + } + currentProductService := productResp.Services + // 查找产品模板 + updateProd, err := GetInitProduct(productName, log) + if err != nil { + log.Errorf("[%s][P:%s] GetProductTemplate error: %v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(e.FindProductTmplErrMsg) + } + productResp.Services = updateProd.Services + + // 设置产品状态为更新中 + if err := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUpdating); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(e.UpdateEnvStatusErrMsg) + } + //对比当前环境中的环境变量和默认的环境变量 + go func() { + err := updateProductGroup(productName, envName, updateType, productResp, currentProductService, log) + if err != nil { + log.Errorf("[%s][P:%s] failed to update product %#v", envName, productName, err) + // 发送更新产品失败消息给用户 + title := fmt.Sprintf("更新 [%s] 的 [%s] 环境失败", productName, envName) + commonservice.SendErrorMessage(username, title, requestID, err, log) + + // 设置产品状态 + log.Infof("[%s][P:%s] update status to => %s", envName, productName, setting.ProductStatusFailed) + if err2 := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusFailed); err2 != nil { + log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err2) + return + } + + log.Infof("[%s][P:%s] update error to => %s", envName, productName, e.String(err)) + if err2 := commonrepo.NewProductColl().UpdateErrors(envName, productName, e.String(err)); err2 != nil { + log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err2) + return + } + } else { + productResp.Status = setting.ProductStatusSuccess + + if err = commonrepo.NewProductColl().UpdateStatus(envName, productName, productResp.Status); err != nil { + log.Errorf("[%s][%s] Product.Update error: %v", envName, productName, err) + return + } + + if err = commonrepo.NewProductColl().UpdateErrors(envName, productName, ""); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err) + return + } + } + }() + return nil +} + +func UpdateHelmProductVariable(productName, envName, username, requestID string, rcs []*template.RenderChart, log *zap.SugaredLogger) error { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + productResp, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("GetProduct envName:%s, productName:%s, err:%+v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(err.Error()) + } + + productResp.ChartInfos = rcs + if err = commonservice.CreateHelmRenderSet( + &commonmodels.RenderSet{ + Name: productResp.Namespace, + EnvName: envName, + ProductTmpl: productName, + UpdateBy: username, + ChartInfos: rcs, + }, + log, + ); err != nil { + log.Errorf("[%s][P:%s] create renderset error: %v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(e.FindProductTmplErrMsg) + } + + if productResp.Render == nil { + productResp.Render = &commonmodels.RenderInfo{ProductTmpl: productResp.ProductName} + } + + renderSet, err := FindHelmRenderSet(productResp.ProductName, productResp.Namespace, log) + if err != nil { + log.Errorf("[%s][P:%s] find product renderset error: %v", productResp.EnvName, productResp.ProductName, err) + return e.ErrCreateEnv.AddDesc(err.Error()) + } + productResp.Render.Revision = renderSet.Revision + + // 设置产品状态为更新中 + if err := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusUpdating); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err) + return e.ErrUpdateEnv.AddDesc(e.UpdateEnvStatusErrMsg) + } + + go func() { + err := updateProductVariable(productName, envName, productResp, log) + if err != nil { + log.Errorf("[%s][P:%s] failed to update product %#v", envName, productName, err) + // 发送更新产品失败消息给用户 + title := fmt.Sprintf("更新 [%s] 的 [%s] 环境失败", productName, envName) + commonservice.SendErrorMessage(username, title, requestID, err, log) + + // 设置产品状态 + log.Infof("[%s][P:%s] update status to => %s", envName, productName, setting.ProductStatusFailed) + if err2 := commonrepo.NewProductColl().UpdateStatus(envName, productName, setting.ProductStatusFailed); err2 != nil { + log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, productName, err2) + return + } + + log.Infof("[%s][P:%s] update error to => %s", envName, productName, e.String(err)) + if err2 := commonrepo.NewProductColl().UpdateErrors(envName, productName, e.String(err)); err2 != nil { + log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err2) + return + } + } else { + productResp.Status = setting.ProductStatusSuccess + + if err = commonrepo.NewProductColl().UpdateStatus(envName, productName, productResp.Status); err != nil { + log.Errorf("[%s][%s] Product.Update error: %v", envName, productName, err) + return + } + + if err = commonrepo.NewProductColl().UpdateErrors(envName, productName, ""); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, productName, err) + return + } + } + }() + + return nil +} + +var mutexUpdateMultiHelm sync.RWMutex + +func UpdateMultiHelmProduct(envNames []string, updateType, productName string, userID int, superUser bool, requestID string, log *zap.SugaredLogger) []*EnvStatus { + mutexUpdateMultiHelm.Lock() + defer func() { + mutexUpdateMultiHelm.Unlock() + }() + + envStatuses := make([]*EnvStatus, 0) + productsRevison, err := ListProductsRevision("", "", userID, superUser, log) + if err != nil { + log.Errorf("UpdateMultiHelmProduct ListProductsRevision err:%v", err) + return envStatuses + } + productMap := make(map[string]*ProductRevision) + for _, productRevison := range productsRevison { + if productRevison.ProductName == productName && sets.NewString(envNames...).Has(productRevison.EnvName) && productRevison.Updatable { + productMap[productRevison.EnvName] = productRevison + if len(productMap) == len(envNames) { + break + } + } + } + + for envName := range productMap { + err = UpdateHelmProduct(productName, envName, updateType, setting.SystemUser, requestID, log) + if err != nil { + log.Errorf("UpdateMultiHelmProduct UpdateProductV2 err:%v", err) + return envStatuses + } + } + + productResps := make([]*ProductResp, 0) + for _, envName := range envNames { + productResp, err := GetProduct(setting.SystemUser, envName, productName, log) + if err == nil && productResp != nil { + productResps = append(productResps, productResp) + } + } + + for _, productResp := range productResps { + if productResp.Error != "" { + envStatuses = append(envStatuses, &EnvStatus{EnvName: productResp.EnvName, Status: setting.ProductStatusFailed, ErrMessage: productResp.Error}) + continue + } + envStatuses = append(envStatuses, &EnvStatus{EnvName: productResp.EnvName, Status: productResp.Status}) + } + return envStatuses +} + func GetProductInfo(username, envName, productName string, log *zap.SugaredLogger) (*commonmodels.Product, error) { opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} prod, err := commonrepo.NewProductColl().Find(opt) @@ -933,6 +1221,56 @@ func ListRenderCharts(productName, envName string, log *zap.SugaredLogger) ([]*t return renderSet.ChartInfos, nil } +func GetHelmChartVersions(productName, envName string, log *zap.SugaredLogger) ([]*commonmodels.HelmVersions, error) { + var ( + helmVersions = make([]*commonmodels.HelmVersions, 0) + chartInfoMap = make(map[string]*template.RenderChart) + ) + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("[EnvName:%s][Product:%s] Product.FindByOwner error: %v", envName, productName, err) + return nil, e.ErrGetEnv + } + + renderSetName := commonservice.GetProductEnvNamespace(envName, productName) + renderSetOpt := &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: prod.Render.Revision} + renderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + if err != nil { + log.Errorf("find helm renderset[%s] error: %v", renderSetName, err) + return helmVersions, err + } + + for _, chartInfo := range renderSet.ChartInfos { + chartInfoMap[chartInfo.ServiceName] = chartInfo + } + + for serviceName, chartInfo := range chartInfoMap { + opt := &commonrepo.ServiceFindOption{ + ServiceName: serviceName, + Type: setting.HelmDeployType, + ProductName: productName, + } + + respService, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + log.Warnf("系统未找到当前类型[%s]的服务[%s]!", setting.HelmDeployType, serviceName) + continue + } + + helmVersion := &commonmodels.HelmVersions{ + ServiceName: serviceName, + CurrentVersion: chartInfo.ChartVersion, + CurrentValuesYaml: chartInfo.ValuesYaml, + LatestVersion: respService.HelmChart.Version, + LatestValuesYaml: respService.HelmChart.ValuesYaml, + } + helmVersions = append(helmVersions, helmVersion) + } + + return helmVersions, nil +} + func createGroups(envName, user, requestID string, args *commonmodels.Product, eventStart int64, renderSet *commonmodels.RenderSet, kubeClient client.Client, log *zap.SugaredLogger) { var err error defer func() { @@ -1560,6 +1898,18 @@ func preCreateProduct(envName string, args *commonmodels.Product, kubeClient cli renderSetName = args.Render.Name } else { switch args.Source { + case setting.HelmDeployType: + err = commonservice.CreateHelmRenderSet( + &commonmodels.RenderSet{ + Name: renderSetName, + Revision: 0, + EnvName: envName, + ProductTmpl: args.ProductName, + UpdateBy: args.UpdateBy, + ChartInfos: args.ChartInfos, + }, + log, + ) default: err = commonservice.CreateRenderSet( &commonmodels.RenderSet{ @@ -1754,6 +2104,111 @@ func ensureKubeEnv(namespace string, kubeClient client.Client, log *zap.SugaredL return nil } +func FindHelmRenderSet(productName, renderName string, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { + resp := &commonmodels.RenderSet{ProductTmpl: productName} + var err error + if renderName != "" { + opt := &commonrepo.RenderSetFindOption{Name: renderName} + resp, err = commonrepo.NewRenderSetColl().Find(opt) + if err != nil { + log.Errorf("find helm renderset[%s] error: %v", renderName, err) + return resp, err + } + } + if renderName != "" && resp.ProductTmpl != productName { + log.Errorf("helm renderset[%s] not match product[%s]", renderName, productName) + return resp, fmt.Errorf("helm renderset[%s] not match product[%s]", renderName, productName) + } + return resp, nil +} + +func installOrUpdateHelmChart(user, envName, requestID string, args *commonmodels.Product, eventStart int64, helmClient helmclient.Client, log *zap.SugaredLogger) { + var ( + err error + wg sync.WaitGroup + errList = &multierror.Error{} + ) + + defer func() { + status := setting.ProductStatusSuccess + errorMsg := "" + if err != nil { + status = setting.ProductStatusFailed + errorMsg = err.Error() + + // 发送创建产品失败消息给用户 + title := fmt.Sprintf("创建 [%s] 的 [%s] 环境失败", args.ProductName, args.EnvName) + commonservice.SendErrorMessage(user, title, requestID, err, log) + } + + commonservice.LogProductStats(envName, setting.CreateProductEvent, args.ProductName, requestID, eventStart, log) + + if err := commonrepo.NewProductColl().UpdateStatus(envName, args.ProductName, status); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateStatus error: %v", envName, args.ProductName, err) + return + } + if err := commonrepo.NewProductColl().UpdateErrors(envName, args.ProductName, errorMsg); err != nil { + log.Errorf("[%s][P:%s] Product.UpdateErrors error: %v", envName, args.ProductName, err) + return + } + }() + chartInfoMap := make(map[string]*template.RenderChart) + for _, renderChart := range args.ChartInfos { + chartInfoMap[renderChart.ServiceName] = renderChart + } + for _, serviceGroups := range args.Services { + for _, service := range serviceGroups { + if renderChart, isExist := chartInfoMap[service.ServiceName]; isExist { + chartSpec := &helmclient.ChartSpec{ + ReleaseName: fmt.Sprintf("%s-%s", args.Namespace, service.ServiceName), + ChartName: fmt.Sprintf("%s/%s", args.Namespace, service.ServiceName), + Namespace: args.Namespace, + Wait: true, + Version: renderChart.ChartVersion, + ValuesYaml: renderChart.ValuesYaml, + Force: false, + SkipCRDs: false, + UpgradeCRDs: true, + Timeout: Timeout * time.Second * 10, + } + // 获取服务详情 + opt := &commonrepo.ServiceFindOption{ + ServiceName: service.ServiceName, + Type: service.Type, + Revision: service.Revision, + ProductName: args.ProductName, + ExcludeStatus: setting.ProductStatusDeleting, + } + serviceObj, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + continue + } + wg.Add(1) + go func(currentChartSpec *helmclient.ChartSpec, currentService *commonmodels.Service) { + defer wg.Done() + + base, err := gerrit.GetGerritWorkspaceBasePath(currentService.RepoName) + _, serviceFileErr := os.Stat(path.Join(base, currentService.LoadPath)) + if err != nil || os.IsNotExist(serviceFileErr) { + if err = commonservice.DownloadService(base, currentService.ServiceName); err != nil { + return + } + } + + if err = helmClient.InstallOrUpgradeChart(context.Background(), currentChartSpec, &helmclient.ChartOption{ + ChartPath: filepath.Join(base, currentService.LoadPath), + }, log); err != nil { + errList = multierror.Append(errList, err) + return + } + }(chartSpec, serviceObj) + } + } + } + wg.Wait() + err = errList.ErrorOrNil() +} + func setServiceRender(args *commonmodels.Product) { for _, serviceGroup := range args.Services { for _, service := range serviceGroup { @@ -1858,6 +2313,268 @@ func getUpdatedProductServices(updateProduct *commonmodels.Product, serviceRevis return updatedAllServices } +func updateProductGroup(productName, envName, updateType string, productResp *commonmodels.Product, currentProductServices [][]*commonmodels.ProductService, log *zap.SugaredLogger) error { + var ( + renderChartMap = make(map[string]*template.RenderChart) + productServiceMap = make(map[string]*commonmodels.ProductService) + productTemplServiceMap = make(map[string]*commonmodels.ProductService) + ) + restConfig, err := kube.GetRESTConfig(productResp.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + + helmClient, err := helmclient.NewClientFromRestConf(restConfig, productResp.Namespace) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + + for _, serviceGroup := range currentProductServices { + for _, service := range serviceGroup { + productServiceMap[service.ServiceName] = service + } + } + + for _, serviceGroup := range productResp.Services { + for _, service := range serviceGroup { + productTemplServiceMap[service.ServiceName] = service + } + } + + // 找到环境里面还存在但是服务编排里面已经删除的服务卸载掉 + for serviceName := range productServiceMap { + if _, isExist := productTemplServiceMap[serviceName]; !isExist { + go func(namespace, serviceName string) { + log.Infof("ready to uninstall release:%s", fmt.Sprintf("%s-%s", namespace, serviceName)) + if err = helmClient.UninstallRelease(&helmclient.ChartSpec{ + ReleaseName: fmt.Sprintf("%s-%s", namespace, serviceName), + Namespace: namespace, + Wait: true, + Force: true, + Timeout: Timeout * time.Second * 10, + }); err != nil { + log.Errorf("helm uninstall release %s err:%v", fmt.Sprintf("%s-%s", namespace, serviceName), err) + } + }(productResp.Namespace, serviceName) + } + } + + //比较当前环境中的变量和系统默认的最新变量 + renderSet, err := diffRenderSet(productName, envName, updateType, productResp, log) + if err != nil { + return e.ErrUpdateEnv.AddDesc("对比环境中的value.yaml和系统默认的value.yaml失败") + } + + for _, renderChart := range renderSet.ChartInfos { + renderChartMap[renderChart.ServiceName] = renderChart + } + + errList := new(multierror.Error) + for groupIndex, services := range productResp.Services { + var wg sync.WaitGroup + groupServices := make([]*commonmodels.ProductService, 0) + for _, service := range services { + opt := &commonrepo.ServiceFindOption{ + ServiceName: service.ServiceName, + Type: setting.HelmDeployType, + ProductName: productName, + Revision: service.Revision, + ExcludeStatus: setting.ProductStatusDeleting, + } + respService, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + log.Warnf("系统未找到当前类型[%s]的服务[%s]!", setting.HelmDeployType, service.ServiceName) + continue + } + if renderChart, isExist := renderChartMap[service.ServiceName]; isExist { + wg.Add(1) + go func(tmpRenderChart *template.RenderChart, currentService *commonmodels.Service) { + defer wg.Done() + chartSpec := helmclient.ChartSpec{ + ReleaseName: fmt.Sprintf("%s-%s", productResp.Namespace, tmpRenderChart.ServiceName), + ChartName: fmt.Sprintf("%s/%s", productResp.Namespace, tmpRenderChart.ServiceName), + Namespace: productResp.Namespace, + Wait: true, + Version: tmpRenderChart.ChartVersion, + ValuesYaml: tmpRenderChart.ValuesYaml, + Force: false, + SkipCRDs: false, + UpgradeCRDs: true, + Timeout: Timeout * time.Second * 10, + } + base, err := gerrit.GetGerritWorkspaceBasePath(currentService.RepoName) + _, serviceFileErr := os.Stat(path.Join(base, currentService.LoadPath)) + if err != nil || os.IsNotExist(serviceFileErr) { + if err = commonservice.DownloadService(base, currentService.ServiceName); err != nil { + return + } + } + err = helmClient.InstallOrUpgradeChart(context.Background(), &chartSpec, + &helmclient.ChartOption{ChartPath: filepath.Join(base, currentService.LoadPath)}, log) + + if err != nil { + log.Errorf("install helm chart %s error :%+v", chartSpec.ReleaseName, err) + errList = multierror.Append(errList, err) + } + }(renderChart, respService) + service.Revision = respService.Revision + groupServices = append(groupServices, service) + } + } + wg.Wait() + err = commonrepo.NewProductColl().UpdateGroup(envName, productName, groupIndex, groupServices) + if err != nil { + log.Errorf("[product.update] err: v%", err) + errList = multierror.Append(errList, err) + } + } + + productResp.Render.Revision = renderSet.Revision + if err = commonrepo.NewProductColl().Update(productResp); err != nil { + log.Errorf("[product.update] err: %v", err) + errList = multierror.Append(errList, err) + } + + return errList.ErrorOrNil() +} + +// diffRenderSet 对比环境中的renderSet的值和服务的最新的renderSet的值 +func diffRenderSet(productName, envName, updateType string, productResp *commonmodels.Product, log *zap.SugaredLogger) (*commonmodels.RenderSet, error) { + productTemp, err := templaterepo.NewProductColl().Find(productName) + if err != nil { + log.Errorf("[ProductTmpl.find] err: %v", err) + return nil, err + } + // 系统默认的变量 + latestRenderSet, err := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{Name: productName}) + if err != nil { + log.Errorf("[RenderSet.find] err: %v", err) + return nil, err + } + + latestRenderSetMap := make(map[string]*template.RenderChart) + for _, renderInfo := range latestRenderSet.ChartInfos { + latestRenderSetMap[renderInfo.ServiceName] = renderInfo + } + + newChartInfos := make([]*template.RenderChart, 0) + renderSetName := commonservice.GetProductEnvNamespace(envName, productName) + switch updateType { + case UpdateTypeSystem: + for _, serviceNameGroup := range productTemp.Services { + for _, serviceName := range serviceNameGroup { + if latestChartInfo, isExist := latestRenderSetMap[serviceName]; isExist { + newChartInfos = append(newChartInfos, latestChartInfo) + } + } + } + case UpdateTypeEnv: + renderSetOpt := &commonrepo.RenderSetFindOption{Name: renderSetName, Revision: productResp.Render.Revision} + currentEnvRenderSet, err := commonrepo.NewRenderSetColl().Find(renderSetOpt) + if err != nil { + log.Errorf("[RenderSet.find] err: %v", err) + return nil, err + } + + // 环境里面的变量 + currentEnvRenderSetMap := make(map[string]*template.RenderChart) + for _, renderInfo := range currentEnvRenderSet.ChartInfos { + currentEnvRenderSetMap[renderInfo.ServiceName] = renderInfo + } + + tmpCurrentChartInfoMap := make(map[string]*template.RenderChart) + tmpLatestChartInfoMap := make(map[string]*template.RenderChart) + //过滤掉服务编排没有的服务,这部分不需要做diff + for _, serviceNameGroup := range productTemp.Services { + for _, serviceName := range serviceNameGroup { + if currentChartInfo, isExist := currentEnvRenderSetMap[serviceName]; isExist { + tmpCurrentChartInfoMap[serviceName] = currentChartInfo + } + if latestChartInfo, isExist := latestRenderSetMap[serviceName]; isExist { + tmpLatestChartInfoMap[serviceName] = latestChartInfo + } + } + } + + for serviceName, latestChartInfo := range tmpLatestChartInfoMap { + if currentChartInfo, ok := tmpCurrentChartInfoMap[serviceName]; ok { + //拿当前环境values.yaml的key的value去替换服务里面的values.yaml的相同的key的value + newValuesYaml, err := overrideValues([]byte(currentChartInfo.ValuesYaml), []byte(latestChartInfo.ValuesYaml)) + if err != nil { + log.Errorf("Failed to override values, err: %s", err) + } else { + latestChartInfo.ValuesYaml = string(newValuesYaml) + } + } + newChartInfos = append(newChartInfos, latestChartInfo) + } + } + + if err = commonservice.CreateHelmRenderSet( + &commonmodels.RenderSet{ + Name: renderSetName, + EnvName: envName, + ProductTmpl: productName, + ChartInfos: newChartInfos, + }, + log, + ); err != nil { + log.Errorf("[RenderSet.create] err: %v", err) + return nil, err + } + + renderSet, err := FindHelmRenderSet(productName, renderSetName, log) + if err != nil { + log.Errorf("[RenderSet.find] err: %v", err) + return nil, err + } + return renderSet, nil +} + +func overrideValues(currentValuesYaml, latestValuesYaml []byte) ([]byte, error) { + currentValuesMap := map[string]interface{}{} + if err := yaml.Unmarshal(currentValuesYaml, ¤tValuesMap); err != nil { + return nil, err + } + + currentValuesFlatMap, err := converter.Flatten(currentValuesMap) + if err != nil { + return nil, err + } + + latestValuesMap := map[string]interface{}{} + if err := yaml.Unmarshal(latestValuesYaml, &latestValuesMap); err != nil { + return nil, err + } + + latestValuesFlatMap, err := converter.Flatten(latestValuesMap) + if err != nil { + return nil, err + } + + replaceMap := make(map[string]interface{}) + for key := range latestValuesFlatMap { + if currentValue, ok := currentValuesFlatMap[key]; ok { + replaceMap[key] = currentValue + } + } + + if len(replaceMap) == 0 { + return nil, nil + } + + var replaceKV []string + for k, v := range replaceMap { + replaceKV = append(replaceKV, fmt.Sprintf("%s=%v", k, v)) + } + + if err := strvals.ParseInto(strings.Join(replaceKV, ","), latestValuesMap); err != nil { + return nil, err + } + + return yaml.Marshal(latestValuesMap) +} + func updateProductVariable(productName, envName string, productResp *commonmodels.Product, log *zap.SugaredLogger) error { renderChartMap := make(map[string]*template.RenderChart) diff --git a/pkg/microservice/aslan/core/environment/service/environment_group.go b/pkg/microservice/aslan/core/environment/service/environment_group.go index 13a7f30114..70e8f6890a 100644 --- a/pkg/microservice/aslan/core/environment/service/environment_group.go +++ b/pkg/microservice/aslan/core/environment/service/environment_group.go @@ -24,13 +24,13 @@ import ( "helm.sh/helm/v3/pkg/releaseutil" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/kube/resource" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/serializer" "github.com/koderover/zadig/pkg/util" diff --git a/pkg/microservice/aslan/core/environment/service/image.go b/pkg/microservice/aslan/core/environment/service/image.go index 0e73b12f4e..d9da78c94f 100644 --- a/pkg/microservice/aslan/core/environment/service/image.go +++ b/pkg/microservice/aslan/core/environment/service/image.go @@ -81,6 +81,18 @@ func UpdateContainerImage(requestID string, args *UpdateContainerImageArgs, log oldImageName := "" for _, group := range product.Services { for _, service := range group { + //如果为helm,serviceName可能不匹配 + if product.Source != setting.HelmDeployType { + if service.ServiceName == args.ServiceName { + for _, container := range service.Containers { + if container.Name == args.ContainerName { + container.Image = args.Image + break + } + } + } + continue + } for _, container := range service.Containers { if container.Name == args.ContainerName { oldImageName = container.Image @@ -144,7 +156,7 @@ func updateImageTagInValues(valuesYaml []byte, repo, newTag string) ([]byte, err var matchingKey string for k, v := range valuesFlatMap { - if repo == v.(string) { + if val, ok := v.(string); ok && repo == val { matchingKey = k } } diff --git a/pkg/microservice/aslan/core/environment/service/kube.go b/pkg/microservice/aslan/core/environment/service/kube.go index 14f05afea5..dbd2e2a2a5 100644 --- a/pkg/microservice/aslan/core/environment/service/kube.go +++ b/pkg/microservice/aslan/core/environment/service/kube.go @@ -27,11 +27,11 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/kube/resource" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/updater" diff --git a/pkg/microservice/aslan/core/environment/service/pm.go b/pkg/microservice/aslan/core/environment/service/pm.go new file mode 100644 index 0000000000..45edcbcb13 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/pm.go @@ -0,0 +1,232 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "errors" + "fmt" + "sort" + "sync" + + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" +) + +type PMService struct { + log *zap.SugaredLogger +} + +func (p *PMService) queryServiceStatus(namespace, envName, productName string, serviceTmpl *commonmodels.Service, kubeClient client.Client) (string, string, []string) { + p.log.Infof("queryServiceStatus of service: %s of product: %s in namespace %s", serviceTmpl.ServiceName, productName, namespace) + pipelineName := fmt.Sprintf("%s-%s-%s", serviceTmpl.ServiceName, envName, "job") + taskObj, err := commonrepo.NewTaskColl().FindTask(pipelineName, config.ServiceType) + if err != nil { + return setting.PodError, setting.PodNotReady, []string{} + } + if taskObj.Status == setting.PodCreated { + return setting.PodPending, setting.PodNotReady, []string{} + } + + return queryPodsStatus(namespace, "", serviceTmpl.ServiceName, kubeClient, p.log) +} + +func (p *PMService) updateService(args *SvcOptArgs) error { + svc := &commonmodels.ProductService{ + ServiceName: args.ServiceName, + Type: args.ServiceType, + Revision: args.ServiceRev.NextRevision, + Containers: args.ServiceRev.Containers, + Configs: make([]*commonmodels.ServiceConfig, 0), + } + opt := &commonrepo.ProductFindOptions{Name: args.ProductName, EnvName: args.EnvName} + exitedProd, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + p.log.Error(err) + return errors.New(e.UpsertServiceErrMsg) + } + // 更新产品服务 + for _, group := range exitedProd.Services { + for i, service := range group { + if service.ServiceName == args.ServiceName && service.Type == args.ServiceType { + group[i] = svc + } + } + } + + if err := commonrepo.NewProductColl().Update(exitedProd); err != nil { + p.log.Errorf("[%s][%s] Product.Update error: %v", args.EnvName, args.ProductName, err) + return e.ErrUpdateProduct + } + return nil +} + +func (p *PMService) listGroupServices(allServices []*commonmodels.ProductService, envName, productName string, kubeClient client.Client, productInfo *commonmodels.Product) []*commonservice.ServiceResp { + var wg sync.WaitGroup + var resp []*commonservice.ServiceResp + var mutex sync.RWMutex + + for _, service := range allServices { + wg.Add(1) + go func(service *commonmodels.ProductService) { + defer wg.Done() + gp := &commonservice.ServiceResp{ + ServiceName: service.ServiceName, + Type: service.Type, + EnvName: envName, + } + serviceTmpl, err := commonservice.GetServiceTemplate( + service.ServiceName, setting.PMDeployType, "", "", service.Revision, p.log, + ) + if err != nil { + gp.Status = setting.PodFailed + mutex.Lock() + resp = append(resp, gp) + mutex.Unlock() + return + } + + gp.ProductName = serviceTmpl.ProductName + if len(serviceTmpl.EnvStatuses) > 0 { + envStatuses := make([]*commonmodels.EnvStatus, 0) + for _, envStatus := range serviceTmpl.EnvStatuses { + if envStatus.EnvName == envName { + envStatuses = append(envStatuses, envStatus) + } + } + if len(envStatuses) > 0 { + gp.EnvStatuses = envStatuses + mutex.Lock() + resp = append(resp, gp) + mutex.Unlock() + return + } + } + + mutex.Lock() + resp = append(resp, gp) + mutex.Unlock() + }(service) + } + + wg.Wait() + + //把数据按照名称排序 + sort.SliceStable(resp, func(i, j int) bool { return resp[i].ServiceName < resp[j].ServiceName }) + + return resp +} + +func (p *PMService) createGroup(envName, productName, username string, group []*commonmodels.ProductService, renderSet *commonmodels.RenderSet, kubeClient client.Client) error { + p.log.Infof("[Namespace:%s][Product:%s] createGroup", envName, productName) + + // 异步创建无依赖的服务 + errList := &multierror.Error{} + + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + errList = multierror.Append(errList, err) + } + for _, productService := range group { + //更新非k8s服务 + if len(productService.EnvConfigs) > 0 { + serviceTempl, err := commonservice.GetServiceTemplate(productService.ServiceName, setting.PMDeployType, productName, setting.ProductStatusDeleting, productService.Revision, p.log) + if err != nil { + errList = multierror.Append(errList, err) + } + if serviceTempl != nil { + oldEnvConfigs := serviceTempl.EnvConfigs + for _, currentEnvConfig := range productService.EnvConfigs { + envConfig := &commonmodels.EnvConfig{ + EnvName: currentEnvConfig.EnvName, + HostIDs: currentEnvConfig.HostIDs, + } + oldEnvConfigs = append(oldEnvConfigs, envConfig) + } + + args := &commonservice.ServiceTmplBuildObject{ + ServiceTmplObject: &commonservice.ServiceTmplObject{ + ProductName: serviceTempl.ProductName, + ServiceName: serviceTempl.ServiceName, + Visibility: serviceTempl.Visibility, + Revision: serviceTempl.Revision, + Type: serviceTempl.Type, + Username: username, + HealthChecks: serviceTempl.HealthChecks, + EnvConfigs: oldEnvConfigs, + EnvStatuses: []*commonmodels.EnvStatus{}, + From: "createEnv", + }, + Build: &commonmodels.Build{Name: serviceTempl.BuildName}, + } + + if err := commonservice.UpdatePmServiceTemplate(username, args, p.log); err != nil { + errList = multierror.Append(errList, err) + } + } + } + var latestRevision int64 = productService.Revision + // 获取最新版本的服务 + if latestServiceTempl, _ := commonservice.GetServiceTemplate(productService.ServiceName, setting.PMDeployType, productName, setting.ProductStatusDeleting, 0, p.log); latestServiceTempl != nil { + latestRevision = latestServiceTempl.Revision + } + // 更新环境 + if latestRevision > productService.Revision { + // 更新产品服务 + for _, serviceGroup := range prod.Services { + for j, service := range serviceGroup { + if service.ServiceName == productService.ServiceName && service.Type == setting.PMDeployType { + serviceGroup[j].Revision = latestRevision + } + } + } + if err := commonrepo.NewProductColl().Update(prod); err != nil { + log.Errorf("[%s][%s] Product.Update error: %v", envName, productName, err) + errList = multierror.Append(errList, err) + } + } + if _, err = workflow.CreateServiceTask(&commonmodels.ServiceTaskArgs{ + ProductName: productName, + ServiceName: productService.ServiceName, + Revision: latestRevision, + EnvNames: []string{envName}, + ServiceTaskCreator: username, + }, p.log); err != nil { + _, messageMap := e.ErrorMessage(err) + if description, ok := messageMap["description"]; ok { + log.Errorf("description :%s", description) + } + errList = multierror.Append(errList, err) + } + } + // 如果创建依赖服务组有返回错误, 停止等待 + if err := errList.ErrorOrNil(); err != nil { + return err + } + + return nil +} diff --git a/pkg/microservice/aslan/core/environment/service/product.go b/pkg/microservice/aslan/core/environment/service/product.go index 53a842421b..2553bce861 100644 --- a/pkg/microservice/aslan/core/environment/service/product.go +++ b/pkg/microservice/aslan/core/environment/service/product.go @@ -31,6 +31,7 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" ) @@ -52,30 +53,32 @@ func CleanProductCronJob(requestID string, log *zap.SugaredLogger) { wl.Insert(config.CleanSkippedList()...) for _, product := range products { if wl.Has(product.EnvName) { - log.Infof("clean up skipped. user %s in whitelist.", product.EnvName) continue } if product.RecycleDay == 0 { - log.Infof("clean up skipped. user %s 不被回收.", product.EnvName) continue } if time.Now().Unix()-product.UpdateTime > int64(60*60*24*product.RecycleDay) { title := "系统清理产品信息" - content := fmt.Sprintf("产品 [%s] 已经连续%d天没有使用, 系统已自动删除该产品, 如有需要请重新创建产品。", product.ProductName, product.RecycleDay) + content := fmt.Sprintf("环境 [%s] 已经连续%d天没有使用, 系统已自动删除该环境, 如有需要请重新创建。", product.EnvName, product.RecycleDay) if err := commonservice.DeleteProduct("robot", product.EnvName, product.ProductName, requestID, log); err != nil { log.Errorf("[%s][P:%s] delete product error: %v", product.EnvName, product.ProductName, err) // 如果有错误,重试删除 if err := commonservice.DeleteProduct("robot", product.EnvName, product.ProductName, requestID, log); err != nil { - content = fmt.Sprintf("产品 [%s] 系统自动清理失败,请手动删除产品。", product.ProductName) + content = fmt.Sprintf("系统自动清理环境 [%s] 失败,请手动删除环境。", product.ProductName) log.Errorf("[%s][P:%s] retry delete product error: %v", product.EnvName, product.ProductName, err) } } - commonservice.SendMessage(product.EnvName, title, content, requestID, log) + poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) + users, _ := poetryClient.ListProductPermissionUsers("", "", log) + for _, user := range users { + commonservice.SendMessage(user, title, content, requestID, log) + } log.Warnf("[%s] product %s deleted", product.EnvName, product.ProductName) } @@ -94,6 +97,8 @@ func GetInitProduct(productTmplName string, log *zap.SugaredLogger) (*commonmode if prodTmpl.ProductFeature == nil || prodTmpl.ProductFeature.DeployType == setting.K8SDeployType { err = commonservice.FillProductTemplateVars([]*templatemodels.Product{prodTmpl}, log) + } else if prodTmpl.ProductFeature.DeployType == setting.HelmDeployType { + err = commonservice.FillProductTemplateValuesYamls(prodTmpl, log) } if err != nil { errMsg := fmt.Sprintf("[ProductTmpl.FillProductTemplate] %s error: %v", productTmplName, err) @@ -143,6 +148,15 @@ func GetInitProduct(productTmplName string, log *zap.SugaredLogger) (*commonmode } serviceResp.Containers = append(serviceResp.Containers, container) } + } else if serviceTmpl.Type == setting.HelmDeployType { + serviceResp.Containers = make([]*commonmodels.Container, 0) + for _, c := range serviceTmpl.Containers { + container := &commonmodels.Container{ + Name: c.Name, + Image: c.Image, + } + serviceResp.Containers = append(serviceResp.Containers, container) + } } servicesResp = append(servicesResp, serviceResp) } @@ -236,8 +250,18 @@ func buildProductResp(envName string, prod *commonmodels.Product, log *zap.Sugar errObj error ) - prodResp.Services = prod.GetGroupServiceNames() - servicesResp, _, errObj = ListGroups("", envName, prod.ProductName, -1, -1, log) + switch prod.Source { + case setting.SourceFromExternal, setting.SourceFromHelm: + servicesResp, _, errObj = commonservice.ListGroupsBySource(envName, prod.ProductName, log) + if len(servicesResp) == 0 && errObj == nil { + prodResp.Status = prod.Status + prodResp.Error = prod.Error + return prodResp + } + default: + prodResp.Services = prod.GetGroupServiceNames() + servicesResp, _, errObj = ListGroups("", envName, prod.ProductName, -1, -1, log) + } if errObj != nil { prodResp.Error = errObj.Error() diff --git a/pkg/microservice/aslan/core/environment/service/revision.go b/pkg/microservice/aslan/core/environment/service/revision.go index 676cd73ad9..e6d02413a2 100644 --- a/pkg/microservice/aslan/core/environment/service/revision.go +++ b/pkg/microservice/aslan/core/environment/service/revision.go @@ -22,13 +22,13 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -155,6 +155,10 @@ func GetProductRevision(product *commonmodels.Product, allServiceTmpls []*common prodRev.ServiceRevisions = make([]*SvcRevision, 0) prodRev.IsPublic = product.IsPublic + if product.Source == setting.SourceFromExternal { + return prodRev, nil + } + // 如果当前产品版本比产品模板小, 则需要更新 if prodRev.NextRevision > prodRev.CurrentRevision { prodRev.Updatable = true @@ -356,6 +360,8 @@ func compareServicesRev(serviceTmplNames []string, services []*commonmodels.Prod serviceRevs = append(serviceRevs, serviceRev) } else if service.Type == setting.HelmDeployType { serviceRevs = append(serviceRevs, serviceRev) + } else if service.Type == setting.PMDeployType { + serviceRevs = append(serviceRevs, serviceRev) } } } @@ -368,12 +374,16 @@ func getMaxServices(services []*commonmodels.Service, serviceName string) ([]*co resp []*commonmodels.Service k8sService = &commonmodels.Service{} helmService = &commonmodels.Service{} + pmService = &commonmodels.Service{} ) for _, service := range services { if service.ServiceName == serviceName && service.Type == setting.K8SDeployType && service.Revision > k8sService.Revision { k8sService = service } + if service.ServiceName == serviceName && service.Type == setting.PMDeployType && service.Revision > pmService.Revision { + pmService = service + } if service.ServiceName == serviceName && service.Type == setting.HelmDeployType && service.Revision > helmService.Revision { helmService = service } @@ -381,6 +391,9 @@ func getMaxServices(services []*commonmodels.Service, serviceName string) ([]*co if k8sService.ServiceName != "" { resp = append(resp, k8sService) } + if pmService.ServiceName != "" { + resp = append(resp, pmService) + } if helmService.ServiceName != "" { resp = append(resp, helmService) } diff --git a/pkg/microservice/aslan/core/environment/service/service.go b/pkg/microservice/aslan/core/environment/service/service.go index 9dc2f018f2..fbf52f39a0 100644 --- a/pkg/microservice/aslan/core/environment/service/service.go +++ b/pkg/microservice/aslan/core/environment/service/service.go @@ -20,19 +20,20 @@ import ( "fmt" "strings" + "github.com/hashicorp/go-multierror" "go.uber.org/zap" "helm.sh/helm/v3/pkg/releaseutil" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - internalresource "github.com/koderover/zadig/pkg/internal/kube/resource" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + internalresource "github.com/koderover/zadig/pkg/shared/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/serializer" @@ -179,6 +180,41 @@ func GetService(envName, productName, serviceName string, log *zap.SugaredLogger namespace := env.Namespace switch env.Source { + case setting.SourceFromExternal, setting.SourceFromHelm: + svc, found, err := getter.GetService(namespace, serviceName, kubeClient) + if err != nil { + return nil, e.ErrGetService.AddErr(err) + } + if !found { + return nil, e.ErrGetService.AddDesc(fmt.Sprintf("service %s not found", serviceName)) + } + ret.Services = append(ret.Services, wrapper.Service(svc).Resource()) + + selector := labels.SelectorFromValidatedSet(svc.Spec.Selector) + //deployment + if deployments, err := getter.ListDeployments(namespace, selector, kubeClient); err == nil { + log.Infof("namespace:%s , serviceName:%s , selector:%s , len(deployments):%d", namespace, serviceName, selector, len(deployments)) + for _, d := range deployments { + scale := getDeploymentWorkloadResource(d, kubeClient, log) + ret.Scales = append(ret.Scales, scale) + } + } + //statefulSets + if statefulSets, err := getter.ListStatefulSets(namespace, selector, kubeClient); err == nil { + log.Infof("namespace:%s , serviceName:%s , selector:%s , len(statefulSets):%d", namespace, serviceName, selector, len(statefulSets)) + for _, sts := range statefulSets { + scale := getStatefulSetWorkloadResource(sts, kubeClient, log) + ret.Scales = append(ret.Scales, scale) + } + } + + //ingress + if ingresses, err := getter.ListIngresses(namespace, selector, kubeClient); err == nil { + log.Infof("namespace:%s , serviceName:%s , selector:%s , len(ingresses):%d", namespace, serviceName, selector, len(ingresses)) + for _, ing := range ingresses { + ret.Ingress = append(ret.Ingress, wrapper.Ingress(ing).Resource()) + } + } default: var service *commonmodels.ProductService for _, svcArray := range env.Services { @@ -300,46 +336,76 @@ func RestartService(envName string, args *SvcOptArgs, log *zap.SugaredLogger) (e return err } - var serviceTmpl *commonmodels.Service - var newRender *commonmodels.RenderSet - var productService *commonmodels.ProductService - for _, group := range productObj.Services { - for _, serviceObj := range group { - if serviceObj.ServiceName == args.ServiceName { - productService = serviceObj - serviceTmpl, err = commonservice.GetServiceTemplate( - serviceObj.ServiceName, setting.K8SDeployType, "", setting.ProductStatusDeleting, serviceObj.Revision, log, - ) - if err != nil { - err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) - return + switch productObj.Source { + case setting.SourceFromHelm, setting.SourceFromExternal: + errList := new(multierror.Error) + serviceObj, found, err := getter.GetService(productObj.Namespace, args.ServiceName, kubeClient) + if err != nil || !found { + return err + } + + selector := labels.SelectorFromValidatedSet(serviceObj.Spec.Selector) + if deployments, err := getter.ListDeployments(productObj.Namespace, selector, kubeClient); err == nil { + log.Infof("namespace:%s , selector:%s , len(deployments):%d", productObj.Namespace, selector, len(deployments)) + for _, deployment := range deployments { + if err = updater.RestartDeployment(productObj.Namespace, deployment.Name, kubeClient); err != nil { + errList = multierror.Append(errList, err) } - opt := &commonrepo.RenderSetFindOption{Name: serviceObj.Render.Name, Revision: serviceObj.Render.Revision} - newRender, err = commonrepo.NewRenderSetColl().Find(opt) - if err != nil { - log.Errorf("[%s][P:%s]renderset Find error: %v", productObj.EnvName, productObj.ProductName, err) - err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) - return + } + } + + //statefulSets + if statefulSets, err := getter.ListStatefulSets(productObj.Namespace, selector, kubeClient); err == nil { + log.Infof("namespace:%s , selector:%s , len(statefulSets):%d", productObj.Namespace, selector, len(statefulSets)) + for _, statefulSet := range statefulSets { + if err = updater.RestartStatefulSet(productObj.Namespace, statefulSet.Name, kubeClient); err != nil { + errList = multierror.Append(errList, err) } - break } } - } - if serviceTmpl != nil && newRender != nil && productService != nil { - go func() { - log.Infof("upsert resource from namespace:%s/serviceName:%s ", productObj.Namespace, args.ServiceName) - _, err := upsertService( - true, - productObj, - productService, - productService, - newRender, kubeClient, log) - - // 如果创建依赖服务组有返回错误, 停止等待 - if err != nil { - log.Errorf(e.DeleteServiceContainerErrMsg+": err:%v", err) + + default: + var serviceTmpl *commonmodels.Service + var newRender *commonmodels.RenderSet + var productService *commonmodels.ProductService + for _, group := range productObj.Services { + for _, serviceObj := range group { + if serviceObj.ServiceName == args.ServiceName { + productService = serviceObj + serviceTmpl, err = commonservice.GetServiceTemplate( + serviceObj.ServiceName, setting.K8SDeployType, "", setting.ProductStatusDeleting, serviceObj.Revision, log, + ) + if err != nil { + err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) + return + } + opt := &commonrepo.RenderSetFindOption{Name: serviceObj.Render.Name, Revision: serviceObj.Render.Revision} + newRender, err = commonrepo.NewRenderSetColl().Find(opt) + if err != nil { + log.Errorf("[%s][P:%s]renderset Find error: %v", productObj.EnvName, productObj.ProductName, err) + err = e.ErrDeleteProduct.AddDesc(e.DeleteServiceContainerErrMsg + ": " + err.Error()) + return + } + break + } } - }() + } + if serviceTmpl != nil && newRender != nil && productService != nil { + go func() { + log.Infof("upsert resource from namespace:%s/serviceName:%s ", productObj.Namespace, args.ServiceName) + _, err := upsertService( + true, + productObj, + productService, + productService, + newRender, kubeClient, log) + + // 如果创建依赖服务组有返回错误, 停止等待 + if err != nil { + log.Errorf(e.DeleteServiceContainerErrMsg+": err:%v", err) + } + }() + } } return nil @@ -378,6 +444,11 @@ func validateServiceContainer(envName, productName, serviceName, container strin func validateServiceContainer2(namespace, envName, productName, serviceName, container, source string, kubeClient client.Client) (string, error) { var selector labels.Selector + //helm类型的服务查询所有标签的pod + if source != setting.SourceFromHelm { + selector = labels.Set{setting.ProductLabel: productName, setting.ServiceLabel: serviceName}.AsSelector() + } + pods, err := getter.ListPods(namespace, selector, kubeClient) if err != nil { return "", fmt.Errorf("[%s] ListPods %s/%s error: %v", namespace, productName, serviceName, err) diff --git a/pkg/microservice/aslan/core/environment/service/types.go b/pkg/microservice/aslan/core/environment/service/types.go index 5d218ba1e3..992fb5eb70 100644 --- a/pkg/microservice/aslan/core/environment/service/types.go +++ b/pkg/microservice/aslan/core/environment/service/types.go @@ -19,9 +19,9 @@ package service import ( "fmt" - internalresource "github.com/koderover/zadig/pkg/internal/kube/resource" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + internalresource "github.com/koderover/zadig/pkg/shared/kube/resource" ) type ProductRevision struct { diff --git a/pkg/microservice/aslan/core/log/handler/log.go b/pkg/microservice/aslan/core/log/handler/log.go index c7f7fc6da0..c9f6a67efb 100644 --- a/pkg/microservice/aslan/core/log/handler/log.go +++ b/pkg/microservice/aslan/core/log/handler/log.go @@ -23,7 +23,7 @@ import ( "github.com/gin-gonic/gin" logservice "github.com/koderover/zadig/pkg/microservice/aslan/core/log/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/log/handler/router.go b/pkg/microservice/aslan/core/log/handler/router.go index 896b3e1464..ca719c183e 100644 --- a/pkg/microservice/aslan/core/log/handler/router.go +++ b/pkg/microservice/aslan/core/log/handler/router.go @@ -19,13 +19,13 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) log := router.Group("log") { @@ -42,5 +42,6 @@ func (*Router) Inject(router *gin.RouterGroup) { sse.GET("/workflow/build/:pipelineName/:taskId/:lines/:serviceName", GetWorkflowBuildJobContainerLogsSSE) sse.GET("/test/:pipelineName/:taskId/:testName/:lines", GetTestJobContainerLogsSSE) sse.GET("/workflow/test/:pipelineName/:taskId/:testName/:lines/:serviceName", GetWorkflowTestJobContainerLogsSSE) + sse.GET("/service/build/:serviceName/:envName/:productName", GetServiceJobContainerLogsSSE) } } diff --git a/pkg/microservice/aslan/core/log/handler/sse.go b/pkg/microservice/aslan/core/log/handler/sse.go index e5b62cd314..7293d4360b 100644 --- a/pkg/microservice/aslan/core/log/handler/sse.go +++ b/pkg/microservice/aslan/core/log/handler/sse.go @@ -25,7 +25,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" logservice "github.com/koderover/zadig/pkg/microservice/aslan/core/log/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/util/ginzap" ) diff --git a/pkg/microservice/aslan/core/multicluster/handler/clusters.go b/pkg/microservice/aslan/core/multicluster/handler/clusters.go new file mode 100644 index 0000000000..555538a479 --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/handler/clusters.go @@ -0,0 +1,173 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type yamlArgs struct { + HubAgentImage string `json:"hub_agent_image"` + HubServerBaseAddr string `json:"hub_server_base_addr"` +} + +func ListClusters(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListClusters(c.Query("clusterType"), + ctx.Logger, + ) +} + +func GetCluster(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetCluster(c.Param("id"), ctx.Logger) +} + +func CreateCluster(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.K8SCluster) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + if err := args.Clean(); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + args.CreatedAt = time.Now().Unix() + args.CreatedBy = ctx.Username + + ctx.Resp, ctx.Err = service.CreateCluster(args, ctx.Logger) +} + +func UpdateCluster(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.K8SCluster) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + if err := args.Clean(); err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = service.UpdateCluster(c.Param("id"), args, ctx.Logger) +} + +func DeleteCluster(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.DeleteCluster(ctx.Username, c.Param("id"), ctx.Logger) +} + +func DisconnectCluster(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.DisconnectCluster(ctx.Username, c.Param("id"), ctx.Logger) +} + +func ReconnectCluster(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.ReconnectCluster(ctx.Username, c.Param("id"), ctx.Logger) +} + +func ClusterConnectFromAgent(c *gin.Context) { + c.Request.URL.Path = strings.TrimPrefix(c.Request.URL.Path, "/api/hub") + service.ProxyAgent(c.Writer, c.Request) + c.Abort() +} + +func GetClusterYaml(hubURI string) func(*gin.Context) { + return func(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { + if ctx.Err != nil { + c.JSON(e.ErrorMessage(ctx.Err)) + c.Abort() + return + } + }() + + yaml, err := service.GetYaml( + c.Param("id"), + hubURI, + strings.HasPrefix(c.Query("type"), "deploy"), + ctx.Logger, + ) + + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + c.Data(200, "text/plain", yaml) + c.Abort() + } +} + +func GetYamlArgs(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + hubAgentImage := config.HubAgentImage() + aslanURL := config.AslanURL() + + var hubBase, err = url.Parse(fmt.Sprintf("%s%s", aslanURL, "/api/hub")) + if err != nil { + ctx.Err = errors.New("parse hubBase failed") + return + } + if strings.ToLower(hubBase.Scheme) == "https" { + hubBase.Scheme = "wss" + } else { + hubBase.Scheme = "ws" + } + hubServerBaseAddr := hubBase.String() + + ctx.Resp = yamlArgs{ + HubAgentImage: hubAgentImage, + HubServerBaseAddr: hubServerBaseAddr, + } +} diff --git a/pkg/microservice/aslan/core/enterprise/handler/router.go b/pkg/microservice/aslan/core/multicluster/handler/router.go similarity index 50% rename from pkg/microservice/aslan/core/enterprise/handler/router.go rename to pkg/microservice/aslan/core/multicluster/handler/router.go index f9cd3619dc..33c59b5aab 100644 --- a/pkg/microservice/aslan/core/enterprise/handler/router.go +++ b/pkg/microservice/aslan/core/multicluster/handler/router.go @@ -19,28 +19,34 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) - - announcement := router.Group("announcement") + Agent := router.Group("agent") { - announcement.POST("/delete", DeleteNotifies) + Agent.GET("/:id/agent.yaml", GetClusterYaml("/api/hub")) } - notification := router.Group("notification") + router.Use(gin2.Auth()) + + Cluster := router.Group("clusters") { - notification.GET("", PullNotify) - notification.PUT("/read", ReadNotify) - notification.POST("/subscribe", UpsertSubscription) - notification.PUT("/subscribe/:type", UpdateSubscribe) - notification.DELETE("/unsubscribe/notifytype/:type", Unsubscribe) - notification.GET("/subscribe", ListSubscriptions) + Cluster.GET("", ListClusters) + Cluster.GET("/:id", GetCluster) + + Cluster.POST("", gin2.RequireSuperAdminAuth, CreateCluster) + Cluster.PUT("/:id", gin2.RequireSuperAdminAuth, UpdateCluster) + Cluster.DELETE("/:id", gin2.RequireSuperAdminAuth, DeleteCluster) + Cluster.PUT("/:id/disconnect", gin2.RequireSuperAdminAuth, DisconnectCluster) + Cluster.PUT("/:id/reconnect", gin2.RequireSuperAdminAuth, ReconnectCluster) } + ClusterYaml := router.Group("yamlArgs") + { + ClusterYaml.GET("", GetYamlArgs) + } } diff --git a/pkg/microservice/aslan/core/multicluster/service/clusters.go b/pkg/microservice/aslan/core/multicluster/service/clusters.go new file mode 100644 index 0000000000..6b13a87177 --- /dev/null +++ b/pkg/microservice/aslan/core/multicluster/service/clusters.go @@ -0,0 +1,96 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListClusters(clusterType string, logger *zap.SugaredLogger) ([]*commonmodels.K8SCluster, error) { + s, _ := kube.NewService("") + + return s.ListClusters(clusterType, logger) +} + +func GetCluster(id string, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { + s, _ := kube.NewService("") + + return s.GetCluster(id, logger) +} + +func CreateCluster(cluster *commonmodels.K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { + s, _ := kube.NewService("") + + return s.CreateCluster(cluster, logger) +} + +func UpdateCluster(id string, cluster *commonmodels.K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { + s, _ := kube.NewService("") + + return s.UpdateCluster(id, cluster, logger) +} + +func DeleteCluster(username, clusterID string, logger *zap.SugaredLogger) error { + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ + ClusterID: clusterID, + }) + + if err != nil { + return e.ErrDeleteCluster.AddErr(err) + } + + if len(products) > 0 { + return e.ErrDeleteCluster.AddDesc("请删除在该集群创建的环境后,再尝试删除该集群") + } + + s, _ := kube.NewService("") + + return s.DeleteCluster(username, clusterID, logger) +} + +func DisconnectCluster(username string, clusterID string, logger *zap.SugaredLogger) error { + s, _ := kube.NewService(config.HubServerAddress()) + + return s.DisconnectCluster(username, clusterID, logger) +} + +func ReconnectCluster(username string, clusterID string, logger *zap.SugaredLogger) error { + s, _ := kube.NewService(config.HubServerAddress()) + + return s.ReconnectCluster(username, clusterID, logger) +} + +func ProxyAgent(writer gin.ResponseWriter, request *http.Request) { + s, _ := kube.NewService(config.HubServerAddress()) + + s.ProxyAgent(writer, request) +} + +func GetYaml(id, hubURI string, useDeployment bool, logger *zap.SugaredLogger) ([]byte, error) { + s, _ := kube.NewService("") + + return s.GetYaml(id, config.HubAgentImage(), config.AslanURL(), hubURI, useDeployment, logger) +} diff --git a/pkg/microservice/aslan/core/project/handler/product.go b/pkg/microservice/aslan/core/project/handler/product.go index e78098b754..72a348d5fb 100644 --- a/pkg/microservice/aslan/core/project/handler/product.go +++ b/pkg/microservice/aslan/core/project/handler/product.go @@ -26,7 +26,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" projectservice "github.com/koderover/zadig/pkg/microservice/aslan/core/project/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/project/handler/render.go b/pkg/microservice/aslan/core/project/handler/render.go index e5640a1e14..86f259fd96 100644 --- a/pkg/microservice/aslan/core/project/handler/render.go +++ b/pkg/microservice/aslan/core/project/handler/render.go @@ -23,7 +23,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/project/handler/router.go b/pkg/microservice/aslan/core/project/handler/router.go index 513b2aa528..e40f4e46f0 100644 --- a/pkg/microservice/aslan/core/project/handler/router.go +++ b/pkg/microservice/aslan/core/project/handler/router.go @@ -19,14 +19,14 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) // 查看自定义变量是否被引用 render := router.Group("renders") @@ -51,11 +51,11 @@ func (*Router) Inject(router *gin.RouterGroup) { product.GET("/:name", GetProductTemplate) product.GET("/:name/services", GetProductTemplateServices) product.GET("", ListProductTemplate) - product.POST("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.SuperUserUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateProductTemplate) - product.PUT("/:name", middleware.IsHavePermission([]string{permission.ServiceTemplateEditUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, UpdateProductTemplate) - product.PUT("/:name/:status", middleware.IsHavePermission([]string{permission.ServiceTemplateEditUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, UpdateProductTmplStatus) - product.PUT("", middleware.StoreProductName, middleware.IsHavePermission([]string{permission.SuperUserUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateProject) - product.DELETE("/:name", middleware.IsHavePermission([]string{permission.SuperUserUUID}, permission.ParamType), middleware.UpdateOperationLogStatus, DeleteProductTemplate) + product.POST("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.SuperUserUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateProductTemplate) + product.PUT("/:name", gin2.IsHavePermission([]string{permission.ServiceTemplateEditUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProductTemplate) + product.PUT("/:name/:status", gin2.IsHavePermission([]string{permission.ServiceTemplateEditUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdateProductTmplStatus) + product.PUT("", gin2.StoreProductName, gin2.IsHavePermission([]string{permission.SuperUserUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateProject) + product.DELETE("/:name", gin2.IsHavePermission([]string{permission.SuperUserUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, DeleteProductTemplate) } openSource := router.Group("opensource") diff --git a/pkg/microservice/aslan/core/project/service/product.go b/pkg/microservice/aslan/core/project/service/product.go index b30bc659ba..423456b82f 100644 --- a/pkg/microservice/aslan/core/project/service/product.go +++ b/pkg/microservice/aslan/core/project/service/product.go @@ -20,22 +20,24 @@ import ( "errors" "fmt" "strconv" + "strings" "sync" "github.com/hashicorp/go-multierror" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" @@ -471,6 +473,18 @@ func DeleteProductTemplate(userName, productName, requestID string, log *zap.Sug return err } + //删除自由编排工作流 + features, err := commonservice.GetFeatures(log) + if err != nil { + log.Errorf("DeleteProductTemplate productName %s getFeatures err: %v", productName, err) + } + if strings.Contains(features, string(config.FreestyleType)) { + collieClient := collie.New(config.CollieAPIAddress(), config.PoetryAPIRootKey()) + if err = collieClient.DeleteCIPipelines(productName, log); err != nil { + log.Errorf("DeleteProductTemplate Delete productName %s freestyle pipeline err: %v", productName, err) + } + } + err = templaterepo.NewProductColl().Delete(productName) if err != nil { log.Errorf("ProductTmpl.Delete error: %v", err) diff --git a/pkg/microservice/aslan/core/service.go b/pkg/microservice/aslan/core/service.go index 2578f971f0..a1a8a6a9df 100644 --- a/pkg/microservice/aslan/core/service.go +++ b/pkg/microservice/aslan/core/service.go @@ -29,6 +29,7 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" systemservice "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" @@ -37,7 +38,35 @@ import ( mongotool "github.com/koderover/zadig/pkg/tool/mongo" ) -func Setup() { +const ( + webhookController = iota +) + +type Controller interface { + Run(workers int, stopCh <-chan struct{}) +} + +func StartControllers(stopCh <-chan struct{}) { + controllerWorkers := map[int]int{ + webhookController: 1, + } + controllers := map[int]Controller{ + webhookController: webhook.NewWebhookController(), + } + + var wg sync.WaitGroup + for name, c := range controllers { + wg.Add(1) + go func(name int, c Controller) { + defer wg.Done() + c.Run(controllerWorkers[name], stopCh) + }(name, c) + } + + wg.Wait() +} + +func Setup(ctx context.Context) { log.Init(&log.Config{ Level: commonconfig.LogLevel(), Filename: commonconfig.LogFile(), @@ -56,10 +85,12 @@ func Setup() { environmentservice.CleanProducts() environmentservice.ResetProductsStatus() + + go StartControllers(ctx.Done()) } -func TearDown() { - mongotool.Close(context.TODO()) +func TearDown(ctx context.Context) { + mongotool.Close(ctx) } func initService() { @@ -96,7 +127,6 @@ func initDatabase() { template.NewProductColl(), commonrepo.NewBasicImageColl(), commonrepo.NewBuildColl(), - commonrepo.NewBuildStatColl(), commonrepo.NewConfigColl(), commonrepo.NewCounterColl(), commonrepo.NewCronjobColl(), @@ -108,18 +138,18 @@ func initDatabase() { commonrepo.NewDeliverySecurityColl(), commonrepo.NewDeliveryTestColl(), commonrepo.NewDeliveryVersionColl(), - commonrepo.NewDeployStatColl(), commonrepo.NewDiffNoteColl(), commonrepo.NewDindCleanColl(), commonrepo.NewFavoriteColl(), commonrepo.NewGithubAppColl(), + commonrepo.NewHelmRepoColl(), commonrepo.NewInstallColl(), commonrepo.NewItReportColl(), commonrepo.NewK8SClusterColl(), commonrepo.NewNotificationColl(), commonrepo.NewNotifyColl(), - commonrepo.NewOperationLogColl(), commonrepo.NewPipelineColl(), + commonrepo.NewPrivateKeyColl(), commonrepo.NewProductColl(), commonrepo.NewProxyColl(), commonrepo.NewQueueColl(), @@ -127,11 +157,13 @@ func initDatabase() { commonrepo.NewRenderSetColl(), commonrepo.NewS3StorageColl(), commonrepo.NewServiceColl(), + commonrepo.NewStrategyColl(), commonrepo.NewStatsColl(), commonrepo.NewSubscriptionColl(), commonrepo.NewTaskColl(), commonrepo.NewTestTaskStatColl(), commonrepo.NewTestingColl(), + commonrepo.NewWebHookColl(), commonrepo.NewWebHookUserColl(), commonrepo.NewWorkflowColl(), commonrepo.NewWorkflowStatColl(), diff --git a/pkg/microservice/aslan/core/service/handler/harbor.go b/pkg/microservice/aslan/core/service/handler/harbor.go new file mode 100644 index 0000000000..02e250bcf2 --- /dev/null +++ b/pkg/microservice/aslan/core/service/handler/harbor.go @@ -0,0 +1,53 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "strconv" + + "github.com/gin-gonic/gin" + + svcservice "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func ListHarborProjects(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("pageSize", "10")) + ctx.Resp, ctx.Err = svcservice.ListHarborProjects(page, pageSize, ctx.Logger) +} + +func ListHarborChartRepos(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = svcservice.ListHarborChartRepos(c.Param("project"), ctx.Logger) +} + +func ListHarborChartVersions(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = svcservice.ListHarborChartVersions(c.Param("project"), c.Param("chart"), ctx.Logger) +} + +func FindHarborChartDetail(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = svcservice.FindHarborChartDetail(c.Param("project"), c.Param("chart"), c.Param("version"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/service/handler/helm.go b/pkg/microservice/aslan/core/service/handler/helm.go new file mode 100644 index 0000000000..26709e6634 --- /dev/null +++ b/pkg/microservice/aslan/core/service/handler/helm.go @@ -0,0 +1,86 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "strconv" + + "github.com/gin-gonic/gin" + + svcservice "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListHelmServices(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = svcservice.ListHelmServices(c.Param("productName"), ctx.Logger) +} + +func GetHelmServiceModule(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + revision, err := strconv.ParseInt(c.DefaultQuery("revision", "0"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid revision number") + return + } + ctx.Resp, ctx.Err = svcservice.GetHelmServiceModule(c.Param("serviceName"), c.Param("productName"), revision, ctx.Logger) +} + +func GetFilePath(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = svcservice.GetFilePath(c.Param("serviceName"), c.Param("productName"), c.Query("dir"), ctx.Logger) +} + +func GetFileContent(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + ctx.Resp, ctx.Err = svcservice.GetFileContent(c.Param("serviceName"), c.Param("productName"), c.Query("filePath"), c.Query("fileName"), ctx.Logger) +} + +func CreateHelmService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(svcservice.HelmServiceReq) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid HelmService json args") + return + } + args.CreateBy = ctx.Username + args.ProductName = c.Param("productName") + + ctx.Err = svcservice.CreateHelmService(args, ctx.Logger) +} + +func UpdateHelmService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(svcservice.HelmServiceArgs) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid HelmServiceArgs json args") + return + } + args.CreateBy = ctx.Username + args.ProductName = c.Param("productName") + + ctx.Err = svcservice.UpdateHelmService(args, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/service/handler/loader.go b/pkg/microservice/aslan/core/service/handler/loader.go index 07968bff56..41010659f5 100644 --- a/pkg/microservice/aslan/core/service/handler/loader.go +++ b/pkg/microservice/aslan/core/service/handler/loader.go @@ -22,7 +22,7 @@ import ( "github.com/gin-gonic/gin" svcservice "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/service/handler/router.go b/pkg/microservice/aslan/core/service/handler/router.go index b5774cdd66..23bc80cb0f 100644 --- a/pkg/microservice/aslan/core/service/handler/router.go +++ b/pkg/microservice/aslan/core/service/handler/router.go @@ -19,24 +19,42 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) + + harbor := router.Group("harbor") + { + harbor.GET("/project", ListHarborProjects) + harbor.GET("/project/:project/charts", ListHarborChartRepos) + harbor.GET("/project/:project/chart/:chart/versions", ListHarborChartVersions) + harbor.GET("/project/:project/chart/:chart/version/:version", FindHarborChartDetail) + } + + helm := router.Group("helm") + { + helm.GET("/:productName", ListHelmServices) + helm.GET("/:productName/:serviceName/serviceModule", GetHelmServiceModule) + helm.GET("/:productName/:serviceName/filePath", GetFilePath) + helm.GET("/:productName/:serviceName/fileContent", GetFileContent) + helm.POST("/:productName", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), CreateHelmService) + helm.PUT("/:productName", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), UpdateHelmService) + } k8s := router.Group("services") { k8s.GET("", ListServiceTemplate) k8s.GET("/:name/:type", GetServiceTemplate) k8s.GET("/:name", GetServiceTemplateOption) - k8s.POST("", GetServiceTemplateProductName, middleware.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateServiceTemplate) - k8s.PUT("", GetServiceTemplateObjectProductName, middleware.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateServiceTemplate) + k8s.POST("", GetServiceTemplateProductName, gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateServiceTemplate) + k8s.PUT("", GetServiceTemplateObjectProductName, gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateServiceTemplate) k8s.PUT("/yaml/validator", YamlValidator) - k8s.DELETE("/:name/:type", middleware.IsHavePermission([]string{permission.ServiceTemplateDeleteUUID}, permission.QueryType), middleware.UpdateOperationLogStatus, DeleteServiceTemplate) + k8s.DELETE("/:name/:type", gin2.IsHavePermission([]string{permission.ServiceTemplateDeleteUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeleteServiceTemplate) k8s.GET("/:name/:type/ports", ListServicePort) } @@ -52,4 +70,9 @@ func (*Router) Inject(router *gin.RouterGroup) { loader.GET("/validateUpdate/:codehostId/:branchName", ValidateServiceUpdate) } + pm := router.Group("pm") + { + pm.POST("/:productName", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, CreatePMService) + pm.PUT("/:productName", gin2.IsHavePermission([]string{permission.ServiceTemplateManageUUID}, permission.ParamType), gin2.UpdateOperationLogStatus, UpdatePmServiceTemplate) + } } diff --git a/pkg/microservice/aslan/core/service/handler/service.go b/pkg/microservice/aslan/core/service/handler/service.go index d3ca400adb..fc8c592452 100644 --- a/pkg/microservice/aslan/core/service/handler/service.go +++ b/pkg/microservice/aslan/core/service/handler/service.go @@ -28,8 +28,8 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" svcservice "github.com/koderover/zadig/pkg/microservice/aslan/core/service/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" @@ -194,3 +194,96 @@ func GetServiceTemplateObjectProductName(c *gin.Context) { c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) c.Next() } + +func CreatePMService(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := new(svcservice.ServiceTmplBuildObject) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreatePMService c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreatePMService json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "新增", "工程管理-物理机部署服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Revision), permission.ServiceTemplateManageUUID, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid service json args") + return + } + if args.Build.Name == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("构建名称不能为空!") + return + } + + for _, heathCheck := range args.ServiceTmplObject.HealthChecks { + if heathCheck.TimeOut < 2 || heathCheck.TimeOut > 60 { + ctx.Err = e.ErrInvalidParam.AddDesc("超时时间必须在2-60之间") + return + } + if heathCheck.Interval != 0 { + if heathCheck.Interval < 2 || heathCheck.Interval > 60 { + ctx.Err = e.ErrInvalidParam.AddDesc("间隔时间必须在2-60之间") + return + } + } + if heathCheck.HealthyThreshold != 0 { + if heathCheck.HealthyThreshold < 2 || heathCheck.HealthyThreshold > 10 { + ctx.Err = e.ErrInvalidParam.AddDesc("健康阈值必须在2-10之间") + return + } + } + if heathCheck.UnhealthyThreshold != 0 { + if heathCheck.UnhealthyThreshold < 2 || heathCheck.UnhealthyThreshold > 10 { + ctx.Err = e.ErrInvalidParam.AddDesc("不健康阈值必须在2-10之间") + return + } + } + } + + ctx.Err = svcservice.CreatePMService(ctx.Username, args, ctx.Logger) +} + +func UpdatePmServiceTemplate(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonservice.ServiceTmplBuildObject) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdatePmServiceTemplate c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdatePmServiceTemplate json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, c.Param("productName"), "更新", "工程管理-非容器服务", fmt.Sprintf("服务名称:%s,版本号:%d", args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Revision), permission.ServiceTemplateManageUUID, "", ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + for _, heathCheck := range args.ServiceTmplObject.HealthChecks { + if heathCheck.TimeOut < 2 || heathCheck.TimeOut > 60 { + ctx.Err = e.ErrInvalidParam.AddDesc("超时时间必须在2-60之间") + return + } + if heathCheck.Interval != 0 { + if heathCheck.Interval < 2 || heathCheck.Interval > 60 { + ctx.Err = e.ErrInvalidParam.AddDesc("间隔时间必须在2-60之间") + return + } + } + if heathCheck.HealthyThreshold != 0 { + if heathCheck.HealthyThreshold < 2 || heathCheck.HealthyThreshold > 10 { + ctx.Err = e.ErrInvalidParam.AddDesc("健康阈值必须在2-10之间") + return + } + } + if heathCheck.UnhealthyThreshold != 0 { + if heathCheck.UnhealthyThreshold < 2 || heathCheck.UnhealthyThreshold > 10 { + ctx.Err = e.ErrInvalidParam.AddDesc("不健康阈值必须在2-10之间") + return + } + } + } + ctx.Err = commonservice.UpdatePmServiceTemplate(ctx.Username, args, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/service/service/harbor.go b/pkg/microservice/aslan/core/service/service/harbor.go new file mode 100644 index 0000000000..2f250a18b1 --- /dev/null +++ b/pkg/microservice/aslan/core/service/service/harbor.go @@ -0,0 +1,198 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "net/url" + "time" + + "go.uber.org/zap" + "sigs.k8s.io/yaml" + + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type HarborProject struct { + UpdateTime string `json:"update_time"` + OwnerName string `json:"owner_name"` + Name string `json:"name"` + Deleted bool `json:"deleted"` + OwnerID int `json:"owner_id"` + RepoCount int `json:"repo_count"` + CreationTime string `json:"creation_time"` + Togglable bool `json:"togglable"` + ProjectID int `json:"project_id"` + CurrentUserRoleID int `json:"current_user_role_id"` + ChartCount int `json:"chart_count"` + CveWhitelist struct { + Items []struct { + CveID string `json:"cve_id"` + } `json:"items"` + ProjectID int `json:"project_id"` + ID int `json:"id"` + ExpiresAt int `json:"expires_at"` + } `json:"cve_whitelist"` + Metadata struct { + EnableContentTrust string `json:"enable_content_trust"` + AutoScan string `json:"auto_scan"` + Severity string `json:"severity"` + ReuseSysCveWhitelist string `json:"reuse_sys_cve_whitelist"` + Public string `json:"public"` + PreventVul string `json:"prevent_vul"` + } `json:"metadata"` +} + +type HarborChartRepo struct { + Updated string `json:"updated"` + Name string `json:"name"` + Created string `json:"created"` + Deprecated bool `json:"deprecated"` + TotalVersions int `json:"total_versions"` + LatestVersion string `json:"latest_version"` + Home string `json:"home"` + Icon string `json:"icon"` +} + +type HarborChartDetail struct { + Metadata struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + APIVersion string `json:"apiVersion"` + AppVersion string `json:"appVersion"` + Urls []string `json:"urls"` + Created time.Time `json:"created"` + Digest string `json:"digest"` + } `json:"metadata"` + Dependencies []string `json:"dependencies"` + YamlValues map[string]interface{} `json:"yaml_values"` + Files struct { + ValuesYaml string `json:"values.yaml"` + } `json:"files"` + Security struct { + Signature struct { + Signed bool `json:"signed"` + ProvFile string `json:"prov_file"` + } `json:"signature"` + } `json:"security"` + Labels []string `json:"labels"` +} + +type HarborChartVersion struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + APIVersion string `json:"apiVersion"` + AppVersion string `json:"appVersion"` + Urls []string `json:"urls"` + Created time.Time `json:"created"` + Digest string `json:"digest"` + Labels []string `json:"labels"` +} + +func GetHarborURL(path string) (string, error) { + if helmIntegrations, err := commonrepo.NewHelmRepoColl().List(); err == nil { + if len(helmIntegrations) > 0 { + helmIntegration := helmIntegrations[0] + uri, err := url.Parse(helmIntegration.URL) + if err != nil { + return "", fmt.Errorf("url prase failed") + } + return fmt.Sprintf("%s://%s:%s@%s/%s", uri.Scheme, helmIntegration.Username, helmIntegration.Password, uri.Host, path), nil + } + } + return "", fmt.Errorf("harbor integration not found") +} + +func ListHarborProjects(page, pageSize int, log *zap.SugaredLogger) ([]*HarborProject, error) { + harborProjects := make([]*HarborProject, 0) + + url, err := GetHarborURL(fmt.Sprintf("api/projects?page=%d&page_size=%d", page, pageSize)) + if err != nil { //前端根据空数组判断显示对应文案,后端不再做错误信息返回 + return harborProjects, nil + } + + _, err = httpclient.Get(url, httpclient.SetResult(&harborProjects)) + if err != nil { + return harborProjects, fmt.Errorf("harbor request url:%s err:%v", url, err) + } + + return harborProjects, nil +} + +func ListHarborChartRepos(project string, log *zap.SugaredLogger) ([]*HarborChartRepo, error) { + harborChartRepos := make([]*HarborChartRepo, 0) + + url, err := GetHarborURL(fmt.Sprintf("api/chartrepo/%s/charts", project)) + if err != nil { //前端根据空数组判断显示对应文案,后端不再做错误信息返回 + return harborChartRepos, nil + } + + _, err = httpclient.Get(url, httpclient.SetResult(&harborChartRepos)) + if err != nil { + return harborChartRepos, fmt.Errorf("harbor request url:%s err:%v", url, err) + } + + return harborChartRepos, nil +} + +func ListHarborChartVersions(project, chartName string, log *zap.SugaredLogger) ([]*HarborChartVersion, error) { + harborChartVersions := make([]*HarborChartVersion, 0) + + url, err := GetHarborURL(fmt.Sprintf("api/chartrepo/%s/charts/%s", project, chartName)) + if err != nil { //前端根据空数组判断显示对应文案,后端不再做错误信息返回 + return harborChartVersions, nil + } + + _, err = httpclient.Get(url, httpclient.SetResult(&harborChartVersions)) + if err != nil { + return harborChartVersions, fmt.Errorf("harbor request url:%s err:%v", url, err) + } + + return harborChartVersions, nil +} + +func FindHarborChartDetail(project, chartName, version string, log *zap.SugaredLogger) (*HarborChartDetail, error) { + harborChartDetail := &HarborChartDetail{} + url, err := GetHarborURL(fmt.Sprintf("api/chartrepo/%s/charts/%s/%s", project, chartName, version)) + if err != nil { + return harborChartDetail, fmt.Errorf("GetHarborUrl err:%v", err) + } + + _, err = httpclient.Get(url, httpclient.SetResult(harborChartDetail)) + if err != nil { + return harborChartDetail, fmt.Errorf("harbor request url:%s err:%v", url, err) + } + + if harborChartDetail != nil { + yamlValuesByte, err := yaml.YAMLToJSON([]byte(harborChartDetail.Files.ValuesYaml)) + if err != nil { + log.Errorf("harbor YAMLToJSON err:%v", err) + return harborChartDetail, nil + } + var valuesMap map[string]interface{} + if err := yaml.Unmarshal(yamlValuesByte, &valuesMap); err != nil { + log.Errorf("harbor Unmarshal err:%v", err) + return harborChartDetail, nil + } + harborChartDetail.YamlValues = valuesMap + } + + return harborChartDetail, nil +} diff --git a/pkg/microservice/aslan/core/service/service/helm.go b/pkg/microservice/aslan/core/service/service/helm.go new file mode 100644 index 0000000000..0e725c8054 --- /dev/null +++ b/pkg/microservice/aslan/core/service/service/helm.go @@ -0,0 +1,633 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/yaml" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gerrit" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" +) + +type HelmService struct { + Services []*models.Service `json:"services"` + FileInfos []*types.FileInfo `json:"file_infos"` +} + +type HelmServiceReq struct { + ProductName string `json:"product_name"` + Visibility string `json:"visibility"` + Type string `json:"type"` + CreateBy string `json:"create_by"` + CodehostID int `json:"codehost_id"` + RepoOwner string `json:"repo_owner"` + RepoName string `json:"repo_name"` + BranchName string `json:"branch_name"` + FilePaths []string `json:"file_paths"` + Revision int64 `json:"revision"` + SrcPath string `json:"src_path"` +} + +type HelmServiceArgs struct { + ProductName string `json:"product_name"` + CreateBy string `json:"create_by"` + HelmServiceInfos []*HelmServiceInfo `json:"helm_service_infos"` +} + +type HelmServiceInfo struct { + ServiceName string `json:"service_name"` + FilePath string `json:"file_path"` + FileName string `json:"file_name"` + FileContent string `json:"file_content"` +} + +type HelmServiceModule struct { + ServiceModules []*ServiceModule `json:"service_modules"` + Service *models.Service `json:"service,omitempty"` +} + +type Chart struct { + APIVersion string `json:"apiVersion"` + Name string `json:"name"` + Version string `json:"version"` + AppVersion string `json:"appVersion"` +} + +func ListHelmServices(productName string, log *zap.SugaredLogger) (*HelmService, error) { + helmService := &HelmService{ + Services: []*models.Service{}, + FileInfos: []*types.FileInfo{}, + } + + opt := &commonrepo.ServiceFindOption{ + ProductName: productName, + Type: setting.HelmDeployType, + ExcludeStatus: setting.ProductStatusDeleting, + } + + services, err := commonrepo.NewServiceColl().List(opt) + if err != nil { + log.Errorf("[helmService.list] err:%v", err) + return nil, e.ErrListTemplate.AddErr(err) + } + helmService.Services = services + + if len(services) > 0 { + chartFilePath := services[0].LoadPath + base, err := gerrit.GetGerritWorkspaceBasePath(services[0].RepoName) + _, serviceFileErr := os.Stat(path.Join(base, chartFilePath)) + if err != nil || os.IsNotExist(serviceFileErr) { + if err = commonservice.DownloadService(base, services[0].ServiceName); err != nil { + return helmService, e.ErrListTemplate.AddErr(err) + } + } + + files, err := ioutil.ReadDir(path.Join(base, chartFilePath)) + if err != nil { + return nil, e.ErrListTemplate.AddDesc(err.Error()) + } + + fis := make([]*types.FileInfo, 0) + for _, file := range files { + if file.Name() == ".git" && file.IsDir() { + continue + } + fi := &types.FileInfo{ + Parent: "/", + Name: file.Name(), + Size: file.Size(), + Mode: file.Mode(), + ModTime: file.ModTime().Unix(), + IsDir: file.IsDir(), + } + fis = append(fis, fi) + } + helmService.FileInfos = fis + } + return helmService, nil +} + +func GetHelmServiceModule(serviceName, productName string, revision int64, log *zap.SugaredLogger) (*HelmServiceModule, error) { + serviceTemplate, err := commonservice.GetServiceTemplate(serviceName, setting.HelmDeployType, productName, setting.ProductStatusDeleting, revision, log) + if err != nil { + return nil, err + } + helmServiceModule := new(HelmServiceModule) + serviceModules := make([]*ServiceModule, 0) + for _, container := range serviceTemplate.Containers { + serviceModule := new(ServiceModule) + serviceModule.Container = container + buildObj, _ := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{ProductName: productName, ServiceName: serviceName, Targets: []string{container.Name}}) + if buildObj != nil { + serviceModule.BuildName = buildObj.Name + } + serviceModules = append(serviceModules, serviceModule) + } + helmServiceModule.Service = serviceTemplate + helmServiceModule.ServiceModules = serviceModules + return helmServiceModule, err +} + +func GetFilePath(serviceName, productName, dir string, log *zap.SugaredLogger) ([]*types.FileInfo, error) { + if dir == "" { + dir = "/" + } + serviceTemplate, err := commonservice.GetServiceTemplate(serviceName, setting.HelmDeployType, productName, setting.ProductStatusDeleting, 0, log) + if err != nil { + return nil, err + } + fis := make([]*types.FileInfo, 0) + base, err := gerrit.GetGerritWorkspaceBasePath(serviceTemplate.RepoName) + _, serviceFileErr := os.Stat(path.Join(base, serviceTemplate.LoadPath)) + if err != nil || os.IsNotExist(serviceFileErr) { + if err = commonservice.DownloadService(base, serviceTemplate.ServiceName); err != nil { + return fis, e.ErrFilePath.AddDesc(err.Error()) + } + } + files, err := ioutil.ReadDir(path.Join(base, serviceTemplate.LoadPath, dir)) + if err != nil { + return fis, e.ErrFilePath.AddDesc(err.Error()) + } + + for _, file := range files { + if file.Name() == ".git" && file.IsDir() { + continue + } + fi := &types.FileInfo{ + Parent: dir, + Name: file.Name(), + Size: file.Size(), + Mode: file.Mode(), + ModTime: file.ModTime().Unix(), + IsDir: file.IsDir(), + } + + fis = append(fis, fi) + } + return fis, nil +} + +func GetFileContent(serviceName, productName, filePath, fileName string, log *zap.SugaredLogger) (string, error) { + serviceTemplate, err := commonservice.GetServiceTemplate(serviceName, setting.HelmDeployType, productName, setting.ProductStatusDeleting, 0, log) + if err != nil { + return "", e.ErrFileContent.AddDesc(err.Error()) + } + base, err := gerrit.GetGerritWorkspaceBasePath(serviceTemplate.RepoName) + _, serviceFileErr := os.Stat(path.Join(base, serviceTemplate.LoadPath)) + if err != nil || os.IsNotExist(serviceFileErr) { + if err = commonservice.DownloadService(base, serviceTemplate.ServiceName); err != nil { + return "", e.ErrFileContent.AddDesc(err.Error()) + } + } + + fileContent, err := ioutil.ReadFile(path.Join(base, serviceTemplate.LoadPath, filePath, fileName)) + if err != nil { + return "", e.ErrFileContent.AddDesc(err.Error()) + } + + return string(fileContent), nil +} + +func CreateHelmService(args *HelmServiceReq, log *zap.SugaredLogger) error { + args.Type = setting.HelmDeployType + args.Visibility = setting.PrivateVisibility + helmRenderCharts := make([]*templatemodels.RenderChart, 0, len(args.FilePaths)) + for _, filePath := range args.FilePaths { + base, err := gerrit.GetGerritWorkspaceBasePath(args.RepoName) + if err != nil { + log.Errorf("GetGerritWorkspaceBasePath err:%v", err) + return e.ErrCreateTemplate.AddErr(err) + } + files, err := ioutil.ReadDir(path.Join(base, filePath)) + if err != nil { + log.Errorf("ReadDir err:%v", err) + return e.ErrCreateTemplate.AddErr(err) + } + var containChartYaml, containValuesYaml, containTemplates bool + var serviceName, valuesYaml, chartVersion string + var valuesMap map[string]interface{} + for _, file := range files { + if file.Name() == setting.ChartYaml { + yamlFile, err := ioutil.ReadFile(path.Join(base, filePath, setting.ChartYaml)) + if err != nil { + return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("读取%s失败", setting.ChartYaml)) + } + chart := new(Chart) + if err = yaml.Unmarshal(yamlFile, chart); err != nil { + return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("解析%s失败", setting.ChartYaml)) + } + serviceName = chart.Name + chartVersion = chart.Version + containChartYaml = true + } else if file.Name() == setting.ValuesYaml { + yamlFileContent, err := ioutil.ReadFile(path.Join(base, filePath, setting.ValuesYaml)) + if err != nil { + return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("读取%s失败", setting.ValuesYaml)) + } + + if err = yaml.Unmarshal(yamlFileContent, &valuesMap); err != nil { + return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("解析%s失败", setting.ValuesYaml)) + } + + valuesYaml = string(yamlFileContent) + containValuesYaml = true + } else if file.Name() == setting.TemplatesDir { + containTemplates = true + } + } + if !containChartYaml || !containValuesYaml || !containTemplates { + return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("%s不是合法的chart目录,目录中必须包含%s/%s/%s目录等请检查!", filePath, setting.ValuesYaml, setting.ChartYaml, setting.TemplatesDir)) + } + + helmRenderCharts = append(helmRenderCharts, &templatemodels.RenderChart{ + ServiceName: serviceName, + ChartVersion: chartVersion, + ValuesYaml: valuesYaml, + }) + + opt := &commonrepo.ServiceFindOption{ + ServiceName: serviceName, + ExcludeStatus: setting.ProductStatusDeleting, + } + serviceTmpl, notFoundErr := commonrepo.NewServiceColl().Find(opt) + if notFoundErr == nil { + if serviceTmpl.ProductName != args.ProductName { + return e.ErrInvalidParam.AddDesc(fmt.Sprintf("项目 [%s] %s", serviceTmpl.ProductName, "有相同的服务名称存在,请检查!")) + } + } + + serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, serviceName, args.Type) + rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) + if err != nil { + return fmt.Errorf("helmService.create get next helm service revision error: %v", err) + } + args.Revision = rev + if err := commonrepo.NewServiceColl().Delete(serviceName, args.Type, "", setting.ProductStatusDeleting, args.Revision); err != nil { + log.Errorf("helmService.create delete %s error: %v", serviceName, err) + } + containerList := recursionGetImage(valuesMap) + if len(containerList) == 0 { + _, containerList = recursionGetImageByColon(valuesMap) + } + serviceObj := &models.Service{ + ServiceName: serviceName, + Type: args.Type, + Revision: args.Revision, + ProductName: args.ProductName, + Visibility: args.Visibility, + CreateTime: time.Now().Unix(), + CreateBy: args.CreateBy, + Containers: containerList, + CodehostID: args.CodehostID, + RepoOwner: args.RepoOwner, + RepoName: args.RepoName, + BranchName: args.BranchName, + LoadPath: filePath, + SrcPath: args.SrcPath, + HelmChart: &models.HelmChart{ + Name: serviceName, + Version: chartVersion, + ValuesYaml: valuesYaml, + }, + } + + // exec lint and dry run + if err = helmChartDryRun(chartVersion, valuesYaml, serviceName, path.Join(base, filePath), log); err != nil { + return e.ErrHelmDryRunFailed.AddDesc(fmt.Sprintf("具体错误信息: %s", err.Error())) + } + + if err := commonrepo.NewServiceColl().Create(serviceObj); err != nil { + log.Errorf("helmService.Create serviceName:%s error:%v", serviceName, err) + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + + if err = uploadService(base, serviceName, filePath); err != nil { + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + + if notFoundErr != nil { + if productTempl, err := commonservice.GetProductTemplate(args.ProductName, log); err == nil { + if len(productTempl.Services) > 0 && !sets.NewString(productTempl.Services[0]...).Has(serviceName) { + productTempl.Services[0] = append(productTempl.Services[0], serviceName) + } else { + productTempl.Services = [][]string{{serviceName}} + } + //更新项目模板 + err = templaterepo.NewProductColl().Update(args.ProductName, productTempl) + if err != nil { + log.Errorf("helmService.Create Update productTmpl error: %v", err) + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + } + } + } + go func() { + compareHelmVariable(helmRenderCharts, args, log) + }() + + return nil +} + +func helmChartDryRun(chartVersion, valuesYaml, serviceName, chartPath string, log *zap.SugaredLogger) error { + restConfig, err := kube.GetRESTConfig("") + if err != nil { + log.Errorf("GetRESTConfig err: %s", err) + return err + } + helmClient, err := helmclient.NewClientFromRestConf(restConfig, config.Namespace()) + if err != nil { + log.Errorf("NewClientFromRestConf err: %s", err) + return err + } + // Add a random to exclude resource conflicts caused by the same name + name := fmt.Sprintf("%s-%s-%s", config.Namespace(), serviceName, util.GetRandomString(6)) + chartSpec := &helmclient.ChartSpec{ + ReleaseName: name, + ChartName: name, + Namespace: config.Namespace(), + Version: chartVersion, + ValuesYaml: valuesYaml, + DryRun: true, + } + + err = helmClient.InstallOrUpgradeChart(context.Background(), chartSpec, + &helmclient.ChartOption{ChartPath: chartPath}, log) + + return err +} + +func UpdateHelmService(args *HelmServiceArgs, log *zap.SugaredLogger) error { + serviceNames := sets.NewString() + modifyServices := make([]*models.Service, 0) + for _, helmServiceInfo := range args.HelmServiceInfos { + opt := &commonrepo.ServiceFindOption{ + ProductName: args.ProductName, + ServiceName: helmServiceInfo.ServiceName, + Type: setting.HelmDeployType, + ExcludeStatus: setting.ProductStatusDeleting, + } + + preServiceTmpl, err := commonrepo.NewServiceColl().Find(opt) + if err != nil { + return e.ErrUpdateTemplate.AddDesc(err.Error()) + } + + if !serviceNames.Has(helmServiceInfo.ServiceName) { + serviceNames.Insert(helmServiceInfo.ServiceName) + modifyServices = append(modifyServices, preServiceTmpl) + } + + base, err := gerrit.GetGerritWorkspaceBasePath(preServiceTmpl.RepoName) + _, serviceFileErr := os.Stat(path.Join(base, preServiceTmpl.LoadPath)) + if err != nil || os.IsNotExist(serviceFileErr) { + if err = commonservice.DownloadService(base, helmServiceInfo.ServiceName); err != nil { + return e.ErrUpdateTemplate.AddDesc(err.Error()) + } + } + if err = ioutil.WriteFile(filepath.Join(base, preServiceTmpl.LoadPath, helmServiceInfo.FilePath, helmServiceInfo.FileName), []byte(helmServiceInfo.FileContent), 0644); err != nil { + log.Errorf("WriteFile filepath:%s err:%v", filepath.Join(base, helmServiceInfo.ServiceName, helmServiceInfo.FilePath, helmServiceInfo.FileName), err) + return e.ErrUpdateTemplate.AddDesc(err.Error()) + } + if helmServiceInfo.FileName == setting.ValuesYaml && preServiceTmpl.HelmChart.ValuesYaml != helmServiceInfo.FileContent { + var valuesMap map[string]interface{} + if err = yaml.Unmarshal([]byte(helmServiceInfo.FileContent), &valuesMap); err != nil { + return e.ErrCreateTemplate.AddDesc("values.yaml解析失败") + } + + containerList := recursionGetImage(valuesMap) + if len(containerList) == 0 { + _, containerList = recursionGetImageByColon(valuesMap) + } + preServiceTmpl.Containers = containerList + preServiceTmpl.HelmChart.ValuesYaml = helmServiceInfo.FileContent + + //修改helm renderset + renderOpt := &commonrepo.RenderSetFindOption{Name: args.ProductName} + if rs, err := commonrepo.NewRenderSetColl().Find(renderOpt); err == nil { + for _, chartInfo := range rs.ChartInfos { + if chartInfo.ServiceName == helmServiceInfo.ServiceName { + chartInfo.ValuesYaml = helmServiceInfo.FileContent + break + } + } + if err = commonrepo.NewRenderSetColl().Update(rs); err != nil { + log.Errorf("[renderset.update] err:%v", err) + } + } + } else if helmServiceInfo.FileName == setting.ChartYaml { + chart := new(Chart) + if err = yaml.Unmarshal([]byte(helmServiceInfo.FileContent), chart); err != nil { + return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("解析%s失败", setting.ChartYaml)) + } + if preServiceTmpl.HelmChart.Version != chart.Version { + preServiceTmpl.HelmChart.Version = chart.Version + + //修改helm renderset + renderOpt := &commonrepo.RenderSetFindOption{Name: args.ProductName} + if rs, err := commonrepo.NewRenderSetColl().Find(renderOpt); err == nil { + for _, chartInfo := range rs.ChartInfos { + if chartInfo.ServiceName == helmServiceInfo.ServiceName { + chartInfo.ChartVersion = chart.Version + break + } + } + if err = commonrepo.NewRenderSetColl().Update(rs); err != nil { + log.Errorf("[renderset.update] err:%v", err) + } + } + } + } + + preServiceTmpl.CreateBy = args.CreateBy + serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, helmServiceInfo.ServiceName, setting.HelmDeployType) + rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) + if err != nil { + return fmt.Errorf("get next helm service revision error: %v", err) + } + + preServiceTmpl.Revision = rev + if err := commonrepo.NewServiceColl().Delete(helmServiceInfo.ServiceName, setting.HelmDeployType, "", setting.ProductStatusDeleting, preServiceTmpl.Revision); err != nil { + log.Errorf("helmService.update delete %s error: %v", helmServiceInfo.ServiceName, err) + } + + if err := commonrepo.NewServiceColl().Create(preServiceTmpl); err != nil { + log.Errorf("helmService.update serviceName:%s error:%v", helmServiceInfo.ServiceName, err) + return e.ErrUpdateTemplate.AddDesc(err.Error()) + } + } + + for _, serviceObj := range modifyServices { + base, _ := gerrit.GetGerritWorkspaceBasePath(serviceObj.RepoName) + if err := uploadService(base, serviceObj.ServiceName, serviceObj.LoadPath); err != nil { + return e.ErrUpdateTemplate.AddDesc(err.Error()) + } + } + + return nil +} + +// compareHelmVariable 比较helm变量是否有改动,是否需要添加新的renderSet +func compareHelmVariable(chartInfos []*templatemodels.RenderChart, args *HelmServiceReq, log *zap.SugaredLogger) { + // 对比上个版本的renderset,新增一个版本 + latestChartInfos := make([]*templatemodels.RenderChart, 0) + renderOpt := &commonrepo.RenderSetFindOption{Name: args.ProductName} + if latestDefaultRenderSet, err := commonrepo.NewRenderSetColl().Find(renderOpt); err == nil { + latestChartInfos = latestDefaultRenderSet.ChartInfos + } + + currentChartInfoMap := make(map[string]*templatemodels.RenderChart) + for _, chartInfo := range chartInfos { + currentChartInfoMap[chartInfo.ServiceName] = chartInfo + } + + mixtureChartInfos := make([]*templatemodels.RenderChart, 0) + for _, latestChartInfo := range latestChartInfos { + //如果新的里面存在就拿新的数据替换,不存在就还使用老的数据 + if currentChartInfo, isExist := currentChartInfoMap[latestChartInfo.ServiceName]; isExist { + mixtureChartInfos = append(mixtureChartInfos, currentChartInfo) + delete(currentChartInfoMap, latestChartInfo.ServiceName) + continue + } + mixtureChartInfos = append(mixtureChartInfos, latestChartInfo) + } + + //把新增的服务添加到新的slice里面 + for _, chartInfo := range currentChartInfoMap { + mixtureChartInfos = append(mixtureChartInfos, chartInfo) + } + + //添加renderset + if err := commonservice.CreateHelmRenderSet( + &models.RenderSet{ + Name: args.ProductName, + Revision: 0, + ProductTmpl: args.ProductName, + UpdateBy: args.CreateBy, + ChartInfos: mixtureChartInfos, + }, log, + ); err != nil { + log.Errorf("helmService.Create CreateHelmRenderSet error: %v", err) + } +} + +// 递归通过repository和tag获取服务组件 +func recursionGetImage(jsonValues map[string]interface{}) []*models.Container { + ret := make([]*models.Container, 0) + for jsonKey, jsonValue := range jsonValues { + if levelMap, ok := jsonValue.(map[string]interface{}); ok { + ret = append(ret, recursionGetImage(levelMap)...) + } else if repository, isStr := jsonValue.(string); isStr { + if strings.Contains(jsonKey, "repository") { + serviceContainer := new(models.Container) + if imageTag, isExist := jsonValues["tag"]; isExist { + if imageTag != "" { + serviceContainer.Image = fmt.Sprintf("%s:%v", repository, imageTag) + imageStr := strings.Split(repository, "/") + if len(imageStr) > 1 { + serviceContainer.Name = imageStr[len(imageStr)-1] + } + ret = append(ret, serviceContainer) + } + } + } + } + } + return ret +} + +func recursionGetImageByColon(jsonValues map[string]interface{}) ([]string, []*models.Container) { + imageRegEx := regexp.MustCompile(config.ImageRegexString) + ret := make([]*models.Container, 0) + banList := sets.NewString() + + for _, jsonValue := range jsonValues { + if levelMap, ok := jsonValue.(map[string]interface{}); ok { + imageList, recursiveRet := recursionGetImageByColon(levelMap) + ret = append(ret, recursiveRet...) + banList.Insert(imageList...) + } else if imageName, isStr := jsonValue.(string); isStr { + if strings.Contains(imageName, ":") && imageRegEx.MatchString(imageName) && + !strings.Contains(imageName, "http") && !strings.Contains(imageName, "https") { + serviceContainer := new(models.Container) + serviceContainer.Image = imageName + imageArr := strings.Split(imageName, "/") + if len(imageArr) >= 1 { + imageTagArr := strings.Split(imageArr[len(imageArr)-1], ":") + serviceContainer.Name = imageTagArr[0] + } + if !banList.Has(imageName) { + banList.Insert(imageName) + ret = append(ret, serviceContainer) + } + } + } + } + return banList.List(), ret +} + +func uploadService(base, serviceName, filePath string) error { + //将当前文件目录压缩上传到s3 + tarFilePath := path.Join(base, fmt.Sprintf("%s.tar.gz", serviceName)) + if err := util.Tar(path.Join(base, filePath), tarFilePath); err != nil { + log.Errorf("%s目录压缩失败", path.Join(base, filePath)) + return err + } + s3Storage, err := s3service.FindDefaultS3() + if err != nil { + log.Errorf("获取默认的s3配置失败 err:%v", err) + return err + } + subFolderName := serviceName + "-" + setting.HelmDeployType + if s3Storage.Subfolder != "" { + s3Storage.Subfolder = fmt.Sprintf("%s/%s/%s", s3Storage.Subfolder, subFolderName, "service") + } else { + s3Storage.Subfolder = fmt.Sprintf("%s/%s", subFolderName, "service") + } + if err = s3service.Upload(context.Background(), s3Storage, tarFilePath, fmt.Sprintf("%s.tar.gz", serviceName)); err != nil { + log.Errorf("s3上传文件失败 err:%v", err) + return err + } + if err = os.Remove(tarFilePath); err != nil { + log.Errorf("remove file err:%v", err) + } + return nil +} diff --git a/pkg/microservice/aslan/core/service/service/loader.go b/pkg/microservice/aslan/core/service/service/loader.go index 504ff0d856..69e95c438d 100644 --- a/pkg/microservice/aslan/core/service/service/loader.go +++ b/pkg/microservice/aslan/core/service/service/loader.go @@ -31,12 +31,12 @@ import ( "go.uber.org/zap" "golang.org/x/oauth2" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/tool/log" diff --git a/pkg/microservice/aslan/core/service/service/pm.go b/pkg/microservice/aslan/core/service/service/pm.go new file mode 100644 index 0000000000..3749084852 --- /dev/null +++ b/pkg/microservice/aslan/core/service/service/pm.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "time" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +type ServiceTmplBuildObject struct { + ServiceTmplObject *commonservice.ServiceTmplObject `json:"pm_service_tmpl"` + Build *commonmodels.Build `json:"build"` +} + +func CreatePMService(username string, args *ServiceTmplBuildObject, log *zap.SugaredLogger) error { + if isAdd, serviceLimit := addService(); !isAdd { + return e.ErrCreateTemplate.AddDesc(fmt.Sprintf("现有服务数量已超过允许的最大值[%d],请联系管理员查看", serviceLimit)) + } + if len(args.ServiceTmplObject.ServiceName) == 0 { + return e.ErrInvalidParam.AddDesc("服务名称为空,请检查") + } + if !config.ServiceNameRegex.MatchString(args.ServiceTmplObject.ServiceName) { + return e.ErrInvalidParam.AddDesc("服务名称格式错误,请检查") + } + + opt := &commonrepo.ServiceFindOption{ + ServiceName: args.ServiceTmplObject.ServiceName, + ExcludeStatus: setting.ProductStatusDeleting, + } + serviceTmpl, notFoundErr := commonrepo.NewServiceColl().Find(opt) + if notFoundErr == nil { + if serviceTmpl.ProductName != args.ServiceTmplObject.ProductName { + return e.ErrInvalidParam.AddDesc(fmt.Sprintf("项目 [%s] %s", serviceTmpl.ProductName, "有相同的服务名称存在,请检查!")) + } + } + + serviceTemplate := fmt.Sprintf(setting.ServiceTemplateCounterName, args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Type) + rev, err := commonrepo.NewCounterColl().GetNextSeq(serviceTemplate) + if err != nil { + return fmt.Errorf("get next pm service revision error: %v", err) + } + args.ServiceTmplObject.Revision = rev + + if err := commonrepo.NewServiceColl().Delete(args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Type, "", setting.ProductStatusDeleting, args.ServiceTmplObject.Revision); err != nil { + log.Errorf("pmService.delete %s error: %v", args.ServiceTmplObject.ServiceName, err) + } + + serviceObj := &commonmodels.Service{ + ServiceName: args.ServiceTmplObject.ServiceName, + Type: args.ServiceTmplObject.Type, + ProductName: args.ServiceTmplObject.ProductName, + Revision: args.ServiceTmplObject.Revision, + Visibility: args.ServiceTmplObject.Visibility, + HealthChecks: args.ServiceTmplObject.HealthChecks, + EnvConfigs: args.ServiceTmplObject.EnvConfigs, + CreateTime: time.Now().Unix(), + CreateBy: username, + BuildName: args.Build.Name, + } + + if err := commonrepo.NewServiceColl().Create(serviceObj); err != nil { + log.Errorf("pmService.Create %s error: %v", args.ServiceTmplObject.ServiceName, err) + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + + //创建构建 + if err := commonservice.CreateBuild(username, args.Build, log); err != nil { + log.Errorf("pmService.Create build %s error: %v", args.Build.Name, err) + if err2 := commonrepo.NewServiceColl().Delete(args.ServiceTmplObject.ServiceName, args.ServiceTmplObject.Type, "", setting.ProductStatusDeleting, args.ServiceTmplObject.Revision); err2 != nil { + log.Errorf("pmService.delete %s error: %v", args.ServiceTmplObject.ServiceName, err2) + } + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + + if notFoundErr != nil { + if productTempl, err := commonservice.GetProductTemplate(args.ServiceTmplObject.ProductName, log); err == nil { + //获取项目里面的所有服务 + if len(productTempl.Services) > 0 && !sets.NewString(productTempl.Services[0]...).Has(args.ServiceTmplObject.ServiceName) { + productTempl.Services[0] = append(productTempl.Services[0], args.ServiceTmplObject.ServiceName) + } else { + productTempl.Services = [][]string{{args.ServiceTmplObject.ServiceName}} + } + //更新项目模板 + err = templaterepo.NewProductColl().Update(args.ServiceTmplObject.ProductName, productTempl) + if err != nil { + log.Errorf("CreatePMService Update %s error: %v", args.ServiceTmplObject.ServiceName, err) + return e.ErrCreateTemplate.AddDesc(err.Error()) + } + } + } + return nil +} diff --git a/pkg/microservice/aslan/core/service/service/service.go b/pkg/microservice/aslan/core/service/service/service.go index dd7494b4a7..7697c0710a 100644 --- a/pkg/microservice/aslan/core/service/service/service.go +++ b/pkg/microservice/aslan/core/service/service/service.go @@ -17,6 +17,7 @@ limitations under the License. package service import ( + "encoding/base64" "encoding/json" "errors" "fmt" @@ -37,10 +38,11 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslanx" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/tool/httpclient" @@ -249,6 +251,9 @@ func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*Serv } func CreateServiceTemplate(userName string, args *commonmodels.Service, log *zap.SugaredLogger) (*ServiceOption, error) { + if isAdd, serviceLimit := addService(); !isAdd { + return nil, e.ErrCreateTemplate.AddDesc(fmt.Sprintf("现有服务数量已超过允许的最大值[%d],请联系管理员查看", serviceLimit)) + } opt := &commonrepo.ServiceFindOption{ ServiceName: args.ServiceName, @@ -385,6 +390,26 @@ func UpdateServiceTemplate(args *commonservice.ServiceTmplObject) error { } envStatuses := make([]*commonmodels.EnvStatus, 0) + // 去掉检查状态中不存在的环境和主机 + for _, envStatus := range args.EnvStatuses { + var existEnv, existHost bool + + envName := envStatus.EnvName + if _, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: envName, + }); err == nil { + existEnv = true + } + + if _, err := commonrepo.NewPrivateKeyColl().Find(envStatus.HostID); err == nil { + existHost = true + } + + if existEnv && existHost { + envStatuses = append(envStatuses, envStatus) + } + } updateArgs := &commonmodels.Service{ ProductName: args.ProductName, @@ -504,6 +529,25 @@ func DeleteServiceTemplate(serviceName, serviceType, productName, isEnvTemplate, } } + // 如果服务是PM类型,删除服务更新build的target信息 + if serviceType == setting.PMDeployType { + if serviceTmpl, err := commonservice.GetServiceTemplate( + serviceName, setting.PMDeployType, productName, setting.ProductStatusDeleting, 0, log, + ); err == nil { + if serviceTmpl.BuildName != "" { + updateTargets := make([]*commonmodels.ServiceModuleTarget, 0) + if preBuild, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: serviceTmpl.BuildName, Version: "stable", ProductName: productName}); err == nil { + for _, target := range preBuild.Targets { + if target.ServiceName != serviceName { + updateTargets = append(updateTargets, target) + } + } + _ = commonrepo.NewBuildColl().UpdateTargets(serviceTmpl.BuildName, productName, updateTargets) + } + } + } + } + err := commonrepo.NewServiceColl().UpdateStatus(serviceName, serviceType, productName, setting.ProductStatusDeleting) if err != nil { errMsg := fmt.Sprintf("[service.UpdateStatus] %s-%s error: %v", serviceName, serviceType, err) @@ -901,6 +945,31 @@ func getCronJobContainers(data string) ([]*commonmodels.Container, error) { return containers, nil } +func addService() (bool, int) { + var ( + totalServiceCount = 0 + limitServiceCount = 0 + ) + totalServices, _ := commonrepo.NewServiceColl().DistinctServices(&commonrepo.ServiceListOption{ExcludeStatus: setting.ProductStatusDeleting}) + totalServiceCount = len(totalServices) + signatures, enabled, _ := aslanx.New(config.AslanURL(), config.PoetryAPIRootKey()).ListSignatures(log.NopSugaredLogger()) + if !enabled { + return true, limitServiceCount + } + if len(signatures) > 0 { + if signatureStr, err := base64.StdEncoding.DecodeString(signatures[0].Token); err == nil { + signatureArr := strings.Split(string(signatureStr), ",") + if len(signatureArr) > 5 { + limitServiceCount, _ = strconv.Atoi(signatureArr[5]) + } + } + } + if limitServiceCount == -1 || limitServiceCount > totalServiceCount { + return true, limitServiceCount + } + return false, limitServiceCount +} + func updateGerritWebhookByService(lastService, currentService *commonmodels.Service) error { var codehostDetail *codehost.Detail var err error diff --git a/pkg/microservice/aslan/core/setting/handler/router.go b/pkg/microservice/aslan/core/setting/handler/router.go index 07512dd6cc..2b4eef2e9f 100644 --- a/pkg/microservice/aslan/core/setting/handler/router.go +++ b/pkg/microservice/aslan/core/setting/handler/router.go @@ -19,13 +19,13 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) user := router.Group("user") { diff --git a/pkg/microservice/aslan/core/setting/handler/user.go b/pkg/microservice/aslan/core/setting/handler/user.go index bf1362a160..c453d691a3 100644 --- a/pkg/microservice/aslan/core/setting/handler/user.go +++ b/pkg/microservice/aslan/core/setting/handler/user.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" settingservice "github.com/koderover/zadig/pkg/microservice/aslan/core/setting/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func GetUserKubeConfig(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/setting/service/user.go b/pkg/microservice/aslan/core/setting/service/user.go index d5e4c12098..a379255bf0 100644 --- a/pkg/microservice/aslan/core/setting/service/user.go +++ b/pkg/microservice/aslan/core/setting/service/user.go @@ -36,12 +36,12 @@ import ( rbacv1beta1 "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" diff --git a/pkg/microservice/aslan/core/system/handler/basic_images.go b/pkg/microservice/aslan/core/system/handler/basic_images.go index e4824e6921..62fd93b76f 100644 --- a/pkg/microservice/aslan/core/system/handler/basic_images.go +++ b/pkg/microservice/aslan/core/system/handler/basic_images.go @@ -27,7 +27,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/system/handler/capacity.go b/pkg/microservice/aslan/core/system/handler/capacity.go new file mode 100644 index 0000000000..ba2623e538 --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/capacity.go @@ -0,0 +1,86 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +const ( + reqParamTarget = "target" +) + +// DryRunFlag indicates whether a run is a dry run or not. +// If it is a dry run, the relevant API is supposed to be no-op except logging. +type DryRunFlag struct { + DryRun bool `json:"dryrun"` +} + +func UpdateStrategy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(models.CapacityStrategy) + if err := c.BindJSON(args); err != nil { + ctx.Err = err + return + } + + if err := service.UpdateSysCapStrategy(args); err != nil { + ctx.Err = err + } +} + +func GarbageCollection(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + flag := new(DryRunFlag) + if err := c.BindJSON(flag); err != nil { + ctx.Err = err + return + } + if err := service.HandleSystemGC(flag.DryRun); err != nil { + ctx.Err = err + } +} + +func CleanCache(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + if err := service.CleanCache(); err != nil { + ctx.Err = err + } +} + +func GetStrategy(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + target := c.Param(reqParamTarget) + resp, err := service.GetCapacityStrategy(models.CapacityTarget(target)) + if err != nil { + ctx.Err = err + return + } + ctx.Resp = resp +} diff --git a/pkg/microservice/aslan/core/system/handler/github_app.go b/pkg/microservice/aslan/core/system/handler/github_app.go index 9f8f4b3509..db49442c0f 100644 --- a/pkg/microservice/aslan/core/system/handler/github_app.go +++ b/pkg/microservice/aslan/core/system/handler/github_app.go @@ -21,7 +21,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/system/handler/helm.go b/pkg/microservice/aslan/core/system/handler/helm.go new file mode 100644 index 0000000000..9287914838 --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/helm.go @@ -0,0 +1,73 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "net/url" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListHelmRepos(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = commonservice.ListHelmRepos(ctx.Logger) +} + +func CreateHelmRepo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.HelmRepo) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid helmRepo json args") + return + } + args.UpdateBy = ctx.Username + if _, err := url.Parse(args.URL); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid url") + return + } + ctx.Err = service.CreateHelmRepo(args, ctx.Logger) +} + +func UpdateHelmRepo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.HelmRepo) + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid helmRepo json args") + return + } + args.UpdateBy = ctx.Username + ctx.Err = service.UpdateHelmRepo(c.Param("id"), args, ctx.Logger) +} + +func DeleteHelmRepo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Err = service.DeleteHelmRepo(c.Param("id"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/install.go b/pkg/microservice/aslan/core/system/handler/install.go index 0d80831272..8117658be0 100644 --- a/pkg/microservice/aslan/core/system/handler/install.go +++ b/pkg/microservice/aslan/core/system/handler/install.go @@ -27,7 +27,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/system/handler/jenkins.go b/pkg/microservice/aslan/core/system/handler/jenkins.go index 0ee92b79ce..20e7962191 100644 --- a/pkg/microservice/aslan/core/system/handler/jenkins.go +++ b/pkg/microservice/aslan/core/system/handler/jenkins.go @@ -23,7 +23,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/enterprise/handler/subscription.go b/pkg/microservice/aslan/core/system/handler/notify.go similarity index 56% rename from pkg/microservice/aslan/core/enterprise/handler/subscription.go rename to pkg/microservice/aslan/core/system/handler/notify.go index c6e46b67fc..5093cea28b 100644 --- a/pkg/microservice/aslan/core/enterprise/handler/subscription.go +++ b/pkg/microservice/aslan/core/system/handler/notify.go @@ -22,11 +22,52 @@ import ( "github.com/gin-gonic/gin" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - enterpriseservice "github.com/koderover/zadig/pkg/microservice/aslan/core/enterprise/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) +func PullNotify(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.PullNotify(ctx.Username, ctx.Logger) +} + +type readNotificationsArgs struct { + IDs []string `json:"ids"` +} + +func ReadNotify(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + args := new(readNotificationsArgs) + + err := c.BindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid readnotificationsargs") + return + } + ctx.Err = service.ReadNotify(ctx.Username, args.IDs, ctx.Logger) +} + +type deleteNotificationsArgs struct { + IDs []string `json:"ids"` +} + +func DeleteNotifies(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deleteNotificationsArgs) + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid args") + return + } + ctx.Err = service.DeleteNotifies(ctx.Username, args.IDs, ctx.Logger) +} + func UpsertSubscription(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -37,7 +78,7 @@ func UpsertSubscription(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid subscription args") return } - ctx.Err = enterpriseservice.UpsertSubscription(ctx.Username, args, ctx.Logger) + ctx.Err = service.UpsertSubscription(ctx.Username, args, ctx.Logger) } func UpdateSubscribe(c *gin.Context) { @@ -55,7 +96,7 @@ func UpdateSubscribe(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid notification type") return } - ctx.Err = enterpriseservice.UpdateSubscribe(ctx.Username, notifytype, args, ctx.Logger) + ctx.Err = service.UpdateSubscribe(ctx.Username, notifytype, args, ctx.Logger) } func Unsubscribe(c *gin.Context) { @@ -66,12 +107,12 @@ func Unsubscribe(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid notification type") return } - ctx.Err = enterpriseservice.Unsubscribe(ctx.Username, notifytype, ctx.Logger) + ctx.Err = service.Unsubscribe(ctx.Username, notifytype, ctx.Logger) } func ListSubscriptions(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = enterpriseservice.ListSubscriptions(ctx.Username, ctx.Logger) + ctx.Resp, ctx.Err = service.ListSubscriptions(ctx.Username, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/private_key.go b/pkg/microservice/aslan/core/system/handler/private_key.go new file mode 100644 index 0000000000..36ce2df13c --- /dev/null +++ b/pkg/microservice/aslan/core/system/handler/private_key.go @@ -0,0 +1,107 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + + "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types/permission" +) + +func ListPrivateKeys(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListPrivateKeys(ctx.Logger) +} + +func GetPrivateKey(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.GetPrivateKey(c.Param("id"), ctx.Logger) +} + +func CreatePrivateKey(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.PrivateKey) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreatePrivateKey c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreatePrivateKey json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, "", "新增", "私钥", fmt.Sprintf("label:%s", args.Label), permission.SuperUserUUID, string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindWith(&args, binding.JSON); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid PrivateKey args") + return + } + args.UpdateBy = ctx.Username + + ctx.Err = service.CreatePrivateKey(args, ctx.Logger) +} + +func UpdatePrivateKey(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.PrivateKey) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdatePrivateKey c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdatePrivateKey json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, "", "更新", "私钥", fmt.Sprintf("id:%s", args.ID), permission.SuperUserUUID, string(data), ctx.Logger) + + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.ShouldBindWith(&args, binding.JSON); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid PrivateKey args") + return + } + args.UpdateBy = ctx.Username + + ctx.Err = service.UpdatePrivateKey(c.Param("id"), args, ctx.Logger) +} + +func DeletePrivateKey(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.Username, "", "删除", "私钥", fmt.Sprintf("id:%s", c.Param("id")), permission.SuperUserUUID, "", ctx.Logger) + + ctx.Err = service.DeletePrivateKey(c.Param("id"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/system/handler/proxy.go b/pkg/microservice/aslan/core/system/handler/proxy.go index a570657b20..517cbd644b 100644 --- a/pkg/microservice/aslan/core/system/handler/proxy.go +++ b/pkg/microservice/aslan/core/system/handler/proxy.go @@ -28,7 +28,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/system/handler/registry.go b/pkg/microservice/aslan/core/system/handler/registry.go index ed65e00823..cd2d69546c 100644 --- a/pkg/microservice/aslan/core/system/handler/registry.go +++ b/pkg/microservice/aslan/core/system/handler/registry.go @@ -28,7 +28,7 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" @@ -41,6 +41,26 @@ func ListRegistries(c *gin.Context) { ctx.Resp, ctx.Err = service.ListRegistries(ctx.Logger) } +func GetDefaultRegistryNamespace(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + reg, err := commonservice.GetDefaultRegistryNamespace(ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + ctx.Resp = &Registry{ + ID: reg.ID.Hex(), + RegAddr: reg.RegAddr, + IsDefault: reg.IsDefault, + Namespace: reg.Namespace, + AccessKey: reg.AccessKey, + SecretyKey: reg.SecretyKey, + } +} + func ListRegistryNamespaces(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() diff --git a/pkg/microservice/aslan/core/system/handler/resp.go b/pkg/microservice/aslan/core/system/handler/resp.go index 9aef1a5e0a..994317fe98 100644 --- a/pkg/microservice/aslan/core/system/handler/resp.go +++ b/pkg/microservice/aslan/core/system/handler/resp.go @@ -15,3 +15,12 @@ limitations under the License. */ package handler + +type Registry struct { + ID string `json:"id"` + RegAddr string `json:"reg_addr"` + IsDefault bool `json:"is_default"` + Namespace string `json:"namespace"` + AccessKey string `json:"access_key"` + SecretyKey string `json:"secret_key"` +} diff --git a/pkg/microservice/aslan/core/system/handler/router.go b/pkg/microservice/aslan/core/system/handler/router.go index 38b7a071e0..8c8061a7ad 100644 --- a/pkg/microservice/aslan/core/system/handler/router.go +++ b/pkg/microservice/aslan/core/system/handler/router.go @@ -19,7 +19,7 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" ) type Router struct{} @@ -30,18 +30,18 @@ func (*Router) Inject(router *gin.RouterGroup) { proxy.GET("/config", GetProxyConfig) } - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) // --------------------------------------------------------------------------------------- // 安装脚本管理接口 // --------------------------------------------------------------------------------------- install := router.Group("install") { - install.POST("", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, CreateInstall) - install.PUT("", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, UpdateInstall) + install.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateInstall) + install.PUT("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateInstall) install.GET("/:name/:version", GetInstall) install.GET("", ListInstalls) - install.PUT("/delete", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, DeleteInstall) + install.PUT("/delete", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteInstall) } // --------------------------------------------------------------------------------------- @@ -51,9 +51,9 @@ func (*Router) Inject(router *gin.RouterGroup) { { proxyManage.GET("", ListProxies) proxyManage.GET("/:id", GetProxy) - proxyManage.POST("", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, CreateProxy) - proxyManage.PUT("/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, UpdateProxy) - proxyManage.DELETE("/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, DeleteProxy) + proxyManage.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateProxy) + proxyManage.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateProxy) + proxyManage.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteProxy) proxyManage.POST("/connectionTest", TestConnection) } @@ -61,11 +61,13 @@ func (*Router) Inject(router *gin.RouterGroup) { registry := router.Group("registry") { registry.GET("", ListRegistries) - registry.GET("/namespaces", middleware.RequireSuperAdminAuth, ListRegistryNamespaces) - registry.POST("/namespaces", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, CreateRegistryNamespace) - registry.PUT("/namespaces/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, UpdateRegistryNamespace) + // 获取默认的镜像仓库配置,用于kodespace CLI调用 + registry.GET("/namespaces/default", GetDefaultRegistryNamespace) + registry.GET("/namespaces", gin2.RequireSuperAdminAuth, ListRegistryNamespaces) + registry.POST("/namespaces", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateRegistryNamespace) + registry.PUT("/namespaces/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateRegistryNamespace) - registry.DELETE("/namespaces/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, DeleteRegistryNamespace) + registry.DELETE("/namespaces/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteRegistryNamespace) registry.GET("/release/repos", ListAllRepos) registry.POST("/images", ListImages) registry.GET("/images/repos/:name", ListRepoImages) @@ -74,10 +76,10 @@ func (*Router) Inject(router *gin.RouterGroup) { s3storage := router.Group("s3storage") { s3storage.GET("", ListS3Storage) - s3storage.POST("", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, CreateS3Storage) + s3storage.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateS3Storage) s3storage.GET("/:id", GetS3Storage) - s3storage.PUT("/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, UpdateS3Storage) - s3storage.DELETE("/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, DeleteS3Storage) + s3storage.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateS3Storage) + s3storage.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteS3Storage) } //系统清理缓存 @@ -93,8 +95,8 @@ func (*Router) Inject(router *gin.RouterGroup) { github := router.Group("githubApp") { github.GET("", GetGithubApp) - github.POST("", middleware.RequireSuperAdminAuth, CreateGithubApp) - github.DELETE("/:id", middleware.RequireSuperAdminAuth, DeleteGithubApp) + github.POST("", gin2.RequireSuperAdminAuth, CreateGithubApp) + github.DELETE("/:id", gin2.RequireSuperAdminAuth, DeleteGithubApp) } // --------------------------------------------------------------------------------------- @@ -102,13 +104,23 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- jenkins := router.Group("jenkins") { - jenkins.POST("/integration", middleware.RequireSuperAdminAuth, CreateJenkinsIntegration) + jenkins.POST("/integration", gin2.RequireSuperAdminAuth, CreateJenkinsIntegration) jenkins.GET("/integration", ListJenkinsIntegration) - jenkins.PUT("/integration/:id", middleware.RequireSuperAdminAuth, UpdateJenkinsIntegration) - jenkins.DELETE("/integration/:id", middleware.RequireSuperAdminAuth, DeleteJenkinsIntegration) - jenkins.POST("/user/connection", middleware.RequireSuperAdminAuth, TestJenkinsConnection) - jenkins.GET("/jobNames", middleware.RequireSuperAdminAuth, ListJobNames) - jenkins.GET("/buildArgs/:jobName", middleware.RequireSuperAdminAuth, ListJobBuildArgs) + jenkins.PUT("/integration/:id", gin2.RequireSuperAdminAuth, UpdateJenkinsIntegration) + jenkins.DELETE("/integration/:id", gin2.RequireSuperAdminAuth, DeleteJenkinsIntegration) + jenkins.POST("/user/connection", gin2.RequireSuperAdminAuth, TestJenkinsConnection) + jenkins.GET("/jobNames", gin2.RequireSuperAdminAuth, ListJobNames) + jenkins.GET("/buildArgs/:jobName", gin2.RequireSuperAdminAuth, ListJobBuildArgs) + } + + //系统配额 + capacity := router.Group("capacity") + { + capacity.POST("", UpdateStrategy) + capacity.GET("/target/:target", GetStrategy) + capacity.POST("/gc", GarbageCollection) + // 清理已被删除的工作流的所有缓存,暂时用于手动调用 + capacity.POST("/clean", CleanCache) } // --------------------------------------------------------------------------------------- @@ -118,9 +130,43 @@ func (*Router) Inject(router *gin.RouterGroup) { { basicImages.GET("", ListBasicImages) basicImages.GET("/:id", GetBasicImage) - basicImages.POST("", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, CreateBasicImage) - basicImages.PUT("/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, UpdateBasicImage) - basicImages.DELETE("/:id", middleware.RequireSuperAdminAuth, middleware.UpdateOperationLogStatus, DeleteBasicImage) + basicImages.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreateBasicImage) + basicImages.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdateBasicImage) + basicImages.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeleteBasicImage) + } + + // --------------------------------------------------------------------------------------- + // helm chart 集成 + // --------------------------------------------------------------------------------------- + integration := router.Group("helm") + { + integration.GET("", ListHelmRepos) + integration.POST("", gin2.RequireSuperAdminAuth, CreateHelmRepo) + integration.PUT("/:id", gin2.RequireSuperAdminAuth, UpdateHelmRepo) + integration.DELETE("/:id", gin2.RequireSuperAdminAuth, DeleteHelmRepo) + } + + // --------------------------------------------------------------------------------------- + // ssh私钥管理接口 + // --------------------------------------------------------------------------------------- + privateKey := router.Group("privateKey") + { + privateKey.GET("", ListPrivateKeys) + privateKey.GET("/:id", GetPrivateKey) + privateKey.POST("", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, CreatePrivateKey) + privateKey.PUT("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, UpdatePrivateKey) + privateKey.DELETE("/:id", gin2.RequireSuperAdminAuth, gin2.UpdateOperationLogStatus, DeletePrivateKey) + } + + notification := router.Group("notification") + { + notification.GET("", PullNotify) + notification.PUT("/read", ReadNotify) + notification.POST("/delete", DeleteNotifies) + notification.POST("/subscribe", UpsertSubscription) + notification.PUT("/subscribe/:type", UpdateSubscribe) + notification.DELETE("/unsubscribe/notifytype/:type", Unsubscribe) + notification.GET("/subscribe", ListSubscriptions) } } diff --git a/pkg/microservice/aslan/core/system/handler/s3.go b/pkg/microservice/aslan/core/system/handler/s3.go index 16dedea0f6..e81ee6cd68 100644 --- a/pkg/microservice/aslan/core/system/handler/s3.go +++ b/pkg/microservice/aslan/core/system/handler/s3.go @@ -27,7 +27,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/system/handler/system_cache.go b/pkg/microservice/aslan/core/system/handler/system_cache.go index dab37aa9af..00e87f1722 100644 --- a/pkg/microservice/aslan/core/system/handler/system_cache.go +++ b/pkg/microservice/aslan/core/system/handler/system_cache.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/system/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func CleanImageCache(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/system/service/basic_images.go b/pkg/microservice/aslan/core/system/service/basic_images.go index 0c43733268..1fe3c19d7c 100644 --- a/pkg/microservice/aslan/core/system/service/basic_images.go +++ b/pkg/microservice/aslan/core/system/service/basic_images.go @@ -25,7 +25,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -139,14 +139,14 @@ func DeleteBasicImage(id string, log *zap.SugaredLogger) error { if err == nil { for _, pipeline := range pipelines { for _, subTask := range pipeline.SubTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { continue } switch pre.TaskType { case config.TaskBuild: // 校验是否被服务工作流的构建使用 - task, err := commonservice.ToBuildTask(subTask) + task, err := base.ToBuildTask(subTask) if err != nil || task == nil { continue } @@ -156,7 +156,7 @@ func DeleteBasicImage(id string, log *zap.SugaredLogger) error { } case config.TaskTestingV2: // 校验是否被服务工作流的测试使用 - testing, err := commonservice.ToTestingTask(subTask) + testing, err := base.ToTestingTask(subTask) if err != nil || testing == nil { continue } diff --git a/pkg/microservice/aslan/core/system/service/capacity.go b/pkg/microservice/aslan/core/system/service/capacity.go new file mode 100644 index 0000000000..1b24e4fe31 --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/capacity.go @@ -0,0 +1,345 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "errors" + "fmt" + "time" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/notify" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/util" +) + +const ( + defaultWorkflowMaxDays int = 365 + //logTag = "SysCap" +) + +var defaultWorkflowTaskRetention = &commonmodels.CapacityStrategy{ + Target: commonmodels.WorkflowTaskRetention, + Retention: &commonmodels.RetentionConfig{ + MaxDays: defaultWorkflowMaxDays, + }, +} + +func UpdateSysCapStrategy(strategy *commonmodels.CapacityStrategy) error { + if err := validateStrategy(strategy); err != nil { + return err + } + + err := commonrepo.NewStrategyColl().Upsert(strategy) + if err != nil { + return err + } + + // 更新成功后,立即按照新的配置清理数据 + go handleWorkflowTaskRetentionCenter(strategy, false) + + return nil +} + +func GetCapacityStrategy(target commonmodels.CapacityTarget) (*commonmodels.CapacityStrategy, error) { + result, err := commonrepo.NewStrategyColl().GetByTarget(target) + if err != nil && target == commonmodels.WorkflowTaskRetention { + return defaultWorkflowTaskRetention, nil // Return default setup + } + return result, err +} + +func HandleSystemGC(dryRun bool) error { + // Find the strategy + strategy, err := commonrepo.NewStrategyColl().GetByTarget(commonmodels.WorkflowTaskRetention) + if err != nil { + strategy = defaultWorkflowTaskRetention + } else if err = validateStrategy(strategy); err != nil { + return err + } + + return handleWorkflowTaskRetentionCenter(strategy, dryRun) +} + +func CleanCache() error { + workflowMap := make(map[string]int) + + workflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{}) + if err != nil { + log.Errorf("list workflows failed, err:%v", err) + return err + } + for _, workflow := range workflows { + // 注意s3的目录名有一个/后缀,处理时需要加一个/ + name := fmt.Sprintf("%s/", workflow.Name) + if _, ok := workflowMap[name]; ok { + continue + } + workflowMap[name] = 1 + } + + pipelines, err := commonrepo.NewPipelineColl().List(&commonrepo.PipelineListOption{}) + if err != nil { + log.Errorf("list pipelines failed, err:%v", err) + return err + } + for _, pipeline := range pipelines { + name := fmt.Sprintf("%s/", pipeline.Name) + if _, ok := workflowMap[name]; ok { + continue + } + workflowMap[name] = 1 + } + + testings, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{TestType: setting.FunctionTest}) + if err != nil { + log.Errorf("list testings failed, err:%v", err) + return err + } + for _, testing := range testings { + name := fmt.Sprintf("%s-%s/", testing.Name, "job") + if _, ok := workflowMap[name]; ok { + continue + } + workflowMap[name] = 1 + } + + s3Server := s3.FindInternalS3() + objects, err := s3.ListFiles(s3Server, "", false /* recursive */) + if err != nil { + log.Errorf("ListFiles failed, err:%v", err) + return err + } + log.Infof("workflow count: %d, pipeline count: %d, testing count: %d", len(workflows), len(pipelines), len(testings)) + log.Infof("total paths in s3: %d, len(workflowMap): %d", len(objects), len(workflowMap)) + + // 找出已被删除的工作流的缓存目录 + paths := make([]string, 0) + for _, object := range objects { + if _, ok := workflowMap[object]; ok { + continue + } + log.Infof("ready to remove file or path:%s\n", object) + paths = append(paths, object) + } + + s3.RemoveFiles(s3Server, paths, false) + + return nil +} + +type MessageCtx struct { + ReqID string `bson:"req_id" json:"req_id"` + Title string `bson:"title" json:"title"` // 消息标题 + Content string `bson:"content" json:"content"` // 消息内容 +} + +func sendSyscapNotify(handleErr error, totalCleanTasks *int) { + content := &MessageCtx{ + ReqID: util.UUID(), + Title: "清理历史工作流数据", + } + now := time.Now().Format("2006-01-02 15:04:05") + content.Content = fmt.Sprintf("清理时间: %s, 状态: 成功, 内容: 成功清理了%d条任务", now, *totalCleanTasks) + if handleErr != nil { + content.Content = fmt.Sprintf("清理时间: %s, 状态: 失败, 内容: %v", now, handleErr) + } + + notifyInfo := &commonmodels.Notify{ + Type: config.Message, + Content: content, + CreateTime: time.Now().Unix(), + IsRead: false, + } + + poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) + users, _ := poetryClient.ListProductPermissionUsers("", "", log.SugaredLogger()) + for _, user := range users { + notifyInfo.Receiver = user + notify.NewNotifyClient().CreateNotify(user, notifyInfo) + } +} + +func handleWorkflowTaskRetentionCenter(strategy *commonmodels.CapacityStrategy, dryRun bool) error { + var handleErr error + var totalCleanTasks int + + // 向管理员发送通知,告知清理结果 + defer sendSyscapNotify(handleErr, &totalCleanTasks) + + // Figuring out starting option based on capacity strategy + var option *commonrepo.ListAllTaskOption + const batch = 100 + retention := strategy.Retention + if retention.MaxDays > 0 { + retentionTime := time.Now().AddDate(0, 0, -retention.MaxDays).Unix() + option = &commonrepo.ListAllTaskOption{ + BeforeCreatTime: true, + CreateTime: retentionTime, + Limit: batch, + } + totalCleanTasks, handleErr = handleWorkflowTaskRetention(dryRun, batch, option) + return handleErr + } + + if retention.MaxItems > 0 { + // 清理产品工作流任务数据 + workflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{}) + if err != nil { + log.Errorf("list workflows failed, err:%v", err) + handleErr = err + return err + } + for _, workflow := range workflows { + option = &commonrepo.ListAllTaskOption{ + ProductName: workflow.ProductTmplName, + PipelineName: workflow.Name, + Type: config.WorkflowType, + Skip: retention.MaxItems, + Limit: batch, + } + count, err := handleWorkflowTaskRetention(dryRun, batch, option) + if err != nil { + continue + } + totalCleanTasks += count + } + + // 清理服务工作流任务数据 + pipelines, err := commonrepo.NewPipelineColl().List(&commonrepo.PipelineListOption{}) + if err != nil { + log.Errorf("list pipelines failed, err:%v", err) + handleErr = err + return err + } + for _, pipeline := range pipelines { + option = &commonrepo.ListAllTaskOption{ + ProductName: pipeline.ProductName, + PipelineName: pipeline.Name, + Type: config.SingleType, + Skip: retention.MaxItems, + Limit: batch, + } + count, err := handleWorkflowTaskRetention(dryRun, batch, option) + if err != nil { + continue + } + totalCleanTasks += count + } + + // 清理测试任务数据 + testings, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{TestType: setting.FunctionTest}) + if err != nil { + log.Errorf("list testings failed, err:%v", err) + handleErr = err + return err + } + for _, testing := range testings { + option = &commonrepo.ListAllTaskOption{ + ProductName: testing.ProductName, + PipelineName: fmt.Sprintf("%s-%s", testing.Name, "job"), + Type: config.TestType, + Skip: retention.MaxItems, + Limit: batch, + } + count, err := handleWorkflowTaskRetention(dryRun, batch, option) + if err != nil { + continue + } + totalCleanTasks += count + } + return nil + } + + return errors.New("no valid strategy for workflow task retention") +} + +func handleWorkflowTaskRetention(dryRun bool, batch int, option *commonrepo.ListAllTaskOption) (int, error) { + s3Server, _ := s3.FindDefaultS3() + // Clean up in batches to prevent pressure of memory spike + var removeIds []string + for { + staleTasks, err := commonrepo.NewTaskColl().ListTasks(option) + if err != nil { + return 0, err + } + if len(staleTasks) == 0 { + break + } + ids := cleanStaleTasks(staleTasks, s3Server, dryRun) + if ids != nil { + removeIds = append(removeIds, ids...) + } + if len(staleTasks) < batch { // last batch + break + } + option.Skip += batch + } + + //if !dryRun { + // if err := s.taskRepo.DeleteByIds(removeIds); err != nil { + // return 0, err + // } + //} + + log.Infof("%d stale workflow tasks will be cleaned up, productName:%s, workflowName:%s, type:%s", len(removeIds), option.ProductName, option.PipelineName, option.Type) + return len(removeIds), nil +} + +// +//func (s *Service) logInfo(format string, args ...interface{}) { +// s.logger.Infof("[%v]: %v", logTag, fmt.Sprintf(format, args...)) +//} +// +// cleanStaleTasks will mark stale tasks as deleted, and remove their relevant S3 files. +// returns: +// []bson.ObjectId, task ides to be marked as deleted +func cleanStaleTasks(tasks []*task.Task, s3Server *s3.S3, dryRun bool) []string { + ids := make([]string, len(tasks)) + paths := make([]string, len(tasks)) + for i, task := range tasks { + ids[i] = task.ID.Hex() + paths[i] = fmt.Sprintf("%s/%d/", task.PipelineName, task.TaskID) + } + go s3.RemoveFiles(s3Server, paths, dryRun) + return ids +} + +func validateStrategy(strategy *commonmodels.CapacityStrategy) error { + if strategy.Target == commonmodels.WorkflowTaskRetention { + retention := strategy.Retention + if retention == nil { + return errors.New("SysCap strategy: nil retention config for WorkflowTaskRetention") + } + if !(retention.MaxDays > 0 && retention.MaxItems == 0) && + !(retention.MaxDays == 0 && retention.MaxItems > 0) { + return fmt.Errorf("SysCap strategy: max days or items value invalid, "+ + "can only set one positive value at a time. days: %v, items: %v", + retention.MaxDays, retention.MaxItems) + } + } else { + // Note: currently doesn't support other strategies yet. + return fmt.Errorf("SysCap strategy target is invalid - passed in value: %v", strategy.Target) + } + return nil +} diff --git a/pkg/microservice/aslan/core/system/service/helm.go b/pkg/microservice/aslan/core/system/service/helm.go new file mode 100644 index 0000000000..585e2f5f3d --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/helm.go @@ -0,0 +1,58 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" +) + +func ListHelmRepos(log *zap.SugaredLogger) ([]*commonmodels.HelmRepo, error) { + helmRepos, err := commonrepo.NewHelmRepoColl().List() + if err != nil { + log.Errorf("ListHelmRepos err:%v", err) + return []*commonmodels.HelmRepo{}, nil + } + + return helmRepos, nil +} + +func CreateHelmRepo(args *commonmodels.HelmRepo, log *zap.SugaredLogger) error { + if err := commonrepo.NewHelmRepoColl().Create(args); err != nil { + log.Errorf("CreateHelmRepo err:%v", err) + return err + } + return nil +} + +func UpdateHelmRepo(id string, args *commonmodels.HelmRepo, log *zap.SugaredLogger) error { + if err := commonrepo.NewHelmRepoColl().Update(id, args); err != nil { + log.Errorf("UpdateHelmRepo err:%v", err) + return err + } + return nil +} + +func DeleteHelmRepo(id string, log *zap.SugaredLogger) error { + if err := commonrepo.NewHelmRepoColl().Delete(id); err != nil { + log.Errorf("DeleteHelmRepo err:%v", err) + return err + } + return nil +} diff --git a/pkg/microservice/aslan/core/enterprise/service/subscription.go b/pkg/microservice/aslan/core/system/service/notify.go similarity index 72% rename from pkg/microservice/aslan/core/enterprise/service/subscription.go rename to pkg/microservice/aslan/core/system/service/notify.go index aa1c50e832..1418ada5ab 100644 --- a/pkg/microservice/aslan/core/enterprise/service/subscription.go +++ b/pkg/microservice/aslan/core/system/service/notify.go @@ -24,6 +24,31 @@ import ( e "github.com/koderover/zadig/pkg/tool/errors" ) +func PullNotify(user string, log *zap.SugaredLogger) ([]*commonmodels.Notify, error) { + resp, err := notify.NewNotifyClient().PullNotify(user) + if err != nil { + log.Errorf("NotifyCli.PullNotify error: %v", err) + return resp, e.ErrPullNotify + } + return resp, nil +} + +func ReadNotify(user string, notifyIDs []string, log *zap.SugaredLogger) error { + if err := notify.NewNotifyClient().Read(user, notifyIDs); err != nil { + log.Errorf("NotifyCli.Read error: %v", err) + return e.ErrReadNotify + } + return nil +} + +func DeleteNotifies(user string, notifyIDs []string, log *zap.SugaredLogger) error { + if err := notify.NewNotifyClient().DeleteNotifies(user, notifyIDs); err != nil { + log.Errorf("NotifyCli.DeleteNotifies error: %v", err) + return e.ErrDeleteNotifies + } + return nil +} + func UpsertSubscription(user string, subscription *commonmodels.Subscription, log *zap.SugaredLogger) error { if err := notify.NewNotifyClient().UpsertSubscription(user, subscription); err != nil { log.Errorf("NotifyCli.Subscribe error: %v", err) diff --git a/pkg/microservice/aslan/core/system/service/private_key.go b/pkg/microservice/aslan/core/system/service/private_key.go new file mode 100644 index 0000000000..46e325229b --- /dev/null +++ b/pkg/microservice/aslan/core/system/service/private_key.go @@ -0,0 +1,81 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListPrivateKeys(log *zap.SugaredLogger) ([]*commonmodels.PrivateKey, error) { + resp, err := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{}) + if err != nil { + log.Errorf("PrivateKey.List error: %v", err) + return resp, e.ErrListPrivateKeys + } + return resp, nil +} + +func GetPrivateKey(id string, log *zap.SugaredLogger) (*commonmodels.PrivateKey, error) { + resp, err := commonrepo.NewPrivateKeyColl().Find(id) + if err != nil { + log.Errorf("PrivateKey.Find %s error: %v", id, err) + return resp, e.ErrGetPrivateKey + } + return resp, nil +} + +func CreatePrivateKey(args *commonmodels.PrivateKey, log *zap.SugaredLogger) error { + if privateKeys, _ := commonrepo.NewPrivateKeyColl().List(&commonrepo.PrivateKeyArgs{Name: args.Name}); len(privateKeys) > 0 { + return e.ErrCreatePrivateKey.AddDesc("Name already exists") + } + err := commonrepo.NewPrivateKeyColl().Create(args) + if err != nil { + log.Errorf("PrivateKey.Create error: %v", err) + return e.ErrCreatePrivateKey + } + return nil +} + +func UpdatePrivateKey(id string, args *commonmodels.PrivateKey, log *zap.SugaredLogger) error { + err := commonrepo.NewPrivateKeyColl().Update(id, args) + if err != nil { + log.Errorf("PrivateKey.Update %s error: %v", id, err) + return e.ErrUpdatePrivateKey + } + return nil +} + +func DeletePrivateKey(id string, log *zap.SugaredLogger) error { + // 检查该私钥是否被引用 + buildOpt := &commonrepo.BuildListOption{PrivateKeyID: id} + builds, err := commonrepo.NewBuildColl().List(buildOpt) + if err == nil && len(builds) != 0 { + log.Errorf("PrivateKey has been used by build, private key id:%s, product name:%s, build name:%s", id, builds[0].ProductName, builds[0].Name) + return e.ErrDeleteUsedPrivateKey + } + + err = commonrepo.NewPrivateKeyColl().Delete(id) + if err != nil { + log.Errorf("PrivateKey.Delete %s error: %v", id, err) + return e.ErrDeletePrivateKey + } + return nil +} diff --git a/pkg/microservice/aslan/core/system/service/system_cache.go b/pkg/microservice/aslan/core/system/service/system_cache.go index 96e8ca0b96..ca4d9ce520 100644 --- a/pkg/microservice/aslan/core/system/service/system_cache.go +++ b/pkg/microservice/aslan/core/system/service/system_cache.go @@ -25,11 +25,11 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/labels" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" kubetool "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" diff --git a/pkg/microservice/aslan/core/workflow/handler/build.go b/pkg/microservice/aslan/core/workflow/handler/build.go index 21e64a0b07..74f0e1e565 100644 --- a/pkg/microservice/aslan/core/workflow/handler/build.go +++ b/pkg/microservice/aslan/core/workflow/handler/build.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func BuildModuleToSubTasks(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go b/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go index e3568ad2a8..fe64869666 100644 --- a/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go +++ b/pkg/microservice/aslan/core/workflow/handler/favorate_pipeline.go @@ -26,11 +26,24 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" ) +func ListFavoritePipelines(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + productName := c.Query("productName") + workflowType := c.Query("type") + if workflowType == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("type can't be empty!") + return + } + ctx.Resp, ctx.Err = workflow.ListFavoritePipelines(&commonrepo.FavoriteArgs{UserID: ctx.User.ID, ProductName: productName, Type: workflowType}) +} + func DeleteFavoritePipeline(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() diff --git a/pkg/microservice/aslan/core/workflow/handler/pipeline.go b/pkg/microservice/aslan/core/workflow/handler/pipeline.go new file mode 100644 index 0000000000..cc7fa21491 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/handler/pipeline.go @@ -0,0 +1,141 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + + "github.com/gin-gonic/gin" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types/permission" +) + +func GetPipelineProductName(c *gin.Context) { + args := new(commonmodels.Pipeline) + data, err := c.GetRawData() + if err != nil { + log.Errorf("c.GetRawData() err : %v", err) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("json.Unmarshal err : %v", err) + return + } + c.Set("productName", args.ProductName) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + c.Next() +} + +func GetProductNameByPipeline(c *gin.Context) { + ctx := internalhandler.NewContext(c) + pipelineName := c.Param("old") + if pipelineName == "" { + pipelineName = c.Param("name") + } + pipeline, err := workflow.GetPipeline(ctx.User.ID, pipelineName, ctx.Logger) + if err != nil { + log.Errorf("GetProductNameByPipeline err : %v", err) + return + } + c.Set("productName", pipeline.ProductName) + c.Next() +} + +// ListPipelines +// @Router /workflow/v2/pipelines [GET] +// @Summary Return all workflows (also called pipelines) +// @Produce json +// @Success 200 {object} interface{} "response type follows list of microservice/aslan/core/common/repository/models#Pipeline" +func ListPipelines(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.ListPipelines(ctx.Logger) +} + +// GetPipeline +// @Router /workflow/v2/pipelines/{name} [GET] +// @Summary Get the relevant workflow (also called pipeline) information with the specified workflow name +// @Param name path string true "Name of the workflow" +// @Produce json +// @Success 200 {object} interface{} "response type follows microservice/aslan/core/common/repository/models#Pipeline" +func GetPipeline(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.GetPipeline(ctx.User.ID, c.Param("name"), ctx.Logger) +} + +// UpsertPipeline create a new pipeline +func UpsertPipeline(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.Pipeline) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpsertPipeline c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpsertPipeline json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, args.ProductName, "新增", "单服务-工作流", args.Name, permission.WorkflowCreateUUID, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil || len(args.Name) == 0 { + log.Error(err) + ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("invalid pipeline json args: %v", err)) + return + } + args.UpdateBy = ctx.Username + ctx.Err = workflow.UpsertPipeline(args, ctx.Logger) +} + +// CopyPipeline duplicate pipeline +func CopyPipeline(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "复制", "单服务-工作流", fmt.Sprintf("old:%s,new:%s", c.Param("old"), c.Param("new")), permission.WorkflowCreateUUID, "", ctx.Logger) + ctx.Err = workflow.CopyPipeline(c.Param("old"), c.Param("new"), ctx.Username, ctx.Logger) +} + +// RenamePipeline rename pipeline +func RenamePipeline(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "修改", "单服务-工作流", fmt.Sprintf("old:%s,new:%s", c.Param("old"), c.Param("new")), permission.WorkflowUpdateUUID, "", ctx.Logger) + ctx.Err = workflow.RenamePipeline(c.Param("old"), c.Param("new"), ctx.Logger) +} + +// DeletePipeline delete pipeline +func DeletePipeline(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "删除", "单服务-工作流", c.Param("name"), permission.WorkflowDeleteUUID, "", ctx.Logger) + ctx.Err = workflow.DeletePipeline(c.Param("name"), ctx.RequestID, false, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/workflow/handler/gerrit.go b/pkg/microservice/aslan/core/workflow/handler/pipeline_status.go similarity index 55% rename from pkg/microservice/aslan/core/workflow/handler/gerrit.go rename to pkg/microservice/aslan/core/workflow/handler/pipeline_status.go index e442b5b3d9..be25decbe2 100644 --- a/pkg/microservice/aslan/core/workflow/handler/gerrit.go +++ b/pkg/microservice/aslan/core/workflow/handler/pipeline_status.go @@ -19,24 +19,22 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/webhook" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" - e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) -// @Router /workflow/webhook/gerritHook [POST] -// @Summary Process webhook for gerrit -// @Accept json -// @Produce json -// @Success 200 {object} map[string]string "map[string]string - {message: success}" -func ProcessGerritHook(c *gin.Context) { +// ListPipelinesPreview return all pipelines task status +func ListPipelinesPreview(c *gin.Context) { ctx := internalhandler.NewContext(c) - log := ctx.Logger - err := webhook.ProcessGerritHook(c.Request, ctx.RequestID, log) - if err != nil { - c.JSON(e.ErrorMessage(err)) - c.Abort() - return - } - c.JSON(200, gin.H{"message": "success"}) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.ListPipelinesPreview(ctx.User.ID, ctx.Logger) +} + +// find task by commitId +func FindTasks(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.FindTasks(c.Query("commitId"), ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go b/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go index aa0a853fe7..9bf556f5e8 100644 --- a/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go +++ b/pkg/microservice/aslan/core/workflow/handler/pipeline_task.go @@ -17,15 +17,172 @@ limitations under the License. package handler import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" "strconv" "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types/permission" ) +func GetProductNameByPipelineTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + args := new(commonmodels.TaskArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("c.GetRawData() err : %v", err) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("json.Unmarshal err : %v", err) + return + } + + pipeline, err := workflow.GetPipeline(ctx.User.ID, args.PipelineName, ctx.Logger) + if err != nil { + log.Errorf("GetProductNameByPipelineTask err : %v", err) + return + } + c.Set("productName", pipeline.ProductName) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + c.Next() +} + +// CreatePipelineTask ... +func CreatePipelineTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(commonmodels.TaskArgs) + data, err := c.GetRawData() + if err != nil { + log.Errorf("CreatePipelineTask c.GetRawData() err : %v", err) + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("CreatePipelineTask json.Unmarshal err : %v", err) + } + internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "新增", "单服务-工作流task", args.PipelineName, permission.WorkflowTaskUUID, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid PipelineTaskArgs") + return + } + if args.TaskCreator == "" { + args.TaskCreator = ctx.Username + } + + args.ReqID = ctx.RequestID + ctx.Resp, ctx.Err = workflow.CreatePipelineTask(args, ctx.Logger) + + // 发送通知 + if ctx.Err != nil { + commonservice.SendFailedTaskMessage(ctx.Username, args.ProductName, args.PipelineName, ctx.RequestID, config.SingleType, ctx.Err, ctx.Logger) + } + +} + +// ListPipelineTasksResult pipelinetask分页信息 +func ListPipelineTasksResult(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + maxResult, err := strconv.Atoi(c.Param("max")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid max result number") + return + } + startAt, err := strconv.Atoi(c.Param("start")) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid start at number") + return + } + ctx.Resp, ctx.Err = workflow.ListPipelineTasksV2Result(c.Param("name"), config.SingleType, maxResult, startAt, ctx.Logger) +} + +func GetPipelineTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + + ctx.Resp, ctx.Err = workflow.GetPipelineTaskV2(taskID, c.Param("name"), config.SingleType, ctx.Logger) +} + +func RestartPipelineTask(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "重启", "单服务-工作流task", c.Param("name"), permission.WorkflowTaskUUID, "", ctx.Logger) + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + + ctx.Err = workflow.RestartPipelineTaskV2(ctx.Username, taskID, c.Param("name"), config.SingleType, ctx.Logger) +} + +func CancelTaskV2(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + internalhandler.InsertOperationLog(c, ctx.Username, c.GetString("productName"), "取消", "单服务-工作流task", c.Param("name"), permission.WorkflowTaskUUID, "", ctx.Logger) + + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid task id") + return + } + ctx.Err = commonservice.CancelTaskV2(ctx.Username, c.Param("name"), taskID, config.SingleType, ctx.RequestID, ctx.Logger) +} + +// ListPipelineUpdatableProductNames 启动任务时检查部署环境 +func ListPipelineUpdatableProductNames(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = workflow.ListPipelineUpdatableProductNames(ctx.Username, c.Param("name"), ctx.Logger) +} + +func GetPackageFile(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + pipelineName := c.Query("pipelineName") + if pipelineName == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid pipelineName") + return + } + + taskID, err := strconv.ParseInt(c.Query("taskId"), 10, 64) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid taskId") + return + } + + resp, pkgFile, err := workflow.GePackageFileContent(pipelineName, taskID, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + c.Writer.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, pkgFile)) + c.Data(200, "application/octet-stream", resp) +} + func GetArtifactFile(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() diff --git a/pkg/microservice/aslan/core/workflow/handler/router.go b/pkg/microservice/aslan/core/workflow/handler/router.go index 5bcb442927..fa511c593c 100644 --- a/pkg/microservice/aslan/core/workflow/handler/router.go +++ b/pkg/microservice/aslan/core/workflow/handler/router.go @@ -19,7 +19,7 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) @@ -31,13 +31,10 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- webhook := router.Group("webhook") { - webhook.POST("/ci/webhook", ProcessGithub) - webhook.POST("/githubWebHook", ProcessGithub) - webhook.POST("/gitlabhook", ProcessGitlabHook) - webhook.POST("/gerritHook", ProcessGerritHook) + webhook.POST("", ProcessWebHook) } - router.Use(middleware.Auth()) + router.Use(gin2.Auth()) build := router.Group("build") { @@ -55,11 +52,40 @@ func (*Router) Inject(router *gin.RouterGroup) { sse.GET("/tasks/id/:id/pipelines/:name", GetPipelineTaskSSE) } + // --------------------------------------------------------------------------------------- + // Pipeline 管理接口 + // --------------------------------------------------------------------------------------- + pipeline := router.Group("v2/pipelines") + { + pipeline.GET("", ListPipelines) + pipeline.GET("/:name", GetPipeline) + pipeline.POST("", GetPipelineProductName, gin2.IsHavePermission([]string{permission.WorkflowCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpsertPipeline) + pipeline.POST("/old/:old/new/:new", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CopyPipeline) + pipeline.PUT("/rename/:old/:new", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RenamePipeline) + pipeline.DELETE("/:name", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowDeleteUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, DeletePipeline) + } + + // --------------------------------------------------------------------------------------- + // Pipeline 状态接口 + // --------------------------------------------------------------------------------------- + statusV2 := router.Group("v2/status") + { + statusV2.GET("/preview", ListPipelinesPreview) + statusV2.GET("/task/info", FindTasks) + } + // --------------------------------------------------------------------------------------- // Pipeline 任务管理接口 // --------------------------------------------------------------------------------------- taskV2 := router.Group("v2/tasks") { + taskV2.POST("", GetProductNameByPipelineTask, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreatePipelineTask) + taskV2.GET("/max/:max/start/:start/pipelines/:name", ListPipelineTasksResult) + taskV2.GET("/id/:id/pipelines/:name", GetPipelineTask) + taskV2.POST("/id/:id/pipelines/:name/restart", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RestartPipelineTask) + taskV2.DELETE("/id/:id/pipelines/:name", GetProductNameByPipeline, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CancelTaskV2) + taskV2.GET("/pipelines/:name/products", ListPipelineUpdatableProductNames) + taskV2.GET("/file", GetPackageFile) taskV2.GET("/workflow/:pipelineName/taskId/:taskId", GetArtifactFile) } @@ -68,9 +94,9 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- favorite := router.Group("favorite") { - favorite.POST("", middleware.UpdateOperationLogStatus, CreateFavoritePipeline) - favorite.DELETE("/:productName/:name/:type", middleware.UpdateOperationLogStatus, DeleteFavoritePipeline) - + favorite.POST("", gin2.UpdateOperationLogStatus, CreateFavoritePipeline) + favorite.DELETE("/:productName/:name/:type", gin2.UpdateOperationLogStatus, DeleteFavoritePipeline) + favorite.GET("", ListFavoritePipelines) } // --------------------------------------------------------------------------------------- @@ -79,12 +105,12 @@ func (*Router) Inject(router *gin.RouterGroup) { workflow := router.Group("workflow") { workflow.POST("/:productName/auto", AutoCreateWorkflow) - workflow.POST("", GetWorkflowProductName, middleware.IsHavePermission([]string{permission.WorkflowCreateUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateWorkflow) - workflow.PUT("", GetWorkflowProductName, middleware.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateWorkflow) + workflow.POST("", GetWorkflowProductName, gin2.IsHavePermission([]string{permission.WorkflowCreateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateWorkflow) + workflow.PUT("", GetWorkflowProductName, gin2.IsHavePermission([]string{permission.WorkflowUpdateUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateWorkflow) workflow.GET("", ListWorkflows) workflow.GET("/testName/:testName", ListAllWorkflows) workflow.GET("/find/:name", FindWorkflow) - workflow.DELETE("/:name", GetProductNameByWorkflow, middleware.IsHavePermission([]string{permission.WorkflowDeleteUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, DeleteWorkflow) + workflow.DELETE("/:name", GetProductNameByWorkflow, gin2.IsHavePermission([]string{permission.WorkflowDeleteUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, DeleteWorkflow) workflow.GET("/preset/:productName", PreSetWorkflow) workflow.PUT("/old/:old/new/:new", CopyWorkflow) @@ -98,12 +124,12 @@ func (*Router) Inject(router *gin.RouterGroup) { //todo 修改权限的uuid workflowtask.GET("/targets/:productName/:namespace", GetWorkflowArgs) workflowtask.GET("/preset/:namespace/:workflowName", PresetWorkflowArgs) - workflowtask.POST("", GetWorkflowTaskProductName, middleware.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateWorkflowTask) - workflowtask.PUT("", GetWorkflowTaskProductName, middleware.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateArtifactWorkflowTask) + workflowtask.POST("", GetWorkflowTaskProductName, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateWorkflowTask) + workflowtask.PUT("", GetWorkflowTaskProductName, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateArtifactWorkflowTask) workflowtask.GET("/max/:max/start/:start/pipelines/:name", ListWorkflowTasksResult) workflowtask.GET("/id/:id/pipelines/:name", GetWorkflowTask) - workflowtask.POST("/id/:id/pipelines/:name/restart", GetWorkflowTaskProductNameByTask, middleware.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, RestartWorkflowTask) - workflowtask.DELETE("/id/:id/pipelines/:name", GetWorkflowTaskProductNameByTask, middleware.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CancelWorkflowTaskV2) + workflowtask.POST("/id/:id/pipelines/:name/restart", GetWorkflowTaskProductNameByTask, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, RestartWorkflowTask) + workflowtask.DELETE("/id/:id/pipelines/:name", GetWorkflowTaskProductNameByTask, gin2.IsHavePermission([]string{permission.WorkflowTaskUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CancelWorkflowTaskV2) } serviceTask := router.Group("servicetask") diff --git a/pkg/microservice/aslan/core/workflow/handler/service_task.go b/pkg/microservice/aslan/core/workflow/handler/service_task.go index 02aa2f1d18..88d39d27cd 100644 --- a/pkg/microservice/aslan/core/workflow/handler/service_task.go +++ b/pkg/microservice/aslan/core/workflow/handler/service_task.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func ListServiceWorkflows(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/workflow/handler/sse.go b/pkg/microservice/aslan/core/workflow/handler/sse.go index 8bcfec118d..a83214fa90 100644 --- a/pkg/microservice/aslan/core/workflow/handler/sse.go +++ b/pkg/microservice/aslan/core/workflow/handler/sse.go @@ -27,7 +27,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/workflow/handler/github.go b/pkg/microservice/aslan/core/workflow/handler/webhook.go similarity index 56% rename from pkg/microservice/aslan/core/workflow/handler/github.go rename to pkg/microservice/aslan/core/workflow/handler/webhook.go index 52b4c62b2a..14da6c08b6 100644 --- a/pkg/microservice/aslan/core/workflow/handler/github.go +++ b/pkg/microservice/aslan/core/workflow/handler/webhook.go @@ -17,56 +17,59 @@ limitations under the License. package handler import ( - "bytes" - "io/ioutil" + "net/http" "github.com/gin-gonic/gin" + "github.com/google/go-github/v35/github" "github.com/hashicorp/go-multierror" + "github.com/xanzy/go-gitlab" + "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/webhook" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" - e "github.com/koderover/zadig/pkg/tool/errors" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) -// @Router /workflow/webhook/githubWebHook [POST] -// @Summary Process webhook for github +// @Router /workflow/webhook [POST] +// @Summary Process webhook // @Accept json // @Produce json // @Success 200 {object} map[string]string "map[string]string - {message: 'success information'}" -func ProcessGithub(c *gin.Context) { +func ProcessWebHook(c *gin.Context) { ctx := internalhandler.NewContext(c) - log := ctx.Logger - errs := &multierror.Error{} + defer func() { internalhandler.JSONResponse(c, ctx) }() - buf, err := ioutil.ReadAll(c.Request.Body) + payload, err := c.GetRawData() if err != nil { - c.JSON(e.ErrorMessage(e.ErrGithubWebHook.AddDesc(errs.Error()))) - c.Abort() + ctx.Err = err return } - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) + if github.WebHookType(c.Request) != "" { + ctx.Err = processGithub(payload, c.Request, ctx.RequestID, ctx.Logger) + } else if gitlab.HookEventType(c.Request) != "" { + ctx.Err = webhook.ProcessGitlabHook(payload, c.Request, ctx.RequestID, ctx.Logger) + } else { + ctx.Err = webhook.ProcessGerritHook(payload, c.Request, ctx.RequestID, ctx.Logger) + } +} + +func processGithub(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { + errs := &multierror.Error{} + // trigger classic pipeline - output, err := webhook.ProcessGithubHook(c.Request, ctx.RequestID, log) + _, err := webhook.ProcessGithubHook(payload, req, requestID, log) if err != nil { log.Errorf("error happens to trigger classic pipeline %v", err) errs = multierror.Append(errs, err) } // trigger workflow - c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) - err = webhook.ProcessGithubWebHook(c.Request, ctx.RequestID, log) + err = webhook.ProcessGithubWebHook(payload, req, requestID, log) if err != nil { log.Errorf("error happens to trigger workflow %v", err) errs = multierror.Append(errs, err) } - if errs.ErrorOrNil() != nil { - c.JSON(e.ErrorMessage(e.ErrGithubWebHook.AddDesc(errs.Error()))) - c.Abort() - return - } - - c.JSON(200, gin.H{"message": output}) + return errs.ErrorOrNil() } diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow.go b/pkg/microservice/aslan/core/workflow/handler/workflow.go index 7826705b20..53e0a12e77 100644 --- a/pkg/microservice/aslan/core/workflow/handler/workflow.go +++ b/pkg/microservice/aslan/core/workflow/handler/workflow.go @@ -26,7 +26,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/workflow/handler/workflow_task.go b/pkg/microservice/aslan/core/workflow/handler/workflow_task.go index c1156a6dfa..c5137f8db9 100644 --- a/pkg/microservice/aslan/core/workflow/handler/workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/handler/workflow_task.go @@ -29,8 +29,8 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go index 2a1d8041ee..66f98be6dc 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit.go @@ -33,10 +33,10 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" gerritservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gerrit" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/util" @@ -52,31 +52,21 @@ type gerritTypeEvent struct { EventCreatedOn int `json:"eventCreatedOn"` } -func ProcessGerritHook(req *http.Request, requestID string, log *zap.SugaredLogger) error { - defer func() { - _ = req.Body.Close() - }() - +func ProcessGerritHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { baseURI := fmt.Sprintf( "%s://%s", req.Header.Get("X-Forwarded-Proto"), req.Header.Get("X-Forwarded-Host"), ) - payload, err := ioutil.ReadAll(req.Body) - if err != nil { - log.Errorf("ProcessGerritHook err:%v", err) - return err - } - gerritTypeEventObj := new(gerritTypeEvent) - if err = json.Unmarshal(payload, gerritTypeEventObj); err != nil { + if err := json.Unmarshal(payload, gerritTypeEventObj); err != nil { log.Errorf("processGerritHook json.Unmarshal err : %v", err) return fmt.Errorf("this event is not supported") } //同步yaml数据 if gerritTypeEventObj.Type == changeMergedEventType { - err = updateServiceTemplateByGerritEvent(req.RequestURI, log) + err := updateServiceTemplateByGerritEvent(req.RequestURI, log) if err != nil { log.Errorf("updateServiceTemplateByGerritEvent err : %v", err) } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go index 3401f5e153..7bbf9e2920 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gerrit_workflow_task.go @@ -31,10 +31,10 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/types/permission" diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github.go b/pkg/microservice/aslan/core/workflow/service/webhook/github.go index d27671ed8c..30fc8c154a 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/github.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "net/http" "net/url" "strconv" @@ -31,16 +30,17 @@ import ( "github.com/hashicorp/go-multierror" "go.uber.org/zap" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" + gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" - githubtool "github.com/koderover/zadig/pkg/tool/github" + githubtool "github.com/koderover/zadig/pkg/tool/git/github" "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/util" ) @@ -50,14 +50,14 @@ const ( payloadFormParam = "payload" ) -func ProcessGithubHook(req *http.Request, requestID string, log *zap.SugaredLogger) (output string, err error) { +func ProcessGithubHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) (output string, err error) { hookType := github.WebHookType(req) if hookType == "integration_installation" || hookType == "installation" || hookType == "ping" { output = fmt.Sprintf("event %s received", hookType) return } - hookSecret := getHookSecret(log) + hookSecret := gitservice.GetHookSecret() if hookSecret == "" { var headers []string @@ -68,8 +68,7 @@ func ProcessGithubHook(req *http.Request, requestID string, log *zap.SugaredLogg log.Infof("[Webhook] hook headers: \n %s", strings.Join(headers, "\n ")) } - var payload []byte - payload, err = ValidatePayload(req, []byte(hookSecret), log) + err = validateSecret(payload, []byte(hookSecret), req) if err != nil { return } @@ -163,71 +162,13 @@ func ProcessGithubHook(req *http.Request, requestID string, log *zap.SugaredLogg return } -var hookSecret string -var hookSecretSet bool - -func getHookSecret(logger *zap.SugaredLogger) string { - if hookSecretSet { - return hookSecret - } - - poetryClient := poetry.New(config.PoetryAPIServer(), config.PoetryAPIRootKey()) - org, err := poetryClient.GetOrganization(poetry.DefaultOrganization) - if err != nil { - logger.Errorf("failed to find default organization: %v", err) - return "--impossible-token--" - } - - hookSecret = org.Token - hookSecretSet = true - - return hookSecret -} - -func ValidatePayload(r *http.Request, secretKey []byte, log *zap.SugaredLogger) (payload []byte, err error) { - var body []byte // Raw body that GitHub uses to calculate the signature. - - switch ct := r.Header.Get("Content-Type"); ct { - case "application/json": - var err error - if body, err = ioutil.ReadAll(r.Body); err != nil { - return nil, err - } - - // If the content type is application/json, - // the JSON payload is just the original body. - payload = body - - case "application/x-www-form-urlencoded": - // payloadFormParam is the name of the form parameter that the JSON payload - // will be in if a webhook has its content type set to application/x-www-form-urlencoded. - - var err error - if body, err = ioutil.ReadAll(r.Body); err != nil { - return nil, err - } - - // If the content type is application/x-www-form-urlencoded, - // the JSON payload will be under the "payload" form param. - form, err := url.ParseQuery(string(body)) - if err != nil { - return nil, err - } - payload = []byte(form.Get(payloadFormParam)) - - default: - return nil, fmt.Errorf("webhook request has unsupported Content-Type %q", ct) - } - +func validateSecret(payload, secretKey []byte, r *http.Request) error { sig := r.Header.Get(signatureHeader) if len(secretKey) > 0 { - if err := github.ValidateSignature(sig, body, secretKey); err != nil { - return nil, err - } - } else { - log.Infof("[Webhook] got payload %s", string(payload)) + return github.ValidateSignature(sig, payload, secretKey) } - return payload, nil + + return nil } func prEventToPipelineTasks(event *github.PullRequestEvent, requestID string, log *zap.SugaredLogger) ([]*commonmodels.TaskArgs, error) { @@ -450,7 +391,7 @@ func pushEventCommitsFiles(e *github.PushEvent) []string { return files } -func ProcessGithubWebHook(req *http.Request, requestID string, log *zap.SugaredLogger) error { +func ProcessGithubWebHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { forwardedProto := req.Header.Get("X-Forwarded-Proto") forwardedHost := req.Header.Get("X-Forwarded-Host") baseURI := fmt.Sprintf("%s://%s", forwardedProto, forwardedHost) @@ -460,7 +401,7 @@ func ProcessGithubWebHook(req *http.Request, requestID string, log *zap.SugaredL return nil } - payload, err := ValidatePayload(req, []byte(getHookSecret(log)), log) + err := validateSecret(payload, []byte(gitservice.GetHookSecret()), req) if err != nil { return err } @@ -538,6 +479,10 @@ func getProductTargetMap(prod *commonmodels.Product) map[string][]commonmodels.D resp[target] = append(resp[target], deployEnv) } + case setting.PMDeployType: + deployEnv := commonmodels.DeployEnv{Type: setting.PMDeployType, Env: serviceObj.ServiceName} + target := fmt.Sprintf("%s%s%s%s%s", prod.ProductName, SplitSymbol, serviceObj.ServiceName, SplitSymbol, serviceObj.ServiceName) + resp[target] = append(resp[target], deployEnv) case setting.HelmDeployType: for _, container := range serviceObj.Containers { env := serviceObj.ServiceName + "/" + container.Name diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go index c1fe8edb2b..a90c906fe6 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github_workflow_task.go @@ -29,10 +29,10 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - ch "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/types/permission" ) @@ -315,7 +315,7 @@ func TriggerWorkflowByGithubEvent(event interface{}, baseURI, deliveryID, reques } func findChangedFilesOfPullRequest(event *github.PullRequestEvent, codehostID int) ([]string, error) { - detail, err := ch.GetCodehostDetail(codehostID) + detail, err := codehost.GetCodehostDetail(codehostID) if err != nil { return nil, fmt.Errorf("failed to find codehost %d: %v", codehostID, err) } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go index 92181565c0..cb6f9dbbda 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go @@ -20,7 +20,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net/http" "strings" "sync" @@ -33,6 +32,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collie" + gitservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/git" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -47,19 +47,14 @@ type EventPush struct { Body string `json:"body"` } -func ProcessGitlabHook(req *http.Request, requestID string, log *zap.SugaredLogger) error { +func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { token := req.Header.Get("X-Gitlab-Token") - secret := getHookSecret(log) + secret := gitservice.GetHookSecret() if secret != "" && token != secret { return errors.New("token is illegal") } - payload, err := ioutil.ReadAll(req.Body) - if err != nil { - return err - } - eventType := gitlab.HookEventType(req) event, err := gitlab.ParseHook(eventType, payload) if err != nil { @@ -146,6 +141,14 @@ func ProcessGitlabHook(req *http.Request, requestID string, log *zap.SugaredLogg } }() + //测试管理webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerTestByGitlabEvent(pushEvent, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() } if mergeEvent != nil { @@ -167,6 +170,14 @@ func ProcessGitlabHook(req *http.Request, requestID string, log *zap.SugaredLogg } }() + //测试管理webhook + wg.Add(1) + go func() { + defer wg.Done() + if err = TriggerTestByGitlabEvent(mergeEvent, baseURI, requestID, log); err != nil { + errorList = multierror.Append(errorList, err) + } + }() } wg.Wait() @@ -249,29 +260,13 @@ func updateServiceTemplateByPushEvent(event *EventPush, log *zap.SugaredLogger) log.Errorf("GetGitlabAddress failed, error: %v", err) return err } - pathWithNamespace := strings.Split(gitlabEvent.Project.PathWithNamespace, "/") - owner := strings.Join(pathWithNamespace[:len(pathWithNamespace)-1], "/") - repo := pathWithNamespace[len(pathWithNamespace)-1] client, err := getGitlabClientByAddress(address) if err != nil { return err } - projectID, err := GitlabGetProjectID(client, owner, repo) - if err != nil { - log.Errorf("Get Project ID of %s/%s failed, error: %v", - owner, repo, err) - return err - } - if projectID != event.ProjectID { - msg := fmt.Sprintf("Push event projectID is: %d, it's not %s/%s, just ignore.", - event.ProjectID, owner, repo) - log.Info(msg) - return nil - } - - diffs, err := client.Compare(projectID, event.Before, event.After) + diffs, err := client.Compare(event.ProjectID, event.Before, event.After) if err != nil { log.Errorf("Failed to get push event diffs, error: %v", err) return err diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_testing_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_testing_task.go new file mode 100644 index 0000000000..082dd8fc96 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_testing_task.go @@ -0,0 +1,235 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "strconv" + + multierror "github.com/hashicorp/go-multierror" + gitlab "github.com/xanzy/go-gitlab" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" + testingservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + "github.com/koderover/zadig/pkg/setting" +) + +type gitEventMatcherForTesting interface { + Match(commonmodels.MainHookRepo) (bool, error) + UpdateTaskArgs(*commonmodels.TestTaskArgs, string) *commonmodels.TestTaskArgs +} + +type testArgsFactory struct { + testing *commonmodels.Testing + reqID string +} + +func (waf *testArgsFactory) Update(args *commonmodels.TestTaskArgs) *commonmodels.TestTaskArgs { + test := waf.testing + args.TestName = test.Name + args.TestTaskCreator = setting.WebhookTaskCreator + args.ProductName = test.ProductName + + return args +} + +type gitlabPushEventMatcherForTesting struct { + log *zap.SugaredLogger + testing *commonmodels.Testing + event *gitlab.PushEvent +} + +func (gpem *gitlabPushEventMatcherForTesting) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { + ev := gpem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.Project.PathWithNamespace { + if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + + return MatchChanges(hookRepo, changedFiles), nil + } + } + + return false, nil +} + +func (gpem *gitlabPushEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gpem.testing, + reqID: requestID, + } + + factory.Update(args) + return args +} + +// TriggerTestByGitlabEvent 测试管理模块的触发器任务 +func TriggerTestByGitlabEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { + // 1. find configured testing + testingList, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{}) + if err != nil { + log.Errorf("failed to list testing %v", err) + return err + } + + mErr := &multierror.Error{} + diffSrv := func(mergeEvent *gitlab.MergeEvent, codehostId int) ([]string, error) { + return findChangedFilesOfMergeRequest(mergeEvent, codehostId) + } + + var notification *commonmodels.Notification + + for _, testing := range testingList { + if testing.HookCtl != nil && testing.HookCtl.Enabled { + log.Infof("find %d hooks in testing %s", len(testing.HookCtl.Items), testing.Name) + for _, item := range testing.HookCtl.Items { + if item.TestArgs == nil { + continue + } + + // 2. match webhook + matcher := createGitlabEventMatcherForTesting(event, diffSrv, testing, log) + if matcher == nil { + continue + } + + if matches, err := matcher.Match(item.MainRepo); err != nil { + mErr = multierror.Append(mErr, err) + } else if matches { + log.Infof("event match hook %v of %s", item.MainRepo, testing.Name) + var mergeRequestID, commitID string + if ev, isPr := event.(*gitlab.MergeEvent); isPr { + // 如果是merge request,且该webhook触发器配置了自动取消, + // 则需要确认该merge request在本次commit之前的commit触发的任务是否处理完,没有处理完则取消掉。 + mergeRequestID = strconv.Itoa(ev.ObjectAttributes.IID) + commitID = ev.ObjectAttributes.LastCommit.ID + autoCancelOpt := &AutoCancelOpt{ + MergeRequestID: mergeRequestID, + CommitID: commitID, + TaskType: config.TestType, + MainRepo: item.MainRepo, + TestArgs: item.TestArgs, + } + err := AutoCancelTask(autoCancelOpt, log) + if err != nil { + log.Errorf("failed to auto cancel testing task when receive event %v due to %v ", event, err) + mErr = multierror.Append(mErr, err) + } + // 发送本次commit的通知 + if notification == nil { + notification, _ = scmnotify.NewService().SendInitWebhookComment( + &item.MainRepo, ev.ObjectAttributes.IID, baseURI, false, true, log, + ) + } + } + + if notification != nil { + item.TestArgs.NotificationID = notification.ID.Hex() + } + + args := matcher.UpdateTaskArgs(item.TestArgs, requestID) + args.MergeRequestID = mergeRequestID + args.CommitID = commitID + args.Source = setting.SourceFromGitlab + args.CodehostID = item.MainRepo.CodehostID + args.RepoOwner = item.MainRepo.RepoOwner + args.RepoName = item.MainRepo.RepoName + + // 3. create task with args + if resp, err := testingservice.CreateTestTask(args, log); err != nil { + log.Errorf("failed to create testing task when receive event %v due to %v ", event, err) + mErr = multierror.Append(mErr, err) + } else { + log.Infof("succeed to create task %v", resp) + } + } else { + log.Debugf("event not matches %v", item.MainRepo) + } + } + } + } + + return mErr.ErrorOrNil() +} + +func createGitlabEventMatcherForTesting( + event interface{}, diffSrv gitlabMergeRequestDiffFunc, testing *commonmodels.Testing, log *zap.SugaredLogger, +) gitEventMatcherForTesting { + switch evt := event.(type) { + case *gitlab.PushEvent: + return &gitlabPushEventMatcherForTesting{ + testing: testing, + log: log, + event: evt, + } + case *gitlab.MergeEvent: + return &gitlabMergeEventMatcherForTesting{ + diffFunc: diffSrv, + log: log, + event: evt, + testing: testing, + } + } + + return nil +} + +type gitlabMergeEventMatcherForTesting struct { + diffFunc gitlabMergeRequestDiffFunc + log *zap.SugaredLogger + testing *commonmodels.Testing + event *gitlab.MergeEvent +} + +func (gmem *gitlabMergeEventMatcherForTesting) Match(hookRepo commonmodels.MainHookRepo) (bool, error) { + ev := gmem.event + // TODO: match codehost + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == ev.ObjectAttributes.Target.PathWithNamespace { + if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { + if ev.ObjectAttributes.State == "opened" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(hookRepo, changedFiles), nil + } + } + } + return false, nil +} + +func (gmem *gitlabMergeEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gmem.testing, + reqID: requestID, + } + + args = factory.Update(args) + + return args +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go index efeb3cbe18..44eef45d53 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflow_task.go @@ -18,7 +18,6 @@ package webhook import ( "fmt" - "math/rand" "strconv" "strings" "sync" @@ -32,13 +31,13 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" environmentservice "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" - gitlab2 "github.com/koderover/zadig/pkg/tool/gitlab" + gitlabtool "github.com/koderover/zadig/pkg/tool/git/gitlab" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/types/permission" @@ -134,9 +133,9 @@ func (gpem *gitlabPushEventMatcher) Match(hookRepo commonmodels.MainHookRepo) (b return false, err } - client, err := gitlab2.NewGitlabClient(detail.Address, detail.OauthToken) + client, err := gitlabtool.NewClient(detail.Address, detail.OauthToken) if err != nil { - gpem.log.Errorf("NewGitlabClient error: %v", err) + gpem.log.Errorf("NewClient error: %v", err) return false, err } @@ -306,13 +305,13 @@ func findChangedFilesOfMergeRequest(event *gitlab.MergeEvent, codehostID int) ([ return nil, fmt.Errorf("failed to find codehost %d: %v", codehostID, err) } - client, err := gitlab2.NewGitlabClient(detail.Address, detail.OauthToken) + client, err := gitlabtool.NewClient(detail.Address, detail.OauthToken) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) } - return client.ListChangedFiles(detail.Address, detail.OauthToken, event) + return client.ListChangedFiles(event) } // InitDiffNote 调用gitlab接口初始化DiffNote,并保存到数据库 @@ -431,7 +430,7 @@ func CreateEnvAndTaskByPR(workflowArgs *commonmodels.WorkflowTaskArgs, prID int, } } - envName := fmt.Sprintf("%s-%d-%s%s", "pr", prID, GetRandomNumString(3, 1), GetRandomNumString(1, 2)) + envName := fmt.Sprintf("%s-%d-%s%s", "pr", prID, util.GetRandomNumString(3), util.GetRandomString(3)) util.Clear(&baseProduct.ID) baseProduct.Namespace = commonservice.GetProductEnvNamespace(envName, workflowArgs.ProductTmplName) baseProduct.UpdateBy = setting.SystemUser @@ -556,20 +555,3 @@ func WaitEnvDelete(timeoutSeconds int, envName string, workflowArgs *commonmodel } return nil } - -func GetRandomNumString(length, returnType int) string { - numStr := "0123456789abcdefghijklmnopqrstuvwxyz" - str := "abcdefghijklmnopqrstuvwxyz" - var bytes []byte - if returnType == 1 { - bytes = []byte(numStr) - } else { - bytes = []byte(str) - } - result := []byte{} - r := rand.New(rand.NewSource(time.Now().UnixNano())) - for i := 0; i < length; i++ { - result = append(result, bytes[r.Intn(len(bytes))]) - } - return string(result) -} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/utils.go b/pkg/microservice/aslan/core/workflow/service/webhook/utils.go index 4f06aa3670..978056e67f 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/utils.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/utils.go @@ -25,19 +25,20 @@ import ( "github.com/google/go-github/v35/github" "github.com/hashicorp/go-multierror" + "github.com/xanzy/go-gitlab" "go.uber.org/zap" "helm.sh/helm/v3/pkg/releaseutil" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" - githubtool "github.com/koderover/zadig/pkg/tool/github" - "github.com/koderover/zadig/pkg/tool/gitlab" + githubtool "github.com/koderover/zadig/pkg/tool/git/github" + gitlabtool "github.com/koderover/zadig/pkg/tool/git/gitlab" "github.com/koderover/zadig/pkg/tool/kube/serializer" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/util" @@ -120,11 +121,7 @@ func syncLatestCommit(service *commonmodels.Service) error { return err } - projectID, err := GitlabGetProjectID(client, owner, repo) - if err != nil { - return err - } - commit, err := GitlabGetLatestCommit(client, projectID, branch, path) + commit, err := GitlabGetLatestCommit(client, owner, repo, branch, path) if err != nil { return err } @@ -135,7 +132,7 @@ func syncLatestCommit(service *commonmodels.Service) error { return nil } -func getGitlabClientByAddress(address string) (*gitlab.Client, error) { +func getGitlabClientByAddress(address string) (*gitlabtool.Client, error) { opt := &codehost.Option{ Address: address, CodeHostType: codehost.GitLabProvider, @@ -145,7 +142,7 @@ func getGitlabClientByAddress(address string) (*gitlab.Client, error) { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc("git client is nil") } - client, err := gitlab.NewGitlabClient(codehost.Address, codehost.AccessToken) + client, err := gitlabtool.NewClient(codehost.Address, codehost.AccessToken) if err != nil { log.Error(err) return nil, e.ErrCodehostListProjects.AddDesc(err.Error()) @@ -154,19 +151,11 @@ func getGitlabClientByAddress(address string) (*gitlab.Client, error) { return client, nil } -func GitlabGetProjectID(client *gitlab.Client, owner, repo string) (projectID int, err error) { - project, err := client.GetProject(owner, repo) - if err != nil { - return -1, fmt.Errorf("failed to get project with %s/%s, error: %v", owner, repo, err) - } - return project.ID, nil -} - -func GitlabGetLatestCommit(client *gitlab.Client, projectID int, ref, path string) (*gitlab.RepoCommit, error) { - commit, err := client.GetLatestCommit(projectID, ref, path) +func GitlabGetLatestCommit(client *gitlabtool.Client, owner, repo string, ref, path string) (*gitlab.Commit, error) { + commit, err := client.GetLatestCommit(owner, repo, ref, path) if err != nil { - return nil, fmt.Errorf("failed to get lastest commit with project id: %d, ref: %s, path:%s, error: %v", - projectID, ref, path, err) + return nil, fmt.Errorf("failed to get lastest commit with project %s/%s, ref: %s, path:%s, error: %v", + owner, repo, ref, path, err) } return commit, nil } @@ -175,11 +164,11 @@ func GitlabGetLatestCommit(client *gitlab.Client, projectID int, ref, path strin // projectID: identity of project, can be retrieved from s.GitlabGetProjectID(owner, repo) // ref: branch (e.g. master) or commit (commit id) or tag // path: file path of raw files, only retrieve leaf node(blob type == file), no recursive get -func GitlabGetRawFiles(client *gitlab.Client, projectID int, ref, path, pathType string) (files []string, err error) { +func GitlabGetRawFiles(client *gitlabtool.Client, owner, repo, ref, path, pathType string) (files []string, err error) { files = make([]string, 0) var errs *multierror.Error if pathType == "tree" { - nodes, err := client.ListTree(projectID, ref, path) + nodes, err := client.ListTree(owner, repo, ref, path) if err != nil { return files, err } @@ -194,7 +183,7 @@ func GitlabGetRawFiles(client *gitlab.Client, projectID int, ref, path, pathType } // if node type is "blob", it is a file // Path is filepath of a node - content, err := client.GetRawFile(projectID, ref, node.Path) + content, err := client.GetRawFile(owner, repo, ref, node.Path) if err != nil { errs = multierror.Append(errs, err) } @@ -204,7 +193,7 @@ func GitlabGetRawFiles(client *gitlab.Client, projectID int, ref, path, pathType } return files, errs.ErrorOrNil() } - content, err := client.GetFileContent(projectID, ref, path) + content, err := client.GetFileContent(owner, repo, ref, path) if err != nil { return files, err } @@ -229,12 +218,7 @@ func syncContentFromGitlab(userName string, args *commonmodels.Service) error { return err } - projectID, _ := GitlabGetProjectID(client, owner, repo) - if owner == "" { - return fmt.Errorf("url format failed") - } - - files, err := GitlabGetRawFiles(client, projectID, branch, path, pathType) + files, err := GitlabGetRawFiles(client, owner, repo, branch, path, pathType) if err != nil { return err } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/convert.go b/pkg/microservice/aslan/core/workflow/service/workflow/convert.go index 6883a78673..4e9324cbea 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/convert.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/convert.go @@ -25,7 +25,7 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -200,7 +200,7 @@ func AddPipelineJiraSubTask(pipeline *commonmodels.Pipeline, log *zap.SugaredLog } for _, subTask := range pipeline.SubTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { return nil, errors.New(e.InterfaceToTaskErrMsg) } @@ -211,7 +211,7 @@ func AddPipelineJiraSubTask(pipeline *commonmodels.Pipeline, log *zap.SugaredLog switch pre.TaskType { case config.TaskBuild: - t, err := commonservice.ToBuildTask(subTask) + t, err := base.ToBuildTask(subTask) if err != nil { log.Error(err) return nil, err diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go b/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go index 12bf1241c9..1aa4b4fa79 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/gerrit.go @@ -23,8 +23,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - ch "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/tool/httpclient" ) @@ -36,10 +36,10 @@ func CreateGerritWebhook(workflow *commonmodels.Workflow, log *zap.SugaredLogger continue } - opt := &ch.Option{ + opt := &codehost.Option{ CodeHostID: workflowWebhook.MainRepo.CodehostID, } - detail, err := ch.GetCodeHostInfo(opt) + detail, err := codehost.GetCodeHostInfo(opt) if err != nil { return err } @@ -88,10 +88,10 @@ func UpdateGerritWebhook(currentWorkflow *commonmodels.Workflow, log *zap.Sugare continue } - opt := &ch.Option{ + opt := &codehost.Option{ CodeHostID: oldWorkflowWebhook.MainRepo.CodehostID, } - detail, err := ch.GetCodeHostInfo(opt) + detail, err := codehost.GetCodeHostInfo(opt) if err != nil { return err } @@ -113,10 +113,10 @@ func UpdateGerritWebhook(currentWorkflow *commonmodels.Workflow, log *zap.Sugare continue } - opt := &ch.Option{ + opt := &codehost.Option{ CodeHostID: workflowWebhook.MainRepo.CodehostID, } - detail, err := ch.GetCodeHostInfo(opt) + detail, err := codehost.GetCodeHostInfo(opt) if err != nil { return err } @@ -155,10 +155,10 @@ func DeleteGerritWebhook(workflow *commonmodels.Workflow, log *zap.SugaredLogger continue } - opt := &ch.Option{ + opt := &codehost.Option{ CodeHostID: workflowWebhook.MainRepo.CodehostID, } - detail, err := ch.GetCodeHostInfo(opt) + detail, err := codehost.GetCodeHostInfo(opt) if err != nil { return err } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/github.go b/pkg/microservice/aslan/core/workflow/service/workflow/github.go index 9d7caf61e7..d41cbe152d 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/github.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/github.go @@ -24,9 +24,9 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go b/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go index 03f95d6982..c4b6cd46f4 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/gitlab.go @@ -22,10 +22,10 @@ import ( "sort" "time" - gitlab "github.com/xanzy/go-gitlab" + "github.com/xanzy/go-gitlab" - ch "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" "github.com/koderover/zadig/pkg/tool/gerrit" ) @@ -39,16 +39,16 @@ type RepoCommit struct { } func QueryByBranch(id int, owner string, name string, branch string) (*RepoCommit, error) { - opt := &ch.Option{ + opt := &codehost.Option{ CodeHostID: id, } - codehost, err := ch.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { return nil, err } - if codehost.Type == setting.SourceFromGitlab { - token, address := codehost.AccessToken, codehost.Address + if ch.Type == setting.SourceFromGitlab { + token, address := ch.AccessToken, ch.Address cli, err := gitlab.NewOAuthClient(token, gitlab.WithBaseURL(address)) if err != nil { @@ -67,8 +67,8 @@ func QueryByBranch(id int, owner string, name string, branch string) (*RepoCommi CreatedAt: br.Commit.CreatedAt, Message: br.Commit.Message, }, nil - } else if codehost.Type == setting.SourceFromGerrit { - cli := gerrit.NewClient(codehost.Address, codehost.AccessToken) + } else if ch.Type == setting.SourceFromGerrit { + cli := gerrit.NewClient(ch.Address, ch.AccessToken) commit, err := cli.GetCommitByBranch(name, branch) if err != nil { return nil, err @@ -85,20 +85,20 @@ func QueryByBranch(id int, owner string, name string, branch string) (*RepoCommi }, nil } - return nil, errors.New(codehost.Type + "is not supported yet") + return nil, errors.New(ch.Type + "is not supported yet") } func QueryByTag(id int, owner string, name string, tag string) (*RepoCommit, error) { - opt := &ch.Option{ + opt := &codehost.Option{ CodeHostID: id, } - codehost, err := ch.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { return nil, err } - if codehost.Type == setting.SourceFromGitlab { - token, address := codehost.AccessToken, codehost.Address + if ch.Type == setting.SourceFromGitlab { + token, address := ch.AccessToken, ch.Address cli, err := gitlab.NewOAuthClient(token, gitlab.WithBaseURL(address)) if err != nil { @@ -117,8 +117,8 @@ func QueryByTag(id int, owner string, name string, tag string) (*RepoCommit, err CreatedAt: br.Commit.CreatedAt, Message: br.Commit.Message, }, nil - } else if codehost.Type == setting.SourceFromGerrit { - cli := gerrit.NewClient(codehost.Address, codehost.AccessToken) + } else if ch.Type == setting.SourceFromGerrit { + cli := gerrit.NewClient(ch.Address, ch.AccessToken) commit, err := cli.GetCommitByTag(name, tag) if err != nil { return nil, err @@ -135,7 +135,7 @@ func QueryByTag(id int, owner string, name string, tag string) (*RepoCommit, err }, nil } - return nil, errors.New(codehost.Type + "is not supported yet") + return nil, errors.New(ch.Type + "is not supported yet") } type PRCommit struct { @@ -153,22 +153,22 @@ func GetLatestPrCommit(codehostID, pr int, namespace, projectName string) (*PRCo // return nil, err //} - opt := &ch.Option{ + opt := &codehost.Option{ CodeHostID: codehostID, } - codehost, err := ch.GetCodeHostInfo(opt) + ch, err := codehost.GetCodeHostInfo(opt) if err != nil { return nil, err } - token, address := codehost.AccessToken, codehost.Address + token, address := ch.AccessToken, ch.Address cli, err := gitlab.NewOAuthClient(token, gitlab.WithBaseURL(address)) if err != nil { return nil, fmt.Errorf("set base url failed, err:%v", err) } - if codehost.Type == gerrit.CodehostTypeGerrit { - cli := gerrit.NewClient(codehost.Address, codehost.AccessToken) + if ch.Type == gerrit.CodehostTypeGerrit { + cli := gerrit.NewClient(ch.Address, ch.AccessToken) change, err := cli.GetCurrentVersionByChangeID(projectName, pr) if err != nil { return nil, err diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go b/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go index ba936917ca..5e1dab244e 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/nsq_handlers.go @@ -35,18 +35,19 @@ import ( "github.com/nsqio/go-nsq" "go.uber.org/zap" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/notify" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/registry" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" - gitlabtool "github.com/koderover/zadig/pkg/tool/gitlab" + "github.com/koderover/zadig/pkg/shared/poetry" + "github.com/koderover/zadig/pkg/tool/git/gitlab" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/util" ) @@ -225,7 +226,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { if taskStatus == config.StatusPassed { subBuildTaskMap := subStage.SubTasks for _, subTask := range subBuildTaskMap { - buildInfo, err := commonservice.ToBuildTask(subTask) + buildInfo, err := base.ToBuildTask(subTask) if err != nil { h.log.Errorf("uploadTaskData get buildInfo ToBuildTask failed ! err:%v", err) continue @@ -285,17 +286,12 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { pathArray := strings.Split(path, "/") dockerfilePath := path[len(pathArray[0])+1:] if strings.Contains(build.Address, "gitlab") { - cli, err := gitlabtool.NewGitlabClient(build.Address, build.OauthToken) + cli, err := gitlab.NewClient(build.Address, build.OauthToken) if err != nil { h.log.Errorf("Failed to get gitlab client, err: %v", err) continue } - project, err := cli.GetProject(build.RepoOwner, build.RepoName) - if err != nil { - h.log.Errorf("uploadTaskData get project err:%v", err) - continue - } - content, err := cli.GetRawFile(project.ID, build.Branch, dockerfilePath) + content, err := cli.GetRawFile(build.RepoOwner, build.RepoName, build.Branch, dockerfilePath) if err != nil { h.log.Errorf("uploadTaskData gitlab GetRawFile err:%v", err) continue @@ -352,7 +348,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { if jiraSubStage.TaskType == config.TaskJira { jiraSubBuildTaskMap := jiraSubStage.SubTasks for _, jiraSubTask := range jiraSubBuildTaskMap { - jiraInfo, _ := commonservice.ToJiraTask(jiraSubTask) + jiraInfo, _ := base.ToJiraTask(jiraSubTask) if jiraInfo != nil { for _, issue := range jiraInfo.Issues { issueURLs = append(issueURLs, issue.URL) @@ -383,7 +379,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { if taskStatus == config.StatusPassed { subDeployTaskMap := subStage.SubTasks for _, subTask := range subDeployTaskMap { - deployInfo, err := commonservice.ToDeployTask(subTask) + deployInfo, err := base.ToDeployTask(subTask) if err != nil { h.log.Errorf("uploadTaskData get deployInfo ToDeployTask failed ! err:%v", err) continue @@ -415,7 +411,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { isNew = false testTaskStat *commonmodels.TestTaskStat ) - testInfo, err := commonservice.ToTestingTask(subTask) + testInfo, err := base.ToTestingTask(subTask) if err != nil { h.log.Errorf("uploadTaskData get testInfo ToTestingTask failed ! err:%v", err) continue @@ -508,7 +504,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { if taskStatus == config.StatusPassed { subDistributeTaskMap := subStage.SubTasks for _, subTask := range subDistributeTaskMap { - releaseImageInfo, err := commonservice.ToReleaseImageTask(subTask) + releaseImageInfo, err := base.ToReleaseImageTask(subTask) if err != nil { h.log.Errorf("uploadTaskData get releaseImage ToReleaseImageTask failed ! err:%v", err) continue @@ -543,7 +539,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { if taskStatus == config.StatusPassed { subDistributeFileTaskMap := subStage.SubTasks for _, subTask := range subDistributeFileTaskMap { - releaseFileInfo, err := commonservice.ToDistributeToS3Task(subTask) + releaseFileInfo, err := base.ToDistributeToS3Task(subTask) if err != nil { h.log.Errorf("uploadTaskData get releaseFile ToDistributeToS3Task failed ! err:%v", err) continue @@ -582,7 +578,7 @@ func (h *TaskAckHandler) uploadTaskData(pt *task.Task) error { case config.TaskTestingV2: subTestTaskMap := subStage.SubTasks for _, subTask := range subTestTaskMap { - testInfo, err := commonservice.ToTestingTask(subTask) + testInfo, err := base.ToTestingTask(subTask) if err != nil { h.log.Errorf("uploadTaskData get testInfo ToTestingTask failed ! err:%v", err) continue @@ -743,7 +739,7 @@ func (h *TaskAckHandler) getDeployTasks(subTasks []map[string]interface{}) ([]*t deploys := make([]*task.Deploy, 0) for _, subTask := range subTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { return nil, errors.New("invalid sub task type") } @@ -751,7 +747,7 @@ func (h *TaskAckHandler) getDeployTasks(subTasks []map[string]interface{}) ([]*t switch pre.TaskType { case config.TaskDeploy: - deploy, err := commonservice.ToDeployTask(subTask) + deploy, err := base.ToDeployTask(subTask) if err != nil { return nil, fmt.Errorf("unmarshal deploy sub task type error: %v", err) } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline.go new file mode 100644 index 0000000000..dc7cf3d5e4 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline.go @@ -0,0 +1,233 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" +) + +func ListPipelines(log *zap.SugaredLogger) ([]*commonmodels.Pipeline, error) { + resp, err := commonrepo.NewPipelineColl().List(&commonrepo.PipelineListOption{}) + if err != nil { + log.Errorf("list PipelineV2 error: %v", err) + return resp, e.ErrListPipeline + } + for i := range resp { + EnsureSubTasksResp(resp[i].SubTasks) + } + return resp, nil +} + +func GetPipeline(userID int, pipelineName string, log *zap.SugaredLogger) (*commonmodels.Pipeline, error) { + resp, err := commonrepo.NewPipelineColl().Find(&commonrepo.PipelineFindOption{Name: pipelineName}) + if err != nil { + log.Error(err) + return resp, e.ErrGetPipeline + } + + EnsureSubTasksResp(resp.SubTasks) + + fPipe, err := commonrepo.NewFavoriteColl().Find(userID, pipelineName, string(config.SingleType)) + if err == nil && fPipe != nil && fPipe.Name == pipelineName { + resp.IsFavorite = true + } + + return resp, nil +} + +func UpsertPipeline(args *commonmodels.Pipeline, log *zap.SugaredLogger) error { + if !checkPipelineSubModules(args) { + errStr := "pipeline没有子模块,请先设置子模块" + return e.ErrCreatePipeline.AddDesc(errStr) + } + + log.Debugf("Start to create or update pipeline %s", args.Name) + if err := ensurePipeline(args, log); err != nil { + return e.ErrCreatePipeline.AddDesc(err.Error()) + } + + var currentHooks, updatedHooks []commonmodels.GitHook + currentPipeline, err := commonrepo.NewPipelineColl().Find(&commonrepo.PipelineFindOption{Name: args.Name}) + if err == nil && currentPipeline != nil && currentPipeline.Hook != nil { + currentHooks = currentPipeline.Hook.GitHooks + } + if args.Hook != nil { + updatedHooks = args.Hook.GitHooks + } + + err = processWebhook(updatedHooks, currentHooks, webhook.PipelinePrefix+args.Name, log) + if err != nil { + log.Errorf("Failed to process webhook, err: %s", err) + return e.ErrCreatePipeline.AddDesc(err.Error()) + } + + if err := commonrepo.NewPipelineColl().Upsert(args); err != nil { + log.Error(err) + return e.ErrCreatePipeline.AddDesc(err.Error()) + } + + return nil +} + +func checkPipelineSubModules(args *commonmodels.Pipeline) bool { + if args.SubTasks != nil && len(args.SubTasks) > 0 { + return true + } + if args.Hook != nil && args.Hook.Enabled { + return true + } + if args.Schedules.Enabled { + return true + } + if args.Slack != nil && args.Slack.Enabled { + return true + } + if args.NotifyCtl != nil && args.NotifyCtl.Enabled { + return true + } + + return false +} + +func CopyPipeline(oldPipelineName, newPipelineName, username string, log *zap.SugaredLogger) error { + oldPipeline, err := commonrepo.NewPipelineColl().Find(&commonrepo.PipelineFindOption{Name: oldPipelineName}) + if err != nil { + log.Error(err) + return e.ErrGetPipeline + } + _, err = commonrepo.NewPipelineColl().Find(&commonrepo.PipelineFindOption{Name: newPipelineName}) + if err == nil { + log.Error("new pipeline already exists") + return e.ErrExistsPipeline + } + oldPipeline.UpdateBy = username + oldPipeline.Name = newPipelineName + return UpsertPipeline(oldPipeline, log) +} + +func RenamePipeline(oldName, newName string, log *zap.SugaredLogger) error { + if len(oldName) == 0 || len(newName) == 0 { + return e.ErrRenamePipeline.AddDesc("pipeline name cannot be empty") + } + + // 检查新名字格式 + if !defaultNameRegex.MatchString(newName) { + log.Errorf("pipeline name must match %s", defaultNameRegexString) + return fmt.Errorf("%s %s", e.InvalidFormatErrMsg, defaultNameRegexString) + } + + taskQueue, err := commonrepo.NewQueueColl().List(&commonrepo.ListQueueOption{}) + if err != nil { + return e.ErrRenamePipeline.AddErr(err) + } + + // 当task还在运行时,不能rename pipeline + for _, task := range taskQueue { + if task.PipelineName == oldName { + return e.ErrRenamePipeline.AddDesc("task still running,can not rename pipeline") + } + } + + // 新名字的pipeline已经存在,不能rename pipeline + opt := &commonrepo.PipelineFindOption{Name: newName} + if _, err := commonrepo.NewPipelineColl().Find(opt); err == nil { + return e.ErrRenamePipeline.AddDesc("newname has already existed, can not rename pipeline") + } + + if err := commonrepo.NewPipelineColl().Rename(oldName, newName); err != nil { + log.Errorf("Pipeline.Rename %s -> %s error: %v", oldName, newName, err) + return e.ErrRenamePipeline.AddErr(err) + } + + if err := commonrepo.NewTaskColl().Rename(oldName, newName, config.SingleType); err != nil { + log.Errorf("PipelineTask.Rename %s -> %s error: %v", oldName, newName, err) + return e.ErrRenamePipeline.AddErr(err) + } + + if err := commonrepo.NewCounterColl().Rename( + fmt.Sprintf(setting.PipelineTaskFmt, oldName), + fmt.Sprintf(setting.PipelineTaskFmt, newName)); err != nil && err.Error() != "not found" { + log.Errorf("Counter.Rename %s -> %s error: %v", oldName, newName, err) + return e.ErrRenamePipeline.AddErr(err) + } + + return nil +} + +func DeletePipeline(pipelineName, requestID string, isDeletingProductTmpl bool, log *zap.SugaredLogger) error { + if !isDeletingProductTmpl { + pipeline, err := commonrepo.NewPipelineColl().Find(&commonrepo.PipelineFindOption{Name: pipelineName}) + if err != nil { + log.Errorf("Pipeline.Find error: %v", err) + return e.ErrDeletePipeline.AddErr(err) + } + prod, err := template.NewProductColl().Find(pipeline.ProductName) + if err != nil { + log.Errorf("ProductTmpl.Find error: %v", err) + return e.ErrDeletePipeline.AddErr(err) + } + if prod.OnboardingStatus != 0 { + return e.ErrDeletePipeline.AddDesc("该工作流所属的项目处于onboarding流程中,不能删除工作流") + } + } + + opt := new(commonrepo.ListQueueOption) + taskQueue, err := commonrepo.NewQueueColl().List(opt) + if err != nil { + log.Errorf("List queued task error: %v", err) + return e.ErrDeletePipeline.AddErr(err) + } + // 当task还在运行时,先取消任务 + for _, task := range taskQueue { + if task.PipelineName == pipelineName && task.Type == config.SingleType { + if err = commonservice.CancelTaskV2("system", task.PipelineName, task.TaskID, config.SingleType, requestID, log); err != nil { + log.Errorf("task still running, cancel pipeline %s task %d", task.PipelineName, task.TaskID) + } + } + } + + err = commonrepo.NewWorkflowStatColl().Delete(pipelineName, string(config.SingleType)) + if err != nil { + log.Errorf("WorkflowStat.Delete failed, error: %v", err) + } + + if err := commonrepo.NewPipelineColl().Delete(pipelineName); err != nil { + log.Errorf("PipelineV2.Delete error: %v", err) + return e.ErrDeletePipeline.AddErr(err) + } + + if err := commonrepo.NewTaskColl().DeleteByPipelineNameAndType(pipelineName, config.SingleType); err != nil { + log.Errorf("PipelineTaskV2.DeleteByPipelineName error: %v", err) + } + + if err := commonrepo.NewCounterColl().Delete("PipelineTask:" + pipelineName); err != nil { + log.Errorf("Counter.Delete error: %v", err) + } + + return nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_cleaner.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_cleaner.go index c16549259c..13a8ddc8f6 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_cleaner.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_cleaner.go @@ -20,7 +20,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/setting" ) @@ -35,7 +35,7 @@ func Clean(task *task.Task) { } func ensureTestTask(subTask map[string]interface{}) (newSub map[string]interface{}, err error) { - t, err := commonservice.ToTestingTask(subTask) + t, err := base.ToTestingTask(subTask) if err != nil { return @@ -65,7 +65,7 @@ func ensureTestTask(subTask map[string]interface{}) (newSub map[string]interface } func ensureBuildTask(subTask map[string]interface{}) (newSub map[string]interface{}, err error) { - t, err := commonservice.ToBuildTask(subTask) + t, err := base.ToBuildTask(subTask) if err != nil { return } @@ -100,7 +100,7 @@ func ensureBuildTask(subTask map[string]interface{}) (newSub map[string]interfac } func ensureJiraTask(subTask map[string]interface{}) (newSub map[string]interface{}, err error) { - t, err := commonservice.ToJiraTask(subTask) + t, err := base.ToJiraTask(subTask) if err != nil { return } @@ -120,7 +120,7 @@ func ensureJiraTask(subTask map[string]interface{}) (newSub map[string]interface // EnsureSubTasksResp 确保SubTask中敏感信息和其他不必要信息不返回给前端 func EnsureSubTasksResp(subTasks []map[string]interface{}) { for i, subTask := range subTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { continue } @@ -147,7 +147,7 @@ func EnsureSubTasksResp(subTasks []map[string]interface{}) { func ensureSubStageResp(stage *commonmodels.Stage) { subTasks := stage.SubTasks for key, subTask := range subTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { continue } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go index feb709105f..d50607630b 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_controller.go @@ -17,9 +17,12 @@ limitations under the License. package workflow import ( + "encoding/base64" "encoding/json" "errors" "fmt" + "strconv" + "strings" "time" "k8s.io/apimachinery/pkg/util/sets" @@ -31,6 +34,7 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" nsqservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslanx" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/log" @@ -368,6 +372,29 @@ func agentCount() int { wdReplicas int ) + signatures, enabled, _ := aslanx.New(config.AslanURL(), config.PoetryAPIRootKey()).ListSignatures(log.NopSugaredLogger()) + if enabled { + if len(signatures) == 0 { + log.Errorf("Not Find Token") + return 0 + } + + signatureStr, err := base64.StdEncoding.DecodeString(signatures[0].Token) + if err != nil { + log.Errorf("token DecodeString error :%v", err) + return 0 + } + + signatureArr := strings.Split(string(signatureStr), ",") + if len(signatureArr) > 4 { + parallelLimit, err = strconv.Atoi(signatureArr[4]) + if err != nil { + log.Errorf("parse parallelLimit error :%v", err) + return 0 + } + } + } + kubeClient := krkubeclient.Client() deployment, _, err := getter.GetDeployment(config.Namespace(), config.ENVWarpdriveService(), kubeClient) if err != nil { @@ -502,7 +529,7 @@ func HandlerWebhookTask(currentTask *task.Task) bool { func updateAgentAndQueue(t *task.Task) error { if err := UpdateTaskAgent(t.TaskID, t.PipelineName, t.CreateTime, config.PodName()); err != nil { - log.Infof("task %v/%v may processed by aonther instance: %v", t.TaskID, t.PipelineName, err) + log.Infof("task %v/%v may processed by another instance: %v", t.TaskID, t.PipelineName, err) return err } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go index 5f7f6f1e2e..b5b92f8dee 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_status.go @@ -18,6 +18,7 @@ package workflow import ( "fmt" + "sort" "strings" "time" @@ -27,10 +28,113 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" e "github.com/koderover/zadig/pkg/tool/errors" ) +type PipelinePreview struct { + ProductName string `bson:"product_name" json:"product_name"` + Name string `bson:"name" json:"name"` + TeamName string `bson:"team_name" json:"team_name"` + SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` + Types []string `bson:"-" json:"types"` + UpdateBy string `bson:"update_by" json:"update_by,omitempty"` + UpdateTime int64 `bson:"update_time" json:"update_time,omitempty"` + IsFavorite bool `bson:"is_favorite" json:"is_favorite"` + LastestTask *commonmodels.TaskInfo `bson:"-" json:"lastest_task"` + LastSucessTask *commonmodels.TaskInfo `bson:"-" json:"last_task_success"` + LastFailureTask *commonmodels.TaskInfo `bson:"-" json:"last_task_failure"` + TotalDuration int64 `bson:"-" json:"total_duration"` + TotalNum int `bson:"-" json:"total_num"` + TotalSuccess int `bson:"-" json:"total_success"` +} + +func ListPipelinesPreview(userID int, log *zap.SugaredLogger) ([]*PipelinePreview, error) { + resp := make([]*PipelinePreview, 0) + + pipelineList, err := commonrepo.NewPipelineColl().List(&commonrepo.PipelineListOption{}) + if err != nil { + log.Errorf("list PipelineV2 error: %v", err) + return resp, e.ErrListPipeline + } + + favoritePipeline, err := commonrepo.NewFavoriteColl().List(&commonrepo.FavoriteArgs{UserID: userID, Type: string(config.SingleType)}) + if err != nil { + log.Errorf("list favorite pipeline error: %v", err) + return resp, e.ErrListPipeline + } + + workflowStats, err := commonrepo.NewWorkflowStatColl().FindWorkflowStat(&commonrepo.WorkflowStatArgs{Type: string(config.SingleType)}) + if err != nil { + log.Errorf("list workflow stat error: %v", err) + return resp, fmt.Errorf("列出工作流统计失败") + } + + for i, pipeline := range pipelineList { + EnsureSubTasksResp(pipelineList[i].SubTasks) + preview := &PipelinePreview{} + preview.Name = pipeline.Name + for _, subtask := range pipeline.SubTasks { + if value, ok := subtask["type"]; ok && value != "" { + task := make(map[string]interface{}) + task["type"] = value + if value, ok := subtask["enabled"]; ok && value != "" { + task["enabled"] = value + if v, ok := value.(bool); v && ok { + if tType, ok := task["type"].(string); ok { + preview.Types = append(preview.Types, tType) + } + } + } + + //返回subtask 部署环境,前端会用到 + if value, ok := subtask["service_name"]; ok && value != "" { + task["service_name"] = value + } + if value, ok := subtask["product_name"]; ok && value != "" { + task["product_name"] = value + } + preview.SubTasks = append(preview.SubTasks, task) + } + } + preview.ProductName = pipeline.ProductName + preview.TeamName = pipeline.TeamName + preview.UpdateBy = pipeline.UpdateBy + preview.UpdateTime = pipeline.UpdateTime + preview.IsFavorite = isFavoratePipeline(favoritePipeline, pipeline.Name) + + latestTask, _ := commonrepo.NewTaskColl().FindLatestTask(&commonrepo.FindTaskOption{PipelineName: pipeline.Name, Type: config.SingleType}) + if latestTask != nil { + preview.LastestTask = &commonmodels.TaskInfo{PipelineName: latestTask.PipelineName, TaskID: latestTask.TaskID, Status: latestTask.Status} + } + + latestPassedTask, _ := commonrepo.NewTaskColl().FindLatestTask(&commonrepo.FindTaskOption{PipelineName: pipeline.Name, Type: config.SingleType, Status: config.StatusPassed}) + if latestPassedTask != nil { + preview.LastSucessTask = &commonmodels.TaskInfo{PipelineName: latestPassedTask.PipelineName, TaskID: latestPassedTask.TaskID} + } + + latestFailedTask, _ := commonrepo.NewTaskColl().FindLatestTask(&commonrepo.FindTaskOption{PipelineName: pipeline.Name, Type: config.SingleType, Status: config.StatusFailed}) + if latestFailedTask != nil { + preview.LastFailureTask = &commonmodels.TaskInfo{PipelineName: latestFailedTask.PipelineName, TaskID: latestFailedTask.TaskID} + } + + preview.TotalDuration, preview.TotalNum, preview.TotalSuccess = findPipelineStat(preview, workflowStats) + + resp = append(resp, preview) + } + sort.SliceStable(resp, func(i, j int) bool { return resp[i].Name < resp[j].Name }) + return resp, nil +} + +func findPipelineStat(pipeline *PipelinePreview, workflowStats []*commonmodels.WorkflowStat) (int64, int, int) { + for _, workflowStat := range workflowStats { + if workflowStat.Name == pipeline.Name { + return workflowStat.TotalDuration, workflowStat.TotalSuccess + workflowStat.TotalFailure, workflowStat.TotalSuccess + } + } + return 0, 0, 0 +} + func isFavoratePipeline(favoritePipelines []*commonmodels.Favorite, pipelineName string) bool { resp := false for _, pipeline := range favoritePipelines { @@ -71,7 +175,7 @@ func FindTasks(commitID string, log *zap.SugaredLogger) ([]*TaskV2Info, error) { subBuildTaskMap := subStage.SubTasks for _, subTask := range subBuildTaskMap { - buildInfo, err := commonservice.ToBuildTask(subTask) + buildInfo, err := base.ToBuildTask(subTask) if err != nil { log.Errorf("get buildInfo ToBuildTask failed ! err:%v", err) continue diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go index 57954bbaa9..9eb4892565 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go @@ -27,17 +27,19 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" @@ -77,14 +79,14 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C // 更新单服务工作的subtask的build_os // 自定义基础镜像的镜像名称可能会被更新,需要使用ID获取最新的镜像名称 for i, subTask := range pipeline.SubTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { log.Errorf("subTask.ToPreview error: %v", err) continue } switch pre.TaskType { case config.TaskBuild: - build, err := commonservice.ToBuildTask(subTask) + build, err := base.ToBuildTask(subTask) if err != nil || build == nil { log.Errorf("subTask.ToBuildTask error: %v", err) continue @@ -127,7 +129,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C continue } case config.TaskTestingV2: - testing, err := commonservice.ToTestingTask(subTask) + testing, err := base.ToTestingTask(subTask) if err != nil || testing == nil { log.Errorf("subTask.ToTestingTask error: %v", err) continue @@ -199,7 +201,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C sort.Sort(ByTaskKind(pt.SubTasks)) for i, t := range pt.SubTasks { - preview, err := commonservice.ToPreview(t) + preview, err := base.ToPreview(t) if err != nil { continue } @@ -207,7 +209,7 @@ func CreatePipelineTask(args *commonmodels.TaskArgs, log *zap.SugaredLogger) (*C continue } - t, err := commonservice.ToDeployTask(t) + t, err := base.ToDeployTask(t) if err == nil && t.Enabled { env, err := commonrepo.NewProductColl().FindEnv(&commonrepo.ProductEnvFindOptions{ Namespace: pt.TaskArgs.Deploy.Namespace, @@ -380,7 +382,7 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t case config.TaskBuild: subBuildTaskMap := subStage.SubTasks for serviceModule, subTask := range subBuildTaskMap { - if buildInfo, err := commonservice.ToBuildTask(subTask); err == nil { + if buildInfo, err := base.ToBuildTask(subTask); err == nil { if newModules, err := commonrepo.NewBuildColl().List(&commonrepo.BuildListOption{Version: "stable", Targets: []string{serviceModule}, ServiceName: buildInfo.Service, ProductName: t.ProductName}); err == nil { newBuildInfo := newModules[0] buildInfo.JobCtx.BuildSteps = []*task.BuildStep{} @@ -442,7 +444,7 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t case config.TaskTestingV2: subTestTaskMap := subStage.SubTasks for testName, subTask := range subTestTaskMap { - if testInfo, err := commonservice.ToTestingTask(subTask); err == nil { + if testInfo, err := base.ToTestingTask(subTask); err == nil { if newTestInfo, err := GetTesting(testInfo.TestModuleName, "", log); err == nil { testInfo.JobCtx.BuildSteps = []*task.BuildStep{} if newTestInfo.Scripts != "" { @@ -498,7 +500,7 @@ func RestartPipelineTaskV2(userName string, taskID int64, pipelineName string, t subDeployTaskMap := subStage.SubTasks for serviceName, subTask := range subDeployTaskMap { - if deployInfo, err := commonservice.ToDeployTask(subTask); err == nil { + if deployInfo, err := base.ToDeployTask(subTask); err == nil { deployInfo.Timeout = timeout deployInfo.IsRestart = true deployInfo.ResetImage = resetImage @@ -711,6 +713,111 @@ type ProductNameWithType struct { Namespace string `json:"namespace"` } +func ListPipelineUpdatableProductNames(userName, pipelineName string, log *zap.SugaredLogger) ([]ProductNameWithType, error) { + resp := make([]ProductNameWithType, 0) + serviceName, err := findDeployServiceName(pipelineName, log) + + if err != nil { + return resp, err + } + + products, err := listPipelineUpdatableProducts(userName, serviceName, log) + if err != nil { + return resp, err + } + + for _, prod := range products { + prodNameWithType := ProductNameWithType{ + Name: prod.EnvName, + Namespace: prod.Namespace, + } + + prodNameWithType.Type = setting.NormalModeProduct + + found := false + for _, r := range resp { + if prodNameWithType == r { + found = true + break + } + } + if found { + continue + } + + resp = append(resp, prodNameWithType) + } + + // adapt for qiniu deployment + if config.OldEnvSupported() { + resp = append(resp, ListOldEnvsByServiceName(serviceName, log)...) + } + + return resp, nil +} + +func findDeployServiceName(pipelineName string, log *zap.SugaredLogger) (resp string, err error) { + pipe, err := commonrepo.NewPipelineColl().Find(&commonrepo.PipelineFindOption{Name: pipelineName}) + if err != nil { + log.Errorf("[%s] PipelineV2.Find error: %v", err) + return resp, e.ErrGetPipeline.AddDesc(err.Error()) + } + + deploy, err := getFirstEnabledDeployTask(pipe.SubTasks) + if err != nil { + log.Errorf("[%s] GetFirstEnabledDeployTask error: %v", err) + return resp, e.ErrGetTask.AddDesc(err.Error()) + } + + if deploy.ServiceName == "" { + return resp, e.ErrGetTask.AddDesc("deploy task has no group name or service name") + } + + return deploy.ServiceName, nil +} + +func getFirstEnabledDeployTask(subTasks []map[string]interface{}) (*task.Deploy, error) { + for _, subTask := range subTasks { + pre, err := base.ToPreview(subTask) + if err != nil { + return nil, err + } + if pre.TaskType == config.TaskDeploy && pre.Enabled { + return base.ToDeployTask(subTask) + } + } + return nil, e.NewErrInvalidTaskType("DeployTask not found") +} + +// ListUpdatableProductNames 列出用户可以deploy的产品环境, 包括自己的产品和被授权的产品 +func listPipelineUpdatableProducts(userName, serviceName string, log *zap.SugaredLogger) ([]*commonmodels.Product, error) { + resp := make([]*commonmodels.Product, 0) + + products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{}) + if err != nil { + log.Errorf("[%s] Collections.Product.List error: %v", userName, err) + return resp, e.ErrListProducts.AddDesc(err.Error()) + } + + //userTeams, err := s.FindUserTeams(userName, log) + //if err != nil { + // log.Errorf("FindUserTeams error: %v", err) + // return resp, err + //} + //userTeams := make([]string, 0) + + for _, prod := range products { + //if prod.EnvName == userName || prod.IsUserAuthed(userName, userTeams, product.ProductWritePermission) { + serviceNames := sets.NewString(GetServiceNames(prod)...) + if serviceNames.Has(serviceName) { + resp = append(resp, prod) + } + //} + } + + return resp, nil +} + func GetServiceNames(p *commonmodels.Product) []string { resp := make([]string, 0) for _, group := range p.Services { @@ -750,6 +857,61 @@ func ListOldEnvsByServiceName(serviceName string, log *zap.SugaredLogger) []Prod return resps } +func GePackageFileContent(pipelineName string, taskID int64, log *zap.SugaredLogger) ([]byte, string, error) { + var packageFile, storageURL string + //获取pipeline task + resp, err := commonrepo.NewTaskColl().Find(taskID, pipelineName, config.SingleType) + if err != nil { + return nil, "", fmt.Errorf("failed to get popeline") + } + + for _, subTask := range resp.SubTasks { + pre, err := base.ToPreview(subTask) + if err != nil { + return nil, "", fmt.Errorf("failed to get preview") + } + switch pre.TaskType { + + case config.TaskBuild: + build, err := base.ToBuildTask(subTask) + if err != nil { + return nil, "", fmt.Errorf("failed to get build") + } + packageFile = build.JobCtx.PackageFile + storageURL = resp.StorageURI + } + } + storage, err := s3.NewS3StorageFromEncryptedURI(storageURL) + if err != nil { + log.Errorf("failed to get s3 storage %s", storageURL) + return nil, packageFile, fmt.Errorf("failed to get s3 storage %s", storageURL) + } + if storage.Subfolder != "" { + storage.Subfolder = fmt.Sprintf("%s/%s/%d/%s", storage.Subfolder, pipelineName, resp.TaskID, "file") + } else { + storage.Subfolder = fmt.Sprintf("%s/%d/%s", pipelineName, resp.TaskID, "file") + } + + tmpfile, err := ioutil.TempFile("", "") + if err != nil { + return nil, packageFile, fmt.Errorf("failed to open file %v", err) + } + + _ = tmpfile.Close() + + defer func() { + _ = os.Remove(tmpfile.Name()) + }() + + err = s3.Download(context.Background(), storage, packageFile, tmpfile.Name()) + + if err != nil { + return nil, packageFile, fmt.Errorf("failed to download %s %v", packageFile, err) + } + fileBytes, err := ioutil.ReadFile(tmpfile.Name()) + return fileBytes, packageFile, err +} + func GetArtifactFileContent(pipelineName string, taskID int64, log *zap.SugaredLogger) ([]byte, error) { s3Storage, artifactFiles, _ := GetTestArtifactInfo(pipelineName, "", taskID, log) tempdir, _ := ioutil.TempDir("", "") diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go index 13a4b929e7..3cf130783c 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go @@ -31,16 +31,17 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehost" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/log" @@ -80,7 +81,7 @@ func ensurePipeline(args *commonmodels.Pipeline, log *zap.SugaredLogger) error { // validateSubTaskSetting Validating subtasks func validateSubTaskSetting(pipeName string, subtasks []map[string]interface{}) error { for i, subTask := range subtasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { return errors.New(e.InterfaceToTaskErrMsg) } @@ -92,7 +93,7 @@ func validateSubTaskSetting(pipeName string, subtasks []map[string]interface{}) switch pre.TaskType { case config.TaskBuild: - t, err := commonservice.ToBuildTask(subTask) + t, err := base.ToBuildTask(subTask) if err != nil { log.Error(err) return err @@ -111,7 +112,7 @@ func validateSubTaskSetting(pipeName string, subtasks []map[string]interface{}) } case config.TaskTestingV2: - t, err := commonservice.ToTestingTask(subTask) + t, err := base.ToTestingTask(subTask) if err != nil { log.Error(err) return err @@ -166,7 +167,7 @@ func getTaskEnvs(pipelineName string) map[string]string { for _, subTask := range pipe.SubTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { log.Error(err) return resp @@ -175,7 +176,7 @@ func getTaskEnvs(pipelineName string) map[string]string { switch pre.TaskType { case config.TaskBuild: - t, err := commonservice.ToBuildTask(subTask) + t, err := base.ToBuildTask(subTask) if err != nil { log.Error(err) return resp @@ -187,7 +188,7 @@ func getTaskEnvs(pipelineName string) map[string]string { } case config.TaskTestingV2: - t, err := commonservice.ToTestingTask(subTask) + t, err := base.ToTestingTask(subTask) if err != nil { log.Error(err) return resp @@ -636,6 +637,13 @@ func (c *ImageIllegal) Error() string { func validateServiceContainer2(namespace, envName, productName, serviceName, container, source string, kubeClient client.Client) (string, error) { var selector labels.Selector + //helm类型的服务查询所有标签的pod + if source != setting.SourceFromHelm { + selector = labels.Set{setting.ProductLabel: productName, setting.ServiceLabel: serviceName}.AsSelector() + //builder := &SelectorBuilder{ProductName: productName, ServiceName: serviceName} + //selector = builder.BuildSelector() + } + pods, err := getter.ListPods(namespace, selector, kubeClient) if err != nil { return "", fmt.Errorf("[%s] ListPods %s/%s error: %v", namespace, productName, serviceName, err) diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go index 48b61e8556..344efb0448 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/service_task.go @@ -19,13 +19,19 @@ package workflow import ( "errors" "fmt" + "sort" + + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + "github.com/koderover/zadig/pkg/types" "go.uber.org/zap" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -115,7 +121,7 @@ func ListServiceWorkflows(productName, envName, serviceName, serviceType string, for _, p := range pipelines { // 单服务工作的部署的服务只会有一个,不需考虑重复插入的问题 for _, subTask := range p.SubTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { log.Errorf("subTask.ToPreview error: %v", err) continue @@ -123,7 +129,7 @@ func ListServiceWorkflows(productName, envName, serviceName, serviceType string, if pre.TaskType != config.TaskDeploy { continue } - deploy, err := commonservice.ToDeployTask(subTask) + deploy, err := base.ToDeployTask(subTask) if err != nil || deploy == nil { log.Errorf("subTask.ToDeployTask error: %v", err) continue @@ -177,3 +183,120 @@ type CreateTaskResp struct { PipelineName string `json:"pipeline_name"` TaskID int64 `json:"task_id"` } + +func CreateServiceTask(args *commonmodels.ServiceTaskArgs, log *zap.SugaredLogger) ([]*CreateTaskResp, error) { + if args.BuildName == "" && args.Revision == 0 { + return nil, fmt.Errorf("服务[%s]的构建名称和服务版本必须有一个存在", args.ServiceName) + } else if args.BuildName == "" && args.Revision > 0 { + serviceTmpl, err := commonservice.GetServiceTemplate( + args.ServiceName, setting.PMDeployType, args.ProductName, setting.ProductStatusDeleting, args.Revision, log, + ) + if err != nil { + return nil, fmt.Errorf("GetServiceTemplate servicename:%s revision:%d err:%v ", args.ServiceName, args.Revision, err) + } + args.BuildName = serviceTmpl.BuildName + if args.BuildName == "" { + return nil, fmt.Errorf("未找到服务[%s]版本对应的构建", args.ServiceName) + } + } + // 获取全局configpayload + configPayload := commonservice.GetConfigPayload(0) + + defaultS3, err := s3.FindDefaultS3() + if err != nil { + err = e.ErrFindDefaultS3Storage.AddDesc("default storage is required by distribute task") + return nil, err + } + + defaultURL, err := defaultS3.GetEncryptedURL() + if err != nil { + err = e.ErrS3Storage.AddErr(err) + return nil, err + } + + stages := make([]*commonmodels.Stage, 0) + subTasks, err := BuildModuleToSubTasks("", "stable", args.ServiceName, args.ServiceName, args.ProductName, nil, nil, log) + if err != nil { + return nil, e.ErrCreateTask.AddErr(err) + } + + task := &task.Task{ + Type: config.ServiceType, + ProductName: args.ProductName, + TaskCreator: args.ServiceTaskCreator, + Status: config.StatusCreated, + SubTasks: subTasks, + TaskArgs: serviceTaskArgsToTaskArgs(args), + ConfigPayload: configPayload, + StorageURI: defaultURL, + } + sort.Sort(ByTaskKind(task.SubTasks)) + + if err := ensurePipelineTask(task, log); err != nil { + log.Errorf("CreateServiceTask ensurePipelineTask err : %v", err) + return nil, err + } + + for _, subTask := range task.SubTasks { + AddSubtaskToStage(&stages, subTask, args.ServiceName) + } + sort.Sort(ByStageKind(stages)) + task.Stages = stages + if len(task.Stages) == 0 { + return nil, e.ErrCreateTask.AddDesc(e.PipelineSubTaskNotFoundErrMsg) + } + + createTaskResps := make([]*CreateTaskResp, 0) + for _, envName := range args.EnvNames { + pipelineName := fmt.Sprintf("%s-%s-%s", args.ServiceName, envName, "job") + + nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.ServiceTaskFmt, pipelineName)) + if err != nil { + log.Errorf("CreateServiceTask Counter.GetNextSeq error: %v", err) + return nil, e.ErrGetCounter.AddDesc(err.Error()) + } + + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: args.ProductName, + EnvName: envName, + }) + if err != nil { + return nil, e.ErrCreateTask.AddDesc( + fmt.Sprintf("找不到 %s 的 %s 环境 ", args.ProductName, args.Namespace), + ) + } + + args.Namespace = envName + args.K8sNamespace = product.Namespace + + task.TaskID = nextTaskID + task.ServiceTaskArgs = args + task.SubTasks = []map[string]interface{}{} + task.PipelineName = pipelineName + + if err := CreateTask(task); err != nil { + log.Error(err) + return nil, e.ErrCreateTask + } + + createTaskResps = append(createTaskResps, &CreateTaskResp{PipelineName: pipelineName, TaskID: nextTaskID}) + } + + return createTaskResps, nil +} + +func serviceTaskArgsToTaskArgs(serviceTaskArgs *commonmodels.ServiceTaskArgs) *commonmodels.TaskArgs { + resp := &commonmodels.TaskArgs{ProductName: serviceTaskArgs.ProductName, TaskCreator: serviceTaskArgs.ServiceTaskCreator} + opt := &commonrepo.BuildFindOption{ + Name: serviceTaskArgs.BuildName, + Version: "stable", + ProductName: serviceTaskArgs.ProductName, + } + buildObj, err := commonrepo.NewBuildColl().Find(opt) + if err != nil { + resp.Builds = make([]*types.Repository, 0) + return resp + } + resp.Builds = buildObj.Repos + return resp +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/sets.go b/pkg/microservice/aslan/core/workflow/service/workflow/sets.go new file mode 100644 index 0000000000..21fe73163f --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/sets.go @@ -0,0 +1,68 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +type hookItem struct { + owner, repo, source string + codeHostID int +} + +type Empty struct{} + +type HookSet map[hookItem]Empty + +// NewHookSet creates a HookSet from a list of values. +func NewHookSet(items ...hookItem) HookSet { + ss := HookSet{} + ss.Insert(items...) + return ss +} + +// Insert adds items to the set. +func (s HookSet) Insert(items ...hookItem) HookSet { + for _, item := range items { + s[item] = Empty{} + } + return s +} + +// Has returns true if and only if item is contained in the set. +func (s HookSet) Has(item hookItem) bool { + _, contained := s[item] + return contained +} + +// Difference returns a set of objects that are not in s2 +// For example: +// s1 = {a1, a2, a3} +// s2 = {a1, a2, a4, a5} +// s1.Difference(s2) = {a3} +// s2.Difference(s1) = {a4, a5} +func (s HookSet) Difference(s2 HookSet) HookSet { + result := NewHookSet() + for key := range s { + if !s2.Has(key) { + result.Insert(key) + } + } + return result +} + +// Len returns the size of the set. +func (s HookSet) Len() int { + return len(s) +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/sort.go b/pkg/microservice/aslan/core/workflow/service/workflow/sort.go index 9887efb623..1d6313ac9b 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/sort.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/sort.go @@ -19,7 +19,7 @@ package workflow import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" ) type ByTaskKind []map[string]interface{} @@ -27,8 +27,8 @@ type ByTaskKind []map[string]interface{} func (a ByTaskKind) Len() int { return len(a) } func (a ByTaskKind) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByTaskKind) Less(i, j int) bool { - iPreview, _ := commonservice.ToPreview(a[i]) - jPreview, _ := commonservice.ToPreview(a[j]) + iPreview, _ := base.ToPreview(a[i]) + jPreview, _ := base.ToPreview(a[j]) return SubtaskOrder[iPreview.TaskType] < SubtaskOrder[jPreview.TaskType] } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go index 2bf0f7d54a..f2fddafe8d 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow.go @@ -27,14 +27,16 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/codehost" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/types" "github.com/koderover/zadig/pkg/types/permission" @@ -244,6 +246,10 @@ func PreSetWorkflow(productName string, log *zap.SugaredLogger) ([]*PreSetResp, target := fmt.Sprintf("%s%s%s%s%s", serviceTmpl.ProductName, SplitSymbol, serviceTmpl.ServiceName, SplitSymbol, container.Name) targets[target] = append(targets[target], deployEnv) } + case setting.PMDeployType: + deployEnv := DeployEnv{Env: service, Type: setting.PMDeployType, ProductName: productName} + target := fmt.Sprintf("%s%s%s%s%s", serviceTmpl.ProductName, SplitSymbol, serviceTmpl.ServiceName, SplitSymbol, serviceTmpl.ServiceName) + targets[target] = append(targets[target], deployEnv) case setting.HelmDeployType: for _, container := range serviceTmpl.Containers { deployEnv := DeployEnv{Env: service + "/" + container.Name, Type: setting.HelmDeployType, ProductName: serviceTmpl.ProductName} @@ -317,6 +323,12 @@ func CreateWorkflow(workflow *commonmodels.Workflow, log *zap.SugaredLogger) err return e.ErrUpsertWorkflow.AddDesc("workflow中没有子模块,请设置子模块") } + err = processWebhook(workflow.HookCtl.Items, nil, webhook.WorkflowPrefix+workflow.Name, log) + if err != nil { + log.Errorf("Failed to process webhook, err: %s", err) + return e.ErrUpsertWorkflow.AddDesc(err.Error()) + } + err = HandleCronjob(workflow, log) if err != nil { return e.ErrUpsertWorkflow.AddDesc(err.Error()) @@ -369,7 +381,19 @@ func UpdateWorkflow(workflow *commonmodels.Workflow, log *zap.SugaredLogger) err return e.ErrUpsertWorkflow.AddDesc("workflow中没有子模块,请设置子模块") } - err := HandleCronjob(workflow, log) + currentWorkflow, err := commonrepo.NewWorkflowColl().Find(workflow.Name) + if err != nil { + log.Errorf("Can not find workflow %s, err: %s", workflow.Name, err) + return e.ErrUpsertWorkflow.AddDesc(err.Error()) + } + + err = processWebhook(workflow.HookCtl.Items, currentWorkflow.HookCtl.Items, webhook.WorkflowPrefix+workflow.Name, log) + if err != nil { + log.Errorf("Failed to process webhook, err: %s", err) + return e.ErrUpsertWorkflow.AddDesc(err.Error()) + } + + err = HandleCronjob(workflow, log) if err != nil { return e.ErrUpsertWorkflow.AddDesc(err.Error()) } @@ -386,7 +410,7 @@ func UpdateWorkflow(workflow *commonmodels.Workflow, log *zap.SugaredLogger) err } } - if err := commonrepo.NewWorkflowColl().Replace(workflow); err != nil { + if err = commonrepo.NewWorkflowColl().Replace(workflow); err != nil { log.Errorf("Workflow.Update error: %v", err) return e.ErrUpsertWorkflow.AddDesc(err.Error()) } @@ -394,6 +418,98 @@ func UpdateWorkflow(workflow *commonmodels.Workflow, log *zap.SugaredLogger) err return nil } +func toHookSet(hooks interface{}) HookSet { + res := NewHookSet() + switch hs := hooks.(type) { + case []*commonmodels.WorkflowHook: + for _, h := range hs { + res.Insert(hookItem{ + owner: h.MainRepo.RepoOwner, + repo: h.MainRepo.RepoName, + codeHostID: h.MainRepo.CodehostID, + source: h.MainRepo.Source, + }) + } + case []commonmodels.GitHook: + for _, h := range hs { + res.Insert(hookItem{ + owner: h.Owner, + repo: h.Repo, + codeHostID: h.CodehostID, + }) + } + } + + return res +} + +func processWebhook(updatedHooks, currentHooks interface{}, name string, logger *zap.SugaredLogger) error { + currentSet := toHookSet(currentHooks) + updatedSet := toHookSet(updatedHooks) + hooksToRemove := currentSet.Difference(updatedSet) + hooksToAdd := updatedSet.Difference(currentSet) + + if hooksToRemove.Len() > 0 { + logger.Debugf("Going to remove webhooks %+v", hooksToRemove) + } + if hooksToAdd.Len() > 0 { + logger.Debugf("Going to add webhooks %+v", hooksToAdd) + } + + var errs *multierror.Error + var wg sync.WaitGroup + + for h := range hooksToRemove { + wg.Add(1) + go func(wh hookItem) { + defer wg.Done() + ch, err := codehost.GetCodeHostInfoByID(wh.codeHostID) + if err != nil { + logger.Errorf("Failed to get codeHost by id %d, err: %s", wh.codeHostID, err) + errs = multierror.Append(errs, err) + return + } + + switch ch.Type { + case setting.SourceFromGithub, setting.SourceFromGitlab: + err = webhook.NewClient().RemoveWebHook(wh.owner, wh.repo, ch.Address, ch.AccessToken, name, ch.Type) + if err != nil { + logger.Errorf("Failed to remove webhook %+v, err: %s", wh, err) + errs = multierror.Append(errs, err) + return + } + } + }(h) + } + + for h := range hooksToAdd { + wg.Add(1) + go func(wh hookItem) { + defer wg.Done() + ch, err := codehost.GetCodeHostInfoByID(wh.codeHostID) + if err != nil { + logger.Errorf("Failed to get codeHost by id %d, err: %s", wh.codeHostID, err) + errs = multierror.Append(errs, err) + return + } + + switch ch.Type { + case setting.SourceFromGithub, setting.SourceFromGitlab: + err = webhook.NewClient().AddWebHook(wh.owner, wh.repo, ch.Address, ch.AccessToken, name, ch.Type) + if err != nil { + logger.Errorf("Failed to add webhook %+v, err: %s", wh, err) + errs = multierror.Append(errs, err) + return + } + } + }(h) + } + + wg.Wait() + + return errs.ErrorOrNil() +} + func ListWorkflows(queryType string, userID int, log *zap.SugaredLogger) ([]*commonmodels.Workflow, error) { workflows, err := commonrepo.NewWorkflowColl().List(&commonrepo.ListWorkflowOption{}) if err != nil { @@ -556,6 +672,12 @@ func DeleteWorkflow(workflowName, requestID string, isDeletingProductTmpl bool, } } + err = processWebhook(nil, workflow.HookCtl.Items, webhook.WorkflowPrefix+workflow.Name, log) + if err != nil { + log.Errorf("Failed to process webhook, err: %s", err) + return e.ErrUpsertWorkflow.AddDesc(err.Error()) + } + go DeleteGerritWebhook(workflow, log) //删除所属的所有定时任务 diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go index 18a5a0a94d..87fd55a33c 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go @@ -30,16 +30,17 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/scmnotify" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types" @@ -134,6 +135,10 @@ func getProductTargetMap(prod *commonmodels.Product) map[string][]commonmodels.D target := fmt.Sprintf("%s%s%s%s%s", prod.ProductName, SplitSymbol, serviceObj.ServiceName, SplitSymbol, container.Name) resp[target] = append(resp[target], deployEnv) } + case setting.PMDeployType: + deployEnv := commonmodels.DeployEnv{Type: setting.PMDeployType, Env: serviceObj.ServiceName} + target := fmt.Sprintf("%s%s%s%s%s", prod.ProductName, SplitSymbol, serviceObj.ServiceName, SplitSymbol, serviceObj.ServiceName) + resp[target] = append(resp[target], deployEnv) case setting.HelmDeployType: for _, container := range serviceObj.Containers { env := serviceObj.ServiceName + "/" + container.Name @@ -170,6 +175,10 @@ func getProductTemplTargetMap(productName string) map[string][]commonmodels.Depl target := fmt.Sprintf("%s%s%s%s%s", productTmpl.ProductName, SplitSymbol, service, SplitSymbol, container.Name) targets[target] = append(targets[target], deployEnv) } + case setting.PMDeployType: + deployEnv := commonmodels.DeployEnv{Env: service, Type: setting.PMDeployType} + target := fmt.Sprintf("%s%s%s%s%s", productTmpl.ProductName, SplitSymbol, service, SplitSymbol, service) + targets[target] = append(targets[target], deployEnv) case setting.HelmDeployType: for _, container := range serviceTmpl.Containers { deployEnv := commonmodels.DeployEnv{Env: service + "/" + container.Name, Type: setting.HelmDeployType, ProductName: serviceTmpl.ProductName} @@ -506,6 +515,9 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, if env != nil { // 生成部署的subtask for _, deployEnv := range target.Deploy { + if deployEnv.Type == setting.PMDeployType { + continue + } deployTask, err := deployEnvToSubTasks(deployEnv, env, productTempl.Timeout) if err != nil { log.Errorf("deploy env to subtask error: %v", err) @@ -677,9 +689,7 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, return nil, e.ErrCreateTask.AddDesc(e.PipelineSubTaskNotFoundErrMsg) } - namespace := config.Namespace() - - endpoint := fmt.Sprintf("%s-%s:9000", namespace, ClusterStorageEP) + endpoint := fmt.Sprintf("%s-%s:9000", config.Namespace(), ClusterStorageEP) task.StorageEndpoint = endpoint @@ -1219,6 +1229,7 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * testTask.JobCtx.TestThreshold = testModule.Threshold testTask.JobCtx.Caches = testModule.Caches testTask.JobCtx.TestResultPath = testModule.TestResultPath + testTask.JobCtx.TestReportPath = testModule.TestReportPath if testTask.Registries == nil { testTask.Registries = registries @@ -1552,6 +1563,12 @@ func BuildModuleToSubTasks(moduleName, version, target, serviceName, productName opt.Targets = []string{target} } + if pro != nil { + serviceTmpl, _ = commonservice.GetServiceTemplate( + target, setting.PMDeployType, productName, setting.ProductStatusDeleting, 0, log, + ) + } + modules, err := commonrepo.NewBuildColl().List(opt) if err != nil { return nil, e.ErrConvertSubTasks.AddErr(err) @@ -1578,10 +1595,25 @@ func BuildModuleToSubTasks(moduleName, version, target, serviceName, productName Registries: registries, } + // 自定义基础镜像的镜像名称可能会被更新,需要使用ID获取最新的镜像名称 + if module.PreBuild.ImageID != "" { + basicImage, err := commonrepo.NewBasicImageColl().Find(module.PreBuild.ImageID) + if err != nil { + log.Errorf("BasicImage.Find failed, id:%s, err:%v", module.PreBuild.ImageID, err) + } else { + build.BuildOS = basicImage.Value + } + } + if build.ImageFrom == "" { build.ImageFrom = commonmodels.ImageFromKoderover } + if serviceTmpl != nil { + build.Namespace = pro.Namespace + build.ServiceType = setting.PMDeployType + } + if pro != nil { build.EnvName = pro.EnvName } @@ -1600,6 +1632,30 @@ func BuildModuleToSubTasks(moduleName, version, target, serviceName, productName build.JobCtx.BuildSteps = append(build.JobCtx.BuildSteps, &task.BuildStep{BuildType: "shell", Scripts: module.Scripts}) } + if module.PMDeployScripts != "" && build.ServiceType == setting.PMDeployType { + build.JobCtx.PMDeployScripts = module.PMDeployScripts + } + + if len(module.SSHs) > 0 && build.ServiceType == setting.PMDeployType { + privateKeys := make([]*task.SSH, 0) + for _, sshID := range module.SSHs { + //私钥信息可能被更新,而构建中存储的信息是旧的,需要根据id获取最新的私钥信息 + latestKeyInfo, err := commonrepo.NewPrivateKeyColl().Find(sshID) + if err != nil || latestKeyInfo == nil { + log.Errorf("PrivateKey.Find failed, id:%s, err:%v", sshID, err) + continue + } + ssh := new(task.SSH) + ssh.Name = latestKeyInfo.Name + ssh.UserName = latestKeyInfo.UserName + ssh.IP = latestKeyInfo.IP + ssh.PrivateKey = latestKeyInfo.PrivateKey + + privateKeys = append(privateKeys, ssh) + } + build.JobCtx.SSHs = privateKeys + } + build.JobCtx.EnvVars = module.PreBuild.Envs if len(module.PreBuild.Envs) == 0 { @@ -1667,7 +1723,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { //设置执行任务时参数 for i, subTask := range pt.SubTasks { - pre, err := commonservice.ToPreview(subTask) + pre, err := base.ToPreview(subTask) if err != nil { return errors.New(e.InterfaceToTaskErrMsg) } @@ -1675,7 +1731,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { switch pre.TaskType { case config.TaskBuild: - t, err := commonservice.ToBuildTask(subTask) + t, err := base.ToBuildTask(subTask) fmtBuildsTask(t, log) if err != nil { log.Error(err) @@ -1783,7 +1839,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } } case config.TaskJenkinsBuild: - t, err := commonservice.ToJenkinsBuildTask(subTask) + t, err := base.ToJenkinsBuildTask(subTask) if err != nil { log.Error(err) return err @@ -1813,7 +1869,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } } case config.TaskArtifact: - t, err := commonservice.ToArtifactTask(subTask) + t, err := base.ToArtifactTask(subTask) if err != nil { log.Error(err) return err @@ -1833,7 +1889,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } } case config.TaskDockerBuild: - t, err := commonservice.ToDockerBuildTask(subTask) + t, err := base.ToDockerBuildTask(subTask) if err != nil { log.Error(err) return err @@ -1862,7 +1918,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } case config.TaskTestingV2: - t, err := commonservice.ToTestingTask(subTask) + t, err := base.ToTestingTask(subTask) if err != nil { log.Error(err) return err @@ -1900,7 +1956,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } } case config.TaskResetImage: - t, err := commonservice.ToDeployTask(subTask) + t, err := base.ToDeployTask(subTask) if err != nil { log.Error(err) return err @@ -1923,7 +1979,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } case config.TaskDeploy: - t, err := commonservice.ToDeployTask(subTask) + t, err := base.ToDeployTask(subTask) if err != nil { log.Error(err) return err @@ -1958,7 +2014,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } case config.TaskDistributeToS3: - task, err := commonservice.ToDistributeToS3Task(subTask) + task, err := base.ToDistributeToS3Task(subTask) if err != nil { log.Error(err) return err @@ -2001,7 +2057,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } case config.TaskReleaseImage: - t, err := commonservice.ToReleaseImageTask(subTask) + t, err := base.ToReleaseImageTask(subTask) if err != nil { log.Error(err) return err @@ -2049,7 +2105,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } case config.TaskJira: - t, err := commonservice.ToJiraTask(subTask) + t, err := base.ToJiraTask(subTask) if err != nil { log.Error(err) return err @@ -2069,7 +2125,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } } case config.TaskSecurity: - t, err := commonservice.ToSecurityTask(subTask) + t, err := base.ToSecurityTask(subTask) if err != nil { log.Error(err) return err @@ -2093,7 +2149,7 @@ func ensurePipelineTask(pt *task.Task, log *zap.SugaredLogger) error { } func AddSubtaskToStage(stages *[]*commonmodels.Stage, subTask map[string]interface{}, target string) { - subTaskPre, err := commonservice.ToPreview(subTask) + subTaskPre, err := base.ToPreview(subTask) if err != nil { log.Errorf("subtask to preview error: %v", err) return diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/it_report.go b/pkg/microservice/aslan/core/workflow/testing/handler/it_report.go index ff168ef9c5..cc5ace33a6 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/it_report.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/it_report.go @@ -24,7 +24,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/router.go b/pkg/microservice/aslan/core/workflow/testing/handler/router.go index 25b7607f52..e0701d22cd 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/router.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/router.go @@ -19,14 +19,20 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" "github.com/koderover/zadig/pkg/types/permission" ) type Router struct{} func (*Router) Inject(router *gin.RouterGroup) { - router.Use(middleware.Auth()) + // 查看html测试报告不做鉴权 + testReport := router.Group("report") + { + testReport.GET("", GetHTMLTestReport) + } + + router.Use(gin2.Auth()) // --------------------------------------------------------------------------------------- // 系统测试接口 @@ -43,11 +49,17 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- tester := router.Group("test") { - tester.POST("", GetTestProductName, middleware.IsHavePermission([]string{permission.TestManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, CreateTestModule) - tester.PUT("", GetTestProductName, middleware.IsHavePermission([]string{permission.TestManageUUID}, permission.ContextKeyType), middleware.UpdateOperationLogStatus, UpdateTestModule) + tester.POST("", GetTestProductName, gin2.IsHavePermission([]string{permission.TestManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, CreateTestModule) + tester.PUT("", GetTestProductName, gin2.IsHavePermission([]string{permission.TestManageUUID}, permission.ContextKeyType), gin2.UpdateOperationLogStatus, UpdateTestModule) tester.GET("", ListTestModules) tester.GET("/:name", GetTestModule) - tester.DELETE("/:name", middleware.IsHavePermission([]string{permission.TestDeleteUUID}, permission.QueryType), middleware.UpdateOperationLogStatus, DeleteTestModule) + tester.DELETE("/:name", gin2.IsHavePermission([]string{permission.TestDeleteUUID}, permission.QueryType), gin2.UpdateOperationLogStatus, DeleteTestModule) + } + + testStat := router.Group("teststat") + { + // 供aslanx的enterprise模块的数据统计调用 + testStat.GET("", ListTestStat) } testDetail := router.Group("testdetail") @@ -60,9 +72,9 @@ func (*Router) Inject(router *gin.RouterGroup) { // --------------------------------------------------------------------------------------- testTask := router.Group("testtask") { - testTask.POST("", middleware.UpdateOperationLogStatus, CreateTestTask) - testTask.POST("/productName/:productName/id/:id/pipelines/:name/restart", middleware.UpdateOperationLogStatus, RestartTestTask) - testTask.DELETE("/productName/:productName/id/:id/pipelines/:name", middleware.UpdateOperationLogStatus, CancelTestTaskV2) + testTask.POST("", gin2.UpdateOperationLogStatus, CreateTestTask) + testTask.POST("/productName/:productName/id/:id/pipelines/:name/restart", gin2.UpdateOperationLogStatus, RestartTestTask) + testTask.DELETE("/productName/:productName/id/:id/pipelines/:name", gin2.UpdateOperationLogStatus, CancelTestTaskV2) } // --------------------------------------------------------------------------------------- diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go b/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go index 4cd66c7f5f..02295c3e80 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/test_detail.go @@ -20,7 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) func ListDetailTestModules(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/workflow/handler/gitlab.go b/pkg/microservice/aslan/core/workflow/testing/handler/test_stat.go similarity index 55% rename from pkg/microservice/aslan/core/workflow/handler/gitlab.go rename to pkg/microservice/aslan/core/workflow/testing/handler/test_stat.go index 10d60d6e4d..215ece4b40 100644 --- a/pkg/microservice/aslan/core/workflow/handler/gitlab.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/test_stat.go @@ -19,24 +19,13 @@ package handler import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/webhook" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" - e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) -// @Router /workflow/webhook/gitlabhook [POST] -// @Summary Process webhook for giblab -// @Accept json -// @Produce json -// @Success 200 {object} map[string]string "map[string]string - {message: success}" -func ProcessGitlabHook(c *gin.Context) { +func ListTestStat(c *gin.Context) { ctx := internalhandler.NewContext(c) - log := ctx.Logger - err := webhook.ProcessGitlabHook(c.Request, ctx.RequestID, log) - if err != nil { - c.JSON(e.ErrorMessage(err)) - c.Abort() - return - } - c.JSON(200, gin.H{"message": "success"}) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListTestStat(ctx.Logger) } diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go b/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go index 90bc7202b3..7f300ca737 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go @@ -31,8 +31,8 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" "github.com/koderover/zadig/pkg/setting" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" ) diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/testing.go b/pkg/microservice/aslan/core/workflow/testing/handler/testing.go index 57596c8f5e..af85f5e980 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/testing.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/testing.go @@ -25,10 +25,11 @@ import ( commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" + "github.com/koderover/zadig/pkg/util/ginzap" ) func GetTestProductName(c *gin.Context) { @@ -68,7 +69,7 @@ func CreateTestModule(c *gin.Context) { return } - ctx.Err = service.Create(ctx.Username, args, ctx.Logger) + ctx.Err = service.CreateTesting(ctx.Username, args, ctx.Logger) } func UpdateTestModule(c *gin.Context) { @@ -92,7 +93,7 @@ func UpdateTestModule(c *gin.Context) { return } - ctx.Err = service.Update(ctx.Username, args, ctx.Logger) + ctx.Err = service.UpdateTesting(ctx.Username, args, ctx.Logger) } func ListTestModules(c *gin.Context) { @@ -112,7 +113,7 @@ func GetTestModule(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("empty Name") return } - ctx.Resp, ctx.Err = service.Get(name, c.Query("productName"), ctx.Logger) + ctx.Resp, ctx.Err = service.GetTesting(name, c.Query("productName"), ctx.Logger) } func DeleteTestModule(c *gin.Context) { @@ -129,3 +130,20 @@ func DeleteTestModule(c *gin.Context) { ctx.Err = service.DeleteTestModule(name, c.Query("productName"), ctx.RequestID, ctx.Logger) } + +func GetHTMLTestReport(c *gin.Context) { + content, err := service.GetHTMLTestReport( + c.Query("pipelineName"), + c.Query("pipelineType"), + c.Query("taskID"), + c.Query("testName"), + ginzap.WithContext(c).Sugar(), + ) + if err != nil { + c.JSON(500, gin.H{"err": err}) + return + } + + c.Header("content-type", "text/html") + c.String(200, content) +} diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go b/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go index cca78c66d9..3c8ef86f61 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/workspace.go @@ -22,7 +22,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" - internalhandler "github.com/koderover/zadig/pkg/microservice/aslan/internal/handler" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" ) diff --git a/pkg/microservice/aslan/core/workflow/testing/service/it_report.go b/pkg/microservice/aslan/core/workflow/testing/service/it_report.go index 5b8658c813..b96a49c04e 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/it_report.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/it_report.go @@ -29,7 +29,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/util" ) @@ -59,7 +59,7 @@ func GetTestLocalTestSuite(serviceName string, log *zap.SugaredLogger) (*commonm case config.TaskTestingV2: subTestTaskMap := subStage.SubTasks for _, subTask := range subTestTaskMap { - testInfo, err := commonservice.ToTestingTask(subTask) + testInfo, err := base.ToTestingTask(subTask) if err != nil { continue } diff --git a/pkg/microservice/aslan/core/workflow/testing/service/test_stat.go b/pkg/microservice/aslan/core/workflow/testing/service/test_stat.go new file mode 100644 index 0000000000..e53441a3a8 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/testing/service/test_stat.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "go.uber.org/zap" + + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" +) + +func ListTestStat(log *zap.SugaredLogger) ([]*commonmodels.TestTaskStat, error) { + testStat, err := commonrepo.NewTestTaskStatColl().ListTestTask() + if err != nil { + log.Errorf("TestTaskStat list err:%v", err) + return nil, err + } + return testStat, nil +} diff --git a/pkg/microservice/aslan/core/workflow/testing/service/testing.go b/pkg/microservice/aslan/core/workflow/testing/service/testing.go index a1e6f88099..733e097694 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/testing.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/testing.go @@ -17,9 +17,12 @@ limitations under the License. package service import ( + "context" "encoding/json" "fmt" + "os" "strconv" + "strings" "time" "go.uber.org/zap" @@ -29,12 +32,14 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/util" ) -func Create(username string, testing *commonmodels.Testing, log *zap.SugaredLogger) error { +func CreateTesting(username string, testing *commonmodels.Testing, log *zap.SugaredLogger) error { if len(testing.Name) == 0 { return e.ErrCreateTestModule.AddDesc("empty Name") } @@ -87,7 +92,7 @@ func HandleCronjob(testing *commonmodels.Testing, log *zap.SugaredLogger) error return nil } -func Update(username string, testing *commonmodels.Testing, log *zap.SugaredLogger) error { +func UpdateTesting(username string, testing *commonmodels.Testing, log *zap.SugaredLogger) error { if len(testing.Name) == 0 { return e.ErrUpdateTestModule.AddDesc("empty Name") } @@ -203,7 +208,7 @@ func ListAllWorkflows(testName string, log *zap.SugaredLogger) ([]*commonmodels. return workflows, nil } -func Get(name, productName string, log *zap.SugaredLogger) (*commonmodels.Testing, error) { +func GetTesting(name, productName string, log *zap.SugaredLogger) (*commonmodels.Testing, error) { resp, err := GetRaw(name, productName, log) if err != nil { return nil, err @@ -313,3 +318,88 @@ func DeleteTestingModule(name, productName string, log *zap.SugaredLogger) error return nil } + +func GetHTMLTestReport(pipelineName, pipelineType, taskIDStr, testName string, log *zap.SugaredLogger) (string, error) { + if err := validateTestReportParam(pipelineName, pipelineType, taskIDStr, testName, log); err != nil { + return "", e.ErrGetTestReport.AddErr(err) + } + + taskID, err := strconv.ParseInt(taskIDStr, 10, 64) + if err != nil { + log.Errorf("invalid taskID: %s, err: %s", taskIDStr, err) + return "", e.ErrGetTestReport.AddDesc("invalid taskID") + } + + task, err := commonrepo.NewTaskColl().Find(taskID, pipelineName, config.WorkflowType) + if err != nil { + log.Errorf("find task failed, pipelineName: %s, type: %s, taskID: %s, err: %s", pipelineName, config.WorkflowType, taskIDStr, err) + return "", e.ErrGetTestReport.AddErr(err) + } + + store, err := s3.NewS3StorageFromEncryptedURI(task.StorageURI) + if err != nil { + log.Errorf("parse storageURI failed, err: %s", err) + return "", e.ErrGetTestReport.AddErr(err) + } + + if store.Subfolder != "" { + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, pipelineName, taskID, "test") + } else { + store.Subfolder = fmt.Sprintf("%s/%d/%s", pipelineName, taskID, "test") + } + + fileName := fmt.Sprintf("%s-%s-%s-%s-%s-html", pipelineType, pipelineName, taskIDStr, config.TaskTestingV2, testName) + fileName = strings.Replace(strings.ToLower(fileName), "_", "-", -1) + + tmpFilename, err := util.GenerateTmpFile() + if err != nil { + log.Errorf("generate temp file error: %s", err) + return "", e.ErrGetTestReport.AddErr(err) + } + defer func() { + _ = os.Remove(tmpFilename) + }() + + err = s3.Download(context.Background(), store, fileName, tmpFilename) + if err != nil { + log.Errorf("download html test report error: %s", err) + return "", e.ErrGetTestReport.AddErr(err) + } + + content, err := os.ReadFile(tmpFilename) + if err != nil { + log.Errorf("parse test report file error: %s", err) + return "", e.ErrGetTestReport.AddErr(err) + } + + return string(content), nil +} + +func validateTestReportParam(pipelineName, pipelineType, taskIDStr, testName string, log *zap.SugaredLogger) error { + if pipelineName == "" { + log.Warn("pipelineName cannot be empty") + return fmt.Errorf("pipelineName cannot be empty") + } + + if pipelineType == "" { + log.Warn("pipelineType cannot be empty") + return fmt.Errorf("pipelineType cannot be empty") + } + + if taskIDStr == "" { + log.Warn("taskID cannot be empty") + return fmt.Errorf("taskID cannot be empty") + } + + if testName == "" { + log.Warn("testName cannot be empty") + return fmt.Errorf("testName cannot be empty") + } + + if pipelineType != string(config.WorkflowType) { + log.Warn("pipelineType is invalid") + return fmt.Errorf("pipelineType is invalid") + } + + return nil +} diff --git a/pkg/microservice/aslan/server/rest/router.go b/pkg/microservice/aslan/server/rest/router.go index 9d05d883cd..5209714c23 100644 --- a/pkg/microservice/aslan/server/rest/router.go +++ b/pkg/microservice/aslan/server/rest/router.go @@ -28,16 +28,16 @@ import ( commonhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/common/handler" cronhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/cron/handler" deliveryhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/handler" - enterprisehandler "github.com/koderover/zadig/pkg/microservice/aslan/core/enterprise/handler" environmenthandler "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/handler" loghandler "github.com/koderover/zadig/pkg/microservice/aslan/core/log/handler" + multiclusterhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/handler" projecthandler "github.com/koderover/zadig/pkg/microservice/aslan/core/project/handler" servicehandler "github.com/koderover/zadig/pkg/microservice/aslan/core/service/handler" settinghandler "github.com/koderover/zadig/pkg/microservice/aslan/core/setting/handler" systemhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/system/handler" workflowhandler "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/handler" testinghandler "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/handler" - "github.com/koderover/zadig/pkg/microservice/aslan/middleware" + gin2 "github.com/koderover/zadig/pkg/middleware/gin" // Note: have to load docs for swagger to work. See https://blog.csdn.net/weixin_43249914/article/details/103035711 _ "github.com/koderover/zadig/pkg/microservice/aslan/server/rest/doc" @@ -60,32 +60,40 @@ func (s *engine) injectRouterGroup(router *gin.RouterGroup) { } store.Options(cookieOption) router.Use(sessions.Sessions("ASLAN", store)) - Auth := middleware.Auth() + Auth := gin2.Auth() // --------------------------------------------------------------------------------------- // 对外公共接口 // --------------------------------------------------------------------------------------- public := router.Group("/api") { + // new webhook url for all + public.POST("/webhook", func(c *gin.Context) { + c.Request.URL.Path = "/api/workflow/webhook" + s.HandleContext(c) + }) // 内部路由重定向。接口路由更新后,需要保证旧路由可用,否则github、gitlab的配置需要手动更新。 public.POST("/ci/webhook", func(c *gin.Context) { - c.Request.URL.Path = "/api/workflow/webhook/ci/webhook" + c.Request.URL.Path = "/api/workflow/webhook" s.HandleContext(c) }) public.POST("/githubWebHook", func(c *gin.Context) { - c.Request.URL.Path = "/api/workflow/webhook/githubWebHook" + c.Request.URL.Path = "/api/workflow/webhook" s.HandleContext(c) }) public.POST("/gitlabhook", func(c *gin.Context) { - c.Request.URL.Path = "/api/workflow/webhook/gitlabhook" + c.Request.URL.Path = "/api/workflow/webhook" s.HandleContext(c) }) public.POST("/gerritHook", func(c *gin.Context) { - c.Request.URL.Path = "/api/workflow/webhook/gerritHook" + c.Request.URL.Path = "/api/workflow/webhook" s.HandleContext(c) }) public.GET("/health", commonhandler.Health) } + // no auth required, should not be exposed via poetry-api-proxy or will fail + router.GET("/api/hub/connect", multiclusterhandler.ClusterConnectFromAgent) + router.GET("/api/kodespace/downloadUrl", Auth, commonhandler.GetToolDownloadURL) jwt := router.Group("/api/token", Auth) @@ -102,10 +110,10 @@ func (s *engine) injectRouterGroup(router *gin.RouterGroup) { "/api/cron": new(cronhandler.Router), "/api/workflow": new(workflowhandler.Router), "/api/build": new(buildhandler.Router), - "/api/enterprise": new(enterprisehandler.Router), "/api/delivery": new(deliveryhandler.Router), "/api/logs": new(loghandler.Router), "/api/testing": new(testinghandler.Router), + "/api/cluster": new(multiclusterhandler.Router), } { r.Inject(router.Group(name)) } diff --git a/pkg/microservice/aslan/server/server.go b/pkg/microservice/aslan/server/server.go index 37a593b5eb..113e349803 100644 --- a/pkg/microservice/aslan/server/server.go +++ b/pkg/microservice/aslan/server/server.go @@ -34,8 +34,8 @@ func Serve(ctx context.Context) error { } }() - core.Setup() - defer core.TearDown() + core.Setup(ctx) + defer core.TearDown(ctx) log.Infof("App Aslan Started at %s", time.Now()) diff --git a/pkg/microservice/cron/config/config.go b/pkg/microservice/cron/config/config.go index 6a1165d909..57b14761f8 100644 --- a/pkg/microservice/cron/config/config.go +++ b/pkg/microservice/cron/config/config.go @@ -30,6 +30,10 @@ func AslanAPI() string { return viper.GetString(setting.ENVAslanAPI) } +func AslanxAPI() string { + return viper.GetString(setting.ENVAslanxAPI) +} + func RootToken() string { return viper.GetString(setting.ENVRootToken) } diff --git a/pkg/microservice/cron/core/service/client/collie_pipeline.go b/pkg/microservice/cron/core/service/client/collie_pipeline.go index 762114ce1c..00daba87f4 100644 --- a/pkg/microservice/cron/core/service/client/collie_pipeline.go +++ b/pkg/microservice/cron/core/service/client/collie_pipeline.go @@ -26,6 +26,19 @@ import ( "github.com/koderover/zadig/pkg/tool/httpclient" ) +func (c *CollieClient) ListColliePipelines(log *zap.SugaredLogger) ([]*service.PipelineResource, error) { + url := "/api/collie/api/pipelines" + + ciPipelines := make([]*service.PipelineResource, 0) + _, err := c.Get(url, httpclient.SetResult(&ciPipelines)) + if err != nil { + log.Errorf("ListColliePipelines from collie failed, err: %+v", err) + return nil, err + } + + return ciPipelines, nil +} + func (c *CollieClient) RunColliePipelineTask(args *service.CreateBuildRequest, log *zap.SugaredLogger) error { // collie pipeline list接口返回的pipelineName的格式为:项目名/pipelineName, 需要对进行encode encodePipelineName := url.QueryEscape(args.PipelineName) diff --git a/pkg/microservice/cron/core/service/client/sonar.go b/pkg/microservice/cron/core/service/client/sonar.go index 59a76d2fef..61f6b15dd5 100644 --- a/pkg/microservice/cron/core/service/client/sonar.go +++ b/pkg/microservice/cron/core/service/client/sonar.go @@ -23,6 +23,7 @@ import ( "go.uber.org/zap" + "github.com/koderover/zadig/pkg/microservice/cron/config" "github.com/koderover/zadig/pkg/setting" ) @@ -37,7 +38,7 @@ func (c *Client) InitPullSonarStatScheduler(log *zap.SugaredLogger) error { func (c *Client) InitPullSonarTestsMeasure(log *zap.SugaredLogger) error { log.Info("start to pull sonar test measure..") - url := fmt.Sprintf("%s/quality/sonar/tests/measure/pull", c.APIBase) + url := fmt.Sprintf("%s/quality/sonar/tests/measure/pull", config.AslanxAPI()) request, err := http.NewRequest("POST", url, nil) if err != nil { @@ -68,7 +69,7 @@ func (c *Client) InitPullSonarTestsMeasure(log *zap.SugaredLogger) error { func (c *Client) InitPullSonarDeliveryMeasure(log *zap.SugaredLogger) error { log.Info("start to pull sonar delivery measure..") - url := fmt.Sprintf("%s/quality/sonar/delivery/measure/pull", c.APIBase) + url := fmt.Sprintf("%s/quality/sonar/delivery/measure/pull", config.AslanxAPI()) request, err := http.NewRequest("POST", url, nil) if err != nil { @@ -99,7 +100,7 @@ func (c *Client) InitPullSonarDeliveryMeasure(log *zap.SugaredLogger) error { func (c *Client) InitPullSonarRepos(log *zap.SugaredLogger) error { log.Info("start to pull sonar repos..") - url := fmt.Sprintf("%s/quality/sonar/repository/pull", c.APIBase) + url := fmt.Sprintf("%s/quality/sonar/repository/pull", config.AslanxAPI()) request, err := http.NewRequest("POST", url, nil) if err != nil { diff --git a/pkg/microservice/cron/core/service/client/stat.go b/pkg/microservice/cron/core/service/client/stat.go index a39f2843ee..97d09e146b 100644 --- a/pkg/microservice/cron/core/service/client/stat.go +++ b/pkg/microservice/cron/core/service/client/stat.go @@ -20,25 +20,27 @@ import ( "fmt" "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/cron/config" ) func (c *Client) InitStatData(log *zap.SugaredLogger) error { //build - url := fmt.Sprintf("%s/quality/stat/initBuildStat", c.APIBase) + url := fmt.Sprintf("%s/quality/stat/initBuildStat", config.AslanxAPI()) log.Info("start init buildStat..") err := c.sendRequest(url) if err != nil { log.Errorf("trigger init buildStat error :%v", err) } //test - url = fmt.Sprintf("%s/quality/stat/initTestStat", c.APIBase) + url = fmt.Sprintf("%s/quality/stat/initTestStat", config.AslanxAPI()) log.Info("start init testStat..") err = c.sendRequest(url) if err != nil { log.Errorf("trigger init testStat error :%v", err) } //deploy - url = fmt.Sprintf("%s/quality/stat/initDeployStat", c.APIBase) + url = fmt.Sprintf("%s/quality/stat/initDeployStat", config.AslanxAPI()) log.Info("start init deployStat..") err = c.sendRequest(url) if err != nil { @@ -50,7 +52,7 @@ func (c *Client) InitStatData(log *zap.SugaredLogger) error { func (c *Client) InitOperationStatData(log *zap.SugaredLogger) error { //operation - url := fmt.Sprintf("%s/operation/stat/initOperationStat", c.APIBase) + url := fmt.Sprintf("%s/operation/stat/initOperationStat", config.AslanxAPI()) log.Info("start init operationStat..") err := c.sendRequest(url) if err != nil { diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_collie_pipeline.go b/pkg/microservice/cron/core/service/scheduler/schedule_collie_pipeline.go new file mode 100644 index 0000000000..ad35b2c8ce --- /dev/null +++ b/pkg/microservice/cron/core/service/scheduler/schedule_collie_pipeline.go @@ -0,0 +1,111 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "reflect" + + "github.com/jasonlvhit/gocron" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/cron/core/service" + "github.com/koderover/zadig/pkg/setting" +) + +// UpsertColliePipelineScheduler ... +func (c *CronClient) UpsertColliePipelineScheduler(log *zap.SugaredLogger) { + pipelines, err := c.CollieCli.ListColliePipelines(log) + if err != nil { + log.Error(err) + return + } + + log.Info("start init collie pipeline task scheduler..") + for _, pipeline := range pipelines { + log.Infof("deal collie pipeline data, name:%s, schedule:%v", pipeline.Metadata.Name, pipeline.Spec.Schedules) + key := "collie-pipeline-timer-" + pipeline.Metadata.Name + + schedules := pipeline.Spec.Schedules + // 如果该pipeline先配置了定时器,然后将定时器全部删除了,这时需要把原来的schedule暂停并删除 + if len(schedules) == 0 { + // 暂停 + if _, ok := c.SchedulerController[key]; ok { + c.SchedulerController[key] <- true + delete(c.SchedulerController, key) + } + // 删除 + if _, ok := c.Schedulers[key]; ok { + log.Warnf("delete expired test from Schedulers, key:%s", key) + delete(c.Schedulers, key) + } + if _, ok := c.lastSchedulers[key]; ok { + log.Warnf("delete expired test from lastSchedulers, key:%s", key) + delete(c.lastSchedulers, key) + } + continue + } + + // 判断该测试的定时器配置是否有更新,没有更新则跳过 + if _, ok := c.lastSchedulers[key]; ok && reflect.DeepEqual(schedules, c.lastSchedulers[key]) { + continue + } else { + c.lastSchedulers[key] = schedules + } + + // 定时器有更新,则重新设置该schedule + newScheduler := gocron.NewScheduler() + for _, schedule := range schedules { + if schedule == nil || !schedule.Enabled { + continue + } + log.Infof("deal collie pipeline schedule, name:%s, item:%v", pipeline.Metadata.Name, schedule) + if err := schedule.Validate(); err != nil { + log.Errorf("[%s] invalid schedule: %v", key, err) + continue + } + BuildScheduledJob(newScheduler, schedule).Do(c.RunColliePipelineScheduledTask, pipeline, log) + } + + c.Schedulers[key] = newScheduler + log.Infof("[%s] building schedulers..", key) + // 停掉旧的scheduler + if _, ok := c.SchedulerController[key]; ok { + c.SchedulerController[key] <- true + } + log.Infof("[%s] lens of scheduler: %d", key, c.Schedulers[key].Len()) + c.SchedulerController[key] = c.Schedulers[key].Start() + } +} + +// RunTestScheduledTask ... +func (c *CronClient) RunColliePipelineScheduledTask(pipeline *service.PipelineResource, log *zap.SugaredLogger) { + log.Infof("start collie pipeline cron job: %s ...", pipeline.Metadata.Name) + + args := &service.CreateBuildRequest{ + UserID: setting.CronTaskCreator, + UserName: setting.CronTaskCreator, + NoCache: false, + ResetVolume: false, + Variables: pipeline.Spec.Variables, + PipelineName: pipeline.Metadata.Name, + ProductName: pipeline.Metadata.Project, + } + + if err := c.CollieCli.RunColliePipelineTask(args, log); err != nil { + log.Errorf("[%s]RunTestScheduledTask err: %v", pipeline.Metadata.Name, err) + } +} diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_env.go b/pkg/microservice/cron/core/service/scheduler/schedule_env.go new file mode 100644 index 0000000000..335ff89117 --- /dev/null +++ b/pkg/microservice/cron/core/service/scheduler/schedule_env.go @@ -0,0 +1,465 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "crypto/tls" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "reflect" + "strconv" + "strings" + "time" + + "github.com/jasonlvhit/gocron" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/cron/core/service" + "github.com/koderover/zadig/pkg/setting" +) + +// UpsertEnvServiceScheduler ... +func (c *CronClient) UpsertEnvServiceScheduler(log *zap.SugaredLogger) { + envs, err := c.AslanCli.ListEnvs(log) + if err != nil { + log.Error(err) + return + } + //当前的环境数据和上次做比较,如果环境有删除或者环境中的服务有删除,要清理掉定时器 + c.compareProductRevision(envs, log) + + log.Info("start init env scheduler..") + taskMap := make(map[string]bool) + for _, env := range envs { + for _, serviceRevision := range env.ServiceRevisions { + if serviceRevision.Type != setting.PMDeployType { + continue + } + envObj, err := c.AslanCli.GetEnvService(env.ProductName, env.EnvName, log) + if err != nil { + log.Error("GetEnvService productName:%s envName:%s err:%v", env.ProductName, env.EnvName, err) + continue + } + envServiceNames := sets.String{} + for _, serviceNames := range envObj.Services { + envServiceNames.Insert(serviceNames...) + } + svc, _ := c.AslanCli.GetService(serviceRevision.ServiceName, env.ProductName, setting.PMDeployType, serviceRevision.CurrentRevision, log) + if svc == nil || len(svc.HealthChecks) == 0 || len(svc.EnvConfigs) == 0 || !envServiceNames.Has(serviceRevision.ServiceName) { + key := "service-" + serviceRevision.ServiceName + "-" + setting.PMDeployType + "-" + + env.EnvName + for scheduleKey := range c.Schedulers { + if strings.Contains(scheduleKey, key) { + c.Schedulers[scheduleKey].Clear() + delete(c.Schedulers, scheduleKey) + } + } + + for lastScheduleKey := range c.lastSchedulers { + if strings.Contains(lastScheduleKey, key) { + delete(c.lastSchedulers, lastScheduleKey) + } + } + continue + } + for _, envConfig := range svc.EnvConfigs { + if envConfig.EnvName != env.EnvName { + continue + } + for _, hostID := range envConfig.HostIDs { + for _, healthCheck := range svc.HealthChecks { + key := "service-" + serviceRevision.ServiceName + "-" + setting.PMDeployType + "-" + + env.EnvName + "-" + hostID + "-" + healthCheck.Protocol + "-" + strconv.Itoa(healthCheck.Port) + "-" + healthCheck.Path + taskMap[key] = true + if _, ok := c.lastServiceSchedulers[key]; ok && reflect.DeepEqual(serviceRevision, c.lastServiceSchedulers[key]) { + continue + } else { + c.lastServiceSchedulers[key] = serviceRevision + } + hostInfo, _ := c.AslanCli.GetHostInfo(hostID, log) + if hostInfo == nil { + continue + } + newScheduler := gocron.NewScheduler() + BuildScheduledEnvJob(newScheduler, healthCheck).Do(c.RunScheduledService, svc, healthCheck, hostInfo.IP, env.EnvName, hostID, log) + c.Schedulers[key] = newScheduler + log.Infof("[%s] service schedulers..", key) + c.Schedulers[key].Start() + } + } + break + } + } + } +} + +func (c *CronClient) RunScheduledService(svc *service.Service, healthCheck *service.PmHealthCheck, address, envName, hostID string, log *zap.SugaredLogger) { + log.Infof("[%s]start to Run ScheduledService...", svc.ServiceName) + var ( + message string + err error + envStatus = new(service.EnvStatus) + ) + + for i := 0; i < MaxProbeRetries; i++ { + if message, err = runProbe(healthCheck, address, log); err == nil { + log.Infof("runProbe message:[%s]", message) + break + } + } + + switch message { + case Success: + healthCheck.CurrentHealthyNum++ + healthCheck.CurrentUnhealthyNum = 0 + case Failure: + healthCheck.CurrentUnhealthyNum++ + healthCheck.CurrentHealthyNum = 0 + } + + envStatus.EnvName = envName + envStatus.Address = address + envStatus.HostID = hostID + envStatus.PmHealthCheck = healthCheck + if healthCheck.CurrentHealthyNum >= healthCheck.HealthyThreshold && healthCheck.CurrentHealthyNum > 0 { + healthCheck.CurrentHealthyNum = 0 + healthCheck.CurrentUnhealthyNum = 0 + envStatus.Status = setting.PodRunning + } + + if healthCheck.CurrentUnhealthyNum >= healthCheck.UnhealthyThreshold && healthCheck.CurrentUnhealthyNum > 0 { + healthCheck.CurrentHealthyNum = 0 + healthCheck.CurrentUnhealthyNum = 0 + envStatus.Status = setting.PodError + } + + if len(svc.EnvStatuses) == 0 { + svc.EnvStatuses = []*service.EnvStatus{envStatus} + } else { + envStatusKeys := sets.String{} + for _, tmpEnvStatus := range svc.EnvStatuses { + //envStatusKeys = append(envStatusKeys, key) + //if tmpEnvStatus.PmHealthCheck.Protocol == healthCheck.Protocol && tmpEnvStatus.Address == envStatus.Address && tmpEnvStatus.PmHealthCheck.Port == healthCheck.Port && + // tmpEnvStatus.PmHealthCheck.Path == healthCheck.Path && tmpEnvStatus.EnvName == envStatus.EnvName { + // tmpEnvStatus.Status = envStatus.Status + //} + if tmpEnvStatus.HostID != hostID { + continue + } + key := fmt.Sprintf("%s-%s-%d-%s-%s", healthCheck.Protocol, tmpEnvStatus.Address, healthCheck.Port, healthCheck.Path, tmpEnvStatus.EnvName) + envStatusKeys.Insert(key) + tmpEnvStatus.Status = envStatus.Status + } + currentEnvStatusKey := fmt.Sprintf("%s-%s-%d-%s-%s", healthCheck.Protocol, envStatus.Address, healthCheck.Port, healthCheck.Path, envStatus.EnvName) + if !envStatusKeys.Has(currentEnvStatusKey) { + svc.EnvStatuses = append(svc.EnvStatuses, envStatus) + } + } + + if err = c.AslanCli.UpdateService(&service.ServiceTmplObject{ + ProductName: svc.ProductName, + ServiceName: svc.ServiceName, + Revision: svc.Revision, + Type: setting.PMDeployType, + EnvStatuses: svc.EnvStatuses, + Username: "system", + }, log); err != nil { + log.Errorf("UpdateService err:%v", err) + } else { + log.Infof("ready to UpdateService serviceName:%s,revision:%d,envName:%s,address:%s,status:%s", svc.ServiceName, svc.Revision, envStatus.EnvName, envStatus.Address, envStatus.Status) + } +} + +// BuildScheduledEnvJob ... +func BuildScheduledEnvJob(scheduler *gocron.Scheduler, healthCheck *service.PmHealthCheck) *gocron.Job { + interval := healthCheck.Interval + if interval < 2 { + interval = 2 + } + return scheduler.Every(interval).Seconds() +} +func runProbe(healthCheck *service.PmHealthCheck, address string, log *zap.SugaredLogger) (string, error) { + var ( + message string + err error + ) + timeout := time.Duration(healthCheck.TimeOut) * time.Second + switch healthCheck.Protocol { + case setting.ProtocolHTTP, setting.ProtocolHTTPS: + if message, err = doHTTPProbe(healthCheck.Protocol, address, healthCheck.Path, healthCheck.Port, timeout, log); err != nil { + log.Errorf("doHttpProbe err:%v", err) + return Failure, err + } + case setting.ProtocolTCP: + if message, err = doTCPProbe(address, healthCheck.Port, timeout, log); err != nil { + log.Errorf("doTCPProbe err:%v", err) + return Failure, err + } + } + + return message, nil +} +func doTCPProbe(addr string, port int, timeout time.Duration, log *zap.SugaredLogger) (string, error) { + var ( + conn net.Conn + err error + ) + if port == 0 { + conn, err = net.DialTimeout("tcp", addr, timeout) + } else { + conn, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%d", addr, port), timeout) + } + + if err != nil { + return Failure, err + } + err = conn.Close() + if err != nil { + log.Errorf("Unexpected error closing TCP socket: %v (%#v)", err, err) + return Failure, err + } + return Success, nil +} + +func doHTTPProbe(protocol, address, path string, port int, timeout time.Duration, log *zap.SugaredLogger) (string, error) { + tlsConfig := &tls.Config{InsecureSkipVerify: true} + transport := &http.Transport{ + TLSClientConfig: tlsConfig, + DisableKeepAlives: true, + Proxy: http.ProxyURL(nil), + } + client := &http.Client{ + Timeout: timeout, + Transport: transport, + CheckRedirect: redirectChecker(false), + } + url, err := formatURL(protocol, address, path, port) + if err != nil { + return Failure, err + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return Failure, err + } + res, err := client.Do(req) + if err != nil { + return Failure, err + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return Failure, err + } + + if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest { + log.Infof("Probe succeeded for %s, Response: %v", url, *res) + return Success, nil + } + log.Infof("Probe failed for %s, response body: %v", url, string(body)) + return Failure, fmt.Errorf("HTTP probe failed with statuscode: %d", res.StatusCode) +} + +func redirectChecker(followNonLocalRedirects bool) func(*http.Request, []*http.Request) error { + if followNonLocalRedirects { + return nil // Use the default http client checker. + } + + return func(req *http.Request, via []*http.Request) error { + if req.URL.Hostname() != via[0].URL.Hostname() { + return http.ErrUseLastResponse + } + // Default behavior: stop after 10 redirects. + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + return nil + } +} + +func formatURL(protocol, address, path string, port int) (string, error) { + if len(strings.Split(address, ":")) > 2 { + return "", fmt.Errorf("illegal address") + } + if path == "" && port == 0 { + return fmt.Sprintf("%s://%s", protocol, address), nil + } + + path = strings.TrimPrefix(path, "/") + + if port == 0 { + return fmt.Sprintf("%s://%s/%s", protocol, address, path), nil + } + return fmt.Sprintf("%s://%s:%d/%s", protocol, address, port, path), nil +} + +// 先比较环境,再比较服务 +func (c *CronClient) compareProductRevision(currentProductRevisions []*service.ProductRevision, log *zap.SugaredLogger) { + if len(c.lastProductRevisions) == 0 { + c.lastProductRevisions = currentProductRevisions + return + } + deleteProductRevisions := make([]*service.ProductRevision, 0) + lastProductSvcRevisionMap := make(map[string][]*service.SvcRevision) + currentProductSvcRevisionMap := make(map[string][]*service.SvcRevision) + for _, lastProductRevision := range c.lastProductRevisions { + isContain := false + for _, currentProductRevision := range currentProductRevisions { + if currentProductRevision.ProductName == lastProductRevision.ProductName && + currentProductRevision.EnvName == lastProductRevision.EnvName { + + currentProductSvcRevisionMap[currentProductRevision.ProductName+"-"+currentProductRevision.EnvName] = currentProductRevision.ServiceRevisions + lastProductSvcRevisionMap[lastProductRevision.ProductName+"-"+lastProductRevision.EnvName] = lastProductRevision.ServiceRevisions + isContain = true + break + } + } + + if !isContain { + deleteProductRevisions = append(deleteProductRevisions, lastProductRevision) + } + } + //已经删除的环境,如果包含非k8s服务,则清除定时器 + for _, env := range deleteProductRevisions { + for _, serviceRevision := range env.ServiceRevisions { + if serviceRevision.Type != setting.PMDeployType { + continue + } + key := "service-" + serviceRevision.ServiceName + "-" + setting.PMDeployType + "-" + + env.EnvName + for scheduleKey := range c.Schedulers { + if strings.Contains(scheduleKey, key) { + c.Schedulers[scheduleKey].Clear() + delete(c.Schedulers, scheduleKey) + } + } + + for lastScheduleKey := range c.lastSchedulers { + if strings.Contains(lastScheduleKey, key) { + delete(c.lastSchedulers, lastScheduleKey) + } + } + + for lastServiceSchedulerKey := range c.lastServiceSchedulers { + if strings.Contains(lastServiceSchedulerKey, key) { + delete(c.lastServiceSchedulers, lastServiceSchedulerKey) + } + } + } + } + // 已经删除的服务,如果是非k8s,则清除定时器 + deleteSvcRevisionsMap := make(map[string][]*service.SvcRevision) + lastServiceMap := make(map[string]*service.SvcRevision) + currentServicesMap := make(map[string]*service.SvcRevision) + for key, lastSvcRevisions := range lastProductSvcRevisionMap { + currentSvcRevisions := currentProductSvcRevisionMap[key] + for _, lastSvcRevision := range lastSvcRevisions { + isContain := false + for _, currentSvcRevision := range currentSvcRevisions { + if lastSvcRevision.ServiceName == currentSvcRevision.ServiceName { + lastServiceMap[key+"-"+lastSvcRevision.ServiceName] = lastSvcRevision + currentServicesMap[key+"-"+currentSvcRevision.ServiceName] = currentSvcRevision + isContain = true + break + } + } + if !isContain { + deleteSvcRevisionsMap[key] = append(deleteSvcRevisionsMap[key], lastSvcRevision) + } + } + } + + for key, deleteSvcRevisions := range deleteSvcRevisionsMap { + envName := strings.Split(key, "-")[1] + for _, deleteSvcRevision := range deleteSvcRevisions { + if deleteSvcRevision.Type != setting.PMDeployType { + continue + } + key := "service-" + deleteSvcRevision.ServiceName + "-" + setting.PMDeployType + "-" + + envName + for scheduleKey := range c.Schedulers { + if strings.Contains(scheduleKey, key) { + c.Schedulers[scheduleKey].Clear() + delete(c.Schedulers, scheduleKey) + } + } + + for lastScheduleKey := range c.lastSchedulers { + if strings.Contains(lastScheduleKey, key) { + delete(c.lastSchedulers, lastScheduleKey) + } + } + + for lastServiceSchedulerKey := range c.lastServiceSchedulers { + if strings.Contains(lastServiceSchedulerKey, key) { + delete(c.lastServiceSchedulers, lastServiceSchedulerKey) + } + } + } + } + + //判断相同的服务,revision是否相同,如果revision相同在判断env_configs和health_checks是否相同 + //找出老的revision的service + oldRevisionServices := make(map[string]*service.SvcRevision) + for lastKey, lastServiceRevison := range lastServiceMap { + currentRevison := currentServicesMap[lastKey] + if lastServiceRevison.CurrentRevision < currentRevison.CurrentRevision { + oldRevisionServices[lastKey] = lastServiceRevison + } + } + //清理掉老版本的探活定时器 + for key, oldRevisionService := range oldRevisionServices { + if oldRevisionService.Type != setting.PMDeployType { + continue + } + envName := strings.Split(key, "-")[1] + key := "service-" + oldRevisionService.ServiceName + "-" + setting.PMDeployType + "-" + + envName + for scheduleKey := range c.Schedulers { + if strings.Contains(scheduleKey, key) { + c.Schedulers[scheduleKey].Clear() + delete(c.Schedulers, scheduleKey) + } + } + + for lastScheduleKey := range c.lastSchedulers { + if strings.Contains(lastScheduleKey, key) { + delete(c.lastSchedulers, lastScheduleKey) + } + } + + for lastServiceSchedulerKey := range c.lastServiceSchedulers { + if strings.Contains(lastServiceSchedulerKey, key) { + delete(c.lastServiceSchedulers, lastServiceSchedulerKey) + } + } + } + + c.lastProductRevisions = currentProductRevisions +} + +const ( + // Success Result + Success string = "success" + // Failure Result + Failure string = "failure" + // maxProbeRetries + MaxProbeRetries = 3 +) diff --git a/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go b/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go index 43b38e00c6..a3444de724 100644 --- a/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go +++ b/pkg/microservice/cron/core/service/scheduler/schedule_workflow.go @@ -123,7 +123,7 @@ func (c *CronClient) UpsertWorkflowScheduler(log *zap.SugaredLogger) { ScheduleNames := sets.NewString( CleanJobScheduler, UpsertWorkflowScheduler, UpsertTestScheduler, InitStatScheduler, InitOperationStatScheduler, - CleanProductScheduler, InitHealthCheckScheduler) + CleanProductScheduler, InitHealthCheckScheduler, UpsertColliePipelineScheduler) // 停掉已被删除的pipeline对应的scheduler for name := range c.Schedulers { diff --git a/pkg/microservice/cron/core/service/scheduler/scheduler.go b/pkg/microservice/cron/core/service/scheduler/scheduler.go index 783d71fdf4..b0c0ef9c57 100644 --- a/pkg/microservice/cron/core/service/scheduler/scheduler.go +++ b/pkg/microservice/cron/core/service/scheduler/scheduler.go @@ -20,6 +20,7 @@ import ( "fmt" stdlog "log" "os" + "strings" "time" "github.com/jasonlvhit/gocron" @@ -31,6 +32,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/cron/core/service" "github.com/koderover/zadig/pkg/microservice/cron/core/service/client" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" "github.com/koderover/zadig/pkg/tool/log" ) @@ -54,6 +56,8 @@ const ( UpsertWorkflowScheduler = "UpsertWorkflowScheduler" // UpsertTestScheduler ... UpsertTestScheduler = "UpsertTestScheduler" + // UpsertColliePipelineScheduler ... + UpsertColliePipelineScheduler = "UpsertColliePipelineScheduler" //CleanProductScheduler ... CleanProductScheduler = "CleanProductScheduler" //InitBuildStatScheduler @@ -68,6 +72,9 @@ const ( SystemCapacityGC = "SystemCapacityGC" //InitHealthCheckScheduler InitHealthCheckScheduler = "InitHealthCheckScheduler" + + // FreestyleType 自由编排工作流 + freestyleType = "freestyle" ) // NewCronClient ... @@ -134,6 +141,13 @@ func (c *CronClient) Init() { c.InitJobScheduler() // 测试管理的定时任务触发 c.InitTestScheduler() + + // 自由编排工作流定时任务触发 + features, _ := getFeatures() + if strings.Contains(features, freestyleType) { + c.InitColliePipelineScheduler() + } + // 定时清理环境 c.InitCleanProductScheduler() // 定时初始化构建数据 @@ -142,6 +156,18 @@ func (c *CronClient) Init() { c.InitOperationStatScheduler() // 定时更新质效看板的统计数据 c.InitPullSonarStatScheduler() + // 定时初始化健康检查 + c.InitHealthCheckScheduler() +} + +func getFeatures() (string, error) { + cl := poetry.New(config.AslanAPI(), config.RootToken()) + fs, err := cl.ListFeatures() + if err != nil { + return "", err + } + + return strings.Join(fs, ","), nil } // InitCleanJobScheduler ... @@ -184,6 +210,16 @@ func (c *CronClient) InitTestScheduler() { c.Schedulers[UpsertTestScheduler].Start() } +// InitJobScheduler ... +func (c *CronClient) InitColliePipelineScheduler() { + + c.Schedulers[UpsertColliePipelineScheduler] = gocron.NewScheduler() + + c.Schedulers[UpsertColliePipelineScheduler].Every(1).Minutes().Do(c.UpsertColliePipelineScheduler, c.log) + + c.Schedulers[UpsertColliePipelineScheduler].Start() +} + // InitBuildStatScheduler ... func (c *CronClient) InitBuildStatScheduler() { @@ -222,3 +258,12 @@ func (c *CronClient) InitSystemCapacityGCScheduler() { c.Schedulers[SystemCapacityGC].Start() } + +func (c *CronClient) InitHealthCheckScheduler() { + + c.Schedulers[InitHealthCheckScheduler] = gocron.NewScheduler() + + c.Schedulers[InitHealthCheckScheduler].Every(1).Minutes().Do(c.UpsertEnvServiceScheduler, c.log) + + c.Schedulers[InitHealthCheckScheduler].Start() +} diff --git a/pkg/microservice/hubagent/config/config.go b/pkg/microservice/hubagent/config/config.go new file mode 100644 index 0000000000..f46e9ccc7a --- /dev/null +++ b/pkg/microservice/hubagent/config/config.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "github.com/spf13/viper" + + // init the config first + _ "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" +) + +func HubAgentToken() string { + return viper.GetString(setting.HubAgentToken) +} + +func HubServerBaseAddr() string { + return viper.GetString(setting.HubServerBaseAddr) +} + +func KubernetesServiceHost() string { + return viper.GetString(setting.KubernetesServiceHost) +} + +func KubernetesServicePort() string { + return viper.GetString(setting.KubernetesServicePort) +} diff --git a/pkg/microservice/hubagent/core/service/client.go b/pkg/microservice/hubagent/core/service/client.go new file mode 100644 index 0000000000..e1b53b71a2 --- /dev/null +++ b/pkg/microservice/hubagent/core/service/client.go @@ -0,0 +1,200 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + "github.com/cenkalti/backoff/v3" + "github.com/gorilla/websocket" + "github.com/pkg/errors" + "go.uber.org/zap" + + commonconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/hubagent/config" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/tool/remotedialer" +) + +const ( + defaultTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + defaultCaPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +) + +type clientConfig struct { + Token string + Server string + TokenPath string + CaPath string + ServiceHost string + ServicePort string +} + +type Client struct { + clientConfig + logger *zap.SugaredLogger +} + +func newClient(config clientConfig) *Client { + return &Client{ + clientConfig: config, + logger: log.SugaredLogger(), + } +} + +type input struct { + Cluster *ClusterInfo `json:"cluster"` +} + +type ClusterInfo struct { + ClusterID string `json:"_"` + Joined time.Time `json:"_"` + + Address string `json:"address"` + Token string `json:"token"` + CACert string `json:"caCert"` +} + +func (c *Client) getParams() (*input, error) { + caData, err := ioutil.ReadFile(c.CaPath) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", c.CaPath) + } + + token, err := ioutil.ReadFile(c.TokenPath) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", c.TokenPath) + } + + return &input{ + Cluster: &ClusterInfo{ + Address: fmt.Sprintf("https://%s:%s", c.ServiceHost, c.ServicePort), + Token: strings.TrimSpace(string(token)), + CACert: base64.StdEncoding.EncodeToString(caData), + }, + }, nil +} + +func Init() error { + log.Init(&log.Config{ + Level: commonconfig.LogLevel(), + Filename: commonconfig.LogFile(), + SendToFile: commonconfig.SendLogToFile(), + Development: commonconfig.Mode() != setting.ReleaseMode, + }) + + token := config.HubAgentToken() + if token == "" { + return fmt.Errorf("token must be configured") + } + + server := config.HubServerBaseAddr() + if server == "" { + return fmt.Errorf("server must be configured") + } + + serviceHost := config.KubernetesServiceHost() + if serviceHost == "" { + return fmt.Errorf("kube service host must be configured") + } + + servicePort := config.KubernetesServicePort() + if servicePort == "" { + return fmt.Errorf("kube service port must be configured") + } + + app := newClient( + clientConfig{ + Server: server, + Token: token, + TokenPath: defaultTokenPath, + CaPath: defaultCaPath, + ServiceHost: serviceHost, + ServicePort: servicePort, + }, + ) + + return app.Start() +} + +func (c *Client) Start() error { + params, err := c.getParams() + if err != nil { + return err + } + + bytes, err := json.Marshal(params) + if err != nil { + return err + } + + headers := map[string][]string{ + setting.Token: {c.Token}, + setting.Params: {base64.StdEncoding.EncodeToString(bytes)}, + } + + connectURL := fmt.Sprintf("%s/connect", c.Server) + c.logger.Infof("connect to %s with token %s", connectURL, c.Token) + + bo := backoff.NewExponentialBackOff() + + bo.InitialInterval = 1 * time.Second + // never stops + bo.MaxElapsedTime = 0 + retries := 0 + timeout := 10 * time.Second + + return backoff.Retry(func() error { + if retries > 0 { + c.logger.Infof("retrying to connect to %s", connectURL) + } + + tm := time.Now() + + remotedialer.ClientConnect( + connectURL, + headers, + &websocket.Dialer{ + HandshakeTimeout: timeout, + Proxy: http.ProxyFromEnvironment, + }, + func(proto, address string) bool { + switch proto { + case "tcp": + return true + case "unix": + return address == "/var/run/docker.sock" + } + return false + }, nil) + + // 如果连接时间超过2倍的超时时间, 则认为已经成功连接,需要立即重连 + if time.Since(tm) > 2*timeout { + bo.Reset() + } + + retries++ + return errors.New("retry") + }, bo) +} diff --git a/pkg/microservice/hubagent/server/server.go b/pkg/microservice/hubagent/server/server.go new file mode 100644 index 0000000000..e76babad5d --- /dev/null +++ b/pkg/microservice/hubagent/server/server.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "github.com/koderover/zadig/pkg/microservice/hubagent/core/service" +) + +func Serve() error { + if err := service.Init(); err != nil { + return err + } + + return nil +} diff --git a/pkg/microservice/hubserver/config/config.go b/pkg/microservice/hubserver/config/config.go new file mode 100644 index 0000000000..23c380fc95 --- /dev/null +++ b/pkg/microservice/hubserver/config/config.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "github.com/spf13/viper" + + // init the config first + _ "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" +) + +func MongoDBAddr() string { + return viper.GetString(setting.ENVMongoDBConnectionString) +} + +func AslanDBName() string { + return viper.GetString(setting.ENVAslanDBName) +} diff --git a/pkg/microservice/hubserver/config/consts.go b/pkg/microservice/hubserver/config/consts.go new file mode 100644 index 0000000000..2963fde2d8 --- /dev/null +++ b/pkg/microservice/hubserver/config/consts.go @@ -0,0 +1,26 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +type K8SClusterStatus string + +const ( + Disconnected K8SClusterStatus = "disconnected" + Pending K8SClusterStatus = "pending" + Normal K8SClusterStatus = "normal" + Abnormal K8SClusterStatus = "abnormal" +) diff --git a/pkg/microservice/hubserver/core/handler/handler.go b/pkg/microservice/hubserver/core/handler/handler.go new file mode 100644 index 0000000000..672326bb8c --- /dev/null +++ b/pkg/microservice/hubserver/core/handler/handler.go @@ -0,0 +1,40 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "net/http" + + "github.com/koderover/zadig/pkg/microservice/hubserver/core/service" + "github.com/koderover/zadig/pkg/tool/remotedialer" +) + +func Disconnect(server *remotedialer.Server, w http.ResponseWriter, r *http.Request) { + service.Disconnect(server, w, r) +} + +func Restore(w http.ResponseWriter, r *http.Request) { + service.Restore(w, r) +} + +func Forward(server *remotedialer.Server, w http.ResponseWriter, r *http.Request) { + service.Forward(server, w, r) +} + +func HasSession(handler *remotedialer.Server, w http.ResponseWriter, r *http.Request) { + service.HasSession(handler, w, r) +} diff --git a/pkg/microservice/hubserver/core/repository/models/models.go b/pkg/microservice/hubserver/core/repository/models/models.go new file mode 100644 index 0000000000..4bfefe013d --- /dev/null +++ b/pkg/microservice/hubserver/core/repository/models/models.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/koderover/zadig/pkg/microservice/hubserver/config" +) + +type K8SClusterInfo struct { + Nodes int `json:"nodes" bson:"nodes"` + Version string `json:"version" bson:"version"` + CPU string `json:"cpu" bson:"cpu"` + Memory string `json:"memory" bson:"memory"` +} + +type K8SCluster struct { + ID primitive.ObjectID `json:"id" bson:"_id"` + Name string `json:"name" bson:"name"` + Tags []string `json:"tags" bson:"tags"` + Description string `json:"description" bson:"description"` + Namespace string `json:"namespace" bson:"namespace"` + Info *K8SClusterInfo `json:"info,omitempty" bson:"info,omitempty"` + Status config.K8SClusterStatus `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Yaml string `json:"yaml" bson:"yaml"` + Production bool `json:"production" bson:"production"` + CreatedAt int64 `json:"createdAt" bson:"createdAt"` + CreatedBy string `json:"createdBy" bson:"createdBy"` + Disconnected bool `json:"-" bson:"disconnected"` + Token string `json:"token" bson:"-"` +} + +func (K8SCluster) TableName() string { + return "k8s_cluster" +} diff --git a/pkg/microservice/hubserver/core/repository/mongodb/k8scluster.go b/pkg/microservice/hubserver/core/repository/mongodb/k8scluster.go new file mode 100644 index 0000000000..9993a2d45f --- /dev/null +++ b/pkg/microservice/hubserver/core/repository/mongodb/k8scluster.go @@ -0,0 +1,110 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + + "github.com/koderover/zadig/pkg/microservice/hubserver/config" + "github.com/koderover/zadig/pkg/microservice/hubserver/core/repository/models" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +type K8sClusterColl struct { + *mongo.Collection + + coll string +} + +func NewK8sClusterColl() *K8sClusterColl { + name := models.K8SCluster{}.TableName() + coll := &K8sClusterColl{Collection: mongotool.Database(config.AslanDBName()).Collection(name), coll: name} + + return coll +} + +func (c *K8sClusterColl) GetCollectionName() string { + return c.coll +} + +func (c *K8sClusterColl) FindConnectedClusters() ([]*models.K8SCluster, error) { + query := bson.M{"disconnected": false} + + ctx := context.Background() + resp := make([]*models.K8SCluster, 0) + + cursor, err := c.Collection.Find(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resp) + if err != nil { + return nil, err + } + + return resp, err +} + +func (c *K8sClusterColl) UpdateStatus(cluster *models.K8SCluster) error { + query := bson.M{"_id": cluster.ID} + + update := bson.M{"$set": bson.M{ + "status": cluster.Status, + }} + + _, err := c.UpdateOne(context.TODO(), query, update) + + return err +} + +func (c *K8sClusterColl) UpdateConnectState(id string, disconnected bool) error { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + + query := bson.M{"_id": oid} + change := bson.M{"disconnected": disconnected} + + if disconnected { + change["status"] = config.Disconnected + } else { + change["status"] = config.Pending + } + update := bson.M{"$set": change} + _, err = c.UpdateOne(context.TODO(), query, update) + + return err +} + +func (c *K8sClusterColl) Get(id string) (*models.K8SCluster, error) { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + + query := bson.M{"_id": oid} + res := &models.K8SCluster{} + + err = c.FindOne(context.TODO(), query).Decode(res) + return res, err +} diff --git a/pkg/microservice/hubserver/core/service/init.go b/pkg/microservice/hubserver/core/service/init.go new file mode 100644 index 0000000000..e11ffc67a4 --- /dev/null +++ b/pkg/microservice/hubserver/core/service/init.go @@ -0,0 +1,40 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + "time" + + "github.com/koderover/zadig/pkg/microservice/hubserver/config" + mongotool "github.com/koderover/zadig/pkg/tool/mongo" +) + +func Init() { + initDatabase() +} + +func initDatabase() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mongotool.Init(ctx, config.MongoDBAddr()) + if err := mongotool.Ping(ctx); err != nil { + panic(fmt.Errorf("failed to connect to mongo, error: %s", err)) + } +} diff --git a/pkg/microservice/hubserver/core/service/service.go b/pkg/microservice/hubserver/core/service/service.go new file mode 100644 index 0000000000..87aba25f07 --- /dev/null +++ b/pkg/microservice/hubserver/core/service/service.go @@ -0,0 +1,309 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "sync" + "time" + + "github.com/gorilla/mux" + "k8s.io/apimachinery/pkg/util/proxy" + + "github.com/koderover/zadig/pkg/microservice/hubserver/config" + "github.com/koderover/zadig/pkg/microservice/hubserver/core/repository/models" + "github.com/koderover/zadig/pkg/microservice/hubserver/core/repository/mongodb" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/crypto" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/tool/remotedialer" +) + +var clusters sync.Map + +func Authorize(req *http.Request) (clientKey string, authed bool, err error) { + log := log.SugaredLogger() + token := req.Header.Get(setting.Token) + + clusterID, err := crypto.AesDecrypt(token) + if err != nil { + err = fmt.Errorf("token is illegal %s: %v", token, err) + return + } + + var cluster *models.K8SCluster + if cluster, err = mongodb.NewK8sClusterColl().Get(clusterID); err != nil { + err = fmt.Errorf("unknown cluster, cluster id:%s, err:%v", token, err) + return + } + + if cluster.Disconnected { + err = fmt.Errorf("cluster %s is marked as disconnected", cluster.Name) + log.Info(err.Error()) + return + } + + params := req.Header.Get(setting.Params) + var input input + bytes, err := base64.StdEncoding.DecodeString(params) + if err != nil { + return + } + + if err = json.Unmarshal(bytes, &input); err != nil { + return + } + + if input.Cluster == nil { + err = fmt.Errorf("no cluster info found") + return + } + + input.Cluster.ClusterID = cluster.ID.Hex() + input.Cluster.Joined = time.Now() + + clusters.Store(cluster.ID.Hex(), input.Cluster) + + if cluster.Status != "normal" { + cluster.Status = "normal" + err = mongodb.NewK8sClusterColl().UpdateStatus(cluster) + if err != nil { + log.Errorf("failed to update clusters status %s %v", cluster.Name, err) + return + } + } + + log.Infof("cluster %s connected", cluster.Name) + return cluster.ID.Hex(), true, nil +} + +func Disconnect(server *remotedialer.Server, w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + clientKey := vars["id"] + var err error + + defer func() { + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + } + }() + + if err = mongodb.NewK8sClusterColl().UpdateConnectState(clientKey, true); err != nil { + return + } + + server.Disconnect(clientKey) + w.WriteHeader(http.StatusOK) +} + +func Restore(w http.ResponseWriter, r *http.Request) { + log := log.SugaredLogger() + vars := mux.Vars(r) + clientKey := vars["id"] + var err error + + defer func() { + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + } + }() + + if err = mongodb.NewK8sClusterColl().UpdateConnectState(clientKey, false); err != nil { + return + } + + log.Infof("cluster %s is restored", clientKey) + w.WriteHeader(http.StatusOK) +} + +var ( + er = &errorResponder{} +) + +type errorResponder struct { +} + +func (e *errorResponder) Error(w http.ResponseWriter, req *http.Request, err error) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) +} + +func Forward(server *remotedialer.Server, w http.ResponseWriter, r *http.Request) { + logger := log.SugaredLogger() + + vars := mux.Vars(r) + clientKey := vars["id"] + path := vars["path"] + + logger.Debugf("got forward request %s %s", clientKey, path) + + var ( + err error + errHandled bool + ) + + defer func() { + if err != nil && !errHandled { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("%#v", err))) + } + }() + + _, err = mongodb.NewK8sClusterColl().Get(clientKey) + if err != nil { + return + } + + clusterInfo, exists := clusters.Load(clientKey) + if !server.HasSession(clientKey) || !exists { + errHandled = true + logger.Infof("waiting for cluster %s to connect", clientKey) + w.WriteHeader(http.StatusNotFound) + return + } + + cluster, ok := clusterInfo.(*ClusterInfo) + if !ok { + errHandled = true + logger.Infof("waiting for cluster %s to connect", clientKey) + w.WriteHeader(http.StatusNotFound) + return + } + + var endpoint *url.URL + + endpoint, err = url.Parse(cluster.Address) + if err != nil { + return + } + + endpoint.Path = path + endpoint.RawQuery = r.URL.RawQuery + r.URL.Host = r.Host + r.Header.Set("authorization", "Bearer "+cluster.Token) + + var certBytes []byte + certBytes, err = base64.StdEncoding.DecodeString(cluster.CACert) + if err != nil { + return + } + + certs := x509.NewCertPool() + certs.AppendCertsFromPEM(certBytes) + + transport := &http.Transport{} + + //noinspection GoDeprecation + transport.Dial = server.Dialer(clientKey, 15*time.Second) + transport.TLSClientConfig = &tls.Config{ + RootCAs: certs, + } + + httpProxy := proxy.NewUpgradeAwareHandler(endpoint, transport, true, false, er) + httpProxy.ServeHTTP(w, r) +} + +func Reset() { + log := log.SugaredLogger() + + clusters, err := mongodb.NewK8sClusterColl().FindConnectedClusters() + if err != nil { + log.Errorf("failed to list clusters %v", clusters) + return + } + + for _, cluster := range clusters { + if cluster.Status == config.Normal { + cluster.Status = config.Abnormal + err := mongodb.NewK8sClusterColl().UpdateStatus(cluster) + if err != nil { + log.Errorf("failed to update clusters status %s %v", cluster.Name, err) + } + } + } +} + +func Sync(server *remotedialer.Server, stopCh <-chan struct{}) { + ticker := time.NewTicker(10 * time.Second) + log := log.SugaredLogger() + + for { + select { + case <-ticker.C: + func() { + clusterInfos, err := mongodb.NewK8sClusterColl().FindConnectedClusters() + if err != nil { + log.Errorf("failed to list clusters %v", clusters) + return + } + + for _, cluster := range clusterInfos { + statusChanged := false + if _, ok := clusters.Load(cluster.ID.Hex()); ok && server.HasSession(cluster.ID.Hex()) { + if cluster.Status != config.Normal { + log.Infof( + "cluster %s connected changed %s => %s", + cluster.Name, cluster.Status, config.Normal, + ) + cluster.Status = config.Normal + statusChanged = true + } + } else { + if cluster.Status == config.Normal { + log.Infof( + "cluster %s disconnected changed %s => %s", + cluster.Name, cluster.Status, config.Abnormal, + ) + cluster.Status = config.Abnormal + statusChanged = true + } + } + if statusChanged { + err := mongodb.NewK8sClusterColl().UpdateStatus(cluster) + if err != nil { + log.Errorf("failed to update clusters status %s %v", cluster.Name, err) + } + } + } + }() + case <-stopCh: + return + } + } +} + +func HasSession(handler *remotedialer.Server, w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + clientKey := vars["id"] + + if handler.HasSession(clientKey) { + if _, ok := clusters.Load(clientKey); ok { + w.WriteHeader(http.StatusOK) + return + } + } + + w.WriteHeader(http.StatusNotFound) +} diff --git a/pkg/microservice/hubserver/core/service/types.go b/pkg/microservice/hubserver/core/service/types.go new file mode 100644 index 0000000000..f12a970173 --- /dev/null +++ b/pkg/microservice/hubserver/core/service/types.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import "time" + +type input struct { + Cluster *ClusterInfo `json:"cluster"` +} + +type ClusterInfo struct { + ClusterID string `json:"_"` + Joined time.Time `json:"_"` + + Address string `json:"address"` + Token string `json:"token"` + CACert string `json:"caCert"` +} diff --git a/pkg/microservice/hubserver/server/rest/server.go b/pkg/microservice/hubserver/server/rest/server.go new file mode 100644 index 0000000000..aa63abb3a4 --- /dev/null +++ b/pkg/microservice/hubserver/server/rest/server.go @@ -0,0 +1,64 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "net/http" + + "github.com/gorilla/mux" + + h "github.com/koderover/zadig/pkg/microservice/hubserver/core/handler" + "github.com/koderover/zadig/pkg/tool/remotedialer" +) + +type engine struct { + *mux.Router +} + +func NewEngine(handler *remotedialer.Server) *engine { + s := &engine{} + s.Router = mux.NewRouter() + s.Router.UseEncodedPath() + + s.injectRouters(handler) + + return s +} + +func (s *engine) injectRouters(handler *remotedialer.Server) { + r := s.Router + + r.Handle("/connect", handler) + + r.HandleFunc("/disconnect/{id}", func(rw http.ResponseWriter, req *http.Request) { + h.Disconnect(handler, rw, req) + }) + + r.HandleFunc("/restore/{id}", func(rw http.ResponseWriter, req *http.Request) { + h.Restore(rw, req) + }) + + r.HandleFunc("/hasSession/{id}", func(rw http.ResponseWriter, req *http.Request) { + h.HasSession(handler, rw, req) + }) + + r.HandleFunc("/kube/{id}{path:.*}", func(rw http.ResponseWriter, req *http.Request) { + h.Forward(handler, rw, req) + }) + + s.Router = r +} diff --git a/pkg/microservice/hubserver/server/server.go b/pkg/microservice/hubserver/server/server.go new file mode 100644 index 0000000000..21492e4b60 --- /dev/null +++ b/pkg/microservice/hubserver/server/server.go @@ -0,0 +1,79 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "net/http" + "sync" + "time" + + commonconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/hubserver/core/service" + "github.com/koderover/zadig/pkg/microservice/hubserver/server/rest" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/tool/remotedialer" +) + +func Serve(ctx context.Context) error { + log.Init(&log.Config{ + Level: commonconfig.LogLevel(), + Filename: commonconfig.LogFile(), + SendToFile: commonconfig.SendLogToFile(), + Development: commonconfig.Mode() != setting.ReleaseMode, + }) + + log.Info("hub server start...") + service.Init() + + handler := remotedialer.New(service.Authorize, remotedialer.DefaultErrorWriter) + engine := rest.NewEngine(handler) + server := &http.Server{Addr: ":26000", Handler: engine} + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + + <-ctx.Done() + + ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { + log.Errorf("Failed to stop server, error: %s\n", err) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + service.Sync(handler, ctx.Done()) + service.Reset() + }() + + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Errorf("Failed to start http server, error: %s\n", err) + return err + } + + wg.Wait() + + return nil +} diff --git a/pkg/microservice/podexec/core/service/auth.go b/pkg/microservice/podexec/core/service/auth.go index f5ee08df0c..18762eae4a 100644 --- a/pkg/microservice/podexec/core/service/auth.go +++ b/pkg/microservice/podexec/core/service/auth.go @@ -26,9 +26,9 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/podexec/config" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/types/permission" ) diff --git a/pkg/microservice/reaper/core/service/meta/types.go b/pkg/microservice/reaper/core/service/meta/types.go index 79f0ed17d3..9ae93c5616 100644 --- a/pkg/microservice/reaper/core/service/meta/types.go +++ b/pkg/microservice/reaper/core/service/meta/types.go @@ -281,14 +281,16 @@ func (r *Repo) Ref() string { // Archive ... type Archive struct { - Dir string `yaml:"dir"` - File string `yaml:"file"` + Dir string `yaml:"dir"` + File string `yaml:"file"` + TestReportFile string `yaml:"test_report_file"` } // GinkgoTest ... type GinkgoTest struct { - ResultPath string `yaml:"result_path"` - ArtifactPaths []string `yaml:"artifact_paths"` + ResultPath string `yaml:"result_path"` + TestReportPath string `yaml:"test_report_path"` + ArtifactPaths []string `yaml:"artifact_paths"` } // DockerRegistry 推送镜像到 docker registry 配置 diff --git a/pkg/microservice/reaper/core/service/reaper/archive.go b/pkg/microservice/reaper/core/service/reaper/archive.go index 1f8ac569c0..2dc084c0fe 100644 --- a/pkg/microservice/reaper/core/service/reaper/archive.go +++ b/pkg/microservice/reaper/core/service/reaper/archive.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/koderover/zadig/pkg/microservice/reaper/internal/s3" + "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" ) @@ -62,43 +63,80 @@ func (r *Reaper) archiveS3Files() (err error) { return nil } -func (r *Reaper) archiveTestFiles() (err error) { - if r.Ctx.Archive != nil && r.Ctx.StorageURI != "" { - var store *s3.S3 +func (r *Reaper) archiveTestFiles() error { + if r.Ctx.Archive == nil || r.Ctx.StorageURI == "" { + return nil + } - if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI); err != nil { - log.Errorf("failed to create s3 storage %s", r.Ctx.StorageURI) - return - } - fileType := "test" - if strings.Contains(r.Ctx.Archive.File, ".tar.gz") { - fileType = "file" - } - if store.Subfolder != "" { - store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, r.Ctx.PipelineName, r.Ctx.TaskID, fileType) - } else { - store.Subfolder = fmt.Sprintf("%s/%d/%s", r.Ctx.PipelineName, r.Ctx.TaskID, fileType) - } + store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI) + if err != nil { + log.Errorf("failed to create s3 storage %s, err: %s", r.Ctx.StorageURI, err) + return err + } - filePath := path.Join(r.Ctx.Archive.Dir, r.Ctx.Archive.File) + fileType := "test" + if strings.Contains(r.Ctx.Archive.File, ".tar.gz") { + fileType = "file" + } + if store.Subfolder != "" { + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, r.Ctx.PipelineName, r.Ctx.TaskID, fileType) + } else { + store.Subfolder = fmt.Sprintf("%s/%d/%s", r.Ctx.PipelineName, r.Ctx.TaskID, fileType) + } - if _, err := os.Stat(filePath); os.IsNotExist(err) { - // no file found, skipped - //log.Warningf("upload filepath not exist") - return nil - } + filePath := path.Join(r.Ctx.Archive.Dir, r.Ctx.Archive.File) - err = s3.Upload( - context.Background(), - store, - filePath, - r.Ctx.Archive.File, - ) + if _, err := os.Stat(filePath); os.IsNotExist(err) { + // no file found, skipped + //log.Warningf("upload filepath not exist") + return nil + } - if err != nil { - log.Errorf("failed to upload package %s, %v", filePath, err) - return err - } + err = s3.Upload(context.Background(), store, filePath, r.Ctx.Archive.File) + if err != nil { + log.Errorf("failed to upload package %s, %v", filePath, err) + return err + } + + return nil +} + +func (r *Reaper) archiveHTMLTestReportFile() error { + // 仅功能测试有HTML测试结果报告 + if r.Ctx.TestType != setting.FunctionTest { + return nil + } + + if r.Ctx.Archive == nil || r.Ctx.StorageURI == "" { + return nil + } + + store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI) + if err != nil { + log.Errorf("failed to create s3 storage %s, err: %s", r.Ctx.StorageURI, err) + return err + } + + if store.Subfolder != "" { + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, r.Ctx.PipelineName, r.Ctx.TaskID, "test") + } else { + store.Subfolder = fmt.Sprintf("%s/%d/%s", r.Ctx.PipelineName, r.Ctx.TaskID, "test") + } + + filePath := r.Ctx.GinkgoTest.TestReportPath + if filePath == "" { + return nil + } + + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return nil + } + + fileName := filepath.Base(r.Ctx.Archive.TestReportFile) + err = s3.Upload(context.Background(), store, filePath, fileName) + if err != nil { + log.Errorf("failed to upload package %s, %s", filePath, err) + return err } return nil diff --git a/pkg/microservice/reaper/core/service/reaper/reaper.go b/pkg/microservice/reaper/core/service/reaper/reaper.go index c31eaa224e..877fd084f3 100644 --- a/pkg/microservice/reaper/core/service/reaper/reaper.go +++ b/pkg/microservice/reaper/core/service/reaper/reaper.go @@ -358,6 +358,7 @@ func (r *Reaper) AfterExec(upStreamErr error) error { } if r.Ctx.TestType == setting.FunctionTest { log.Info("merging test result") + // 解析功能测试的测试结果目录的文件,对数据进行统计,将最终的统计结果写入到一个本地文件中 if err = mergeGinkgoTestResults( r.Ctx.Archive.File, resultPath, @@ -369,6 +370,7 @@ func (r *Reaper) AfterExec(upStreamErr error) error { } } else if r.Ctx.TestType == setting.PerformanceTest { log.Info("performance test result") + // 解析性能测试的测试结果目录的文件,对数据进行统计,将最终的统计结果写入到一个本地文件中 if err = JmeterTestResults( r.Ctx.Archive.File, resultPath, @@ -378,17 +380,24 @@ func (r *Reaper) AfterExec(upStreamErr error) error { return err } } - //处理artifact + // 将测试文件导出地址的文件上传到S3 if len(r.Ctx.GinkgoTest.ArtifactPaths) > 0 { if err = artifactsUpload(r.Ctx, r.ActiveWorkspace); err != nil { log.Errorf("artifactsUpload err %v", err) return err } } + // 将上面生成的统计结果文件上传到S3 if err = r.archiveTestFiles(); err != nil { log.Errorf("archiveFiles err %v", err) return err } + // 将HTML测试报告上传到S3 + if err = r.archiveHTMLTestReportFile(); err != nil { + log.Errorf("archiveFiles err %v", err) + return err + } + } // should archive file first, since compress cache will clean the workspace @@ -403,6 +412,12 @@ func (r *Reaper) AfterExec(upStreamErr error) error { return err } + // 运行物理机部署脚本 + if err = r.RunPMDeployScripts(); err != nil { + log.Errorf("RunPMDeployScripts err %v", err) + return err + } + // create dog food file to tell wd that task is finished dogFoodErr := ioutil.WriteFile(setting.DogFood, []byte(time.Now().Format(time.RFC3339)), 0644) if dogFoodErr != nil { diff --git a/pkg/microservice/reaper/core/service/reaper/reaper_test.go b/pkg/microservice/reaper/core/service/reaper/reaper_test.go index 0ca3fb1a6f..cac753c9c0 100644 --- a/pkg/microservice/reaper/core/service/reaper/reaper_test.go +++ b/pkg/microservice/reaper/core/service/reaper/reaper_test.go @@ -28,6 +28,7 @@ import ( "gopkg.in/mholt/archiver.v3" "github.com/koderover/zadig/pkg/microservice/reaper/core/service/meta" + _ "github.com/koderover/zadig/pkg/util/testing" ) func TestReaper_CompressAndDecompressCache(t *testing.T) { diff --git a/pkg/microservice/reaper/core/service/reaper/script.go b/pkg/microservice/reaper/core/service/reaper/script.go index a7603cc080..b8c3e99431 100644 --- a/pkg/microservice/reaper/core/service/reaper/script.go +++ b/pkg/microservice/reaper/core/service/reaper/script.go @@ -19,6 +19,7 @@ package reaper import ( "bufio" "context" + "encoding/base64" "fmt" "io/ioutil" "os" @@ -303,3 +304,62 @@ func (r *Reaper) RunPostScripts() error { return cmd.Run() } + +func (r *Reaper) RunPMDeployScripts() error { + if len(r.Ctx.PMDeployScripts) == 0 { + return nil + } + + scripts := r.Ctx.PMDeployScripts + pmDeployScriptFile := "pm_deploy_script.sh" + if err := ioutil.WriteFile(filepath.Join(os.TempDir(), pmDeployScriptFile), []byte(strings.Join(scripts, "\n")), 0700); err != nil { + return fmt.Errorf("write script file error: %v", err) + } + + cmd := exec.Command("/bin/bash", filepath.Join(os.TempDir(), pmDeployScriptFile)) + cmd.Dir = r.ActiveWorkspace + + cmd.Env = r.getUserEnvs() + // ssh连接参数 + for _, ssh := range r.Ctx.SSHs { + decodeBytes, err := base64.StdEncoding.DecodeString(ssh.PrivateKey) + if err != nil { + return fmt.Errorf("decode private_key failed, error: %v", err) + } + if err = ioutil.WriteFile(filepath.Join(os.TempDir(), ssh.Name+"_PK"), decodeBytes, 0600); err != nil { + return fmt.Errorf("write private_key file error: %v", err) + } + + cmd.Env = append(cmd.Env, fmt.Sprintf("%s_PK=%s", ssh.Name, filepath.Join(os.TempDir(), ssh.Name+"_PK"))) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s_IP=%s", ssh.Name, ssh.IP)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s_USERNAME=%s", ssh.Name, ssh.UserName)) + + r.Ctx.SecretEnvs = append(r.Ctx.SecretEnvs, fmt.Sprintf("%s_PK=%s", ssh.Name, filepath.Join(os.TempDir(), ssh.Name+"_PK"))) + } + + cmdOutReader, err := cmd.StdoutPipe() + if err != nil { + return err + } + + outScanner := bufio.NewScanner(cmdOutReader) + go func() { + for outScanner.Scan() { + fmt.Printf("%s\n", r.maskSecretEnvs(outScanner.Text())) + } + }() + + cmdErrReader, err := cmd.StderrPipe() + if err != nil { + return err + } + + errScanner := bufio.NewScanner(cmdErrReader) + go func() { + for errScanner.Scan() { + fmt.Printf("%s\n", r.maskSecretEnvs(errScanner.Text())) + } + }() + + return cmd.Run() +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/build.go b/pkg/microservice/warpdrive/core/service/taskplugin/build.go index 7c236fe8a2..26ca8022eb 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/build.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/build.go @@ -26,12 +26,13 @@ import ( "go.uber.org/zap" "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/updater" ) @@ -115,6 +116,13 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe envNameVar := &task.KeyVal{Key: "ENV_NAME", Value: envName, IsCredential: false} p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envNameVar) } + privateKeys := sets.String{} + for _, privateKey := range pipelineTask.ConfigPayload.PrivateKeys { + privateKeys.Insert(privateKey.Name) + } + + privateKeysVar := &task.KeyVal{Key: "AGENTS", Value: strings.Join(privateKeys.List(), ","), IsCredential: false} + p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, privateKeysVar) p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace for _, repo := range p.Task.JobCtx.Builds { repoName := strings.Replace(repo.RepoName, "-", "_", -1) diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go b/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go index f58672bbde..74374194e2 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go @@ -34,12 +34,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/httpclient" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/distribute_kodo.go b/pkg/microservice/warpdrive/core/service/taskplugin/distribute_kodo.go index b84fc090bb..06d42d54b5 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/distribute_kodo.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/distribute_kodo.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "time" "go.uber.org/zap" @@ -84,7 +85,22 @@ func (p *Distribute2S3TaskPlugin) TaskTimeout() int { return p.Task.Timeout } +//是否是kodo +func isKODO(storage *s3.S3) bool { + return strings.Contains(storage.Endpoint, "qiniucs.com") +} + func upload(ctx context.Context, log *zap.SugaredLogger, storage *s3.S3, localfile, destfile string) error { + if isKODO(storage) { + err := uploadFileToS3(storage.Ak, storage.Sk, storage.Bucket, destfile, localfile) + if err != nil { + log.Warnf("failed to upload file to s3 %s=>%s %v", localfile, destfile, err) + } else { + log.Infof("succeed to upload file to s3 %s=>%s", localfile, destfile) + } + + return err + } return s3.Upload(ctx, storage, localfile, destfile) } @@ -157,6 +173,10 @@ func (p *Distribute2S3TaskPlugin) Run(ctx context.Context, pipelineTask *task.Ta p.Task.ServiceName, fmt.Sprintf("%d", pipelineTask.TaskID)) + if isKODO(destStorage) { + remoteFileKey = filepath.Join(remoteFileKey, p.Task.PackageFile) + } + if destStorage.Subfolder != "" { destStorage.Subfolder = fmt.Sprintf("%s/%s", destStorage.Subfolder, remoteFileKey) } else { @@ -165,6 +185,12 @@ func (p *Distribute2S3TaskPlugin) Run(ctx context.Context, pipelineTask *task.Ta remoteFileName := p.Task.PackageFile p.Task.RemoteFileKey = filepath.Join(destStorage.Subfolder, p.Task.PackageFile) + + if isKODO(destStorage) { + remoteFileName = remoteFileKey + p.Task.RemoteFileKey = destStorage.Subfolder + } + err = upload(ctx, p.Log, destStorage, localFile, remoteFileName) if err != nil { p.Log.Errorf("failed to upload file to dest storage %s %v", destStorage.GetURI(), err) @@ -207,6 +233,11 @@ func (p *Distribute2S3TaskPlugin) Run(ctx context.Context, pipelineTask *task.Ta }() remoteMd5File := fmt.Sprintf("%s.md5", p.Task.PackageFile) + + if isKODO(destStorage) { + remoteMd5File = fmt.Sprintf("%s.md5", remoteFileKey) + } + err = upload(ctx, p.Log, destStorage, localMd5File, remoteMd5File) if err != nil { p.Log.Errorf("failed to upload md5 file to %s %v", destStorage.GetURI(), err) diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/github/client.go b/pkg/microservice/warpdrive/core/service/taskplugin/github/client.go index 51d595c4a1..4dc8be271f 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/github/client.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/github/client.go @@ -17,7 +17,7 @@ limitations under the License. package github import ( - "github.com/koderover/zadig/pkg/tool/github" + "github.com/koderover/zadig/pkg/tool/git/github" ) type Client struct { diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/jira.go b/pkg/microservice/warpdrive/core/service/taskplugin/jira.go index 19fb59cf64..910a3f756e 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/jira.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/jira.go @@ -26,11 +26,11 @@ import ( "github.com/xanzy/go-gitlab" "go.uber.org/zap" - "github.com/koderover/zadig/pkg/internal/poetry" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" wdgithub "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/github" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" "github.com/koderover/zadig/pkg/tool/jira" "github.com/koderover/zadig/pkg/tool/log" "github.com/koderover/zadig/pkg/util" diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/job.go b/pkg/microservice/warpdrive/core/service/taskplugin/job.go index a0c6cfcfc5..c9ce17ad30 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/job.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/job.go @@ -37,12 +37,12 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/koderover/zadig/pkg/internal/kube/wrapper" "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/taskplugin/s3" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/containerlog" "github.com/koderover/zadig/pkg/tool/kube/getter" @@ -145,11 +145,12 @@ func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLa // JobCtxBuilder ... type JobCtxBuilder struct { - JobName string - ArchiveFile string - PipelineCtx *task.PipelineCtx - JobCtx task.JobCtx - Installs []*task.Install + JobName string + ArchiveFile string + TestReportFile string + PipelineCtx *task.PipelineCtx + JobCtx task.JobCtx + Installs []*task.Install } func replaceWrapLine(script string) string { @@ -190,6 +191,7 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName }, Scripts: make([]string, 0), PostScripts: make([]string, 0), + PMDeployScripts: make([]string, 0), SSHs: b.JobCtx.SSHs, TestType: b.JobCtx.TestType, ClassicBuild: pipelineTask.ConfigPayload.ClassicBuild, @@ -252,9 +254,14 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName ctx.PostScripts = append(ctx.PostScripts, strings.Split(replaceWrapLine(b.JobCtx.PostScripts), "\n")...) } + if b.JobCtx.PMDeployScripts != "" { + ctx.PMDeployScripts = append(ctx.PMDeployScripts, strings.Split(replaceWrapLine(b.JobCtx.PMDeployScripts), "\n")...) + } + ctx.Archive = &types.Archive{ - Dir: b.PipelineCtx.DistDir, - File: b.ArchiveFile, + Dir: b.PipelineCtx.DistDir, + File: b.ArchiveFile, + TestReportFile: b.TestReportFile, //StorageUri: b.JobCtx.StorageUri, } @@ -281,8 +288,9 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName if b.JobCtx.TestResultPath != "" { ctx.GinkgoTest = &types.GinkgoTest{ - ResultPath: b.JobCtx.TestResultPath, - ArtifactPaths: b.JobCtx.ArtifactPaths, + ResultPath: b.JobCtx.TestResultPath, + TestReportPath: b.JobCtx.TestReportPath, + ArtifactPaths: b.JobCtx.ArtifactPaths, } } diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/testing.go b/pkg/microservice/warpdrive/core/service/taskplugin/testing.go index 50140e48eb..5cfdff60ed 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/testing.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/testing.go @@ -21,7 +21,6 @@ import ( "encoding/csv" "encoding/xml" "fmt" - "io/ioutil" "os" "strings" "time" @@ -139,23 +138,26 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, linkedNamespaceEnvVar) p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, envNameEnvVar) - testJobName := strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", config.SingleType, - pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, pipelineTask.ServiceName)), "_", "-", -1) + var testReportFile string // html 测试报告 + fileName := fmt.Sprintf("%s-%s-%d-%s-%s", config.SingleType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, pipelineTask.ServiceName) // 测试结果保存名称,两种模式需要区分开 - fileName := testJobName if pipelineTask.Type == config.WorkflowType { - fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", - config.WorkflowType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName)), "_", "-", -1) + fileName = fmt.Sprintf("%s-%s-%d-%s-%s", config.WorkflowType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName) + testReportFile = fmt.Sprintf("%s-%s-%d-%s-%s-html", config.WorkflowType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, p.Task.TestModuleName) } else if pipelineTask.Type == config.TestType { - fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", - config.TestType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName)), "_", "-", -1) + fileName = fmt.Sprintf("%s-%s-%d-%s-%s", config.TestType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName) } + + fileName = strings.Replace(strings.ToLower(fileName), "_", "-", -1) + testReportFile = strings.Replace(strings.ToLower(testReportFile), "_", "-", -1) + jobCtx := JobCtxBuilder{ - JobName: p.JobName, - PipelineCtx: pipelineCtx, - ArchiveFile: fileName, - JobCtx: p.Task.JobCtx, - Installs: p.Task.InstallCtx, + JobName: p.JobName, + PipelineCtx: pipelineCtx, + ArchiveFile: fileName, + TestReportFile: testReportFile, + JobCtx: p.Task.JobCtx, + Installs: p.Task.InstallCtx, } jobCtxBytes, err := yaml.Marshal(jobCtx.BuildReaperContext(pipelineTask, serviceName)) @@ -271,157 +273,163 @@ func (p *TestPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serv } p.Task.LogFile = p.JobName - testJobName := strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", config.SingleType, - pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, pipelineTask.ServiceName)), "_", "-", -1) + fileName := fmt.Sprintf("%s-%s-%d-%s-%s", config.SingleType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, pipelineTask.ServiceName) // 测试结果保存名称,两种模式需要区分开 - fileName := testJobName if pipelineTask.Type == config.WorkflowType { - fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", - config.WorkflowType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName)), "_", "-", -1) + fileName = fmt.Sprintf("%s-%s-%d-%s-%s", config.WorkflowType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName) } else if pipelineTask.Type == config.TestType { - fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", - config.TestType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName)), "_", "-", -1) + fileName = fmt.Sprintf("%s-%s-%d-%s-%s", config.TestType, pipelineTask.PipelineName, pipelineTask.TaskID, config.TaskTestingV2, serviceName) } + fileName = strings.Replace(strings.ToLower(fileName), "_", "-", -1) + //如果用户配置了测试结果目录需要收集,则下载测试结果,发送到aslan server //Note here: p.Task.TestName目前只有默认值test - if p.Task.JobCtx.TestResultPath != "" { - testReport := new(types.TestReport) - if pipelineTask.TestReports == nil { - pipelineTask.TestReports = make(map[string]interface{}) + if p.Task.JobCtx.TestResultPath == "" { + return + } + + testReport := new(types.TestReport) + if pipelineTask.TestReports == nil { + pipelineTask.TestReports = make(map[string]interface{}) + } + + if p.Task.JobCtx.TestType == "" { + p.Task.JobCtx.TestType = setting.FunctionTest + } + + store, err := s3.NewS3StorageFromEncryptedURI(pipelineTask.StorageURI) + if err != nil { + return + } + + if store.Subfolder != "" { + store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, pipelineName, pipelineTaskID, "artifact") + } else { + store.Subfolder = fmt.Sprintf("%s/%d/%s", pipelineName, pipelineTaskID, "artifact") + } + if files, err := s3.ListFiles(store, "/", true); err == nil { + if len(files) > 0 { + p.Task.JobCtx.IsHasArtifact = true } + } + + tmpFilename, err := util.GenerateTmpFile() + if err != nil { + p.Log.Error(fmt.Sprintf("generate temp file error: %v", err)) + } + defer func() { + _ = os.Remove(tmpFilename) + }() + + store.Subfolder = strings.Replace(store.Subfolder, fmt.Sprintf("%s/%d/%s", pipelineName, pipelineTaskID, "artifact"), fmt.Sprintf("%s/%d/%s", pipelineName, pipelineTaskID, "test"), -1) + err = s3.Download(context.Background(), store, fileName, tmpFilename) + if err != nil { + return + } - if p.Task.JobCtx.TestType == "" { - p.Task.JobCtx.TestType = setting.FunctionTest + if p.Task.JobCtx.TestType == setting.FunctionTest { + b, err := os.ReadFile(tmpFilename) + if err != nil { + p.Log.Error(fmt.Sprintf("get test result file error: %v", err)) + + msg := "no test result is found" + p.Task.Error = msg + p.Task.TaskStatus = config.StatusFailed + return } - var store *s3.S3 - if store, err = s3.NewS3StorageFromEncryptedURI(pipelineTask.StorageURI); err == nil { - if store.Subfolder != "" { - store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, pipelineName, pipelineTaskID, "artifact") - } else { - store.Subfolder = fmt.Sprintf("%s/%d/%s", pipelineName, pipelineTaskID, "artifact") - } - if files, err := s3.ListFiles(store, "/", true); err == nil { - if len(files) > 0 { - p.Task.JobCtx.IsHasArtifact = true - } - } + err = xml.Unmarshal(b, &testReport.FunctionTestSuite) + if err != nil { + msg := fmt.Sprintf("unmarshal result file test suite summary error: %v", err) + p.Log.Error(msg) + p.Task.Error = msg + p.Task.TaskStatus = config.StatusFailed + return + } + p.Task.ReportReady = true + testReport.FunctionTestSuite.TestCases = []types.TestCase{} + //测试报告 + pipelineTask.TestReports[serviceName] = testReport + + if testReport.FunctionTestSuite.Errors > 0 || testReport.FunctionTestSuite.Failures > 0 { + msg := fmt.Sprintf( + "%d failure case(s) found", + testReport.FunctionTestSuite.Errors+testReport.FunctionTestSuite.Failures, + ) + p.Log.Error(msg) + p.Task.Error = msg + p.Task.TaskStatus = config.StatusFailed + return + } - tmpFilename, err := util.GenerateTmpFile() - if err != nil { - p.Log.Error(fmt.Sprintf("generate temp file error: %v", err)) - } - defer func() { - _ = os.Remove(tmpFilename) - }() - - store.Subfolder = strings.Replace(store.Subfolder, fmt.Sprintf("%s/%d/%s", pipelineName, pipelineTaskID, "artifact"), fmt.Sprintf("%s/%d/%s", pipelineName, pipelineTaskID, "test"), -1) - if err = s3.Download(context.Background(), store, fileName, tmpFilename); err == nil { - if p.Task.JobCtx.TestType == setting.FunctionTest { - - b, err := ioutil.ReadFile(tmpFilename) - if err != nil { - p.Log.Error(fmt.Sprintf("get test result file error: %v", err)) - - msg := "none test result found" - p.Task.Error = msg - p.Task.TaskStatus = config.StatusFailed - return - } - - err = xml.Unmarshal(b, &testReport.FunctionTestSuite) - if err != nil { - msg := fmt.Sprintf("unmarshal result file test suite summary error: %v", err) - p.Log.Error(msg) - p.Task.Error = msg - p.Task.TaskStatus = config.StatusFailed - return - } - p.Task.ReportReady = true - testReport.FunctionTestSuite.TestCases = []types.TestCase{} - //测试报告 - pipelineTask.TestReports[serviceName] = testReport - - if testReport.FunctionTestSuite.Errors > 0 || testReport.FunctionTestSuite.Failures > 0 { - msg := fmt.Sprintf( - "%d failure case(s) found", - testReport.FunctionTestSuite.Errors+testReport.FunctionTestSuite.Failures, - ) - p.Log.Error(msg) - p.Task.Error = msg - p.Task.TaskStatus = config.StatusFailed - return - } - - } else if p.Task.JobCtx.TestType == setting.PerformanceTest { - csvFile, err := os.Open(tmpFilename) - if err != nil { - msg := fmt.Sprintf("get performance test result file error: %v", err) - p.Log.Error(msg) - p.Task.Error = msg - p.Task.TaskStatus = config.StatusFailed - return - } - defer csvFile.Close() - csvReader := csv.NewReader(csvFile) - row, err := csvReader.Read() - if len(row) != 11 { - msg := "csv file type match error" - p.Log.Error(msg) - p.Task.Error = msg - p.Task.TaskStatus = config.StatusFailed - return - } - - if err != nil { - msg := fmt.Sprintf("read performance csv first row error: %v", err) - p.Log.Error(msg) - p.Task.Error = msg - p.Task.TaskStatus = config.StatusFailed - return - } - - rows, err := csvReader.ReadAll() - if err != nil { - msg := fmt.Sprintf("read csv readAll error: %v", err) - p.Log.Error(msg) - p.Task.Error = msg - p.Task.TaskStatus = config.StatusFailed - return - } - performanceTestSuites := make([]*types.PerformanceTestSuite, 0) - for _, row := range rows { - performanceTestSuite := new(types.PerformanceTestSuite) - performanceTestSuite.Label = row[0] - performanceTestSuite.Samples = row[1] - performanceTestSuite.Average = row[2] - performanceTestSuite.Min = row[3] - performanceTestSuite.Max = row[4] - performanceTestSuite.Line = row[5] - performanceTestSuite.StdDev = row[6] - performanceTestSuite.Error = row[7] - performanceTestSuite.Throughput = row[8] - performanceTestSuite.ReceivedKb = row[9] - performanceTestSuite.AvgByte = row[10] - - performanceTestSuites = append(performanceTestSuites, performanceTestSuite) - } - p.Task.ReportReady = true - testReport.PerformanceTestSuites = performanceTestSuites - //测试报告 - pipelineTask.TestReports[serviceName] = testReport - } - } + } else if p.Task.JobCtx.TestType == setting.PerformanceTest { + csvFile, err := os.Open(tmpFilename) + if err != nil { + msg := fmt.Sprintf("get performance test result file error: %v", err) + p.Log.Error(msg) + p.Task.Error = msg + p.Task.TaskStatus = config.StatusFailed + return + } + defer csvFile.Close() + csvReader := csv.NewReader(csvFile) + row, err := csvReader.Read() + if len(row) != 11 { + msg := "csv file type match error" + p.Log.Error(msg) + p.Task.Error = msg + p.Task.TaskStatus = config.StatusFailed + return } - // compare failure percent with threshold - // Integer operation: FAIL/TOTAL > %V => FAIL*100 > TOTAL*V - //if itReport.TestSuiteSummary.Failures*100 > itReport.TestSuiteSummary.Tests*p.Task.JobCtx.TestThreshold { - // p.Task.TaskStatus = task.StatusFailed - // return - //} + + if err != nil { + msg := fmt.Sprintf("read performance csv first row error: %v", err) + p.Log.Error(msg) + p.Task.Error = msg + p.Task.TaskStatus = config.StatusFailed + return + } + + rows, err := csvReader.ReadAll() + if err != nil { + msg := fmt.Sprintf("read csv readAll error: %v", err) + p.Log.Error(msg) + p.Task.Error = msg + p.Task.TaskStatus = config.StatusFailed + return + } + performanceTestSuites := make([]*types.PerformanceTestSuite, 0) + for _, row := range rows { + performanceTestSuite := new(types.PerformanceTestSuite) + performanceTestSuite.Label = row[0] + performanceTestSuite.Samples = row[1] + performanceTestSuite.Average = row[2] + performanceTestSuite.Min = row[3] + performanceTestSuite.Max = row[4] + performanceTestSuite.Line = row[5] + performanceTestSuite.StdDev = row[6] + performanceTestSuite.Error = row[7] + performanceTestSuite.Throughput = row[8] + performanceTestSuite.ReceivedKb = row[9] + performanceTestSuite.AvgByte = row[10] + + performanceTestSuites = append(performanceTestSuites, performanceTestSuite) + } + p.Task.ReportReady = true + testReport.PerformanceTestSuites = performanceTestSuites + //测试报告 + pipelineTask.TestReports[serviceName] = testReport } + + // compare failure percent with threshold + // Integer operation: FAIL/TOTAL > %V => FAIL*100 > TOTAL*V + //if itReport.TestSuiteSummary.Failures*100 > itReport.TestSuiteSummary.Tests*p.Task.JobCtx.TestThreshold { + // p.Task.TaskStatus = task.StatusFailed + // return + //} + } func (p *TestPlugin) SetTask(t map[string]interface{}) error { diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/utils.go b/pkg/microservice/warpdrive/core/service/taskplugin/utils.go index b8b93933ae..6d41110151 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/utils.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/utils.go @@ -30,10 +30,22 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/tool/kodo" ) func int32Ptr(i int32) *int32 { return &i } +func uploadFileToS3(access, secret, bucket, remote, local string) error { + s3Cli, err := kodo.NewUploadClient(access, secret, bucket) + if err != nil { + return err + } + if _, _, err := s3Cli.UploadFile(remote, local); err != nil { + return err + } + return nil +} + // 选择最合适的dockerhost func GetBestDockerHost(hostList []string, pipelineType, namespace string, log *zap.SugaredLogger) (string, error) { bestHosts := []string{} diff --git a/pkg/microservice/warpdrive/core/service/types/reaper.go b/pkg/microservice/warpdrive/core/service/types/reaper.go index 7a179a9dbd..627d3d68c0 100644 --- a/pkg/microservice/warpdrive/core/service/types/reaper.go +++ b/pkg/microservice/warpdrive/core/service/types/reaper.go @@ -57,6 +57,9 @@ type Context struct { // PostScripts 后置编译脚本 PostScripts []string `yaml:"post_scripts"` + // PMDeployScripts 物理机部署脚本 + PMDeployScripts []string `yaml:"pm_deploy_scripts"` + // SSH ssh连接参数 SSHs []*task.SSH `yaml:"sshs"` @@ -241,14 +244,16 @@ func (r *Repo) Ref() string { // Archive ... type Archive struct { - Dir string `yaml:"dir"` - File string `yaml:"file"` + Dir string `yaml:"dir"` + File string `yaml:"file"` + TestReportFile string `yaml:"test_report_file"` } // GinkgoTest ... type GinkgoTest struct { - ResultPath string `yaml:"result_path"` - ArtifactPaths []string `yaml:"artifact_paths"` + ResultPath string `yaml:"result_path"` + TestReportPath string `yaml:"test_report_path"` + ArtifactPaths []string `yaml:"artifact_paths"` } // DockerRegistry 推送镜像到 docker registry 配置 diff --git a/pkg/microservice/warpdrive/core/service/types/task/build.go b/pkg/microservice/warpdrive/core/service/types/task/build.go index cd09723c60..29cd5b7c82 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/build.go +++ b/pkg/microservice/warpdrive/core/service/types/task/build.go @@ -155,6 +155,7 @@ type JobCtx struct { // TestJobCtx TestThreshold int `bson:"test_threshold" json:"test_threshold"` TestResultPath string `bson:"test_result_path,omitempty" json:"test_result_path,omitempty"` + TestReportPath string `bson:"test_report_path,omitempty" json:"test_report_path,omitempty"` TestJobName string `bson:"test_job_name,omitempty" json:"test_job_name,omitempty"` // DockerBuildCtx DockerBuildCtx *DockerBuildCtx `bson:"docker_build_ctx,omitempty" json:"docker_build_ctx,omitempty"` @@ -169,8 +170,9 @@ type JobCtx struct { //StorageUri string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` // ClassicBuild used by qbox build - ClassicBuild bool `bson:"classic_build" json:"classic_build"` - PostScripts string `bson:"post_scripts,omitempty" json:"post_scripts"` + ClassicBuild bool `bson:"classic_build" json:"classic_build"` + PostScripts string `bson:"post_scripts,omitempty" json:"post_scripts"` + PMDeployScripts string `bson:"pm_deploy_scripts,omitempty" json:"pm_deploy_scripts"` } type SSH struct { diff --git a/pkg/microservice/warpdrive/core/service/types/task/config_payload.go b/pkg/microservice/warpdrive/core/service/types/task/config_payload.go index 87f165c532..b349b9ef60 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/config_payload.go +++ b/pkg/microservice/warpdrive/core/service/types/task/config_payload.go @@ -47,7 +47,8 @@ type ConfigPayload struct { IgnoreCache bool `json:"ignore_cache"` // ResetCache means ignore workspace cache - ResetCache bool `json:"reset_cache"` + ResetCache bool `json:"reset_cache"` + PrivateKeys []*PrivateKey `json:"private_keys"` } func (cp *ConfigPayload) GetGitKnownHost() string { @@ -147,3 +148,7 @@ type DockerConfig struct { type JenkinsBuildConfig struct { JenkinsBuildImage string } + +type PrivateKey struct { + Name string `json:"name"` +} diff --git a/pkg/microservice/aslan/middleware/audit_log.go b/pkg/middleware/gin/audit_log.go similarity index 74% rename from pkg/microservice/aslan/middleware/audit_log.go rename to pkg/middleware/gin/audit_log.go index 5c31d20574..df97ef9670 100644 --- a/pkg/microservice/aslan/middleware/audit_log.go +++ b/pkg/middleware/gin/audit_log.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package middleware +package gin import ( "github.com/gin-gonic/gin" - commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/shared/client/aslanx" "github.com/koderover/zadig/pkg/util/ginzap" ) @@ -30,8 +31,9 @@ func UpdateOperationLogStatus(c *gin.Context) { if c.GetString("operationLogID") == "" { return } - err := commonservice.UpdateOperation(c.GetString("operationLogID"), c.Writer.Status(), log) + + err := aslanx.New(config.AslanURL(), config.PoetryAPIRootKey()).UpdateAuditLog(c.GetString("operationLogID"), c.Writer.Status(), log) if err != nil { - log.Errorf("UpdateOperation err :%v", err) + log.Errorf("UpdateOperation err:%v", err) } } diff --git a/pkg/microservice/aslan/middleware/auth.go b/pkg/middleware/gin/auth.go similarity index 94% rename from pkg/microservice/aslan/middleware/auth.go rename to pkg/middleware/gin/auth.go index 4f2783830d..6f671343bc 100644 --- a/pkg/microservice/aslan/middleware/auth.go +++ b/pkg/middleware/gin/auth.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package middleware +package gin import ( "net/http" @@ -22,9 +22,9 @@ import ( "github.com/gin-gonic/gin" - "github.com/koderover/zadig/pkg/internal/poetry" - "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" "github.com/koderover/zadig/pkg/tool/log" ) diff --git a/pkg/microservice/aslan/middleware/permission.go b/pkg/middleware/gin/permission.go similarity index 97% rename from pkg/microservice/aslan/middleware/permission.go rename to pkg/middleware/gin/permission.go index 7e25c24678..d3071b4e59 100644 --- a/pkg/microservice/aslan/middleware/permission.go +++ b/pkg/middleware/gin/permission.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package middleware +package gin import ( "fmt" @@ -23,9 +23,9 @@ import ( "github.com/gin-gonic/gin" "k8s.io/apimachinery/pkg/util/sets" - "github.com/koderover/zadig/pkg/internal/poetry" - "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/poetry" "github.com/koderover/zadig/pkg/types/permission" "github.com/koderover/zadig/pkg/util/ginzap" ) diff --git a/pkg/microservice/aslan/middleware/product_name.go b/pkg/middleware/gin/product_name.go similarity index 98% rename from pkg/microservice/aslan/middleware/product_name.go rename to pkg/middleware/gin/product_name.go index cbb018ac2a..d66bff4044 100644 --- a/pkg/microservice/aslan/middleware/product_name.go +++ b/pkg/middleware/gin/product_name.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package middleware +package gin import ( "bytes" diff --git a/pkg/setting/consts.go b/pkg/setting/consts.go index 8491106dac..88b2f15835 100644 --- a/pkg/setting/consts.go +++ b/pkg/setting/consts.go @@ -74,6 +74,7 @@ const ( // cron ENVAslanAPI = "ASLAN_API" + ENVAslanxAPI = "ASLANX_API" ENVRootToken = "ROOT_TOKEN" ENVAslanAddr = "ASLAN_ADDRESS" ENVNsqdAddr = "NSQD_ADDR" @@ -280,6 +281,16 @@ const ( CookieHeader = "Cookie" ) +//install script constants +const ( + StandardScriptName = "install.sh" + AllInOneScriptName = "install_with_k8s.sh" + UninstallScriptName = "uninstall.sh" + StandardInstallType = "standard" + AllInOneInstallType = "all-in-one" + UninstallInstallType = "uninstall" +) + // Pod Status const ( StatusRunning = "Running" @@ -371,8 +382,6 @@ const ( ProxyHTTPSAddr = "PROXY_HTTPS_ADDR" ProxyHTTPAddr = "PROXY_HTTP_ADDR" ProxySocks5Addr = "PROXY_SOCKS_ADDR" - - EnableGitCheck = "ENABLE_GIT_CHECK" ) const ( diff --git a/pkg/shared/client/aslan/client.go b/pkg/shared/client/aslan/client.go new file mode 100644 index 0000000000..0c99f65a28 --- /dev/null +++ b/pkg/shared/client/aslan/client.go @@ -0,0 +1,43 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string + token string +} + +func New(host, token string) *Client { + c := httpclient.New( + httpclient.SetAuthScheme(setting.RootAPIKey), + httpclient.SetAuthToken(token), + httpclient.SetHostURL(host), + ) + + return &Client{ + Client: c, + host: host, + token: token, + } +} diff --git a/pkg/shared/client/aslan/testing.go b/pkg/shared/client/aslan/testing.go new file mode 100644 index 0000000000..99b1075b62 --- /dev/null +++ b/pkg/shared/client/aslan/testing.go @@ -0,0 +1,64 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Testing struct { + Name string `bson:"name" json:"name"` + TestType string `bson:"test_type" json:"test_type"` +} + +func (c *Client) ListTestings(log *zap.SugaredLogger) ([]*Testing, error) { + url := "/aslan/testing/test" + resp := make([]*Testing, 0) + + _, err := c.Get(url, httpclient.SetResult(&resp)) + if err != nil { + log.Errorf("ListTestings error: %s", err) + return nil, err + } + + return resp, nil +} + +type TestTaskStat struct { + Name string `bson:"name" json:"name"` + TotalSuccess int `bson:"total_success" json:"totalSuccess"` + TotalFailure int `bson:"total_failure" json:"totalFailure"` + TotalDuration int64 `bson:"total_duration" json:"totalDuration"` + TestCaseNum int `bson:"test_case_num" json:"testCaseNum"` + CreateTime int64 `bson:"create_time" json:"createTime"` + UpdateTime int64 `bson:"update_time" json:"updateTime"` +} + +func (c *Client) ListTestTaskStats(log *zap.SugaredLogger) ([]*TestTaskStat, error) { + url := "/aslan/testing/teststat" + resp := make([]*TestTaskStat, 0) + + _, err := c.Get(url, httpclient.SetResult(&resp)) + if err != nil { + log.Errorf("list test task stat error: %s", err) + return nil, err + } + + return resp, nil +} diff --git a/pkg/shared/client/aslanx/audit_log.go b/pkg/shared/client/aslanx/audit_log.go new file mode 100644 index 0000000000..58845357b6 --- /dev/null +++ b/pkg/shared/client/aslanx/audit_log.go @@ -0,0 +1,83 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslanx + +import ( + "fmt" + "time" + + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type operationLog struct { + Username string `json:"username"` + ProductName string `json:"product_name"` + Method string `json:"method"` + PermissionUUID string `json:"permission_uuid"` + Function string `json:"function"` + Name string `json:"name"` + RequestBody string `json:"request_body"` + Status int `json:"status"` + CreatedAt int64 `json:"created_at"` +} + +func (c *Client) AddAuditLog(username, productName, method, function, detail, permissionUUID, requestBody string, log *zap.SugaredLogger) (string, error) { + url := "/api/aslanx/enterprise/operation" + req := operationLog{ + Username: username, + ProductName: productName, + Method: method, + PermissionUUID: permissionUUID, + Function: function, + Name: detail, + RequestBody: requestBody, + Status: 0, + CreatedAt: time.Now().Unix(), + } + + var operationLogID string + _, err := c.Post(url, httpclient.SetBody(req), httpclient.SetResult(&operationLogID)) + // ignore not found error + if err != nil && !httpclient.IsNotFound(err) { + log.Errorf("Failed to add audit log, error: %s", err) + return "", err + } + + return operationLogID, nil +} + +type updateOperationArgs struct { + Status int `json:"status"` +} + +func (c *Client) UpdateAuditLog(id string, status int, log *zap.SugaredLogger) error { + url := fmt.Sprintf("/api/aslanx/enterprise/operation/%s", id) + req := updateOperationArgs{ + Status: status, + } + + _, err := c.Put(url, httpclient.SetBody(req)) + // ignore not found error + if err != nil && !httpclient.IsNotFound(err) { + log.Errorf("Failed to update audit log, error: %s", err) + return err + } + + return nil +} diff --git a/pkg/shared/client/aslanx/client.go b/pkg/shared/client/aslanx/client.go new file mode 100644 index 0000000000..05caf54c1d --- /dev/null +++ b/pkg/shared/client/aslanx/client.go @@ -0,0 +1,43 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslanx + +import ( + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Client struct { + *httpclient.Client + + host string + token string +} + +func New(host, token string) *Client { + c := httpclient.New( + httpclient.SetAuthScheme(setting.RootAPIKey), + httpclient.SetAuthToken(token), + httpclient.SetHostURL(host), + ) + + return &Client{ + Client: c, + host: host, + token: token, + } +} diff --git a/pkg/shared/client/aslanx/signature.go b/pkg/shared/client/aslanx/signature.go new file mode 100644 index 0000000000..c3501e8d82 --- /dev/null +++ b/pkg/shared/client/aslanx/signature.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslanx + +import ( + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type Signature struct { + ID string `json:"id,omitempty"` + Token string `json:"token"` + UpdateBy string `json:"update_by"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +// ListSignatures returns all signatures, enabled and err. +// enabled means signature control is enabled. +// If the status code is 404, mark enabled as false. +func (c *Client) ListSignatures(log *zap.SugaredLogger) ([]*Signature, bool, error) { + url := "/api/aslanx/enterprise/license" + + signatures := make([]*Signature, 0) + _, err := c.Get(url, httpclient.SetResult(&signatures)) + if err != nil { + if !httpclient.IsNotFound(err) { + log.Errorf("Failed to list signatures, error: %s", err) + return nil, true, err + } + + return nil, false, err + } + + return signatures, true, nil +} diff --git a/pkg/microservice/aslan/core/common/service/codehost/codehost.go b/pkg/shared/codehost/codehost.go similarity index 96% rename from pkg/microservice/aslan/core/common/service/codehost/codehost.go rename to pkg/shared/codehost/codehost.go index 9f0cd969c3..c5b6a2e742 100644 --- a/pkg/microservice/aslan/core/common/service/codehost/codehost.go +++ b/pkg/shared/codehost/codehost.go @@ -20,8 +20,8 @@ import ( "errors" "strings" - "github.com/koderover/zadig/pkg/internal/poetry" - "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/shared/poetry" ) const ( diff --git a/pkg/microservice/aslan/internal/handler/base.go b/pkg/shared/handler/base.go similarity index 90% rename from pkg/microservice/aslan/internal/handler/base.go rename to pkg/shared/handler/base.go index 0166efb637..0743e8b1cc 100644 --- a/pkg/microservice/aslan/internal/handler/base.go +++ b/pkg/shared/handler/base.go @@ -22,8 +22,10 @@ import ( "github.com/gin-gonic/gin" "go.uber.org/zap" - "github.com/koderover/zadig/pkg/internal/poetry" + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslanx" + "github.com/koderover/zadig/pkg/shared/poetry" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/types/permission" "github.com/koderover/zadig/pkg/util/ginzap" @@ -78,8 +80,13 @@ func JSONResponse(c *gin.Context, ctx *Context) { } } -// 插入操作日志 -func InsertOperationLog(c *gin.Context, username, productName, method, function, detail, permissionUUID string, requestBody string, logger *zap.SugaredLogger) { +// InsertOperationLog 插入操作日志 +func InsertOperationLog(c *gin.Context, username, productName, method, function, detail, permissionUUID, requestBody string, logger *zap.SugaredLogger) { + operationLogID, err := aslanx.New(config.AslanURL(), config.PoetryAPIRootKey()).AddAuditLog(username, productName, method, function, detail, permissionUUID, requestBody, logger) + if err != nil { + logger.Errorf("InsertOperation err:%v", err) + } + c.Set("operationLogID", operationLogID) } // responseHelper recursively finds all nil slice in the given interface, diff --git a/pkg/microservice/aslan/internal/handler/handler_suite_test.go b/pkg/shared/handler/handler_suite_test.go similarity index 100% rename from pkg/microservice/aslan/internal/handler/handler_suite_test.go rename to pkg/shared/handler/handler_suite_test.go diff --git a/pkg/microservice/aslan/internal/handler/options_test.go b/pkg/shared/handler/options_test.go similarity index 100% rename from pkg/microservice/aslan/internal/handler/options_test.go rename to pkg/shared/handler/options_test.go diff --git a/pkg/microservice/aslan/internal/handler/sse.go b/pkg/shared/handler/sse.go similarity index 100% rename from pkg/microservice/aslan/internal/handler/sse.go rename to pkg/shared/handler/sse.go diff --git a/pkg/internal/kube/resource/configmap.go b/pkg/shared/kube/resource/configmap.go similarity index 100% rename from pkg/internal/kube/resource/configmap.go rename to pkg/shared/kube/resource/configmap.go diff --git a/pkg/internal/kube/resource/event.go b/pkg/shared/kube/resource/event.go similarity index 100% rename from pkg/internal/kube/resource/event.go rename to pkg/shared/kube/resource/event.go diff --git a/pkg/internal/kube/resource/ingress.go b/pkg/shared/kube/resource/ingress.go similarity index 100% rename from pkg/internal/kube/resource/ingress.go rename to pkg/shared/kube/resource/ingress.go diff --git a/pkg/internal/kube/resource/job.go b/pkg/shared/kube/resource/job.go similarity index 100% rename from pkg/internal/kube/resource/job.go rename to pkg/shared/kube/resource/job.go diff --git a/pkg/internal/kube/resource/namespace.go b/pkg/shared/kube/resource/namespace.go similarity index 100% rename from pkg/internal/kube/resource/namespace.go rename to pkg/shared/kube/resource/namespace.go diff --git a/pkg/internal/kube/resource/pod.go b/pkg/shared/kube/resource/pod.go similarity index 100% rename from pkg/internal/kube/resource/pod.go rename to pkg/shared/kube/resource/pod.go diff --git a/pkg/internal/kube/resource/service.go b/pkg/shared/kube/resource/service.go similarity index 100% rename from pkg/internal/kube/resource/service.go rename to pkg/shared/kube/resource/service.go diff --git a/pkg/internal/kube/resource/workload.go b/pkg/shared/kube/resource/workload.go similarity index 100% rename from pkg/internal/kube/resource/workload.go rename to pkg/shared/kube/resource/workload.go diff --git a/pkg/internal/kube/wrapper/configmap.go b/pkg/shared/kube/wrapper/configmap.go similarity index 97% rename from pkg/internal/kube/wrapper/configmap.go rename to pkg/shared/kube/wrapper/configmap.go index f3bbdd5b3f..14a08e0074 100644 --- a/pkg/internal/kube/wrapper/configmap.go +++ b/pkg/shared/kube/wrapper/configmap.go @@ -23,8 +23,8 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/resource" kubeutil "github.com/koderover/zadig/pkg/tool/kube/util" "github.com/koderover/zadig/pkg/util" ) diff --git a/pkg/internal/kube/wrapper/deployment.go b/pkg/shared/kube/wrapper/deployment.go similarity index 96% rename from pkg/internal/kube/wrapper/deployment.go rename to pkg/shared/kube/wrapper/deployment.go index aef9ba5ecb..d5928f9f3d 100644 --- a/pkg/internal/kube/wrapper/deployment.go +++ b/pkg/shared/kube/wrapper/deployment.go @@ -20,8 +20,8 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/resource" ) // deployment is the wrapper for appsv1.Deployment type. diff --git a/pkg/internal/kube/wrapper/event.go b/pkg/shared/kube/wrapper/event.go similarity index 95% rename from pkg/internal/kube/wrapper/event.go rename to pkg/shared/kube/wrapper/event.go index aaee269ba8..a2b758f59c 100644 --- a/pkg/internal/kube/wrapper/event.go +++ b/pkg/shared/kube/wrapper/event.go @@ -19,7 +19,7 @@ package wrapper import ( corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/resource" ) // event is the wrapper for corev1.Event type. diff --git a/pkg/internal/kube/wrapper/ingress.go b/pkg/shared/kube/wrapper/ingress.go similarity index 97% rename from pkg/internal/kube/wrapper/ingress.go rename to pkg/shared/kube/wrapper/ingress.go index 1396c3aaba..e6e250a6f7 100644 --- a/pkg/internal/kube/wrapper/ingress.go +++ b/pkg/shared/kube/wrapper/ingress.go @@ -19,7 +19,7 @@ package wrapper import ( extensionsv1beta1 "k8s.io/api/extensions/v1beta1" - "github.com/koderover/zadig/pkg/internal/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" ) diff --git a/pkg/internal/kube/wrapper/job.go b/pkg/shared/kube/wrapper/job.go similarity index 96% rename from pkg/internal/kube/wrapper/job.go rename to pkg/shared/kube/wrapper/job.go index faea9c9275..a1fcdc0567 100644 --- a/pkg/internal/kube/wrapper/job.go +++ b/pkg/shared/kube/wrapper/job.go @@ -20,7 +20,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" ) diff --git a/pkg/internal/kube/wrapper/namespace.go b/pkg/shared/kube/wrapper/namespace.go similarity index 96% rename from pkg/internal/kube/wrapper/namespace.go rename to pkg/shared/kube/wrapper/namespace.go index 53d5cffc2c..4faa5d727b 100644 --- a/pkg/internal/kube/wrapper/namespace.go +++ b/pkg/shared/kube/wrapper/namespace.go @@ -19,7 +19,7 @@ package wrapper import ( corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" ) diff --git a/pkg/internal/kube/wrapper/pod.go b/pkg/shared/kube/wrapper/pod.go similarity index 98% rename from pkg/internal/kube/wrapper/pod.go rename to pkg/shared/kube/wrapper/pod.go index 190b3744c2..32902ec2a4 100644 --- a/pkg/internal/kube/wrapper/pod.go +++ b/pkg/shared/kube/wrapper/pod.go @@ -21,7 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" ) diff --git a/pkg/internal/kube/wrapper/service.go b/pkg/shared/kube/wrapper/service.go similarity index 96% rename from pkg/internal/kube/wrapper/service.go rename to pkg/shared/kube/wrapper/service.go index afdd819314..a42c121d13 100644 --- a/pkg/internal/kube/wrapper/service.go +++ b/pkg/shared/kube/wrapper/service.go @@ -19,7 +19,7 @@ package wrapper import ( corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" + "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" ) diff --git a/pkg/internal/kube/wrapper/statefulset.go b/pkg/shared/kube/wrapper/statefulset.go similarity index 96% rename from pkg/internal/kube/wrapper/statefulset.go rename to pkg/shared/kube/wrapper/statefulset.go index 8048712468..8c39a5645f 100644 --- a/pkg/internal/kube/wrapper/statefulset.go +++ b/pkg/shared/kube/wrapper/statefulset.go @@ -20,8 +20,8 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "github.com/koderover/zadig/pkg/internal/kube/resource" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/resource" ) // statefulSet is the wrapper for appsv1.StatefulSet type. diff --git a/pkg/internal/poetry/actions.go b/pkg/shared/poetry/actions.go similarity index 84% rename from pkg/internal/poetry/actions.go rename to pkg/shared/poetry/actions.go index b08f650a43..bca860415b 100644 --- a/pkg/internal/poetry/actions.go +++ b/pkg/shared/poetry/actions.go @@ -26,6 +26,29 @@ import ( "github.com/koderover/zadig/pkg/tool/httpclient" ) +func (c *Client) AddEnvRolePermission(productName, envName string, permissionUUIDs []string, roleID int, log *zap.SugaredLogger) (int, error) { + url := "/directory/roleEnv" + req := map[string]interface{}{ + "productName": productName, + "envName": envName, + "permissionUUIDs": permissionUUIDs, + "roleId": roleID, + } + + rm := &ResponseMessage{} + _, err := c.Post(url, httpclient.SetBody(req), httpclient.SetResult(rm)) + if err != nil { + log.Errorf("AddEnvRolePermission error: %v", err) + return 0, e.ErrListUsers.AddDesc(err.Error()) + } + + if rm.ResultCode == 0 { + return 1, nil + } + + return 0, e.ErrListUsers.AddDesc(fmt.Sprintf("ResultCode: %d", rm.ResultCode)) +} + func (c *Client) DeleteEnvRolePermission(productName, envName string, log *zap.SugaredLogger) (int, error) { url := "/directory/roleEnv" qs := map[string]string{ diff --git a/pkg/internal/poetry/auth.go b/pkg/shared/poetry/auth.go similarity index 100% rename from pkg/internal/poetry/auth.go rename to pkg/shared/poetry/auth.go diff --git a/pkg/internal/poetry/check.go b/pkg/shared/poetry/check.go similarity index 100% rename from pkg/internal/poetry/check.go rename to pkg/shared/poetry/check.go diff --git a/pkg/internal/poetry/client.go b/pkg/shared/poetry/client.go similarity index 100% rename from pkg/internal/poetry/client.go rename to pkg/shared/poetry/client.go diff --git a/pkg/internal/poetry/codehost.go b/pkg/shared/poetry/codehost.go similarity index 100% rename from pkg/internal/poetry/codehost.go rename to pkg/shared/poetry/codehost.go diff --git a/pkg/internal/poetry/jira.go b/pkg/shared/poetry/jira.go similarity index 100% rename from pkg/internal/poetry/jira.go rename to pkg/shared/poetry/jira.go diff --git a/pkg/internal/poetry/org.go b/pkg/shared/poetry/org.go similarity index 100% rename from pkg/internal/poetry/org.go rename to pkg/shared/poetry/org.go diff --git a/pkg/internal/poetry/permission.go b/pkg/shared/poetry/permission.go similarity index 100% rename from pkg/internal/poetry/permission.go rename to pkg/shared/poetry/permission.go diff --git a/pkg/internal/poetry/team.go b/pkg/shared/poetry/team.go similarity index 100% rename from pkg/internal/poetry/team.go rename to pkg/shared/poetry/team.go diff --git a/pkg/shared/poetry/token.go b/pkg/shared/poetry/token.go new file mode 100644 index 0000000000..bf31b540fc --- /dev/null +++ b/pkg/shared/poetry/token.go @@ -0,0 +1,36 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poetry + +import "github.com/koderover/zadig/pkg/tool/httpclient" + +type signature struct { + Token string `json:"token"` + Username string `json:"username"` +} + +func (c *Client) SignatureCheck(username, token string) error { + url := "/directory/token/check" + + s := &signature{Username: username, Token: token} + _, err := c.Post(url, httpclient.SetBody(s)) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/internal/poetry/types.go b/pkg/shared/poetry/types.go similarity index 100% rename from pkg/internal/poetry/types.go rename to pkg/shared/poetry/types.go diff --git a/pkg/internal/poetry/user.go b/pkg/shared/poetry/user.go similarity index 100% rename from pkg/internal/poetry/user.go rename to pkg/shared/poetry/user.go diff --git a/pkg/tool/errors/http_errors.go b/pkg/tool/errors/http_errors.go index d512bc752f..2c920d954c 100644 --- a/pkg/tool/errors/http_errors.go +++ b/pkg/tool/errors/http_errors.go @@ -106,6 +106,8 @@ var ( ErrUpdateServiceGroupTemplate = NewHTTPError(6056, "更新服务组失败") // ErrValidateServiceUpdate ErrValidateServiceUpdate = NewHTTPError(6057, "更新服务配置失败") + // ErrChartDryRun + ErrHelmDryRunFailed = NewHTTPError(6058, "helm chart --dry-run 失败,服务保存不成功") //----------------------------------------------------------------------------------------------- // Product APIs Range: 6060 - 6079 @@ -450,6 +452,8 @@ var ( ErrGetTestModule = NewHTTPError(6532, "获取测试模块失败") // ErrDeleteTestModule ... ErrDeleteTestModule = NewHTTPError(6533, "删除测试模块失败") + // ErrGetTestReport ... + ErrGetTestReport = NewHTTPError(6534, "获取html测试报告失败") // Workflow APIs Range: 6540 - 6550 //----------------------------------------------------------------------------------------------- diff --git a/pkg/tool/github/apps.go b/pkg/tool/git/github/apps.go similarity index 100% rename from pkg/tool/github/apps.go rename to pkg/tool/git/github/apps.go diff --git a/pkg/tool/github/checks.go b/pkg/tool/git/github/checks.go similarity index 100% rename from pkg/tool/github/checks.go rename to pkg/tool/git/github/checks.go diff --git a/pkg/tool/github/client.go b/pkg/tool/git/github/client.go similarity index 96% rename from pkg/tool/github/client.go rename to pkg/tool/git/github/client.go index 17c33ea2ca..e2f7bc5137 100644 --- a/pkg/tool/github/client.go +++ b/pkg/tool/git/github/client.go @@ -62,6 +62,9 @@ type Client struct { } func NewClient(cfg *Config) *Client { + if cfg == nil { + return &Client{Client: github.NewClient(nil)} + } httpClient := cfg.HttpClient if httpClient == nil { dc := http.DefaultClient @@ -210,7 +213,7 @@ func wrapError(res *github.Response, err error) error { if res.StatusCode > 399 { body, _ := io.ReadAll(res.Body) - return &httpclient.Error{Code: res.StatusCode, Status: res.Status, Detail: body} + return httpclient.NewGenericServerResponse(res.StatusCode, res.Request.Method, string(body)) } return nil diff --git a/pkg/tool/git/github/organizations.go b/pkg/tool/git/github/organizations.go new file mode 100644 index 0000000000..414785d1eb --- /dev/null +++ b/pkg/tool/git/github/organizations.go @@ -0,0 +1,45 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "context" + + "github.com/google/go-github/v35/github" +) + +func (c *Client) ListOrganizationsForAuthenticatedUser(ctx context.Context, opts *ListOptions) ([]*github.Organization, error) { + organizations, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { + os, r, err := c.Organizations.List(ctx, "", o) + var res []interface{} + for _, o := range os { + res = append(res, o) + } + return res, r, err + }, opts)) + + var res []*github.Organization + os, ok := organizations.([]interface{}) + if !ok { + return nil, nil + } + for _, o := range os { + res = append(res, o.(*github.Organization)) + } + + return res, err +} diff --git a/pkg/tool/github/pull_requests.go b/pkg/tool/git/github/pull_requests.go similarity index 100% rename from pkg/tool/github/pull_requests.go rename to pkg/tool/git/github/pull_requests.go diff --git a/pkg/tool/git/github/repositories.go b/pkg/tool/git/github/repositories.go new file mode 100644 index 0000000000..9b76568aab --- /dev/null +++ b/pkg/tool/git/github/repositories.go @@ -0,0 +1,171 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "context" + + "github.com/google/go-github/v35/github" + + "github.com/koderover/zadig/pkg/tool/git" +) + +func (c *Client) ListRepositoriesForAuthenticatedUser(ctx context.Context, opts *ListOptions) ([]*github.Repository, error) { + repositories, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { + rs, r, err := c.Repositories.List(ctx, "", &github.RepositoryListOptions{ListOptions: *o}) + var res []interface{} + for _, r := range rs { + res = append(res, r) + } + return res, r, err + }, opts)) + + var res []*github.Repository + rs, ok := repositories.([]interface{}) + if !ok { + return nil, nil + } + for _, r := range rs { + res = append(res, r.(*github.Repository)) + } + + return res, err +} + +func (c *Client) ListBranches(ctx context.Context, owner, repo string, opts *ListOptions) ([]*github.Branch, error) { + branches, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { + bs, r, err := c.Repositories.ListBranches(ctx, owner, repo, &github.BranchListOptions{ListOptions: *o}) + var res []interface{} + for _, b := range bs { + res = append(res, b) + } + return res, r, err + }, opts)) + + var res []*github.Branch + bs, ok := branches.([]interface{}) + if !ok { + return nil, nil + } + for _, b := range bs { + res = append(res, b.(*github.Branch)) + } + + return res, err +} + +func (c *Client) ListTags(ctx context.Context, owner, repo string, opts *ListOptions) ([]*github.RepositoryTag, error) { + tags, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { + ts, r, err := c.Repositories.ListTags(ctx, owner, repo, o) + var res []interface{} + for _, t := range ts { + res = append(res, t) + } + return res, r, err + }, opts)) + + var res []*github.RepositoryTag + ts, ok := tags.([]interface{}) + if !ok { + return nil, nil + } + for _, t := range ts { + res = append(res, t.(*github.RepositoryTag)) + } + + return res, err +} + +func (c *Client) ListHooks(ctx context.Context, owner, repo string, opts *ListOptions) ([]*github.Hook, error) { + hooks, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { + hs, r, err := c.Repositories.ListHooks(ctx, owner, repo, o) + var res []interface{} + for _, h := range hs { + res = append(res, h) + } + return res, r, err + }, opts)) + + var res []*github.Hook + hs, ok := hooks.([]interface{}) + if !ok { + return nil, nil + } + for _, hook := range hs { + res = append(res, hook.(*github.Hook)) + } + + return res, err +} + +func (c *Client) ListReleases(ctx context.Context, owner, repo string, opts *ListOptions) ([]*github.RepositoryRelease, error) { + releases, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { + hs, r, err := c.Repositories.ListReleases(ctx, owner, repo, o) + var res []interface{} + for _, h := range hs { + res = append(res, h) + } + return res, r, err + }, opts)) + + var res []*github.RepositoryRelease + hs, ok := releases.([]interface{}) + if !ok { + return nil, nil + } + for _, hook := range hs { + res = append(res, hook.(*github.RepositoryRelease)) + } + + return res, err +} + +func (c *Client) DeleteHook(ctx context.Context, owner, repo string, id int64) error { + return wrapError(c.Repositories.DeleteHook(ctx, owner, repo, id)) +} + +func (c *Client) CreateHook(ctx context.Context, owner, repo string, hook *git.Hook) (*github.Hook, error) { + h := &github.Hook{ + Config: map[string]interface{}{ + "url": hook.URL, + "content_type": "json", + "secret": hook.Secret, + }, + Events: hook.Events, + Active: hook.Active, + } + created, err := wrap(c.Repositories.CreateHook(ctx, owner, repo, h)) + if h, ok := created.(*github.Hook); ok { + return h, err + } + + return nil, err +} + +func (c *Client) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, error) { + created, err := wrap(c.Repositories.CreateStatus(ctx, owner, repo, ref, status)) + if s, ok := created.(*github.RepoStatus); ok { + return s, err + } + + return nil, err +} + +func (c *Client) GetContents(ctx context.Context, owner, repo, path string, opts *github.RepositoryContentGetOptions) (*github.RepositoryContent, []*github.RepositoryContent, error) { + fileContent, directoryContent, resp, err := c.Repositories.GetContents(ctx, owner, repo, path, opts) + return fileContent, directoryContent, wrapError(resp, err) +} diff --git a/pkg/tool/git/github/users.go b/pkg/tool/git/github/users.go new file mode 100644 index 0000000000..59dfcce89c --- /dev/null +++ b/pkg/tool/git/github/users.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "context" + + "github.com/google/go-github/v35/github" +) + +func (c *Client) GetAuthenticatedUser(ctx context.Context) (*github.User, error) { + ur, err := wrap(c.Users.Get(ctx, "")) + if u, ok := ur.(*github.User); ok { + return u, err + } + + return nil, err +} diff --git a/pkg/tool/git/gitlab/branch.go b/pkg/tool/git/gitlab/branch.go new file mode 100644 index 0000000000..40763dd06a --- /dev/null +++ b/pkg/tool/git/gitlab/branch.go @@ -0,0 +1,42 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import "github.com/xanzy/go-gitlab" + +// ListBranches lists branches by projectID <- urlEncode(namespace/projectName) +func (c *Client) ListBranches(owner, repo string, opts *ListOptions) ([]*gitlab.Branch, error) { + branches, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { + bs, r, err := c.Branches.ListBranches(generateProjectName(owner, repo), &gitlab.ListBranchesOptions{ListOptions: *o}) + var res []interface{} + for _, b := range bs { + res = append(res, b) + } + return res, r, err + }, opts)) + + var res []*gitlab.Branch + bs, ok := branches.([]interface{}) + if !ok { + return nil, nil + } + for _, b := range bs { + res = append(res, b.(*gitlab.Branch)) + } + + return res, err +} diff --git a/pkg/tool/git/gitlab/client.go b/pkg/tool/git/gitlab/client.go new file mode 100644 index 0000000000..f58270b6b6 --- /dev/null +++ b/pkg/tool/git/gitlab/client.go @@ -0,0 +1,108 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "fmt" + "io" + + "github.com/xanzy/go-gitlab" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +// TODO: LOU: unify the github/gitlab helpers + +type listFunc func(options *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) + +type ListOptions struct { + // Page number of the results to fetch. Default: 1 + Page int + // Results per page (max 100). Default: 30 + PerPage int + + // NoPaginated indicates if we need to fetch all result or just one page. True means fetching just one page + NoPaginated bool +} + +type Client struct { + *gitlab.Client +} + +func NewClient(address, accessToken string) (*Client, error) { + cli, err := gitlab.NewOAuthClient(accessToken, gitlab.WithBaseURL(address)) + if err != nil { + return nil, fmt.Errorf("failed to create gitlab client, err: %s", err) + } + + return &Client{Client: cli}, nil +} + +func generateProjectName(owner, repo string) string { + return fmt.Sprintf("%s/%s", owner, repo) +} + +func paginated(f listFunc, opts *ListOptions) ([]interface{}, *gitlab.Response, error) { + if opts == nil { + opts = &ListOptions{} + } + if opts.Page == 0 { + opts.Page = 1 + } + if opts.PerPage == 0 { + opts.PerPage = 100 + } + ghOpts := &gitlab.ListOptions{Page: opts.Page, PerPage: opts.PerPage} + + if opts.NoPaginated { + return f(ghOpts) + } + + var all, result []interface{} + var response *gitlab.Response + var err error + + for ghOpts.Page > 0 { + result, response, err = f(ghOpts) + err = wrapError(response, err) + if err != nil { + return nil, response, err + } + + all = append(all, result...) + ghOpts.Page = response.NextPage + } + + return all, response, err +} + +func wrap(obj interface{}, res *gitlab.Response, err error) (interface{}, error) { + return obj, wrapError(res, err) +} + +func wrapError(res *gitlab.Response, err error) error { + if err != nil { + return err + } + + if res.StatusCode > 399 { + body, _ := io.ReadAll(res.Body) + return httpclient.NewGenericServerResponse(res.StatusCode, res.Request.Method, string(body)) + } + + return nil +} diff --git a/pkg/tool/git/gitlab/commit.go b/pkg/tool/git/gitlab/commit.go new file mode 100644 index 0000000000..a27ea21c34 --- /dev/null +++ b/pkg/tool/git/gitlab/commit.go @@ -0,0 +1,55 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "github.com/xanzy/go-gitlab" +) + +func (c *Client) GetLatestCommit(owner, repo string, branch, path string) (*gitlab.Commit, error) { + // List commits with specified ref and path + opts := &gitlab.ListCommitsOptions{ + Path: &path, + RefName: &branch, + ListOptions: gitlab.ListOptions{ + PerPage: 1, + Page: 1, + }, + } + commits, err := wrap(c.Commits.ListCommits(generateProjectName(owner, repo), opts)) + if err != nil { + return nil, err + } + cs, ok := commits.([]*gitlab.Commit) + if !ok || len(cs) == 0 { + return nil, nil + } + + return cs[0], nil +} + +func (c *Client) GetSingleCommitOfProject(owner, repo, commitSha string) (*gitlab.Commit, error) { + commit, err := wrap(c.Commits.GetCommit(generateProjectName(owner, repo), commitSha)) + if err != nil { + return nil, err + } + if ct, ok := commit.(*gitlab.Commit); ok { + return ct, nil + } + + return nil, err +} diff --git a/pkg/tool/git/gitlab/group_project.go b/pkg/tool/git/gitlab/group_project.go new file mode 100644 index 0000000000..17ebe83b41 --- /dev/null +++ b/pkg/tool/git/gitlab/group_project.go @@ -0,0 +1,56 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "github.com/xanzy/go-gitlab" +) + +func (c *Client) ListGroupProjects(namespace, keyword string, opts *ListOptions) ([]*gitlab.Project, error) { + projects, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { + gopts := &gitlab.ListGroupProjectsOptions{ + ListOptions: *o, + } + + orderBy := "name" + sort := "asc" + gopts.OrderBy = &orderBy + gopts.Sort = &sort + + // gitlab search works only when character length > 2 + if keyword != "" && len(keyword) > 2 { + gopts.Search = &keyword + } + ps, r, err := c.Groups.ListGroupProjects(namespace, gopts) + var res []interface{} + for _, p := range ps { + res = append(res, p) + } + return res, r, err + }, opts)) + + var res []*gitlab.Project + ps, ok := projects.([]interface{}) + if !ok { + return nil, nil + } + for _, p := range ps { + res = append(res, p.(*gitlab.Project)) + } + + return res, err +} diff --git a/pkg/tool/git/gitlab/merge_request.go b/pkg/tool/git/gitlab/merge_request.go new file mode 100644 index 0000000000..baa84c00ac --- /dev/null +++ b/pkg/tool/git/gitlab/merge_request.go @@ -0,0 +1,74 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import "github.com/xanzy/go-gitlab" + +func (c *Client) ListOpenedProjectMergeRequests(owner, repo, targetBranch string, opts *ListOptions) ([]*gitlab.MergeRequest, error) { + mergeRequests, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { + state := "opened" + + mopts := &gitlab.ListProjectMergeRequestsOptions{ + State: &state, + ListOptions: *o, + } + if targetBranch != "" { + mopts.TargetBranch = &targetBranch + } + mrs, r, err := c.MergeRequests.ListProjectMergeRequests(generateProjectName(owner, repo), mopts) + var res []interface{} + for _, mr := range mrs { + res = append(res, mr) + } + return res, r, err + }, opts)) + + var res []*gitlab.MergeRequest + mrs, ok := mergeRequests.([]interface{}) + if !ok { + return nil, nil + } + for _, mr := range mrs { + res = append(res, mr.(*gitlab.MergeRequest)) + } + + return res, err +} + +func (c *Client) ListChangedFiles(event *gitlab.MergeEvent) ([]string, error) { + files := make([]string, 0) + mergeRequest, err := wrap(c.MergeRequests.GetMergeRequestChanges(event.ObjectAttributes.TargetProjectID, event.ObjectAttributes.IID, nil)) + if err != nil || mergeRequest == nil { + return nil, err + } + mr, ok := mergeRequest.(*gitlab.MergeRequest) + if !ok { + return nil, nil + } + for _, change := range mr.Changes { + files = append(files, change.NewPath) + files = append(files, change.OldPath) + } + + return files, nil +} + +func (c *Client) CreateCommitDiscussion(owner, repo, commitHash, comment string) error { + args := &gitlab.CreateCommitDiscussionOptions{Body: &comment} + _, err := wrap(c.Discussions.CreateCommitDiscussion(generateProjectName(owner, repo), commitHash, args)) + return err +} diff --git a/pkg/tool/git/gitlab/namespace.go b/pkg/tool/git/gitlab/namespace.go new file mode 100644 index 0000000000..cb7f594693 --- /dev/null +++ b/pkg/tool/git/gitlab/namespace.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "github.com/xanzy/go-gitlab" +) + +func (c *Client) ListNamespaces(keyword string, opts *ListOptions) ([]*gitlab.Namespace, error) { + namespaces, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { + mopts := &gitlab.ListNamespacesOptions{ + ListOptions: *o, + } + // gitlab search works only when character length > 2 + if keyword != "" && len(keyword) > 2 { + mopts.Search = &keyword + } + ns, r, err := c.Namespaces.ListNamespaces(mopts) + var res []interface{} + for _, n := range ns { + res = append(res, n) + } + return res, r, err + }, opts)) + + var res []*gitlab.Namespace + ns, ok := namespaces.([]interface{}) + if !ok { + return nil, nil + } + for _, n := range ns { + res = append(res, n.(*gitlab.Namespace)) + } + + return res, err +} diff --git a/pkg/tool/git/gitlab/project.go b/pkg/tool/git/gitlab/project.go new file mode 100644 index 0000000000..10874fc5bd --- /dev/null +++ b/pkg/tool/git/gitlab/project.go @@ -0,0 +1,137 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "strings" + + "github.com/xanzy/go-gitlab" + + "github.com/koderover/zadig/pkg/tool/git" + "github.com/koderover/zadig/pkg/util/boolptr" +) + +func (c *Client) ListUserProjects(owner, keyword string, opts *ListOptions) ([]*gitlab.Project, error) { + projects, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { + encodeOwner := strings.Replace(owner, ".", "%2e", -1) + popts := &gitlab.ListProjectsOptions{ + ListOptions: *o, + } + + if keyword == "" { + popts.Search = &keyword + } + ps, r, err := c.Projects.ListUserProjects(encodeOwner, popts) + var res []interface{} + for _, p := range ps { + res = append(res, p) + } + return res, r, err + }, opts)) + + var res []*gitlab.Project + ps, ok := projects.([]interface{}) + if !ok { + return nil, nil + } + for _, p := range ps { + res = append(res, p.(*gitlab.Project)) + } + + return res, err +} + +func (c *Client) AddProjectHook(owner, repo string, hook *git.Hook) (*gitlab.ProjectHook, error) { + opts := &gitlab.AddProjectHookOptions{ + URL: &hook.URL, + Token: &hook.Secret, + } + opts = addEventsToProjectHookOptions(hook.Events, opts) + created, err := wrap(c.Projects.AddProjectHook(generateProjectName(owner, repo), opts)) + if err != nil { + return nil, err + } + + if h, ok := created.(*gitlab.ProjectHook); ok { + return h, nil + } + + return nil, err +} + +func (c *Client) DeleteProjectHook(owner, repo string, id int) error { + return wrapError(c.Projects.DeleteProjectHook(generateProjectName(owner, repo), id)) +} + +func (c *Client) ListProjectHooks(owner, repo string, opts *ListOptions) ([]*gitlab.ProjectHook, error) { + hooks, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { + hs, r, err := c.Projects.ListProjectHooks(generateProjectName(owner, repo), (*gitlab.ListProjectHooksOptions)(o)) + var res []interface{} + for _, h := range hs { + res = append(res, h) + } + return res, r, err + }, opts)) + + var res []*gitlab.ProjectHook + hs, ok := hooks.([]interface{}) + if !ok { + return nil, nil + } + for _, hook := range hs { + res = append(res, hook.(*gitlab.ProjectHook)) + } + + return res, err +} + +func (c *Client) GetProjectID(owner, repo string) (int, error) { + p, err := c.getProject(owner, repo) + if err != nil { + return 0, err + } + + return p.ID, nil +} + +func (c *Client) getProject(owner, repo string) (*gitlab.Project, error) { + project, err := wrap(c.Projects.GetProject(generateProjectName(owner, repo), nil)) + if err != nil { + return nil, err + } + + if p, ok := project.(*gitlab.Project); ok { + return p, nil + } + + return nil, err +} + +func addEventsToProjectHookOptions(events []string, opts *gitlab.AddProjectHookOptions) *gitlab.AddProjectHookOptions { + for _, evt := range events { + switch evt { + case git.PushEvent: + opts.PushEvents = boolptr.True() + case git.PullRequestEvent: + opts.MergeRequestsEvents = boolptr.True() + case git.BranchOrTagCreateEvent: + opts.TagPushEvents = boolptr.True() + } + } + + return opts +} diff --git a/pkg/tool/git/gitlab/repo.go b/pkg/tool/git/gitlab/repo.go new file mode 100644 index 0000000000..2d9c3f6e50 --- /dev/null +++ b/pkg/tool/git/gitlab/repo.go @@ -0,0 +1,92 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "encoding/base64" + + "github.com/xanzy/go-gitlab" +) + +func (c *Client) ListTree(owner, repo string, ref string, path string) ([]*gitlab.TreeNode, error) { + // Recursive default value is false, + opts := &gitlab.ListTreeOptions{ + Ref: &ref, + Path: &path, + } + + tn, err := wrap(c.Repositories.ListTree(generateProjectName(owner, repo), opts)) + if t, ok := tn.([]*gitlab.TreeNode); ok { + return t, err + } + + return nil, err +} + +func (c *Client) GetRawFile(owner, repo string, sha string, fileName string) ([]byte, error) { + opts := &gitlab.GetFileOptions{ + Ref: &sha, + } + + f, err := wrap(c.RepositoryFiles.GetFile(generateProjectName(owner, repo), fileName, opts)) + if err != nil { + return nil, err + } + file, ok := f.(*gitlab.File) + if !ok { + return nil, err + } + ct, err := wrap(c.Repositories.RawBlobContent(generateProjectName(owner, repo), file.BlobID)) + if t, ok := ct.([]byte); ok { + return t, err + } + + return nil, err +} + +func (c *Client) GetFileContent(owner, repo string, ref, path string) ([]byte, error) { + opts := &gitlab.GetFileOptions{ + Ref: gitlab.String(ref), + } + f, err := wrap(c.RepositoryFiles.GetFile(generateProjectName(owner, repo), path, opts)) + if err != nil { + return nil, err + } + file, ok := f.(*gitlab.File) + if !ok { + return nil, err + } + + return base64.StdEncoding.DecodeString(file.Content) +} + +func (c *Client) Compare(projectID int, from, to string) ([]*gitlab.Diff, error) { + opts := &gitlab.CompareOptions{ + From: &from, + To: &to, + } + + compare, err := wrap(c.Repositories.Compare(projectID, opts)) + if err != nil { + return nil, err + } + if cp, ok := compare.(*gitlab.Compare); ok { + return cp.Diffs, nil + } + + return nil, err +} diff --git a/pkg/tool/git/gitlab/tag.go b/pkg/tool/git/gitlab/tag.go new file mode 100644 index 0000000000..0ffb86e5fb --- /dev/null +++ b/pkg/tool/git/gitlab/tag.go @@ -0,0 +1,42 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import "github.com/xanzy/go-gitlab" + +// ListTags lists branches by projectID <- urlEncode(namespace/projectName) +func (c *Client) ListTags(owner, repo string, opts *ListOptions) ([]*gitlab.Tag, error) { + tags, err := wrap(paginated(func(o *gitlab.ListOptions) ([]interface{}, *gitlab.Response, error) { + ts, r, err := c.Tags.ListTags(generateProjectName(owner, repo), &gitlab.ListTagsOptions{ListOptions: *o}) + var res []interface{} + for _, t := range ts { + res = append(res, t) + } + return res, r, err + }, opts)) + + var res []*gitlab.Tag + ts, ok := tags.([]interface{}) + if !ok { + return nil, nil + } + for _, t := range ts { + res = append(res, t.(*gitlab.Tag)) + } + + return res, err +} diff --git a/pkg/tool/git/hook.go b/pkg/tool/git/hook.go new file mode 100644 index 0000000000..d7780b0ad4 --- /dev/null +++ b/pkg/tool/git/hook.go @@ -0,0 +1,31 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +const ( + PushEvent = "push" + PullRequestEvent = "pull_request" + CheckRunEvent = "check_run" + BranchOrTagCreateEvent = "create" +) + +type Hook struct { + URL string `json:"url,omitempty"` + Secret string `json:"secret,omitempty"` + Events []string `json:"events,omitempty"` + Active *bool `json:"active,omitempty"` +} diff --git a/pkg/tool/github/repositories.go b/pkg/tool/github/repositories.go deleted file mode 100644 index 81d8b7035b..0000000000 --- a/pkg/tool/github/repositories.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package github - -import ( - "context" - - "github.com/google/go-github/v35/github" -) - -func (c *Client) ListHooks(ctx context.Context, owner, repo string, opts *ListOptions) ([]*github.Hook, error) { - hooks, err := wrap(paginated(func(o *github.ListOptions) ([]interface{}, *github.Response, error) { - hs, r, err := c.Repositories.ListHooks(ctx, owner, repo, o) - var res []interface{} - for _, h := range hs { - res = append(res, h) - } - return res, r, err - }, opts)) - - var res []*github.Hook - hs, ok := hooks.([]interface{}) - if !ok { - return nil, nil - } - for _, hook := range hs { - res = append(res, hook.(*github.Hook)) - } - - return res, err -} - -func (c *Client) DeleteHook(ctx context.Context, owner, repo string, id int64) error { - return wrapError(c.Repositories.DeleteHook(ctx, owner, repo, id)) -} - -func (c *Client) CreateHook(ctx context.Context, owner, repo string, hook *github.Hook) (*github.Hook, error) { - created, err := wrap(c.Repositories.CreateHook(ctx, owner, repo, hook)) - if h, ok := created.(*github.Hook); ok { - return h, err - } - - return nil, err -} - -func (c *Client) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, error) { - created, err := wrap(c.Repositories.CreateStatus(ctx, owner, repo, ref, status)) - if s, ok := created.(*github.RepoStatus); ok { - return s, err - } - - return nil, err -} - -func (c *Client) GetContents(ctx context.Context, owner, repo, path string, opts *github.RepositoryContentGetOptions) (*github.RepositoryContent, []*github.RepositoryContent, error) { - fileContent, directoryContent, resp, err := c.Repositories.GetContents(ctx, owner, repo, path, opts) - return fileContent, directoryContent, wrapError(resp, err) -} diff --git a/pkg/tool/gitlab/branch.go b/pkg/tool/gitlab/branch.go deleted file mode 100644 index 77058e793a..0000000000 --- a/pkg/tool/gitlab/branch.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import "github.com/xanzy/go-gitlab" - -type Branch struct { - Name string `json:"name"` - Protected bool `json:"protected"` - Merged bool `json:"merged"` -} - -// List branches by projectID <- urlEncode(namespace/projectName) -func (c *Client) ListBranches(projectID string) ([]*Branch, error) { - opts := &gitlab.ListBranchesOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 100, - Page: 1, - }, - } - - branches, _, err := c.Branches.ListBranches(projectID, opts) - if err != nil { - return nil, err - } - - var respBs []*Branch - for _, branch := range branches { - respBs = append(respBs, &Branch{ - Name: branch.Name, - Protected: branch.Protected, - Merged: branch.Merged, - }) - } - return respBs, nil -} diff --git a/pkg/tool/gitlab/commit.go b/pkg/tool/gitlab/commit.go deleted file mode 100644 index c76594f472..0000000000 --- a/pkg/tool/gitlab/commit.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import ( - "time" - - "github.com/xanzy/go-gitlab" -) - -// RepoCommit : Repository commit struct -type RepoCommit struct { - ID string `json:"id"` - Title string `json:"title"` - AuthorName string `json:"author_name"` - CreatedAt *time.Time `json:"created_at"` - Message string `json:"message"` -} - -func (c *Client) GetLatestCommit(projectID int, branch, path string) (*RepoCommit, error) { - commit := &RepoCommit{} - // List commits with specified ref and path - opts := &gitlab.ListCommitsOptions{ - Path: &path, - RefName: &branch, - ListOptions: gitlab.ListOptions{ - PerPage: 1, - Page: 1, - }, - } - commits, _, err := c.Commits.ListCommits(projectID, opts) - if err != nil { - return commit, err - } - if len(commits) > 0 { - commit.ID = commits[0].ID - commit.Title = commits[0].Title - commit.AuthorName = commits[0].AuthorName - commit.CreatedAt = commits[0].CreatedAt - commit.Message = commits[0].Message - } - return commit, nil -} - -// GetLatestCommit By string pro -func (c *Client) GetLatestCommitByProName(projectID, branch, path string) (*RepoCommit, error) { - commit := &RepoCommit{} - // List commits with specified ref and path - opts := &gitlab.ListCommitsOptions{ - RefName: &branch, - ListOptions: gitlab.ListOptions{ - PerPage: 1, - Page: 1, - }, - } - commits, _, err := c.Commits.ListCommits(projectID, opts) - if err != nil { - return commit, err - } - if len(commits) > 0 { - commit.ID = commits[0].ID - commit.Title = commits[0].Title - commit.AuthorName = commits[0].AuthorName - commit.CreatedAt = commits[0].CreatedAt - commit.Message = commits[0].Message - } - return commit, nil -} - -func (c *Client) GetSingleCommitOfProject(address, token, projectPath, commitSha string) (*gitlab.Commit, error) { - commit, _, err := c.Commits.GetCommit(projectPath, commitSha) - if err != nil { - return nil, err - } - - return commit, nil -} diff --git a/pkg/tool/gitlab/group_project.go b/pkg/tool/gitlab/group_project.go deleted file mode 100644 index bc4af7c5c5..0000000000 --- a/pkg/tool/gitlab/group_project.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import ( - "strings" - - "github.com/xanzy/go-gitlab" -) - -func (c *Client) ListGroupProjects(namespace, keyword string) ([]*Project, error) { - respProjs := make([]*Project, 0) - - opts := &gitlab.ListGroupProjectsOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 100, - Page: 1, - }, - } - - orderBy := "name" - sort := "asc" - opts.OrderBy = &orderBy - opts.Sort = &sort - - // gitlab search works only when character length > 2 - if keyword != "" && len(keyword) > 2 { - opts.Search = &keyword - } - - projs, _, err := c.Groups.ListGroupProjects(namespace, opts) - if err != nil { - return nil, err - } - - for _, proj := range projs { - respProj := &Project{ - ID: proj.ID, - Name: proj.Path, - Description: proj.Description, - DefaultBranch: proj.DefaultBranch, - Namespace: proj.Namespace.FullPath, - } - - if len(keyword) > 0 && !strings.Contains(strings.ToLower(proj.Path), strings.ToLower(keyword)) { - // filter - } else { - respProjs = append(respProjs, respProj) - } - } - - return filterProjectsByNamespace(respProjs, namespace), nil -} - -func filterProjectsByNamespace(pros []*Project, namespace string) []*Project { - r := make([]*Project, 0) - for _, project := range pros { - if project.Namespace == namespace { - r = append(r, project) - } - } - return r -} diff --git a/pkg/tool/gitlab/merge_request.go b/pkg/tool/gitlab/merge_request.go deleted file mode 100644 index ce45cc35dc..0000000000 --- a/pkg/tool/gitlab/merge_request.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import "github.com/xanzy/go-gitlab" - -type MergeRequest struct { - MRID int `json:"id"` - TargetBranch string `json:"targetBranch"` - SourceBranch string `json:"sourceBranch"` - ProjectID int `json:"projectId"` - Title string `json:"title"` - State string `json:"state"` - CreatedAt int64 `json:"createdAt"` - UpdatedAt int64 `json:"updatedAt"` - AuthorUsername string `json:"authorUsername"` - Number int `json:"number"` - User string `json:"user"` - Base string `json:"base,omitempty"` -} - -func (c *Client) ListOpened(projectID, targetBranch string) ([]*MergeRequest, error) { - state := "opened" - - opts := &gitlab.ListProjectMergeRequestsOptions{ - State: &state, - ListOptions: gitlab.ListOptions{ - PerPage: 100, - Page: 1, - }, - } - if targetBranch != "" { - opts.TargetBranch = &targetBranch - } - - var respMRs []*MergeRequest - - for opts.ListOptions.Page > 0 { - - mrs, resp, err := c.MergeRequests.ListProjectMergeRequests(projectID, opts) - if err != nil { - return nil, err - } - - for _, mr := range mrs { - req := &MergeRequest{ - MRID: mr.IID, - TargetBranch: mr.TargetBranch, - SourceBranch: mr.SourceBranch, - ProjectID: mr.ProjectID, - Title: mr.Title, - State: mr.State, - CreatedAt: mr.CreatedAt.Unix(), - UpdatedAt: mr.UpdatedAt.Unix(), - AuthorUsername: mr.Author.Username, - } - respMRs = append(respMRs, req) - } - - opts.ListOptions.Page = resp.NextPage - } - return respMRs, nil -} - -func (c *Client) ListChangedFiles(address, token string, event *gitlab.MergeEvent) ([]string, error) { - files := make([]string, 0) - mergeRequest, _, err := c.MergeRequests.GetMergeRequestChanges(event.ObjectAttributes.TargetProjectID, event.ObjectAttributes.IID, nil) - if err != nil || mergeRequest == nil { - return nil, err - } - for _, change := range mergeRequest.Changes { - files = append(files, change.NewPath) - files = append(files, change.OldPath) - } - - return files, nil -} - -func (c *Client) CreateCommitDiscussion(projectPath, commitHash, comment string) error { - args := &gitlab.CreateCommitDiscussionOptions{Body: &comment} - _, _, err := c.Discussions.CreateCommitDiscussion(projectPath, commitHash, args) - return err -} diff --git a/pkg/tool/gitlab/namespace.go b/pkg/tool/gitlab/namespace.go deleted file mode 100644 index 6b413cb163..0000000000 --- a/pkg/tool/gitlab/namespace.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import ( - "strings" - - "github.com/xanzy/go-gitlab" -) - -type Namespace struct { - Name string `json:"name"` - Path string `json:"path"` - Kind string `json:"kind"` -} - -func (c *Client) ListNamespaces(keyword string) ([]*Namespace, error) { - opts := &gitlab.ListNamespacesOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 100, - Page: 1, - }, - } - - // gitlab search works only when character length > 2 - if keyword != "" && len(keyword) > 2 { - opts.Search = &keyword - } - - var respNs []*Namespace - namespaces, _, err := c.Namespaces.ListNamespaces(opts) - if err != nil { - return nil, err - } - - for _, ns := range namespaces { - respN := &Namespace{ - Name: ns.Path, - Path: ns.FullPath, - Kind: ns.Kind, - } - - if len(keyword) > 0 && !strings.Contains(strings.ToLower(ns.Path), strings.ToLower(keyword)) { - // filter - } else { - respNs = append(respNs, respN) - } - } - - return respNs, nil -} diff --git a/pkg/tool/gitlab/project.go b/pkg/tool/gitlab/project.go deleted file mode 100644 index 8b247fe5ab..0000000000 --- a/pkg/tool/gitlab/project.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import ( - "fmt" - "strings" - - "github.com/xanzy/go-gitlab" -) - -type Project struct { - ID int `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - DefaultBranch string `json:"defaultBranch"` - Namespace string `json:"namespace"` -} - -const DefaultOrgID = 1 - -func (c *Client) ListUserProjects(owner, keyword string) ([]*Project, error) { - encodeOwner := strings.Replace(owner, ".", "%2e", -1) - opts := &gitlab.ListProjectsOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 100, - Page: 1, - }, - } - - if keyword == "" { - opts.Search = &keyword - } - - respProjs := make([]*Project, 0) - - projs, _, err := c.Projects.ListUserProjects(encodeOwner, opts) - if err != nil { - return nil, err - } - - for _, proj := range projs { - respProj := &Project{ - ID: proj.ID, - Name: proj.Path, - Namespace: proj.Namespace.FullPath, - Description: proj.Description, - DefaultBranch: proj.DefaultBranch, - } - if len(keyword) > 0 && !strings.Contains(strings.ToLower(proj.Path), strings.ToLower(keyword)) { - // filter - } else { - respProjs = append(respProjs, respProj) - } - } - - return filterProjectsByNamespace(respProjs, owner), nil -} - -func (c *Client) GetProject(owner, repo string) (*Project, error) { - proj, _, err := c.Projects.GetProject(fmt.Sprintf("%s/%s", owner, repo), nil) - if err != nil { - return nil, err - } - resp := &Project{ - ID: proj.ID, - Name: proj.Name, - Description: proj.Description, - DefaultBranch: proj.DefaultBranch, - } - return resp, nil -} diff --git a/pkg/tool/gitlab/repo.go b/pkg/tool/gitlab/repo.go deleted file mode 100644 index 6b6ab21965..0000000000 --- a/pkg/tool/gitlab/repo.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import ( - "encoding/base64" - - "github.com/xanzy/go-gitlab" -) - -func (c *Client) ListTree(projectID int, ref string, path string) ([]*gitlab.TreeNode, error) { - // Recursive default value is false, - opts := &gitlab.ListTreeOptions{ - Ref: &ref, - Path: &path, - } - - nodes, _, err := c.Repositories.ListTree(projectID, opts) - return nodes, err -} - -func (c *Client) GetRawFile(projectID int, sha string, fileName string) (content []byte, err error) { - content = make([]byte, 0) - opts := &gitlab.GetFileOptions{ - Ref: &sha, - } - - file, _, err := c.RepositoryFiles.GetFile(projectID, fileName, opts) - if err != nil { - return content, err - } - content, _, err = c.Repositories.RawBlobContent(projectID, file.BlobID) - return content, err -} - -func (c *Client) GetFileContent(projectID int, ref, path string) ([]byte, error) { - opts := &gitlab.GetFileOptions{ - Ref: gitlab.String(ref), - } - fileContent, _, err := c.RepositoryFiles.GetFile(projectID, path, opts) - if err != nil { - return nil, err - } - content, err := base64.StdEncoding.DecodeString(fileContent.Content) - return content, err -} - -func (c *Client) Compare(projectID int, from, to string) ([]*gitlab.Diff, error) { - opts := &gitlab.CompareOptions{ - From: &from, - To: &to, - } - compare, _, err := c.Repositories.Compare(projectID, opts) - if err != nil { - return nil, err - } - - return compare.Diffs, nil -} diff --git a/pkg/tool/gitlab/tag.go b/pkg/tool/gitlab/tag.go deleted file mode 100644 index 9b3c896967..0000000000 --- a/pkg/tool/gitlab/tag.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gitlab - -import "github.com/xanzy/go-gitlab" - -type Tag struct { - Name string `json:"name"` - ZipballURL string `json:"zipball_url"` - TarballURL string `json:"tarball_url"` - Message string `json:"message"` -} - -// List branches by projectID <- urlEncode(namespace/projectName) -func (c *Client) ListTags(projectID string) ([]*Tag, error) { - opts := &gitlab.ListTagsOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 100, - Page: 1, - }, - } - - respTags := make([]*Tag, 0) - - for opts.Page > 0 { - tags, resp, err := c.Tags.ListTags(projectID, opts) - if err != nil { - return nil, err - } - - for _, branch := range tags { - respB := &Tag{ - Name: branch.Name, - Message: branch.Message, - } - respTags = append(respTags, respB) - } - opts.Page = resp.NextPage - } - return respTags, nil -} diff --git a/pkg/tool/helmclient/helmclient.go b/pkg/tool/helmclient/helmclient.go index 601f63cc84..d540cc9bcf 100644 --- a/pkg/tool/helmclient/helmclient.go +++ b/pkg/tool/helmclient/helmclient.go @@ -605,6 +605,7 @@ func mergeInstallOptions(chartSpec *ChartSpec, installOptions *action.Install) { installOptions.Atomic = chartSpec.Atomic installOptions.SkipCRDs = chartSpec.SkipCRDs installOptions.SubNotes = chartSpec.SubNotes + installOptions.DryRun = chartSpec.DryRun } // mergeUpgradeOptions merges values of the provided chart to helm upgrade options used by the client @@ -622,6 +623,7 @@ func mergeUpgradeOptions(chartSpec *ChartSpec, upgradeOptions *action.Upgrade) { upgradeOptions.Atomic = chartSpec.Atomic upgradeOptions.CleanupOnFail = chartSpec.CleanupOnFail upgradeOptions.SubNotes = chartSpec.SubNotes + upgradeOptions.DryRun = chartSpec.DryRun } // mergeUninstallReleaseOptions merges values of the provided chart to helm uninstall options used by the client diff --git a/pkg/tool/helmclient/types.go b/pkg/tool/helmclient/types.go index 8c85001a1f..b5134ec2c0 100644 --- a/pkg/tool/helmclient/types.go +++ b/pkg/tool/helmclient/types.go @@ -53,6 +53,7 @@ type RESTClientGetter struct { namespace string kubeConfig []byte restConfig *rest.Config + DryRun bool } // Client defines the values of a helm client @@ -62,6 +63,7 @@ type HelmClient struct { storage *repo.File ActionConfig *action.Configuration linting bool + DryRun bool } // ChartSpec defines the values of a helm chart @@ -129,6 +131,9 @@ type ChartSpec struct { // +optional CleanupOnFail bool `json:"cleanupOnFail,omitempty"` + + // +optional + DryRun bool `json:"dry_run,omitempty"` } type ChartOption struct { diff --git a/pkg/tool/httpclient/client.go b/pkg/tool/httpclient/client.go index 1474d46ff5..66c6844026 100644 --- a/pkg/tool/httpclient/client.go +++ b/pkg/tool/httpclient/client.go @@ -145,7 +145,7 @@ func wrapError(res *resty.Response, err error) (*resty.Response, error) { } if res.IsError() { - return nil, &Error{Code: res.StatusCode(), Status: res.Status(), Detail: res.Body()} + return nil, NewErrorFromRestyResponse(res) } return res, nil diff --git a/pkg/tool/httpclient/errors.go b/pkg/tool/httpclient/errors.go index fd30a6e400..47ba984dce 100644 --- a/pkg/tool/httpclient/errors.go +++ b/pkg/tool/httpclient/errors.go @@ -16,16 +16,204 @@ limitations under the License. package httpclient -import "fmt" +import ( + "errors" + "fmt" + "net/http" + + "github.com/go-resty/resty/v2" +) + +// StatusReason is an enumeration of possible failure causes. Each StatusReason +// must map to a single HTTP status code, but multiple reasons may map +// to the same HTTP status code. +type StatusReason string + +const ( + // StatusReasonUnknown means the server has declined to indicate a specific reason. + StatusReasonUnknown StatusReason = "" + + // StatusReasonBadRequest means that the request itself was invalid, because the request + // doesn't make any sense, for example deleting a read-only object. This is different than + // StatusReasonInvalid above which indicates that the API call could possibly succeed, but the + // data was invalid. API calls that return BadRequest can never succeed. + // Status code 400 + StatusReasonBadRequest StatusReason = "BadRequest" + + // StatusReasonUnauthorized means the server can be reached and understood the request, but requires + // the user to present appropriate authorization credentials (identified by the WWW-Authenticate header) + // in order for the action to be completed. If the user has specified credentials on the request, the + // server considers them insufficient. + // Status code 401 + StatusReasonUnauthorized StatusReason = "Unauthorized" + + // StatusReasonForbidden means the server can be reached and understood the request, but refuses + // to take any further action. It is the result of the server being configured to deny access for some reason + // to the requested resource by the client. + // Status code 403 + StatusReasonForbidden StatusReason = "Forbidden" + + // StatusReasonNotFound means one or more resources required for this operation + // could not be found. + // Status code 404 + StatusReasonNotFound StatusReason = "NotFound" + + // StatusReasonMethodNotAllowed means that the action the client attempted to perform on the + // resource was not supported by the code - for instance, attempting to delete a resource that + // can only be created. API calls that return MethodNotAllowed can never succeed. + // Status code 405 + StatusReasonMethodNotAllowed StatusReason = "MethodNotAllowed" + + // StatusReasonNotAcceptable means that the accept types indicated by the client were not acceptable + // to the server - for instance, attempting to receive protobuf for a resource that supports only json and yaml. + // API calls that return NotAcceptable can never succeed. + // Status code 406 + StatusReasonNotAcceptable StatusReason = "NotAcceptable" + + // StatusReasonAlreadyExists means the resource you are creating already exists. + // Status code 409 + StatusReasonAlreadyExists StatusReason = "AlreadyExists" + + // StatusReasonConflict means the requested operation cannot be completed + // due to a conflict in the operation. The client may need to alter the + // request. Each resource may define custom details that indicate the + // nature of the conflict. + // Status code 409 + StatusReasonConflict StatusReason = "Conflict" + + // StatusReasonGone means the item is no longer available at the server and no + // forwarding address is known. + // Status code 410 + StatusReasonGone StatusReason = "Gone" + + // StatusReasonUnsupportedMediaType means that the content type sent by the client is not acceptable + // to the server - for instance, attempting to send protobuf for a resource that supports only json and yaml. + // API calls that return UnsupportedMediaType can never succeed. + // Status code 415 + StatusReasonUnsupportedMediaType StatusReason = "UnsupportedMediaType" + + // StatusReasonInvalid means the requested create or update operation cannot be + // completed due to invalid data provided as part of the request. The client may + // need to alter the request. + // Status code 422 + StatusReasonInvalid StatusReason = "Invalid" + + // StatusReasonTooManyRequests means the server experienced too many requests within a + // given window and that the client must wait to perform the action again. A client may + // always retry the request that led to this error, although the client should wait at least + // the number of seconds specified by the retryAfterSeconds field. + // Status code 429 + StatusReasonTooManyRequests StatusReason = "TooManyRequests" + + // StatusReasonInternalError indicates that an internal error occurred, it is unexpected + // and the outcome of the call is unknown. + // Status code 500 + StatusReasonInternalError StatusReason = "InternalError" + + // StatusReasonServiceUnavailable means that the request itself was valid, + // but the requested service is unavailable at this time. + // Retrying the request after some time might succeed. + // Status code 503 + StatusReasonServiceUnavailable StatusReason = "ServiceUnavailable" +) + +type httpStatus interface { + Status() StatusReason +} type Error struct { - Code int - Status string - Detail []byte + Code int + ErrStatus StatusReason + Message string + Detail string } -func (e Error) Error() string { - return fmt.Sprintf("[%s] %s", e.Status, e.Detail) +func (e *Error) Error() string { + return fmt.Sprintf("[%d %s] %s", e.Code, e.ErrStatus, e.Detail) +} + +func (e *Error) Status() StatusReason { + return e.ErrStatus } var _ error = &Error{} +var _ httpStatus = &Error{} + +func IsNotFound(err error) bool { + return ReasonForError(err) == StatusReasonNotFound +} + +func ReasonForError(err error) StatusReason { + if status := httpStatus(nil); errors.As(err, &status) { + return status.Status() + } + return StatusReasonUnknown +} + +func NewErrorFromRestyResponse(res *resty.Response) *Error { + return NewGenericServerResponse(res.StatusCode(), res.Request.Method, res.String()) +} + +// NewGenericServerResponse returns a new error for server responses. +func NewGenericServerResponse(code int, method string, detail string) *Error { + reason := StatusReasonUnknown + message := fmt.Sprintf("the server responded with the status code %d but did not return more information", code) + switch code { + case http.StatusConflict: + if method == resty.MethodPost { + reason = StatusReasonAlreadyExists + } else { + reason = StatusReasonConflict + } + message = "the server reported a conflict" + case http.StatusNotFound: + reason = StatusReasonNotFound + message = "the server could not find the requested resource" + case http.StatusBadRequest: + reason = StatusReasonBadRequest + message = "the server rejected our request for an unknown reason" + case http.StatusUnauthorized: + reason = StatusReasonUnauthorized + message = "the server has asked for the client to provide credentials" + case http.StatusForbidden: + reason = StatusReasonForbidden + // the server message has details about who is trying to perform what action. Keep its message. + message = detail + case http.StatusNotAcceptable: + reason = StatusReasonNotAcceptable + // the server message has details about what types are acceptable + if len(detail) == 0 || detail == "unknown" { + message = "the server was unable to respond with a content type that the client supports" + } else { + message = detail + } + case http.StatusUnsupportedMediaType: + reason = StatusReasonUnsupportedMediaType + // the server message has details about what types are acceptable + message = detail + case http.StatusMethodNotAllowed: + reason = StatusReasonMethodNotAllowed + message = "the server does not allow this method on the requested resource" + case http.StatusUnprocessableEntity: + reason = StatusReasonInvalid + message = "the server rejected our request due to an error in our request" + case http.StatusServiceUnavailable: + reason = StatusReasonServiceUnavailable + message = "the server is currently unable to handle the request" + case http.StatusTooManyRequests: + reason = StatusReasonTooManyRequests + message = "the server has received too many requests and has asked us to try again later" + default: + if code >= 500 { + reason = StatusReasonInternalError + message = "an error on the server has prevented the request from succeeding" + } + } + + return &Error{ + Code: code, + ErrStatus: reason, + Message: message, + Detail: detail, + } +} diff --git a/pkg/tool/kodo/config.go b/pkg/tool/kodo/config.go new file mode 100644 index 0000000000..1dc0857570 --- /dev/null +++ b/pkg/tool/kodo/config.go @@ -0,0 +1,24 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kodo + +// Config 为文件上传,资源管理等配置 +type Config struct { + Zone *Zone //空间所在的机房 + UseHTTPS bool //是否使用https域名 + UseCdnDomains bool //是否使用cdn加速域名 +} diff --git a/pkg/tool/kodo/form_upload.go b/pkg/tool/kodo/form_upload.go new file mode 100644 index 0000000000..456f61390c --- /dev/null +++ b/pkg/tool/kodo/form_upload.go @@ -0,0 +1,301 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kodo + +import ( + "bytes" + "context" + "fmt" + "hash" + "hash/crc32" + "io" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "strings" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +// PutExtra 为表单上传的额外可选项 +type PutExtra struct { + // 可选,用户自定义参数,必须以 "x:" 开头。若不以x:开头,则忽略。 + Params map[string]string + + // 可选,当为 "" 时候,服务端自动判断。 + MimeType string + + // 上传事件:进度通知。这个事件的回调函数应该尽可能快地结束。 + OnProgress func(fsize, uploaded int64) +} + +// PutRet 为七牛标准的上传回复内容。 +// 如果使用了上传回调或者自定义了returnBody,那么需要根据实际情况,自己自定义一个返回值结构体 +type PutRet struct { + Hash string `json:"hash"` + PersistentID string `json:"persistentId"` + Key string `json:"key"` +} + +// FormUploader 表示一个表单上传的对象 +type FormUploader struct { + cfg *Config +} + +// NewFormUploader 用来构建一个表单上传的对象 +func NewFormUploader(cfg *Config) *FormUploader { + if cfg == nil { + cfg = &Config{} + } + + return &FormUploader{ + cfg: cfg, + } +} + +// PutFile 用来以表单方式上传一个文件,和 Put 不同的只是一个通过提供文件路径来访问文件内容,一个通过 io.Reader 来访问。 +// +// ctx 是请求的上下文。 +// ret 是上传成功后返回的数据。如果 uptoken 中没有设置 callbackUrl 或 returnBody,那么返回的数据结构是 PutRet 结构。 +// uptoken 是由业务服务器颁发的上传凭证。 +// key 是要上传的文件访问路径。比如:"foo/bar.jpg"。注意我们建议 key 不要以 '/' 开头。另外,key 为空字符串是合法的。 +// localFile 是要上传的文件的本地路径。 +// extra 是上传的一些可选项,可以指定为nil。详细见 PutExtra 结构的描述。 +// +func (p *FormUploader) PutFile( + ctx context.Context, ret interface{}, uptoken, key, localFile string, extra *PutExtra) (err error) { + return p.putFile(ctx, ret, uptoken, key, true, localFile, extra) +} + +func (p *FormUploader) putFile( + ctx context.Context, ret interface{}, uptoken string, + key string, hasKey bool, localFile string, extra *PutExtra) (err error) { + + f, err := os.Open(localFile) + if err != nil { + return + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return + } + fsize := fi.Size() + + if extra == nil { + extra = &PutExtra{} + } + + return p.put(ctx, ret, uptoken, key, hasKey, f, fsize, extra, filepath.Base(localFile)) +} + +func (p *FormUploader) put( + ctx context.Context, ret interface{}, uptoken string, + key string, hasKey bool, data io.Reader, size int64, extra *PutExtra, fileName string) (err error) { + + ak, bucket, gErr := getAkBucketFromUploadToken(uptoken) + if gErr != nil { + err = gErr + return + } + + var upHost string + upHost, err = p.upHost(ak, bucket) + if err != nil { + return + } + + var b bytes.Buffer + writer := multipart.NewWriter(&b) + + if extra == nil { + extra = &PutExtra{} + } + + if extra.OnProgress != nil { + data = &readerWithProgress{reader: data, fsize: size, onProgress: extra.OnProgress} + } + + err = writeMultipart(writer, uptoken, key, hasKey, extra, fileName) + if err != nil { + return + } + + var dataReader io.Reader + + h := crc32.NewIEEE() + dataReader = io.TeeReader(data, h) + crcReader := newCrc32Reader(writer.Boundary(), h) + //write file + head := make(textproto.MIMEHeader) + head.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, + escapeQuotes(fileName))) + if extra.MimeType != "" { + head.Set("Content-Type", extra.MimeType) + } + + _, err = writer.CreatePart(head) + if err != nil { + return + } + + lastLine := fmt.Sprintf("\r\n--%s--\r\n", writer.Boundary()) + r := strings.NewReader(lastLine) + + bodyLen := int64(-1) + if size >= 0 { + bodyLen = int64(b.Len()) + size + int64(len(lastLine)) + bodyLen += crcReader.length() + } + + mr := io.MultiReader(&b, dataReader, crcReader, r) + + contentType := writer.FormDataContentType() + _, err = httpclient.Post( + upHost, + httpclient.SetResult(ret), + httpclient.SetHeader("Content-Type", contentType), + httpclient.SetBody(mr), + ) + // err = p.client.CallWith64(ctx, ret, "POST", upHost, contentType, mr, bodyLen) + if err != nil { + return + } + if extra.OnProgress != nil { + extra.OnProgress(size, size) + } + + return +} + +func (p *FormUploader) upHost(ak, bucket string) (upHost string, err error) { + var zone *Zone + if p.cfg.Zone != nil { + zone = p.cfg.Zone + } else { + if v, zoneErr := GetZone(ak, bucket); zoneErr != nil { + err = zoneErr + return + } else { + zone = v + } + } + + scheme := "http://" + if p.cfg.UseHTTPS { + scheme = "https://" + } + + host := zone.SrcUpHosts[0] + if p.cfg.UseCdnDomains { + host = zone.CdnUpHosts[0] + } + + upHost = fmt.Sprintf("%s%s", scheme, host) + return +} + +type crc32Reader struct { + h hash.Hash32 + boundary string + r io.Reader + flag bool + nlDashBoundaryNl string + header string + crc32PadLen int64 +} + +func newCrc32Reader(boundary string, h hash.Hash32) *crc32Reader { + nlDashBoundaryNl := fmt.Sprintf("\r\n--%s\r\n", boundary) + header := `Content-Disposition: form-data; name="crc32"` + "\r\n\r\n" + return &crc32Reader{ + h: h, + boundary: boundary, + nlDashBoundaryNl: nlDashBoundaryNl, + header: header, + crc32PadLen: 10, + } +} + +func (r *crc32Reader) Read(p []byte) (int, error) { + if r.flag == false { + crc32 := r.h.Sum32() + crc32Line := r.nlDashBoundaryNl + r.header + fmt.Sprintf("%010d", crc32) //padding crc32 results to 10 digits + r.r = strings.NewReader(crc32Line) + r.flag = true + } + return r.r.Read(p) +} + +func (r crc32Reader) length() (length int64) { + return int64(len(r.nlDashBoundaryNl+r.header)) + r.crc32PadLen +} + +type readerWithProgress struct { + reader io.Reader + uploaded int64 + fsize int64 + onProgress func(fsize, uploaded int64) +} + +func (p *readerWithProgress) Read(b []byte) (n int, err error) { + if p.uploaded > 0 { + p.onProgress(p.fsize, p.uploaded) + } + + n, err = p.reader.Read(b) + p.uploaded += int64(n) + return +} + +func writeMultipart(writer *multipart.Writer, uptoken, key string, hasKey bool, + extra *PutExtra, fileName string) (err error) { + + //token + if err = writer.WriteField("token", uptoken); err != nil { + return + } + + //key + if hasKey { + if err = writer.WriteField("key", key); err != nil { + return + } + } + + //extra.Params + if extra.Params != nil { + for k, v := range extra.Params { + if strings.HasPrefix(k, "x:") && v != "" { + err = writer.WriteField(k, v) + if err != nil { + return + } + } + } + } + + return err +} + +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} diff --git a/pkg/tool/kodo/qbox/qbox_auth.go b/pkg/tool/kodo/qbox/qbox_auth.go new file mode 100644 index 0000000000..84d127614f --- /dev/null +++ b/pkg/tool/kodo/qbox/qbox_auth.go @@ -0,0 +1,45 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qbox + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "fmt" +) + +// Mac 七牛AK/SK的对象,AK/SK可以从 https://portal.qiniu.com/user/key 获取。 +type Mac struct { + AccessKey string + SecretKey []byte +} + +// NewMac 构建一个新的拥有AK/SK的对象 +func NewMac(accessKey, secretKey string) (mac *Mac) { + return &Mac{accessKey, []byte(secretKey)} +} + +// SignWithData 对数据进行签名,一般用于上传凭证的生成用途 +func (mac *Mac) SignWithData(b []byte) (token string) { + encodedData := base64.URLEncoding.EncodeToString(b) + h := hmac.New(sha1.New, mac.SecretKey) + h.Write([]byte(encodedData)) + digest := h.Sum(nil) + sign := base64.URLEncoding.EncodeToString(digest) + return fmt.Sprintf("%s:%s:%s", mac.AccessKey, sign, encodedData) +} diff --git a/pkg/tool/kodo/token.go b/pkg/tool/kodo/token.go new file mode 100644 index 0000000000..4300005a02 --- /dev/null +++ b/pkg/tool/kodo/token.go @@ -0,0 +1,89 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kodo + +import ( + "encoding/base64" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/koderover/zadig/pkg/tool/kodo/qbox" +) + +// PutPolicy 表示文件上传的上传策略 +type PutPolicy struct { + Scope string `json:"scope"` + Expires uint32 `json:"deadline"` // 截止时间(以秒为单位) + IsPrefixalScope int `json:"isPrefixalScope,omitempty"` + InsertOnly uint16 `json:"insertOnly,omitempty"` // 若非0, 即使Scope为 Bucket:Key 的形式也是insert only + DetectMime uint8 `json:"detectMime,omitempty"` // 若非0, 则服务端根据内容自动确定 MimeType + FsizeLimit int64 `json:"fsizeLimit,omitempty"` + MimeLimit string `json:"mimeLimit,omitempty"` + SaveKey string `json:"saveKey,omitempty"` + CallbackFetchKey uint8 `json:"callbackFetchKey,omitempty"` + CallbackURL string `json:"callbackUrl,omitempty"` + CallbackHost string `json:"callbackHost,omitempty"` + CallbackBody string `json:"callbackBody,omitempty"` + CallbackBodyType string `json:"callbackBodyType,omitempty"` + ReturnURL string `json:"returnUrl,omitempty"` + ReturnBody string `json:"returnBody,omitempty"` + PersistentOps string `json:"persistentOps,omitempty"` + PersistentNotifyURL string `json:"persistentNotifyUrl,omitempty"` + PersistentPipeline string `json:"persistentPipeline,omitempty"` + EndUser string `json:"endUser,omitempty"` + DeleteAfterDays int `json:"deleteAfterDays,omitempty"` + FileType int `json:"fileType,omitempty"` +} + +// UploadToken 方法用来进行上传凭证的生成 +func (p *PutPolicy) UploadToken(mac *qbox.Mac) (token string) { + if p.Expires == 0 { + p.Expires = 3600 // 1 hour + } + p.Expires += uint32(time.Now().Unix()) + + putPolicyJSON, _ := json.Marshal(p) + token = mac.SignWithData(putPolicyJSON) + return +} + +func getAkBucketFromUploadToken(token string) (ak, bucket string, err error) { + items := strings.Split(token, ":") + if len(items) != 3 { + err = errors.New("invalid upload token, format error") + return + } + + ak = items[0] + policyBytes, dErr := base64.URLEncoding.DecodeString(items[2]) + if dErr != nil { + err = errors.New("invalid upload token, invalid put policy") + return + } + + putPolicy := PutPolicy{} + uErr := json.Unmarshal(policyBytes, &putPolicy) + if uErr != nil { + err = errors.New("invalid upload token, invalid put policy") + return + } + + bucket = strings.Split(putPolicy.Scope, ":")[0] + return +} diff --git a/pkg/tool/kodo/upload_client.go b/pkg/tool/kodo/upload_client.go new file mode 100644 index 0000000000..15cd04869b --- /dev/null +++ b/pkg/tool/kodo/upload_client.go @@ -0,0 +1,71 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kodo + +import ( + "context" + "fmt" + + "github.com/koderover/zadig/pkg/tool/kodo/qbox" +) + +// UploadClient struct +type UploadClient struct { + AccessKey string + SecretKey string + Bucket string +} + +// NewUploadClient constructor +func NewUploadClient(ak, sk, bucket string) (UploadClient, error) { + + cli := UploadClient{ + AccessKey: ak, + SecretKey: sk, + Bucket: bucket, + } + + if len(ak) == 0 || len(sk) == 0 || len(bucket) == 0 { + return cli, fmt.Errorf("empty access key, secret key or bucket name") + } + + return cli, nil +} + +// UploadFile ... +func (cli UploadClient) UploadFile(key, localFile string) (retKey, retHash string, err error) { + + putPolicy := PutPolicy{ + Scope: fmt.Sprintf("%s:%s", cli.Bucket, key), + } + + mac := qbox.NewMac(cli.AccessKey, cli.SecretKey) + upToken := putPolicy.UploadToken(mac) + cfg := Config{ + Zone: &ZoneHuadong, + UseHTTPS: false, + UseCdnDomains: false, + } + + formUploader := NewFormUploader(&cfg) + ret := PutRet{} + err = formUploader.PutFile(context.Background(), &ret, upToken, key, localFile, nil) + if err != nil { + return + } + return ret.Key, ret.Hash, nil +} diff --git a/pkg/tool/kodo/zone.go b/pkg/tool/kodo/zone.go new file mode 100644 index 0000000000..28fd11e739 --- /dev/null +++ b/pkg/tool/kodo/zone.go @@ -0,0 +1,153 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kodo + +import ( + "fmt" + "strings" + "sync" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +// 资源管理相关的默认域名 +const ( + DefaultRsHost = "rs.qiniu.com" + DefaultRsfHost = "rsf.qiniu.com" + DefaultAPIHost = "api.qiniu.com" +) + +// Zone 为空间对应的机房属性,主要包括了上传,资源管理等操作的域名 +type Zone struct { + SrcUpHosts []string + CdnUpHosts []string + RsHost string + RsfHost string + ApiHost string + IovipHost string +} + +// ZoneHuadong 表示华东机房 +var ZoneHuadong = Zone{ + SrcUpHosts: []string{ + "up.qiniup.com", + "up-nb.qiniup.com", + "up-xs.qiniup.com", + }, + CdnUpHosts: []string{ + "upload.qiniup.com", + "upload-nb.qiniup.com", + "upload-xs.qiniup.com", + }, + RsHost: "rs.qiniu.com", + RsfHost: "rsf.qiniu.com", + ApiHost: "api.qiniu.com", + IovipHost: "iovip.qbox.me", +} + +// UcHost 为查询空间相关域名的API服务地址 +const UcHost = "https://uc.qbox.me" + +// UcQueryRet 为查询请求的回复 +type UcQueryRet struct { + TTL int `json:"ttl"` + Io map[string]map[string][]string `json:"io"` + Up map[string]UcQueryUp `json:"up"` +} + +// UcQueryUp 为查询请求回复中的上传域名信息 +type UcQueryUp struct { + Main []string `json:"main,omitempty"` + Backup []string `json:"backup,omitempty"` + Info string `json:"info,omitempty"` +} + +var ( + zoneMutext sync.RWMutex + zoneCache = make(map[string]*Zone) +) + +// GetZone 用来根据ak和bucket来获取空间相关的机房信息 +func GetZone(ak, bucket string) (zone *Zone, err error) { + zoneID := fmt.Sprintf("%s:%s", ak, bucket) + //check from cache + zoneMutext.RLock() + if v, ok := zoneCache[zoneID]; ok { + zone = v + } + zoneMutext.RUnlock() + if zone != nil { + return + } + + //query from server + reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", UcHost, ak, bucket) + var ret UcQueryRet + _, qErr := httpclient.Get( + reqURL, + httpclient.SetResult(&ret), + httpclient.SetHeader("Content-Type", "application/x-www-form-urlencoded"), + ) + //qErr := rpc.DefaultClient.CallWithForm(ctx, &ret, "GET", reqURL, nil) + if qErr != nil { + err = fmt.Errorf("query zone error, %s", qErr.Error()) + return + } + + ioHost := ret.Io["src"]["main"][0] + srcUpHosts := ret.Up["src"].Main + if ret.Up["src"].Backup != nil { + srcUpHosts = append(srcUpHosts, ret.Up["src"].Backup...) + } + cdnUpHosts := ret.Up["acc"].Main + if ret.Up["acc"].Backup != nil { + cdnUpHosts = append(cdnUpHosts, ret.Up["acc"].Backup...) + } + + zone = &Zone{ + SrcUpHosts: srcUpHosts, + CdnUpHosts: cdnUpHosts, + IovipHost: ioHost, + RsHost: DefaultRsHost, + RsfHost: DefaultRsfHost, + ApiHost: DefaultAPIHost, + } + + //set specific hosts if possible + setSpecificHosts(ioHost, zone) + + zoneMutext.Lock() + zoneCache[zoneID] = zone + zoneMutext.Unlock() + return +} + +func setSpecificHosts(ioHost string, zone *Zone) { + if strings.Contains(ioHost, "-z1") { + zone.RsHost = "rs-z1.qiniu.com" + zone.RsfHost = "rsf-z1.qiniu.com" + zone.ApiHost = "api-z1.qiniu.com" + } else if strings.Contains(ioHost, "-z2") { + zone.RsHost = "rs-z2.qiniu.com" + zone.RsfHost = "rsf-z2.qiniu.com" + zone.ApiHost = "api-z2.qiniu.com" + } else if strings.Contains(ioHost, "-na0") { + zone.RsHost = "rs-na0.qiniu.com" + zone.RsfHost = "rsf-na0.qiniu.com" + zone.ApiHost = "api-na0.qiniu.com" + } +} diff --git a/pkg/tool/kube/client/cluster.go b/pkg/tool/kube/client/cluster.go index 19c9643377..927cd41152 100644 --- a/pkg/tool/kube/client/cluster.go +++ b/pkg/tool/kube/client/cluster.go @@ -82,6 +82,10 @@ func NewClientFromAPIConfig(cfg *api.Config) (client.Client, error) { return newAPIClient(cls.GetClient(), cls.GetAPIReader()), nil } +func NewAPIClient(c client.Client, r client.Reader) client.Client { + return newAPIClient(c, r) +} + // apiClient is similar with the default Client(), but it always gets objects from API server. type apiClient struct { client.Client diff --git a/pkg/tool/kube/patcher/patcher.go b/pkg/tool/kube/patcher/patcher.go index 9ba66418d8..dc7fb8b7b2 100644 --- a/pkg/tool/kube/patcher/patcher.go +++ b/pkg/tool/kube/patcher/patcher.go @@ -19,11 +19,8 @@ package patcher import ( "fmt" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/jsonmergepatch" "k8s.io/apimachinery/pkg/util/mergepatch" @@ -33,44 +30,6 @@ import ( "github.com/koderover/zadig/pkg/tool/kube/util" ) -var metadataAccessor = meta.NewAccessor() - -// GetOriginalConfiguration retrieves the original configuration of the object -// from the annotation, or nil if no annotation was found. -func GetOriginalConfiguration(obj runtime.Object) ([]byte, error) { - annots, err := metadataAccessor.Annotations(obj) - if err != nil { - return nil, err - } - - if annots == nil { - return nil, nil - } - - original, ok := annots[corev1.LastAppliedConfigAnnotation] - if !ok { - return nil, nil - } - - return []byte(original), nil -} - -func GetGroupVersionKind(obj runtime.Object) (*schema.GroupVersionKind, error) { - apiVersion, err := metadataAccessor.APIVersion(obj) - if err != nil { - return nil, err - } - - kind, err := metadataAccessor.Kind(obj) - if err != nil { - return nil, err - } - - gvk := schema.FromAPIVersionAndKind(apiVersion, kind) - - return &gvk, nil -} - // GeneratePatchBytes generate the patchBytes inspired by the patcher in kubectl/pkg/cmd/apply/patcher.go func GeneratePatchBytes(obj, modifiedObj runtime.Object) ([]byte, types.PatchType, error) { // Serialize the current configuration of the object from the server. @@ -84,13 +43,13 @@ func GeneratePatchBytes(obj, modifiedObj runtime.Object) ([]byte, types.PatchTyp return nil, "", fmt.Errorf("get modified configuration is failed, err: %v", err) } - gvk, err := GetGroupVersionKind(obj) + gvk, err := util.GetGroupVersionKind(obj) if err != nil { return nil, "", fmt.Errorf("retrieving gvk is failed, err: %v", err) } // Retrieve the original configuration of the object from the annotation. - original, err := GetOriginalConfiguration(obj) + original, err := util.GetOriginalConfiguration(obj) if err != nil { return nil, "", fmt.Errorf("retrieving original configuration from:\n%v\nis failed, err: %v", obj, err) } diff --git a/pkg/tool/kube/updater/base.go b/pkg/tool/kube/updater/base.go index d3c297b694..0c138e6273 100644 --- a/pkg/tool/kube/updater/base.go +++ b/pkg/tool/kube/updater/base.go @@ -93,15 +93,23 @@ func updateOrCreateObject(obj client.Object, cl client.Client) error { } func createOrPatchObject(modified client.Object, cl client.Client) error { - c := modified.DeepCopyObject() - current := c.(client.Object) - found, err := getter.GetResourceInCache(modified.GetNamespace(), modified.GetName(), current, cl) + c := modified.DeepCopyObject().(client.Object) + found, err := getter.GetResourceInCache(modified.GetNamespace(), modified.GetName(), c, cl) if err != nil { return err } else if !found { return createObject(modified, cl) } + // fill gvk in case it is missing. + // for objects retrieved from APIServer, gvk is not set, so we need to set it + // for objects in cache, gvk is set, nothing will be changed here + gvk := modified.GetObjectKind().GroupVersionKind() + current, err := util.SetGroupVersionKind(c, &gvk) + if err != nil { + return err + } + patchBytes, patchType, err := patcher.GeneratePatchBytes(current, modified) if err != nil { return err @@ -111,7 +119,7 @@ func createOrPatchObject(modified client.Object, cl client.Client) error { return nil } - return patchObjectWithType(current, patchBytes, patchType, cl) + return patchObjectWithType(current.(client.Object), patchBytes, patchType, cl) } func patchObjectWithType(obj client.Object, patchBytes []byte, patchType types.PatchType, cl client.Client) error { diff --git a/pkg/tool/kube/util/apply.go b/pkg/tool/kube/util/apply.go index 150c6fe592..4ad4e4da24 100644 --- a/pkg/tool/kube/util/apply.go +++ b/pkg/tool/kube/util/apply.go @@ -17,11 +17,15 @@ limitations under the License. package util import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubectl/pkg/util" ) +var metadataAccessor = meta.NewAccessor() + func CreateApplyAnnotation(obj runtime.Object) error { return util.CreateApplyAnnotation(obj, unstructured.UnstructuredJSONScheme) } @@ -29,3 +33,23 @@ func CreateApplyAnnotation(obj runtime.Object) error { func GetModifiedConfiguration(obj runtime.Object) ([]byte, error) { return util.GetModifiedConfiguration(obj, true, unstructured.UnstructuredJSONScheme) } + +// GetOriginalConfiguration retrieves the original configuration of the object +// from the annotation, or nil if no annotation was found. +func GetOriginalConfiguration(obj runtime.Object) ([]byte, error) { + annots, err := metadataAccessor.Annotations(obj) + if err != nil { + return nil, err + } + + if annots == nil { + return nil, nil + } + + original, ok := annots[corev1.LastAppliedConfigAnnotation] + if !ok { + return nil, nil + } + + return []byte(original), nil +} diff --git a/pkg/tool/kube/util/meta.go b/pkg/tool/kube/util/meta.go new file mode 100644 index 0000000000..d1f7cdd306 --- /dev/null +++ b/pkg/tool/kube/util/meta.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func GetGroupVersionKind(obj runtime.Object) (*schema.GroupVersionKind, error) { + t, err := meta.TypeAccessor(obj) + if err != nil { + return nil, err + } + + gvk := schema.FromAPIVersionAndKind(t.GetAPIVersion(), t.GetKind()) + + return &gvk, nil +} + +func SetGroupVersionKind(obj runtime.Object, gvk *schema.GroupVersionKind) (runtime.Object, error) { + t, err := meta.TypeAccessor(obj) + if err != nil { + return nil, err + } + + t.SetAPIVersion(gvk.GroupVersion().String()) + t.SetKind(gvk.Kind) + + return obj, nil +} diff --git a/pkg/util/boolptr/boolptr.go b/pkg/util/boolptr/boolptr.go new file mode 100644 index 0000000000..2a414301df --- /dev/null +++ b/pkg/util/boolptr/boolptr.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package boolptr + +// IsTrue returns true if and only if the bool pointer is non-nil and set to true. +func IsTrue(b *bool) bool { + return b != nil && *b +} + +// IsFalse returns true if and only if the bool pointer is non-nil and set to false. +func IsFalse(b *bool) bool { + return b != nil && !*b +} + +// True returns a *bool whose underlying value is true. +func True() *bool { + t := true + return &t +} + +// False returns a *bool whose underlying value is false. +func False() *bool { + t := false + return &t +} + +// Equal returns true if and only if both values are set and equal. +func Equal(a, b *bool) bool { + if a == nil || b == nil { + return false + } else { + return *a == *b + } +} + +// NilOrEqual returns true if both values are set and equal or both are nil. +func NilOrEqual(a, b *bool) bool { + if a == nil && b == nil { + return true + } else if a == nil || b == nil { + return false + } else { + return *a == *b + } +} diff --git a/pkg/util/file.go b/pkg/util/file.go index 6bfb8eb6a9..09bf9a6749 100644 --- a/pkg/util/file.go +++ b/pkg/util/file.go @@ -20,7 +20,6 @@ import ( "archive/tar" "compress/gzip" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -29,7 +28,7 @@ import ( func GenerateTmpFile() (string, error) { var tmpFile *os.File - tmpFile, err := ioutil.TempFile("", "") + tmpFile, err := os.CreateTemp("", "") if err != nil { return "", err } diff --git a/pkg/util/random.go b/pkg/util/random.go index c4a9958bdf..1cbd6b4dc2 100644 --- a/pkg/util/random.go +++ b/pkg/util/random.go @@ -17,9 +17,33 @@ limitations under the License. package util import ( + "math/rand" + "time" + "github.com/google/uuid" ) func UUID() string { return uuid.New().String() } + +const str = "abcdefghijklmnopqrstuvwxyz" +const numStr = "0123456789abcdefghijklmnopqrstuvwxyz" + +func GetRandomNumString(length int) string { + res := make([]byte, length) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for i := range res { + res[i] = numStr[r.Intn(len(numStr))] + } + return string(res) +} + +func GetRandomString(length int) string { + res := make([]byte, length) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for i := range res { + res[i] = numStr[r.Intn(len(str))] + } + return string(res) +} diff --git a/ut.file b/ut.file index 2803d690ca..046285034f 100644 --- a/ut.file +++ b/ut.file @@ -1,2 +1,6 @@ ./pkg/microservice/cron/core/service/client +./pkg/microservice/reaper/core/service/cmd +./pkg/microservice/reaper/core/service/meta +./pkg/microservice/reaper/core/service/reaper +./pkg/microservice/aslan/core/common/service/registry ./pkg/util/converter diff --git a/zadig-portal/src/api/index.js b/zadig-portal/src/api/index.js index 82b11d5f24..1e4acfd9c0 100644 --- a/zadig-portal/src/api/index.js +++ b/zadig-portal/src/api/index.js @@ -245,7 +245,7 @@ export function envRevisionsAPI (projectName, envName) { } export function productServicesAPI (projectName, envName, envSource, searchName = '', perPage = 20, page = 1) { - if (envSource === 'external') { + if (envSource === 'helm' || envSource === 'external') { return http.get(`/api/aslan/environment/environments/${projectName}/groups/${envSource}?envName=${envName}`) } else { return http.get(`/api/aslan/environment/environments/${projectName}/groups?envName=${envName}&serviceName=${searchName}&perPage=${perPage}&page=${page}`) @@ -302,6 +302,54 @@ export function deleteServiceTemplateAPI (name, type, projectName, visibility) { return http.delete(`/api/aslan/service/services/${name}/${type}?productName=${projectName}&visibility=${visibility}`) } +export function createPmServiceAPI (projectName, payload) { + return http.post(`/api/aslan/service/pm/${projectName}`, payload) +} + +export function updatePmServiceAPI (projectName, payload) { + return http.put(`/api/aslan/service/pm/${projectName}`, payload) +} + +export function getHelmChartProjectChartsAPI (project, projectName = '') { + return http.get(`/api/aslan/service/harbor/project/${project}/charts?productName=${projectName}`) +} + +export function getHelmChartProjectAPI (projectName = '') { + return http.get(`/api/aslan/service/harbor/project`) +} + +export function addHelmChartAPI (projectName = '', payload) { + return http.post(`/api/aslan/service/helm/${projectName}`, payload) +} + +export function updateHelmChartAPI (projectName = '', payload) { + return http.put(`/api/aslan/service/helm/${projectName}`, payload) +} + +export function getHelmChartVersionAPI (project, chart) { + return http.get(`/api/aslan/service/harbor/project/${project}/chart/${chart}/versions`) +} + +export function helmChartWithConfigAPI (serviceName, projectName) { + return http.get(`/api/aslan/service/helm/${projectName}/${serviceName}`) +} + +export function getHelmChartService (projectName) { + return http.get(`/api/aslan/service/helm/${projectName}`) +} + +export function getHelmChartServiceFilePath (projectName, serviceName, path) { + return http.get(`/api/aslan/service/helm/${projectName}/${serviceName}/filePath?dir=${path}`) +} + +export function getHelmChartServiceFileContent (projectName, serviceName, path, fileName) { + return http.get(`/api/aslan/service/helm/${projectName}/${serviceName}/fileContent?filePath=${path}&fileName=${fileName}`) +} + +export function getHelmChartServiceModule (projectName, serviceName) { + return http.get(`/api/aslan/service/helm/${projectName}/${serviceName}/serviceModule`) +} + export function imagesAPI (payload, registry = '') { return http.post(`/api/aslan/system/registry/images?registryId=${registry}`, { names: payload }) } @@ -619,6 +667,61 @@ export function deleteGithubAppAPI (id) { return http.delete(`/api/aslan/system/githubApp/${id}`) } +// Account +export function registrationChangeAPI (payload) { + return http.put(`/api/directory/isOpenRegistry`, { openRegistry: payload }) +} + +export function getSSOAPI (organization_id) { + return http.get(`/api/directory/sso?orgId=${organization_id}`) +} + +export function updateSSOAPI (organization_id, payload) { + return http.post(`/api/directory/sso?orgId=${organization_id}`, payload) +} + +export function deleteSSOAPI (organization_id) { + return http.delete(`/api/directory/sso?orgId=${organization_id}`) +} + +export function createSSOAPI (organization_id, payload) { + return http.post(`/api/directory/sso?orgId=${organization_id}`, payload) +} + +export function getAccountAPI (organization_id) { + return http.get(`/api/directory/ad/default?orgId=${organization_id}`) +} + +export function deleteAccountAPI (organization_id) { + return http.delete(`/api/directory/ad/default?orgId=${organization_id}`) +} + +export function updateAccountAPI (organization_id, payload) { + return http.put(`/api/directory/ad/default?orgId=${organization_id}`, payload) +} + +export function createAccountAPI (organization_id, payload) { + return http.post(`/api/directory/ad?orgId=${organization_id}`, payload) +} + +export function syncAccountAPI (organization_id) { + return http.post(`/api/directory/ads/sync?orgId=${organization_id}`) +} + +// Jira +export function getJiraAPI (organization_id) { + return http.get(`/api/directory/jira?orgId=${organization_id}`) +} +export function updateJiraAPI (organization_id, payload) { + return http.post(`/api/directory/jira?orgId=${organization_id}`, payload) +} +export function deleteJiraAPI (organization_id) { + return http.delete(`/api/directory/jira?orgId=${organization_id}`) +} +export function createJiraAPI (organization_id, payload) { + return http.post(`/api/directory/jira?orgId=${organization_id}`, payload) +} + // Jenkins export function addJenkins (payload) { return http.post('/api/aslan/system/jenkins/integration', payload) @@ -684,20 +787,7 @@ export function deleteAppAPI (data) { return http.put('/api/aslan/system/install/delete', data) } -export function getJiraAPI (organization_id) { - return http.get(`/api/directory/jira?orgId=${organization_id}`) -} -export function updateJiraAPI (organization_id, payload) { - return http.post(`/api/directory/jira?orgId=${organization_id}`, payload) -} -export function deleteJiraAPI (organization_id) { - return http.delete(`/api/directory/jira?orgId=${organization_id}`) -} -export function createJiraAPI (organization_id, payload) { - return http.post(`/api/directory/jira?orgId=${organization_id}`, payload) -} - -// custom image +// Custom image export function addImgAPI (payload) { return http.post(`/api/aslan/system/basicImages`, payload) } @@ -722,6 +812,14 @@ export function updateProxyConfigAPI (id, payload) { return http.put(`/api/aslan/system/proxyManage/${id}`, payload) } +// Quota +export function getCapacityAPI (target) { + return http.get(`/api/aslan/system/capacity/target/${target}`) +} +export function setCapacityAPI (payload) { + return http.post(`/api/aslan/system/capacity`, payload) +} + // Cache export function cleanCacheAPI () { return http.post('/api/aslan/system/cleanCache/oneClick') @@ -764,6 +862,48 @@ export function deleteStorageAPI (id) { return http.delete(`/api/aslan/system/s3storage/${id}`) } +// Cluster +export function getClusterListAPI () { + return http.get(`/api/aslan/cluster/clusters`) +} + +export function createClusterAPI (payload) { + return http.post(`/api/aslan/cluster/clusters`, payload) +} + +export function updateClusterAPI (id, payload) { + return http.put(`/api/aslan/cluster/clusters/${id}`, payload) +} + +export function recoverClusterAPI (id) { + return http.put(`/api/aslan/cluster/clusters/${id}/reconnect`) +} + +export function disconnectClusterAPI (id) { + return http.put(`/api/aslan/cluster/clusters/${id}/disconnect`) +} + +export function deleteClusterAPI (id) { + return http.delete(`/api/aslan/cluster/clusters/${id}`) +} + +// Host +export function getHostListAPI () { + return http.get(`/api/aslan/system/privateKey`) +} + +export function createHostAPI (payload) { + return http.post(`/api/aslan/system/privateKey`, payload) +} + +export function updateHostAPI (id, payload) { + return http.put(`/api/aslan/system/privateKey/${id}`, payload) +} + +export function deleteHostAPI (id) { + return http.delete(`/api/aslan/system/privateKey/${id}`) +} + // Delivery Center export function getArtifactsAPI (per_page, page, name = '', type = '', image = '', repoName = '', branch = '') { @@ -832,6 +972,22 @@ export function updateK8sEnvAPI (product_name, env_name, payload, envType = '', return http.post(`/api/aslan/environment/environments/${product_name}?envName=${env_name}&envType=${envType}&force=${force}`, payload) } +export function updateHelmEnvAPI (projectName, envName, updateType = '') { + return http.put(`/api/aslan/environment/environments/${projectName}/helmEnv?envName=${envName}&updateType=${updateType}`) +} + +export function getHelmEnvVarAPI (projectName, envName) { + return http.get(`/api/aslan/environment/environments/${projectName}/helmRenderCharts?envName=${envName}`) +} + +export function updateHelmEnvVarAPI (projectName, envName, payload) { + return http.put(`/api/aslan/environment/environments/${projectName}/helmEnvVariable?envName=${envName}`, payload) +} + +export function getHelmEnvChartDiffAPI (projectName, envName) { + return http.get(`/api/aslan/environment/environments/${projectName}/helmChartVersions?envName=${envName}`) +} + export function recycleEnvAPI (product_name, env_name, recycle_day) { return http.put(`/api/aslan/environment/environments/${product_name}/envRecycle?envName=${env_name}&recycleDay=${recycle_day}`) } @@ -864,6 +1020,10 @@ export function restartServiceAPI (productName, serviceName, envName = '', scale return http.post(`/api/aslan/environment/environments/${productName}/services/${serviceName}/restartNew?envName=${envName}&type=${type}&name=${scaleName}&envType=${envType}`) } +export function restartPmServiceAPI (payload) { + return http.post(`/api/aslan/workflow/servicetask`, payload) +} + export function scaleServiceAPI (productName, serviceName, envName = '', scaleName, scaleNumber, type, envType = '') { return http.post( `/api/aslan/environment/environments/${productName}/services/${serviceName}/scaleNew/${scaleNumber}?envName=${envName}&type=${type}&name=${scaleName}&envType=${envType}` @@ -892,6 +1052,10 @@ export function autoUpgradeEnvAPI (projectName, payload, force = '') { return http.put(`/api/aslan/environment/environments/${projectName}/autoUpdate?force=${force}`, payload) } +export function autoUpgradeHelmEnvAPI (projectName, payload) { + return http.put(`/api/aslan/environment/environments/${projectName}/updateMultiEnv`, payload) +} + // Login export function userLoginAPI (organization_id, payload) { return http.post(`/api/directory/user/login?orgId=${organization_id}`, payload) @@ -914,11 +1078,11 @@ export function getJwtTokenAPI () { } export function getSubscribeAPI () { - return http.get('/api/aslan/enterprise/notification/subscribe') + return http.get('/api/aslan/system/notification/subscribe') } export function saveSubscribeAPI (payload) { - return http.post('/api/aslan/enterprise/notification/subscribe', payload) + return http.post('/api/aslan/system/notification/subscribe', payload) } export function downloadPubKeyAPI () { @@ -931,15 +1095,15 @@ export function updateServiceImageAPI (payload, type, envType = '') { // Notification export function getNotificationAPI () { - return http.get('/api/aslan/enterprise/notification') + return http.get('/api/aslan/system/notification') } export function deleteAnnouncementAPI (payload) { - return http.post('/api/aslan/enterprise/announcement/delete', payload) + return http.post('/api/aslan/system/notification/delete', payload) } export function markNotiReadAPI (payload) { - return http.put('/api/aslan/enterprise/notification/read', payload) + return http.put('/api/aslan/system/notification/read', payload) } // Onboarding diff --git a/zadig-portal/src/common/xterm_debug.vue b/zadig-portal/src/common/xterm_debug.vue index 27d5435f76..fd849b9652 100644 --- a/zadig-portal/src/common/xterm_debug.vue +++ b/zadig-portal/src/common/xterm_debug.vue @@ -36,7 +36,7 @@ export default { initTerm () { let wsLink = false const hostname = this.getLogWSUrl() - const url = `/api/podexec/${this.productName}/${this.namespace}/${this.podName}/${this.containerName}/podExec` + const url = `/api/podexec/${this.productName}/${this.namespace}/${this.podName}/${this.containerName}/podExec?clusterId=${this.clusterId}` this.ws = new WebSocket(hostname + url) this.$nextTick(() => { @@ -119,6 +119,11 @@ export default { namespace: { required: true, type: String + }, + clusterId: { + required: false, + type: String, + default: '' } }, mounted () { diff --git a/zadig-portal/src/common/xterm_log.vue b/zadig-portal/src/common/xterm_log.vue index 641da6628f..b0fb0be644 100644 --- a/zadig-portal/src/common/xterm_log.vue +++ b/zadig-portal/src/common/xterm_log.vue @@ -19,6 +19,19 @@ export default { methods: { clearCurrentTerm () { this.term.clear() + }, + scroll (item) { + const height = item.offsetHeight + const top = + item.getBoundingClientRect() && item.getBoundingClientRect().top + const viewPortHeight = + window.innerHeight || + document.documentElement.clientHeight || + document.body.clientHeight + if (top + height <= viewPortHeight && top >= 60) { + const el = document.querySelector('.workflow-task-detail').style + el.overflow = 'hidden' + } } }, props: { @@ -50,6 +63,12 @@ export default { term.open(document.getElementById(this.id)) fitAddon.fit() this.term = term + const list = document.querySelectorAll('.xterm-viewport'); + [].forEach.call(list, (item) => { + item.addEventListener('scroll', () => { + this.scroll(item) + }) + }) } } diff --git a/zadig-portal/src/components/entry/home/common/notification.vue b/zadig-portal/src/components/entry/home/common/notification.vue index c5432d0eeb..58e046e8af 100644 --- a/zadig-portal/src/components/entry/home/common/notification.vue +++ b/zadig-portal/src/components/entry/home/common/notification.vue @@ -39,7 +39,7 @@ -

+

{{wordTranslation(notification.content.status,'pipeline','task')}} @@ -78,7 +78,7 @@
-

+

{{notification.content.title}}

@@ -385,14 +385,6 @@ export default { color: #ff1949; } } - - .truncate { - display: block; - max-width: 100%; - overflow: hidden; - white-space: nowrap; - text-overflow: ellipsis; - } } } } diff --git a/zadig-portal/src/components/entry/home/sidebar.vue b/zadig-portal/src/components/entry/home/sidebar.vue index 162d39becf..e4bb3264cd 100644 --- a/zadig-portal/src/components/entry/home/sidebar.vue +++ b/zadig-portal/src/components/entry/home/sidebar.vue @@ -169,6 +169,19 @@ export default { url: 'system/storage' }] }, + { + category_name: '资源管理', + items: [{ + name: '集群管理', + icon: 'iconfont iconjiqun', + url: 'system/cluster' + }, + { + name: '主机管理', + icon: 'iconfont iconzhuji', + url: 'system/host' + }] + }, { category_name: '系统配置', items: [{ diff --git a/zadig-portal/src/components/entry/login.vue b/zadig-portal/src/components/entry/login.vue index ce927f6de6..c40446d321 100644 --- a/zadig-portal/src/components/entry/login.vue +++ b/zadig-portal/src/components/entry/login.vue @@ -36,6 +36,13 @@ class="btn-md btn-theme btn-block login-btn"> 登录 + + +