From d36c3b2404d45a2c7af0814f9f96e45c6203a300 Mon Sep 17 00:00:00 2001 From: yangk Date: Mon, 12 Jun 2023 11:49:41 +0800 Subject: [PATCH 1/4] fix: collect log nil points --- node/nodem/logger/manage.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/node/nodem/logger/manage.go b/node/nodem/logger/manage.go index c08411d3d9..04ecdd67d7 100644 --- a/node/nodem/logger/manage.go +++ b/node/nodem/logger/manage.go @@ -35,10 +35,10 @@ import ( _ "github.com/containerd/containerd/api/events" ) -//RFC3339NanoFixed time format +// RFC3339NanoFixed time format var RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" -//ContainerLogManage container log manage +// ContainerLogManage container log manage type ContainerLogManage struct { ctx context.Context cancel context.CancelFunc @@ -47,7 +47,7 @@ type ContainerLogManage struct { containerLogs sync.Map } -//CreatContainerLogManage create a container log manage +// CreatContainerLogManage create a container log manage func CreatContainerLogManage(conf *option.Conf) *ContainerLogManage { ctx, cancel := context.WithCancel(context.Background()) return &ContainerLogManage{ @@ -58,7 +58,7 @@ func CreatContainerLogManage(conf *option.Conf) *ContainerLogManage { } } -//Start start +// Start start func (c *ContainerLogManage) Start() error { errchan := make(chan error) go func() { @@ -73,7 +73,7 @@ func (c *ContainerLogManage) Start() error { return nil } -//Stop stop all logger +// Stop stop all logger func (c *ContainerLogManage) Stop() { c.containerLogs.Range(func(k, v interface{}) bool { cl := v.(*ContainerLog) @@ -185,8 +185,8 @@ func (c *ContainerLogManage) loollist() { return case <-ticker.C: for _, container := range c.listContainer() { - cj, _ := c.getContainer(container.GetId()) - if cj.GetLogPath() == "" { + cj, err := c.getContainer(container.GetId()) + if err != nil || cj.GetLogPath() == "" { continue } if cj.ContainerRuntime == sources.ContainerRuntimeDocker { @@ -227,7 +227,7 @@ func (c *ContainerLogManage) listAndWatchContainer(errchan chan error) { } } -//Out - +// Out - type Out struct { Timestamp time.Time Namespace string @@ -254,7 +254,7 @@ func createContainerLog(ctx context.Context, container *sources.ContainerDesc, r } } -//ContainerLog container log struct +// ContainerLog container log struct type ContainerLog struct { ctx context.Context cancel context.CancelFunc @@ -267,7 +267,7 @@ type ContainerLog struct { stoped *bool } -//StartLogging start copy log +// StartLogging start copy log func (container *ContainerLog) StartLogging() error { loggers, err := container.startLogger() if err != nil { @@ -285,7 +285,7 @@ func (container *ContainerLog) StartLogging() error { return nil } -//ContainerLoggerConfig logger config +// ContainerLoggerConfig logger config type ContainerLoggerConfig struct { Name string Options map[string]string @@ -321,7 +321,7 @@ func getLoggerConfig(envs []string) []*ContainerLoggerConfig { return re } -//ErrNeglectedContainer not define logger name +// ErrNeglectedContainer not define logger name var ErrNeglectedContainer = fmt.Errorf("Neglected container") // containerInfo is extra info returned by containerd grpc api @@ -475,7 +475,7 @@ func (container *ContainerLog) startLogger() ([]Logger, error) { return loggers, nil } -//Restart restart +// Restart restart func (container *ContainerLog) Restart() { if container == nil { return @@ -488,7 +488,7 @@ func (container *ContainerLog) Restart() { } } -//Stop stop copy container log +// Stop stop copy container log func (container *ContainerLog) Stop() { if container.LogCopier != nil { container.LogCopier.Close() @@ -499,7 +499,7 @@ func (container *ContainerLog) Stop() { logrus.Debugf("rainbond logger stop for container %s", container.ContainerStatus.GetMetadata().GetName()) } -//Close close +// Close close func (container *ContainerLog) Close() { if container.LogCopier != nil { container.LogCopier.Close() From 0cdc9068e9c847098d0630fb3cecd8a798955d68 Mon Sep 17 00:00:00 2001 From: yangk Date: Wed, 14 Jun 2023 17:39:51 +0800 Subject: [PATCH 2/4] fix: metrics conflict --- worker/master/controller/thirdcomponent/controller.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/worker/master/controller/thirdcomponent/controller.go b/worker/master/controller/thirdcomponent/controller.go index c8be10bff9..c98d2ebb48 100644 --- a/worker/master/controller/thirdcomponent/controller.go +++ b/worker/master/controller/thirdcomponent/controller.go @@ -20,6 +20,7 @@ package thirdcomponent import ( "context" + "os" "reflect" "time" @@ -404,7 +405,11 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { // Collect - func (r *Reconciler) Collect(ch chan<- prometheus.Metric) { - ch <- prometheus.MustNewConstMetric(r.discoverNum.Desc(), prometheus.GaugeValue, r.discoverPool.GetSize()) + hostname, err := os.Hostname() + if err != nil { + hostname = os.Getenv("POD_IP") + } + ch <- prometheus.MustNewConstMetric(r.discoverNum.Desc(), prometheus.GaugeValue, r.discoverPool.GetSize(), hostname) } // Setup adds a controller that reconciles AppDeployment. From 123f2de0bad689b599a3f1fe2166ac496bc30d05 Mon Sep 17 00:00:00 2001 From: yangk Date: Wed, 14 Jun 2023 19:19:19 +0800 Subject: [PATCH 3/4] fix: metrics conflict2 --- .../master/controller/thirdcomponent/controller.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/worker/master/controller/thirdcomponent/controller.go b/worker/master/controller/thirdcomponent/controller.go index c98d2ebb48..5ca5d49d4d 100644 --- a/worker/master/controller/thirdcomponent/controller.go +++ b/worker/master/controller/thirdcomponent/controller.go @@ -60,7 +60,7 @@ type Reconciler struct { concurrentReconciles int applyer apply.Applicator discoverPool *DiscoverPool - discoverNum prometheus.Gauge + discoverNum *prometheus.Desc informer runtimecache.Informer lister rainbondlistersv1alpha1.ThirdComponentLister @@ -409,7 +409,7 @@ func (r *Reconciler) Collect(ch chan<- prometheus.Metric) { if err != nil { hostname = os.Getenv("POD_IP") } - ch <- prometheus.MustNewConstMetric(r.discoverNum.Desc(), prometheus.GaugeValue, r.discoverPool.GetSize(), hostname) + ch <- prometheus.MustNewConstMetric(r.discoverNum, prometheus.GaugeValue, r.discoverPool.GetSize(), hostname) } // Setup adds a controller that reconciles AppDeployment. @@ -427,11 +427,9 @@ func Setup(ctx context.Context, mgr ctrl.Manager) (*Reconciler, error) { restConfig: mgr.GetConfig(), Scheme: mgr.GetScheme(), applyer: apply.NewAPIApplicator(mgr.GetClient()), - discoverNum: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "controller", - Name: "third_component_discover_number", - Help: "Number of running endpoint discover worker of third component.", - }), + discoverNum: prometheus.NewDesc(prometheus.BuildFQName("controller", "", "third_component_discover_number"), + "Number of running endpoint discover worker of third component.", + []string{"hostname"}, nil), informer: informer, lister: lister, recorder: recorder, From 57f1775cfff8b77c91a454f837b1704326012cf0 Mon Sep 17 00:00:00 2001 From: yangk Date: Thu, 15 Jun 2023 11:06:46 +0800 Subject: [PATCH 4/4] fix: build task not scheduled to build node --- builder/build/code_build.go | 22 +++++++++--------- builder/sources/image.go | 45 +++++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/builder/build/code_build.go b/builder/build/code_build.go index 8c4900e4ec..c31da42753 100644 --- a/builder/build/code_build.go +++ b/builder/build/code_build.go @@ -365,20 +365,18 @@ func (s *slugBuild) runBuildJob(re *Request) error { } podSpec := corev1.PodSpec{RestartPolicy: corev1.RestartPolicyOnFailure} // only support never and onfailure // schedule builder - if re.CacheMode == "hostpath" { - logrus.Debugf("builder cache mode using hostpath, schedule job into current node") - hostIP := os.Getenv("HOST_IP") - if hostIP != "" { - podSpec.NodeSelector = map[string]string{ - "kubernetes.io/hostname": hostIP, - } - podSpec.Tolerations = []corev1.Toleration{ - { - Operator: "Exists", - }, - } + hostIP := os.Getenv("HOST_IP") + if hostIP != "" { + podSpec.NodeSelector = map[string]string{ + "kubernetes.io/hostname": hostIP, + } + podSpec.Tolerations = []corev1.Toleration{ + { + Operator: "Exists", + }, } } + logrus.Debugf("request is: %+v", re) volumes, mounts := s.createVolumeAndMount(re, sourceTarFileName) diff --git a/builder/sources/image.go b/builder/sources/image.go index 370f446f93..d8d192bf05 100644 --- a/builder/sources/image.go +++ b/builder/sources/image.go @@ -62,16 +62,16 @@ import ( "time" ) -//ErrorNoAuth error no auth +// ErrorNoAuth error no auth var ErrorNoAuth = fmt.Errorf("pull image require docker login") -//ErrorNoImage error no image +// ErrorNoImage error no image var ErrorNoImage = fmt.Errorf("image not exist") -//Namespace containerd image namespace +// Namespace containerd image namespace var Namespace = "k8s.io" -//ImagePull pull docker image +// ImagePull pull docker image // Deprecated: use sources.ImageClient.ImagePull instead func ImagePull(client *containerd.Client, ref string, username, password string, logger event.Logger, timeout int) (*containerd.Image, error) { printLog(logger, "info", fmt.Sprintf("start get image:%s", ref), map[string]string{"step": "pullimage"}) @@ -169,7 +169,7 @@ func ImageTag(containerdClient *containerd.Client, source, target string, logger return nil } -//ImageNameHandle 解析imagename +// ImageNameHandle 解析imagename func ImageNameHandle(imageName string) *model.ImageName { var i model.ImageName if strings.Contains(imageName, "/") { @@ -197,7 +197,7 @@ func ImageNameHandle(imageName string) *model.ImageName { return &i } -//ImageNameWithNamespaceHandle if have namespace,will parse namespace +// ImageNameWithNamespaceHandle if have namespace,will parse namespace func ImageNameWithNamespaceHandle(imageName string) *model.ImageName { var i model.ImageName if strings.Contains(imageName, "/") { @@ -229,8 +229,8 @@ func ImageNameWithNamespaceHandle(imageName string) *model.ImageName { return &i } -//ImagePush push image to registry -//timeout minutes of the unit +// ImagePush push image to registry +// timeout minutes of the unit // Deprecated: use sources.ImageClient.ImagePush instead func ImagePush(client *containerd.Client, rawRef, user, pass string, logger event.Logger, timeout int) error { printLog(logger, "info", fmt.Sprintf("start push image：%s", rawRef), map[string]string{"step": "pushimage"}) @@ -323,7 +323,7 @@ func ImagePush(client *containerd.Client, rawRef, user, pass string, logger even return nil } -//TrustedImagePush push image to trusted registry +// TrustedImagePush push image to trusted registry func TrustedImagePush(containerdClient *containerd.Client, image, user, pass string, logger event.Logger, timeout int) error { if err := CheckTrustedRepositories(image, user, pass); err != nil { return err @@ -331,7 +331,7 @@ func TrustedImagePush(containerdClient *containerd.Client, image, user, pass str return ImagePush(containerdClient, image, user, pass, logger, timeout) } -//CheckTrustedRepositories check Repositories is exist ,if not create it. +// CheckTrustedRepositories check Repositories is exist ,if not create it. func CheckTrustedRepositories(image, user, pass string) error { ref, err := reference.ParseNormalizedNamed(image) if err != nil { @@ -389,7 +389,7 @@ func EncodeAuthToBase64(authConfig types.AuthConfig) (string, error) { return base64.URLEncoding.EncodeToString(buf), nil } -//ImageBuild use kaniko build image +// ImageBuild use kaniko build image func ImageBuild(contextDir, cachePVCName, cacheMode, RbdNamespace, ServiceID, DeployVersion string, logger event.Logger, buildType, plugImageName, KanikoImage string, KanikoArgs []string) error { // create image name var buildImageName string @@ -421,6 +421,17 @@ func ImageBuild(contextDir, cachePVCName, cacheMode, RbdNamespace, ServiceID, De }, } podSpec := corev1.PodSpec{RestartPolicy: corev1.RestartPolicyOnFailure} // only support never and onfailure + hostIP := os.Getenv("HOST_IP") + if hostIP != "" { + podSpec.NodeSelector = map[string]string{ + "kubernetes.io/hostname": hostIP, + } + podSpec.Tolerations = []corev1.Toleration{ + { + Operator: "Exists", + }, + } + } volumes, volumeMounts := CreateVolumesAndMounts(contextDir, buildType, cacheMode, cachePVCName) podSpec.Volumes = volumes // container config @@ -460,7 +471,7 @@ func ImageBuild(contextDir, cachePVCName, cacheMode, RbdNamespace, ServiceID, De return nil } -//ImageInspectWithRaw get image inspect +// ImageInspectWithRaw get image inspect func ImageInspectWithRaw(dockerCli *client.Client, image string) (*types.ImageInspect, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -471,7 +482,7 @@ func ImageInspectWithRaw(dockerCli *client.Client, image string) (*types.ImageIn return &ins, nil } -//ImageSave save image to tar file +// ImageSave save image to tar file // destination destination file name eg. /tmp/xxx.tar func ImageSave(dockerCli *client.Client, image, destination string, logger event.Logger) error { ctx, cancel := context.WithCancel(context.Background()) @@ -484,7 +495,7 @@ func ImageSave(dockerCli *client.Client, image, destination string, logger event return CopyToFile(destination, rc) } -//MultiImageSave save multi image to tar file +// MultiImageSave save multi image to tar file // destination destination file name eg. /tmp/xxx.tar func MultiImageSave(ctx context.Context, dockerCli *client.Client, destination string, logger event.Logger, images ...string) error { rc, err := dockerCli.ImageSave(ctx, images) @@ -495,7 +506,7 @@ func MultiImageSave(ctx context.Context, dockerCli *client.Client, destination s return CopyToFile(destination, rc) } -//ImageLoad load image from tar file +// ImageLoad load image from tar file // destination destination file name eg. /tmp/xxx.tar func ImageLoad(dockerCli *client.Client, tarFile string, logger event.Logger) error { ctx, cancel := context.WithCancel(context.Background()) @@ -537,7 +548,7 @@ func ImageLoad(dockerCli *client.Client, tarFile string, logger event.Logger) er return nil } -//ImageImport save image to tar file +// ImageImport save image to tar file // source source file name eg. /tmp/xxx.tar func ImageImport(dockerCli *client.Client, image, source string, logger event.Logger) error { ctx, cancel := context.WithCancel(context.Background()) @@ -608,7 +619,7 @@ func CopyToFile(outfile string, r io.Reader) error { return nil } -//ImageRemove remove image +// ImageRemove remove image func ImageRemove(containerdClient *containerd.Client, image string) error { ctx := namespaces.WithNamespace(context.Background(), Namespace) imageStore := containerdClient.ImageService()