Skip to content

refactor(backend): Split the checkBusinessWithBizID function into sin… #15843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ylgeeker wants to merge 5 commits into
base: v1.5.0
Choose a base branch
from
Draft

refactor(backend): Split the checkBusinessWithBizID function into sin… #15843

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
70e1b6b
refactor(backend): Split the checkBusinessWithBizID function into sin…
ylgeeker Jan 20, 2026
2bb13f0
refactor(backend): Extract FailureCollector and use group-based switc…
ylgeeker Jan 29, 2026
6f3499f
feat(backend): workflow failure collection, strategy matching, and sl…
ylgeeker Feb 3, 2026
7a0e434
refactor(backend): Split the workflow into some separate logic. issue…
ylgeeker Feb 3, 2026
db5f10c
feat(backend): Return read-only watch chan, use self prefix for workf…
ylgeeker Feb 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
pre-*-bkcodeai
.vscode
.python-version

**/CODE_REVIEW_SUGGESTIONS.md
bkcodeai.json
package-lock.json
*.swp
Expand Down
2 changes: 2 additions & 0 deletions dbm-services/common/dbha-v2/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,7 @@ build
logs
pids

**/CODE_REVIEW_SUGGESTIONS.md

docs/swagger.json
docs/swagger.yaml
13 changes: 12 additions & 1 deletion dbm-services/common/dbha-v2/internal/analysis/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ type Service struct {
engine *gin.Engine
httpApmSvr *http.Server
discoveryCli *discovery.Client
discovery *discovery.Discovery
regCli *discovery.Registry
wflow *workflow.Workflow
db *hamysql.GormDB
Expand Down Expand Up @@ -138,6 +139,10 @@ func (s *Service) Run(ctx context.Context) error {

func (s *Service) Close() {
s.wflow.Close()
if s.discovery != nil {
s.discovery.Close()
s.discovery = nil
}
if s.httpApmSvr != nil {
timeout := max(config.Cfg.Apm.ReadTimeout, config.Cfg.Apm.WriteTimeout)

Expand Down Expand Up @@ -167,6 +172,12 @@ func (s *Service) createDiscovery() error {
}
s.discoveryCli = cli

disc, err := cli.CreateDiscovery()
if err != nil {
return err
}
s.discovery = disc

s.regCli = cli.CreateRegistry()
s.updateInfo()
return nil
Expand Down Expand Up @@ -257,7 +268,7 @@ func (s *Service) createNotifier() error {
}

func (s *Service) createWorkflow(ctx context.Context) error {
wflow, err := workflow.New(s.discoveryCli, s.db)
wflow, err := workflow.New(s.discoveryCli, s.db, s.discovery, s.discoveryCli.GetSelfPrefix(), s.info.ID)
if err != nil {
return err
}
Expand Down
94 changes: 94 additions & 0 deletions dbm-services/common/dbha-v2/internal/analysis/workflow/alarm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/**
* MIT License
*
* Copyright (c) 2023 腾讯蓝鲸
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package workflow

import (
"fmt"
"time"

"dbm-services/common/dbha-v2/internal/analysis/config"
"dbm-services/common/dbha-v2/internal/analysis/detector"
"dbm-services/common/dbha-v2/pkg/logger"
"dbm-services/common/dbha-v2/pkg/monitor"
"dbm-services/common/dbha-v2/pkg/process"
)

// AlarmNotifier posts alarm events to the monitoring platform.
type AlarmNotifier struct{}

// NewAlarmNotifier creates an AlarmNotifier.
func NewAlarmNotifier() *AlarmNotifier {
return &AlarmNotifier{}
}

// TriggerWithDetectorResponse posts an alarm event with detector response dimensions.
func (n *AlarmNotifier) TriggerWithDetectorResponse(procName string, status process.Status,
content string, exitCode int, resp *detector.Response) {
target := instanceKey(resp.Meta.BkCloudID, resp.Meta.IP, resp.Meta.Port)
monitorEvent := &monitor.EventData{
Name: resp.DbEventName.String(),
Target: target,
Timestamp: uint64(time.Now().UnixMilli()),
}

monitorEvent.Content.Content = content

monitorEvent.Dimension.DetectorExitCode = exitCode
monitorEvent.Dimension.IP = resp.Meta.IP
monitorEvent.Dimension.Port = resp.Meta.Port
monitorEvent.Dimension.BkBizId = resp.Meta.BkBizID
monitorEvent.Dimension.DbClusterType = resp.Meta.ClusterType
monitorEvent.Dimension.DbMachineType = resp.Meta.MachineType
monitorEvent.Dimension.DetectorProcName = procName
monitorEvent.Dimension.DetectorProcStatus = status
monitorEvent.Dimension.DbEventName = resp.DbEventName
monitorEvent.Dimension.DbEventNameReason = resp.DbEventNameReason.Str()

logger.Info("the workflow triggers an alarm, db-inst: %d:%s:%d content: %s",
resp.Meta.BkCloudID, resp.Meta.IP, resp.Meta.Port, content)

if err := monitor.PostBKMonitor(config.Cfg.Monitor.Timeout, monitorEvent); err != nil {
logger.Warn("failed to post the alarm event to BkMonitor, errmsg: %s", err)
}
}

// TriggerWithBizId posts an alarm event for a business ID.
func (n *AlarmNotifier) TriggerWithBizId(bizId int, content string) {
target := fmt.Sprintf("BizId: %d", bizId)

monitorEvent := &monitor.EventData{
Name: "",
Target: target,
Timestamp: uint64(time.Now().UnixMilli()),
}

monitorEvent.Content.Content = content

monitorEvent.Dimension.BkBizId = bizId

if err := monitor.PostBKMonitor(config.Cfg.Monitor.Timeout, monitorEvent); err != nil {
logger.Warn("%s", err)
}
}
178 changes: 178 additions & 0 deletions dbm-services/common/dbha-v2/internal/analysis/workflow/checker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
/**
* MIT License
*
* Copyright (c) 2023 腾讯蓝鲸
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package workflow

import (
"sync"

"dbm-services/common/dbha-v2/internal/analysis/detector"
"dbm-services/common/dbha-v2/internal/analysis/workflow/parser"
"dbm-services/common/dbha-v2/pkg/logger"
"dbm-services/common/dbha-v2/pkg/storage/hamodel"
"dbm-services/common/dbha-v2/pkg/storage/haprobe"
)

// BusinessChecker runs event, host, status, and missed-probe checks for a business and triggers detector/switch via DetectorHandler.
type BusinessChecker struct {
parser *StatusParser
detector *DetectorHandler
}

// NewBusinessChecker creates a BusinessChecker.
func NewBusinessChecker(parser *StatusParser, detector *DetectorHandler) *BusinessChecker {
return &BusinessChecker{parser: parser, detector: detector}
}

// CheckEventWithBizId builds double-check tasks from dbEvents and runs liveness double-check.
func (c *BusinessChecker) CheckEventWithBizId(bizId int, dbEvents []*haprobe.DbEvent,
skipDbInsts map[string]*hamodel.SkipDbInstance, metaInsts map[string]*hamodel.DbmMetadata) {
_ = bizId

badInsts := []detector.DoubleCheckTask{}

for _, event := range dbEvents {
key := instanceKey(event.BkCloudID, event.Endpoint.Host, event.Endpoint.Port)

if _, exists := skipDbInsts[key]; exists {
logger.Info("skip the db-inst: %s", key)
continue
}

meta, exists := metaInsts[key]
if !exists {
logger.Warn("not found the meta for the db-inst: %s", key)
continue
}

logger.Warn("recheck the db-inst: %s", key)
badInsts = append(badInsts, detector.DoubleCheckTask{
Meta: meta,
DbType: event.DbTypeName,
})
}

c.detector.LivenessDoubleCheck(badInsts)
}

// CheckDbHosts parses host status and invokes checkDbEventsFunc with the resulting events.
func (c *BusinessChecker) CheckDbHosts(dbHosts []*haprobe.HostMetric, checkDbEventsFunc func(dbEvents []*haprobe.DbEvent)) {
dbEvents, err := c.parser.ParseHostStatus(dbHosts)
if err != nil {
logger.Warn("failed to parse the host status, errmsg: %s", err)
return
}

if len(dbEvents) == 0 {
return
}

checkDbEventsFunc(dbEvents)
}

// CheckDbStatus parses DB status and invokes checkDbEventsFunc with the resulting events.
func (c *BusinessChecker) CheckDbStatus(dbStatusVals []parser.DBTyperWrapper,
checkDbEventsFunc func(dbEvents []*haprobe.DbEvent)) {
dbEvents, err := c.parser.ParseDbStatus(dbStatusVals)
if err != nil {
logger.Warn("failed to parse the DB status, errmsg: %s", err)
return
}

checkDbEventsFunc(dbEvents)
}

// CheckMissedProbe finds instances with no probe and runs liveness double check.
func (c *BusinessChecker) CheckMissedProbe(dbStatus []*hamodel.DbhaDataStatus, skipDbInsts map[string]*hamodel.SkipDbInstance,
metaInsts map[string]*hamodel.DbmMetadata) {
dbMetricKeys := map[string]struct{}{}
for _, dbStat := range dbStatus {
key := instanceKey(dbStat.BkCloudID, dbStat.DbIp, dbStat.DbPort)
dbMetricKeys[key] = struct{}{}
}

missedProbeInsts := []detector.DoubleCheckTask{}
for _, dbMeta := range metaInsts {
key := instanceKey(dbMeta.BkCloudID, dbMeta.IP, dbMeta.Port)

if _, exists := skipDbInsts[key]; exists {
logger.Info("skip the db instance: %s", key)
continue
}

if _, exists := dbMetricKeys[key]; exists {
logger.Debug("db instance(%s) has probe", key)
continue
}

logger.Debug("missed probe instances: %#v", *dbMeta)
missedProbeInsts = append(missedProbeInsts, detector.DoubleCheckTask{
Meta: dbMeta,
DbType: dbMeta.GetDbType(),
})
}

c.detector.LivenessDoubleCheck(missedProbeInsts)
}

// RunBusinessChecks runs all check tasks concurrently for a business.
func (c *BusinessChecker) RunBusinessChecks(
bizId int,
dbStatus []*hamodel.DbhaDataStatus,
statusData *DbStatusData,
skipInsts map[string]*hamodel.SkipDbInstance,
metaInsts map[string]*hamodel.DbmMetadata,
) {
checkDbEventFunc := func(dbEvents []*haprobe.DbEvent) {
c.CheckEventWithBizId(bizId, dbEvents, skipInsts, metaInsts)
}

var wg sync.WaitGroup

wg.Add(1)
go func() {
defer wg.Done()
c.CheckMissedProbe(dbStatus, skipInsts, metaInsts)
}()

wg.Add(1)
go func() {
defer wg.Done()
c.CheckEventWithBizId(bizId, statusData.DbEvents, skipInsts, metaInsts)
}()

wg.Add(1)
go func() {
defer wg.Done()
c.CheckDbHosts(statusData.DbHosts, checkDbEventFunc)
}()

wg.Add(1)
go func() {
defer wg.Done()
c.CheckDbStatus(statusData.DbStatusVals, checkDbEventFunc)
}()

wg.Wait()
}
Loading