diff --git a/backend/modules/evaluation/domain/service/evaluator.go b/backend/modules/evaluation/domain/service/evaluator.go index 00b5baf282..d2cc03547f 100644 --- a/backend/modules/evaluation/domain/service/evaluator.go +++ b/backend/modules/evaluation/domain/service/evaluator.go @@ -31,6 +31,8 @@ type EvaluatorService interface { DeleteEvaluator(ctx context.Context, evaluatorIDs []int64, userID string) error // RunEvaluator evaluator_version 运行 RunEvaluator(ctx context.Context, request *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) + // CreateEvaluatorRunFailRecord creates a failed evaluator record for a run attempt that failed before a normal record was persisted. + CreateEvaluatorRunFailRecord(ctx context.Context, request *entity.RunEvaluatorRequest, runErr error) (*entity.EvaluatorRecord, error) // ShouldInterceptEvaluator 判断评估器是否应劫持本次评估，劫持时创建记录并返回 ShouldInterceptEvaluator(ctx context.Context, request *entity.RunEvaluatorRequest) (record *entity.EvaluatorRecord, intercepted bool, err error) // AsyncRunEvaluator Agent evaluator_version 异步运行 diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index 8d97d68d80..32098b01af 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -848,6 +848,68 @@ func (e *EvaluatorServiceImpl) RunEvaluator(ctx context.Context, request *entity return recordDO, nil } +// CreateEvaluatorRunFailRecord creates a failed evaluator record for an evaluator run attempt that failed +// before RunEvaluator/AsyncRunEvaluator could persist its normal record. This keeps experiment turn results +// complete and preserves the original evaluator-level failure reason for users. +func (e *EvaluatorServiceImpl) CreateEvaluatorRunFailRecord(ctx context.Context, request *entity.RunEvaluatorRequest, runErr error) (*entity.EvaluatorRecord, error) { + if request == nil { + return nil, errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("run evaluator request is nil")) + } + if runErr == nil { + runErr = errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("evaluator run failed")) + } + + recordID, err := e.idgen.GenID(ctx) + if err != nil { + return nil, err + } + + code := int32(errno.CommonInternalErrorCode) + statusErr, isStatusErr := errorx.FromStatusError(runErr) + if isStatusErr && statusErr.Code() > 0 { + code = statusErr.Code() + } + errMsg := errorx.ErrorWithoutStack(runErr) + if e.cConfiger != nil && (!isStatusErr || statusErr.Code() != errno.CustomRPCEvaluatorRunFailedCode) { + if converted := e.cConfiger.GetErrCtrl(ctx).ConvertErrMsg(errMsg); converted != "" { + errMsg = converted + } + } + + userIDInContext := session.UserIDInCtxOrEmpty(ctx) + now := time.Now().UnixMilli() + recordDO := &entity.EvaluatorRecord{ + ID: recordID, + SpaceID: request.SpaceID, + ExperimentID: request.ExperimentID, + ExperimentRunID: request.ExperimentRunID, + ItemID: request.ItemID, + TurnID: request.TurnID, + EvaluatorVersionID: request.EvaluatorVersionID, + LogID: logs.GetLogID(ctx), + EvaluatorInputData: request.InputData, + EvaluatorOutputData: &entity.EvaluatorOutputData{ + EvaluatorRunError: &entity.EvaluatorRunError{ + Code: code, + Message: errMsg, + }, + }, + Status: entity.EvaluatorRunStatusFail, + Ext: request.Ext, + BaseInfo: &entity.BaseInfo{ + CreatedBy: &entity.UserInfo{UserID: gptr.Of(userIDInContext)}, + UpdatedBy: &entity.UserInfo{UserID: gptr.Of(userIDInContext)}, + CreatedAt: gptr.Of(now), + UpdatedAt: gptr.Of(now), + }, + } + + if err := e.evaluatorRecordRepo.CreateEvaluatorRecord(ctx, recordDO); err != nil { + return nil, err + } + return recordDO, nil +} + // AsyncRunEvaluator Agent evaluator_version 异步运行 func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { evaluatorDOList, err := e.evaluatorRepo.BatchGetEvaluatorByVersionID(ctx, nil, []int64{request.EvaluatorVersionID}, false, false) diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index beb9644423..9cb07f58f7 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4325,3 +4325,159 @@ func TestEvaluatorServiceImpl_ShouldInterceptEvaluator(t *testing.T) { }) } } + +func TestEvaluatorServiceImpl_CreateEvaluatorRunFailRecord(t *testing.T) { + tests := []struct { + name string + runErr error + setupConfig func(*componentMocks.MockIConfiger) + wantCode int32 + wantMsg string + }{ + { + name: "status error uses code and strips stack", + runErr: errorx.NewByCode(errno.EvaluatorQPSLimitCode, errorx.WithExtraMsg("evaluator throttled due to space-level rate limit")), + setupConfig: func(mockConfiger *componentMocks.MockIConfiger) { + mockConfiger.EXPECT().GetErrCtrl(gomock.Any()).Return(entity.DefaultExptErrCtrl()) + }, + wantCode: int32(errno.EvaluatorQPSLimitCode), + wantMsg: "evaluator throttled", + }, + { + name: "config conversion overrides non custom rpc message", + runErr: errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("raw evaluator failure")), + setupConfig: func(mockConfiger *componentMocks.MockIConfiger) { + mockConfiger.EXPECT().GetErrCtrl(gomock.Any()).Return(&entity.ExptErrCtrl{ + ResultErrConverts: []*entity.ResultErrConvert{ + {MatchedText: "raw evaluator failure", ToErrMsg: "converted evaluator failure"}, + }, + }) + }, + wantCode: int32(errno.CommonInternalErrorCode), + wantMsg: "converted evaluator failure", + }, + { + name: "custom rpc error skips config conversion and strips stack", + runErr: errorx.NewByCode(errno.CustomRPCEvaluatorRunFailedCode, errorx.WithExtraMsg("custom rpc failed")), + wantCode: int32(errno.CustomRPCEvaluatorRunFailedCode), + wantMsg: "custom rpc failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl) + mockEvaluatorRecordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + mockConfiger := componentMocks.NewMockIConfiger(ctrl) + + s := &EvaluatorServiceImpl{ + idgen: mockIDGen, + evaluatorRecordRepo: mockEvaluatorRecordRepo, + cConfiger: mockConfiger, + } + + req := &entity.RunEvaluatorRequest{ + SpaceID: 1, + EvaluatorVersionID: 2, + InputData: &entity.EvaluatorInputData{}, + ExperimentID: 3, + ExperimentRunID: 4, + ItemID: 5, + TurnID: 6, + Ext: map[string]string{"k": "v"}, + } + + if tt.setupConfig != nil { + tt.setupConfig(mockConfiger) + } + mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(100), nil) + mockEvaluatorRecordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, record *entity.EvaluatorRecord) error { + require.NotNil(t, record) + assert.Equal(t, int64(100), record.ID) + assert.Equal(t, req.SpaceID, record.SpaceID) + assert.Equal(t, req.EvaluatorVersionID, record.EvaluatorVersionID) + assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) + require.NotNil(t, record.EvaluatorOutputData) + require.NotNil(t, record.EvaluatorOutputData.EvaluatorRunError) + assert.Equal(t, tt.wantCode, record.EvaluatorOutputData.EvaluatorRunError.Code) + assert.Contains(t, record.EvaluatorOutputData.EvaluatorRunError.Message, tt.wantMsg) + assert.NotContains(t, record.EvaluatorOutputData.EvaluatorRunError.Message, "stack=") + return nil + }, + ) + + record, err := s.CreateEvaluatorRunFailRecord(context.Background(), req, tt.runErr) + require.NoError(t, err) + require.NotNil(t, record) + assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) + }) + } +} + +func TestEvaluatorServiceImpl_CreateEvaluatorRunFailRecord_Errors(t *testing.T) { + t.Run("nil request returns error", func(t *testing.T) { + s := &EvaluatorServiceImpl{} + record, err := s.CreateEvaluatorRunFailRecord(context.Background(), nil, errors.New("run failed")) + assert.Error(t, err) + assert.Nil(t, record) + }) + + t.Run("id generation error", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl) + s := &EvaluatorServiceImpl{idgen: mockIDGen} + + mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(0), errors.New("idgen failed")) + + record, err := s.CreateEvaluatorRunFailRecord(context.Background(), &entity.RunEvaluatorRequest{EvaluatorVersionID: 1}, errors.New("run failed")) + assert.Error(t, err) + assert.Nil(t, record) + }) + + t.Run("nil run error uses default failure", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl) + mockEvaluatorRecordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + s := &EvaluatorServiceImpl{idgen: mockIDGen, evaluatorRecordRepo: mockEvaluatorRecordRepo} + + mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(100), nil) + mockEvaluatorRecordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, record *entity.EvaluatorRecord) error { + require.NotNil(t, record) + require.NotNil(t, record.EvaluatorOutputData) + require.NotNil(t, record.EvaluatorOutputData.EvaluatorRunError) + assert.Equal(t, int32(errno.CommonInternalErrorCode), record.EvaluatorOutputData.EvaluatorRunError.Code) + assert.Contains(t, record.EvaluatorOutputData.EvaluatorRunError.Message, "evaluator run failed") + return nil + }, + ) + + record, err := s.CreateEvaluatorRunFailRecord(context.Background(), &entity.RunEvaluatorRequest{EvaluatorVersionID: 1}, nil) + require.NoError(t, err) + require.NotNil(t, record) + }) + + t.Run("create record error bubbles up", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl) + mockEvaluatorRecordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + s := &EvaluatorServiceImpl{idgen: mockIDGen, evaluatorRecordRepo: mockEvaluatorRecordRepo} + + mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(100), nil) + mockEvaluatorRecordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(errors.New("create record failed")) + + record, err := s.CreateEvaluatorRunFailRecord(context.Background(), &entity.RunEvaluatorRequest{EvaluatorVersionID: 1}, errors.New("run failed")) + assert.Error(t, err) + assert.Nil(t, record) + }) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index ae3fdd48e6..530995da6e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -7,9 +7,9 @@ import ( "context" "fmt" "strconv" + "strings" "time" - "github.com/bytedance/gg/gcond" "github.com/bytedance/gg/gptr" "github.com/jinzhu/copier" @@ -163,7 +163,11 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * clone.EvaluatorResultIds = &entity.EvaluatorResults{ EvalVerIDToResID: make(map[int64]int64, len(result.EvaluatorResults)), } - for _, er := range result.EvaluatorResults { + for evID, er := range result.EvaluatorResults { + if er == nil { + logs.CtxWarn(ctx, "[ExptTurnEval] nil evaluator record, evaluator_version_id: %v", evID) + continue + } clone.EvaluatorResultIds.EvalVerIDToResID[er.EvaluatorVersionID] = er.ID if er.EvaluatorOutputData != nil && er.EvaluatorOutputData.EvaluatorRunError != nil && er.EvaluatorOutputData.EvaluatorRunError.Code > 0 { evalErr = errno.NewEvaluatorResultErr(er.EvaluatorOutputData.EvaluatorRunError.Message) @@ -172,6 +176,8 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * if result.EvalErr != nil { evalErr = result.EvalErr + } else if evalErr == nil { + evalErr = e.validateEvaluatorResultsComplete(etec, result) } if evalErr != nil { @@ -196,7 +202,10 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * clone.Status = entity.TurnRunState_Fail clone.ErrMsg = errno.SerializeErr(evalErr) } else { - clone.Status = gcond.If(result.AsyncAbort, clone.Status, entity.TurnRunState_Success) + if !result.AsyncAbort { + clone.Status = entity.TurnRunState_Success + clone.ErrMsg = "" + } } result.SetEvalErr(evalErr) @@ -211,6 +220,35 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * return nil } +func (e *ExptItemEvalCtxExecutor) validateEvaluatorResultsComplete(etec *entity.ExptTurnEvalCtx, result *entity.ExptTurnRunResult) error { + if etec == nil || etec.Expt == nil || result == nil || result.AsyncAbort { + return nil + } + if etec.Expt.EvalConf == nil || etec.Expt.EvalConf.ConnectorConf.EvaluatorsConf == nil || len(etec.Expt.Evaluators) == 0 { + return nil + } + + missing := make([]string, 0) + for _, evaluator := range etec.Expt.Evaluators { + if evaluator == nil { + continue + } + evaluatorVersionID := evaluator.GetEvaluatorVersionID() + if evaluatorVersionID == 0 { + continue + } + record := result.GetEvaluatorRecord(evaluatorVersionID) + if record == nil || record.ID == 0 { + missing = append(missing, strconv.FormatInt(evaluatorVersionID, 10)) + } + } + if len(missing) == 0 { + return nil + } + + return errno.NewEvaluatorResultErr(fmt.Sprintf("evaluator result missing, evaluator_version_ids: %s", strings.Join(missing, ","))) +} + func (e *ExptItemEvalCtxExecutor) SetItemRunProcessing(ctx context.Context, exptID, exptRunID, itemID, spaceID int64, session *entity.Session) error { return e.ItemResultRepo.UpdateItemRunLog(ctx, exptID, exptRunID, []int64{itemID}, map[string]any{"status": int32(entity.ItemRunState_Processing)}, spaceID) } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index b5612830bc..dcfff336e8 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -444,7 +444,7 @@ func Test_ExptItemEvalCtxExecutor_storeTurnRunResult(t *testing.T) { }) t.Run("正常流程", func(t *testing.T) { - turnResultLog := &entity.ExptTurnResultRunLog{ID: 1, TurnID: 1} + turnResultLog := &entity.ExptTurnResultRunLog{ID: 1, TurnID: 1, ErrMsg: "old error"} etec := &entity.ExptTurnEvalCtx{ Turn: &entity.Turn{ID: 1}, ExptItemEvalCtx: &entity.ExptItemEvalCtx{ @@ -458,9 +458,50 @@ func Test_ExptItemEvalCtxExecutor_storeTurnRunResult(t *testing.T) { TargetResult: &entity.EvalTargetRecord{ID: 10}, EvaluatorResults: map[int64]*entity.EvaluatorRecord{1: {ID: 100, EvaluatorVersionID: 1}}, } - mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil) + mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, logs []*entity.ExptTurnResultRunLog) error { + require.Len(t, logs, 1) + assert.Equal(t, entity.TurnRunState_Success, logs[0].Status) + assert.Empty(t, logs[0].ErrMsg) + return nil + }) + err := executor.storeTurnRunResult(context.Background(), etec, result) + assert.NoError(t, err) + }) + + t.Run("缺少评估器结果时落失败状态", func(t *testing.T) { + turnResultLog := &entity.ExptTurnResultRunLog{ID: 1, TurnID: 1} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ + ID: 1, + SpaceID: 2, + Evaluators: []*entity.Evaluator{ + {EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: 1}}, + {EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: 2}}, + }, + EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{EvaluatorsConf: &entity.EvaluatorsConf{}}}, + }, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 2}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{1: turnResultLog}}, + }, + } + result := &entity.ExptTurnRunResult{ + TargetResult: &entity.EvalTargetRecord{ID: 10}, + EvaluatorResults: map[int64]*entity.EvaluatorRecord{1: {ID: 100, EvaluatorVersionID: 1}}, + } + mockConfiger.EXPECT().GetErrCtrl(gomock.Any()).Return(entity.DefaultExptErrCtrl()) + mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, logs []*entity.ExptTurnResultRunLog) error { + require.Len(t, logs, 1) + assert.Equal(t, entity.TurnRunState_Fail, logs[0].Status) + assert.Contains(t, logs[0].ErrMsg, "evaluator result missing") + return nil + }) + err := executor.storeTurnRunResult(context.Background(), etec, result) assert.NoError(t, err) + assert.Error(t, result.GetEvalErr()) }) t.Run("ctx取消后仍落turn失败状态", func(t *testing.T) { diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index 7dfca6f335..b481c57946 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -461,27 +461,30 @@ func (e *DefaultExptTurnEvaluationImpl) callEvaluators(ctx context.Context, exec continue } + baseRunReq := &entity.RunEvaluatorRequest{ + SpaceID: spaceID, + Name: "", + EvaluatorVersionID: evForCapture.GetEvaluatorVersionID(), + InputData: inputDataForCapture, + ExperimentID: etec.Event.ExptID, + ExperimentRunID: etec.Event.ExptRunID, + ItemID: item.ItemID, + TurnID: turn.ID, + Ext: etec.Ext, + EvaluatorRunConf: ecForCapture.RunConf, + } + if evForCapture.IsAsync() { pool.Add(func() error { - return e.asyncCallEvaluator(ctx, evForCapture, ecForCapture, etec, inputDataForCapture, &recordMap) + return e.asyncCallEvaluator(ctx, evForCapture, ecForCapture, etec, inputDataForCapture, baseRunReq, &recordMap) }) } else { pool.Add(func() error { var err error defer e.metric.EmitTurnExecEvaluatorResult(spaceID, err != nil) - evaluatorRecord, err := e.evaluatorService.RunEvaluator(ctx, &entity.RunEvaluatorRequest{ - SpaceID: spaceID, - Name: "", - EvaluatorVersionID: evForCapture.GetEvaluatorVersionID(), - InputData: inputDataForCapture, - ExperimentID: etec.Event.ExptID, - ExperimentRunID: etec.Event.ExptRunID, - ItemID: item.ItemID, - TurnID: turn.ID, - Ext: etec.Ext, - EvaluatorRunConf: ecForCapture.RunConf, - }) + evaluatorRecord, err := e.evaluatorService.RunEvaluator(ctx, baseRunReq) if err != nil { + e.createAndStoreFailedEvaluatorRecord(ctx, baseRunReq, err, &recordMap) return err } @@ -502,12 +505,29 @@ func (e *DefaultExptTurnEvaluationImpl) callEvaluators(ctx context.Context, exec return records, err } +func (e *DefaultExptTurnEvaluationImpl) createAndStoreFailedEvaluatorRecord(ctx context.Context, req *entity.RunEvaluatorRequest, runErr error, recordMap *sync.Map) { + if e == nil || e.evaluatorService == nil || req == nil || recordMap == nil { + return + } + failedRecord, createErr := e.evaluatorService.CreateEvaluatorRunFailRecord(ctx, req, runErr) + if createErr != nil { + logs.CtxError(ctx, "[CallEvaluators] create failed evaluator record fail, evaluator_version_id: %v, origin_err: %v, create_err: %v", req.EvaluatorVersionID, runErr, createErr) + return + } + if failedRecord == nil { + logs.CtxWarn(ctx, "[CallEvaluators] create failed evaluator record got nil, evaluator_version_id: %v, origin_err: %v", req.EvaluatorVersionID, runErr) + return + } + recordMap.Store(req.EvaluatorVersionID, failedRecord) +} + func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluator( ctx context.Context, ev *entity.Evaluator, ec *entity.EvaluatorConf, etec *entity.ExptTurnEvalCtx, inputData *entity.EvaluatorInputData, + baseRunReq *entity.RunEvaluatorRequest, recordMap *sync.Map, ) error { var err error @@ -527,6 +547,7 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluator( EvaluatorRunConf: ec.RunConf, }) if err != nil { + e.createAndStoreFailedEvaluatorRecord(ctx, baseRunReq, err, recordMap) return err } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index f6547beae7..a49875df57 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -12,6 +12,7 @@ import ( "github.com/bytedance/gg/gptr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "github.com/coze-dev/coze-loop/backend/infra/external/benefit" @@ -21,6 +22,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" svcmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" ) // mock DenyReason implementation @@ -405,7 +407,8 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent(t *testing.T) { }, ) - err := service.asyncCallEvaluator(context.Background(), ev, ec, etec, inputData, &recordMap) + baseRunReq := &entity.RunEvaluatorRequest{SpaceID: 1, EvaluatorVersionID: 101, InputData: inputData, ExperimentID: 2, ExperimentRunID: 3, ItemID: 4, TurnID: 5, Ext: etec.Ext, EvaluatorRunConf: ec.RunConf} + err := service.asyncCallEvaluator(context.Background(), ev, ec, etec, inputData, baseRunReq, &recordMap) assert.NoError(t, err) // verify recordMap @@ -458,8 +461,6 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent_Errors(t *testin inputData := &entity.EvaluatorInputData{ InputFields: map[string]*entity.Content{}, } - var recordMap sync.Map - tests := []struct { name string mockSetup func() @@ -469,7 +470,23 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent_Errors(t *testin name: "AsyncRunEvaluator error", mockSetup: func() { mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), true) - mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(nil, errors.New("async run error")) + runErr := errors.New("async run error") + mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(nil, runErr) + mockEvaluatorService.EXPECT().CreateEvaluatorRunFailRecord(gomock.Any(), gomock.Any(), runErr).Return(&entity.EvaluatorRecord{ + ID: 303, + EvaluatorVersionID: 101, + Status: entity.EvaluatorRunStatusFail, + }, nil) + }, + wantErr: true, + }, + { + name: "AsyncRunEvaluator error and failed record creation error", + mockSetup: func() { + mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), true) + runErr := errors.New("async run error") + mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(nil, runErr) + mockEvaluatorService.EXPECT().CreateEvaluatorRunFailRecord(gomock.Any(), gomock.Any(), runErr).Return(nil, errors.New("create failed record error")) }, wantErr: true, }, @@ -478,7 +495,8 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent_Errors(t *testin mockSetup: func() { mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), true) mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ - ID: 202, + ID: 202, + EvaluatorVersionID: 101, }, nil) mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("set ctx error")) }, @@ -488,13 +506,26 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent_Errors(t *testin for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + var recordMap sync.Map tt.mockSetup() - err := service.asyncCallEvaluator(context.Background(), ev, ec, etec, inputData, &recordMap) + baseRunReq := &entity.RunEvaluatorRequest{SpaceID: 1, EvaluatorVersionID: 101, InputData: inputData, ExperimentID: 2, ExperimentRunID: 3, ItemID: 4, TurnID: 5, Ext: etec.Ext, EvaluatorRunConf: ec.RunConf} + err := service.asyncCallEvaluator(context.Background(), ev, ec, etec, inputData, baseRunReq, &recordMap) if tt.wantErr { assert.Error(t, err) } else { assert.NoError(t, err) } + switch tt.name { + case "AsyncRunEvaluator error": + val, ok := recordMap.Load(int64(101)) + require.True(t, ok) + record, ok := val.(*entity.EvaluatorRecord) + require.True(t, ok) + assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) + case "AsyncRunEvaluator error and failed record creation error": + _, ok := recordMap.Load(int64(101)) + assert.False(t, ok) + } }) } } @@ -1468,7 +1499,17 @@ func TestDefaultExptTurnEvaluationImpl_CallEvaluators(t *testing.T) { prepare: func() { mockBenefitService.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil) mockEvaluatorService.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil) - mockEvaluatorService.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).Return(nil, errors.New("run evaluator failed")) + runErr := errors.New("run evaluator failed") + mockEvaluatorService.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).Return(nil, runErr) + mockEvaluatorService.EXPECT().CreateEvaluatorRunFailRecord(gomock.Any(), gomock.Any(), runErr).Return(&entity.EvaluatorRecord{ + ID: 999, + EvaluatorVersionID: 1, + Status: entity.EvaluatorRunStatusFail, + EvaluatorOutputData: &entity.EvaluatorOutputData{EvaluatorRunError: &entity.EvaluatorRunError{ + Code: int32(errno.CommonInternalErrorCode), + Message: "run evaluator failed", + }}, + }, nil) mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), gomock.Any()) }, etec: &entity.ExptTurnEvalCtx{ @@ -3054,7 +3095,6 @@ func TestDefaultExptTurnEvaluationImpl_callTarget_EdgeCases(t *testing.T) { } func TestDefaultExptTurnEvaluationImpl_callEvaluators_EdgeCases(t *testing.T) { - t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -3112,7 +3152,9 @@ func TestDefaultExptTurnEvaluationImpl_callEvaluators_EdgeCases(t *testing.T) { etec: &entity.ExptTurnEvalCtx{ ExptItemEvalCtx: &entity.ExptItemEvalCtx{ EvalSetItem: &entity.EvaluationSetItem{ItemID: 1}, + Event: &entity.ExptItemEvalEvent{ExptID: 10, ExptRunID: 20, SpaceID: 2}, Expt: &entity.Experiment{ + SpaceID: 2, Evaluators: []*entity.Evaluator{ {ID: 1, EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: 999}}, // Non-existent evaluator }, @@ -3144,7 +3186,9 @@ func TestDefaultExptTurnEvaluationImpl_callEvaluators_EdgeCases(t *testing.T) { etec: &entity.ExptTurnEvalCtx{ ExptItemEvalCtx: &entity.ExptItemEvalCtx{ EvalSetItem: &entity.EvaluationSetItem{ItemID: 1}, + Event: &entity.ExptItemEvalEvent{ExptID: 10, ExptRunID: 20, SpaceID: 2}, Expt: &entity.Experiment{ + SpaceID: 2, Evaluators: []*entity.Evaluator{ {ID: 1, EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: 1}}, }, @@ -3155,6 +3199,7 @@ func TestDefaultExptTurnEvaluationImpl_callEvaluators_EdgeCases(t *testing.T) { EvaluatorConf: []*entity.EvaluatorConf{ { EvaluatorVersionID: 1, + RunConf: &entity.EvaluatorRunConfig{}, IngressConf: &entity.EvaluatorIngressConf{ EvalSetAdapter: &entity.FieldAdapter{ FieldConfs: []*entity.FieldConf{{FieldName: "field1", FromField: "[invalid_json_path"}}, @@ -3184,7 +3229,6 @@ func TestDefaultExptTurnEvaluationImpl_callEvaluators_EdgeCases(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Parallel() tt.prepare() // Check if targetResult is nil to avoid panic if tt.target != nil && tt.target.EvalTargetOutputData == nil { @@ -3289,6 +3333,15 @@ func TestDefaultExptTurnEvaluationImpl_callEvaluators_ExecAll(t *testing.T) { } return &entity.EvaluatorRecord{ID: 2, Status: entity.EvaluatorRunStatusSuccess}, nil }).AnyTimes() + mockEvaluatorService.EXPECT().CreateEvaluatorRunFailRecord(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *entity.RunEvaluatorRequest, runErr error) (*entity.EvaluatorRecord, error) { + return &entity.EvaluatorRecord{ + ID: 100 + req.EvaluatorVersionID, + EvaluatorVersionID: req.EvaluatorVersionID, + Status: entity.EvaluatorRunStatusFail, + EvaluatorOutputData: &entity.EvaluatorOutputData{EvaluatorRunError: &entity.EvaluatorRunError{Code: int32(errno.CommonInternalErrorCode), Message: runErr.Error()}}, + }, nil + }).AnyTimes() records, err := service.callEvaluators(context.Background(), []int64{1, 2}, newEtec(), mockTargetResult, []*entity.Message{}) @@ -3323,6 +3376,15 @@ func TestDefaultExptTurnEvaluationImpl_callEvaluators_ExecAll(t *testing.T) { } return nil, errors.New(err2Msg) }).AnyTimes() + mockEvaluatorService.EXPECT().CreateEvaluatorRunFailRecord(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *entity.RunEvaluatorRequest, runErr error) (*entity.EvaluatorRecord, error) { + return &entity.EvaluatorRecord{ + ID: 100 + req.EvaluatorVersionID, + EvaluatorVersionID: req.EvaluatorVersionID, + Status: entity.EvaluatorRunStatusFail, + EvaluatorOutputData: &entity.EvaluatorOutputData{EvaluatorRunError: &entity.EvaluatorRunError{Code: int32(errno.CommonInternalErrorCode), Message: runErr.Error()}}, + }, nil + }).AnyTimes() _, err := service.callEvaluators(context.Background(), []int64{1, 2}, newEtec(), mockTargetResult, []*entity.Message{}) diff --git a/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go b/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go index b6cc8946af..82fd663ba0 100644 --- a/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go +++ b/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go @@ -161,6 +161,21 @@ func (mr *MockEvaluatorServiceMockRecorder) CreateEvaluator(ctx, evaluator, cid return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEvaluator", reflect.TypeOf((*MockEvaluatorService)(nil).CreateEvaluator), ctx, evaluator, cid) } +// CreateEvaluatorRunFailRecord mocks base method. +func (m *MockEvaluatorService) CreateEvaluatorRunFailRecord(ctx context.Context, request *entity.RunEvaluatorRequest, runErr error) (*entity.EvaluatorRecord, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateEvaluatorRunFailRecord", ctx, request, runErr) + ret0, _ := ret[0].(*entity.EvaluatorRecord) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateEvaluatorRunFailRecord indicates an expected call of CreateEvaluatorRunFailRecord. +func (mr *MockEvaluatorServiceMockRecorder) CreateEvaluatorRunFailRecord(ctx, request, runErr any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEvaluatorRunFailRecord", reflect.TypeOf((*MockEvaluatorService)(nil).CreateEvaluatorRunFailRecord), ctx, request, runErr) +} + // DebugEvaluator mocks base method. func (m *MockEvaluatorService) DebugEvaluator(ctx context.Context, evaluatorDO *entity.Evaluator, inputData *entity.EvaluatorInputData, evaluatorRunConf *entity.EvaluatorRunConfig, exptSpaceID int64) (*entity.EvaluatorOutputData, error) { m.ctrl.T.Helper()