[feat][evaluation] add failed evaluator record creation and handling

backend/modules/evaluation/domain/service/evaluator.go

@@ -31,6 +31,8 @@ type EvaluatorService interface {
 	DeleteEvaluator(ctx context.Context, evaluatorIDs []int64, userID string) error
 	// RunEvaluator evaluator_version 运行
 	RunEvaluator(ctx context.Context, request *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error)
+	// CreateEvaluatorRunFailRecord creates a failed evaluator record for a run attempt that failed before a normal record was persisted.
+	CreateEvaluatorRunFailRecord(ctx context.Context, request *entity.RunEvaluatorRequest, runErr error) (*entity.EvaluatorRecord, error)
 	// ShouldInterceptEvaluator 判断评估器是否应劫持本次评估，劫持时创建记录并返回
 	ShouldInterceptEvaluator(ctx context.Context, request *entity.RunEvaluatorRequest) (record *entity.EvaluatorRecord, intercepted bool, err error)
 	// AsyncRunEvaluator Agent evaluator_version 异步运行

backend/modules/evaluation/domain/service/evaluator_impl.go

@@ -848,6 +848,68 @@ func (e *EvaluatorServiceImpl) RunEvaluator(ctx context.Context, request *entity
 	return recordDO, nil
 }
 
+// CreateEvaluatorRunFailRecord creates a failed evaluator record for an evaluator run attempt that failed
+// before RunEvaluator/AsyncRunEvaluator could persist its normal record. This keeps experiment turn results
+// complete and preserves the original evaluator-level failure reason for users.
+func (e *EvaluatorServiceImpl) CreateEvaluatorRunFailRecord(ctx context.Context, request *entity.RunEvaluatorRequest, runErr error) (*entity.EvaluatorRecord, error) {
+	if request == nil {
+		return nil, errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("run evaluator request is nil"))
+	}
+	if runErr == nil {
+		runErr = errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("evaluator run failed"))
+	}
+
+	recordID, err := e.idgen.GenID(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	code := int32(errno.CommonInternalErrorCode)
+	statusErr, isStatusErr := errorx.FromStatusError(runErr)
+	if isStatusErr && statusErr.Code() > 0 {
+		code = statusErr.Code()
+	}
+	errMsg := errorx.ErrorWithoutStack(runErr)
+	if e.cConfiger != nil && (!isStatusErr || statusErr.Code() != errno.CustomRPCEvaluatorRunFailedCode) {
+		if converted := e.cConfiger.GetErrCtrl(ctx).ConvertErrMsg(errMsg); converted != "" {
+			errMsg = converted
+		}
+	}
+
+	userIDInContext := session.UserIDInCtxOrEmpty(ctx)
+	now := time.Now().UnixMilli()
+	recordDO := &entity.EvaluatorRecord{
+		ID:                 recordID,
+		SpaceID:            request.SpaceID,
+		ExperimentID:       request.ExperimentID,
+		ExperimentRunID:    request.ExperimentRunID,
+		ItemID:             request.ItemID,
+		TurnID:             request.TurnID,
+		EvaluatorVersionID: request.EvaluatorVersionID,
+		LogID:              logs.GetLogID(ctx),
+		EvaluatorInputData: request.InputData,
+		EvaluatorOutputData: &entity.EvaluatorOutputData{
+			EvaluatorRunError: &entity.EvaluatorRunError{
+				Code:    code,
+				Message: errMsg,
+			},
+		},
+		Status: entity.EvaluatorRunStatusFail,
+		Ext:    request.Ext,
+		BaseInfo: &entity.BaseInfo{
+			CreatedBy: &entity.UserInfo{UserID: gptr.Of(userIDInContext)},
+			UpdatedBy: &entity.UserInfo{UserID: gptr.Of(userIDInContext)},
+			CreatedAt: gptr.Of(now),
+			UpdatedAt: gptr.Of(now),
+		},
+	}
+
+	if err := e.evaluatorRecordRepo.CreateEvaluatorRecord(ctx, recordDO); err != nil {
+		return nil, err
+	}
+	return recordDO, nil
+}
+
 // AsyncRunEvaluator Agent evaluator_version 异步运行
 func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) {
 	evaluatorDOList, err := e.evaluatorRepo.BatchGetEvaluatorByVersionID(ctx, nil, []int64{request.EvaluatorVersionID}, false, false)

backend/modules/evaluation/domain/service/evaluator_impl_test.go

@@ -4325,3 +4325,159 @@ func TestEvaluatorServiceImpl_ShouldInterceptEvaluator(t *testing.T) {
 		})
 	}
 }
+
+func TestEvaluatorServiceImpl_CreateEvaluatorRunFailRecord(t *testing.T) {
+	tests := []struct {
+		name        string
+		runErr      error
+		setupConfig func(*componentMocks.MockIConfiger)
+		wantCode    int32
+		wantMsg     string
+	}{
+		{
+			name:   "status error uses code and strips stack",
+			runErr: errorx.NewByCode(errno.EvaluatorQPSLimitCode, errorx.WithExtraMsg("evaluator throttled due to space-level rate limit")),
+			setupConfig: func(mockConfiger *componentMocks.MockIConfiger) {
+				mockConfiger.EXPECT().GetErrCtrl(gomock.Any()).Return(entity.DefaultExptErrCtrl())
+			},
+			wantCode: int32(errno.EvaluatorQPSLimitCode),
+			wantMsg:  "evaluator throttled",
+		},
+		{
+			name:   "config conversion overrides non custom rpc message",
+			runErr: errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("raw evaluator failure")),
+			setupConfig: func(mockConfiger *componentMocks.MockIConfiger) {
+				mockConfiger.EXPECT().GetErrCtrl(gomock.Any()).Return(&entity.ExptErrCtrl{
+					ResultErrConverts: []*entity.ResultErrConvert{
+						{MatchedText: "raw evaluator failure", ToErrMsg: "converted evaluator failure"},
+					},
+				})
+			},
+			wantCode: int32(errno.CommonInternalErrorCode),
+			wantMsg:  "converted evaluator failure",
+		},
+		{
+			name:     "custom rpc error skips config conversion and strips stack",
+			runErr:   errorx.NewByCode(errno.CustomRPCEvaluatorRunFailedCode, errorx.WithExtraMsg("custom rpc failed")),
+			wantCode: int32(errno.CustomRPCEvaluatorRunFailedCode),
+			wantMsg:  "custom rpc failed",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+
+			mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl)
+			mockEvaluatorRecordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl)
+			mockConfiger := componentMocks.NewMockIConfiger(ctrl)
+
+			s := &EvaluatorServiceImpl{
+				idgen:               mockIDGen,
+				evaluatorRecordRepo: mockEvaluatorRecordRepo,
+				cConfiger:           mockConfiger,
+			}
+
+			req := &entity.RunEvaluatorRequest{
+				SpaceID:            1,
+				EvaluatorVersionID: 2,
+				InputData:          &entity.EvaluatorInputData{},
+				ExperimentID:       3,
+				ExperimentRunID:    4,
+				ItemID:             5,
+				TurnID:             6,
+				Ext:                map[string]string{"k": "v"},
+			}
+
+			if tt.setupConfig != nil {
+				tt.setupConfig(mockConfiger)
+			}
+			mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(100), nil)
+			mockEvaluatorRecordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).DoAndReturn(
+				func(ctx context.Context, record *entity.EvaluatorRecord) error {
+					require.NotNil(t, record)
+					assert.Equal(t, int64(100), record.ID)
+					assert.Equal(t, req.SpaceID, record.SpaceID)
+					assert.Equal(t, req.EvaluatorVersionID, record.EvaluatorVersionID)
+					assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status)
+					require.NotNil(t, record.EvaluatorOutputData)
+					require.NotNil(t, record.EvaluatorOutputData.EvaluatorRunError)
+					assert.Equal(t, tt.wantCode, record.EvaluatorOutputData.EvaluatorRunError.Code)
+					assert.Contains(t, record.EvaluatorOutputData.EvaluatorRunError.Message, tt.wantMsg)
+					assert.NotContains(t, record.EvaluatorOutputData.EvaluatorRunError.Message, "stack=")
+					return nil
+				},
+			)
+
+			record, err := s.CreateEvaluatorRunFailRecord(context.Background(), req, tt.runErr)
+			require.NoError(t, err)
+			require.NotNil(t, record)
+			assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status)
+		})
+	}
+}
+
+func TestEvaluatorServiceImpl_CreateEvaluatorRunFailRecord_Errors(t *testing.T) {
+	t.Run("nil request returns error", func(t *testing.T) {
+		s := &EvaluatorServiceImpl{}
+		record, err := s.CreateEvaluatorRunFailRecord(context.Background(), nil, errors.New("run failed"))
+		assert.Error(t, err)
+		assert.Nil(t, record)
+	})
+
+	t.Run("id generation error", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+
+		mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl)
+		s := &EvaluatorServiceImpl{idgen: mockIDGen}
+
+		mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(0), errors.New("idgen failed"))
+
+		record, err := s.CreateEvaluatorRunFailRecord(context.Background(), &entity.RunEvaluatorRequest{EvaluatorVersionID: 1}, errors.New("run failed"))
+		assert.Error(t, err)
+		assert.Nil(t, record)
+	})
+
+	t.Run("nil run error uses default failure", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+
+		mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl)
+		mockEvaluatorRecordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl)
+		s := &EvaluatorServiceImpl{idgen: mockIDGen, evaluatorRecordRepo: mockEvaluatorRecordRepo}
+
+		mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(100), nil)
+		mockEvaluatorRecordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).DoAndReturn(
+			func(ctx context.Context, record *entity.EvaluatorRecord) error {
+				require.NotNil(t, record)
+				require.NotNil(t, record.EvaluatorOutputData)
+				require.NotNil(t, record.EvaluatorOutputData.EvaluatorRunError)
+				assert.Equal(t, int32(errno.CommonInternalErrorCode), record.EvaluatorOutputData.EvaluatorRunError.Code)
+				assert.Contains(t, record.EvaluatorOutputData.EvaluatorRunError.Message, "evaluator run failed")
+				return nil
+			},
+		)
+
+		record, err := s.CreateEvaluatorRunFailRecord(context.Background(), &entity.RunEvaluatorRequest{EvaluatorVersionID: 1}, nil)
+		require.NoError(t, err)
+		require.NotNil(t, record)
+	})
+
+	t.Run("create record error bubbles up", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+
+		mockIDGen := idgenmocks.NewMockIIDGenerator(ctrl)
+		mockEvaluatorRecordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl)
+		s := &EvaluatorServiceImpl{idgen: mockIDGen, evaluatorRecordRepo: mockEvaluatorRecordRepo}
+
+		mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(100), nil)
+		mockEvaluatorRecordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(errors.New("create record failed"))
+
+		record, err := s.CreateEvaluatorRunFailRecord(context.Background(), &entity.RunEvaluatorRequest{EvaluatorVersionID: 1}, errors.New("run failed"))
+		assert.Error(t, err)
+		assert.Nil(t, record)
+	})
+}

backend/modules/evaluation/domain/service/expt_run_item_impl.go

@@ -7,9 +7,9 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
-	"github.com/bytedance/gg/gcond"
 	"github.com/bytedance/gg/gptr"
 	"github.com/jinzhu/copier"
 
@@ -163,7 +163,11 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec *
 	clone.EvaluatorResultIds = &entity.EvaluatorResults{
 		EvalVerIDToResID: make(map[int64]int64, len(result.EvaluatorResults)),
 	}
-	for _, er := range result.EvaluatorResults {
+	for evID, er := range result.EvaluatorResults {
+		if er == nil {
+			logs.CtxWarn(ctx, "[ExptTurnEval] nil evaluator record, evaluator_version_id: %v", evID)
+			continue
+		}
 		clone.EvaluatorResultIds.EvalVerIDToResID[er.EvaluatorVersionID] = er.ID
 		if er.EvaluatorOutputData != nil && er.EvaluatorOutputData.EvaluatorRunError != nil && er.EvaluatorOutputData.EvaluatorRunError.Code > 0 {
 			evalErr = errno.NewEvaluatorResultErr(er.EvaluatorOutputData.EvaluatorRunError.Message)
@@ -172,6 +176,8 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec *
 
 	if result.EvalErr != nil {
 		evalErr = result.EvalErr
+	} else if evalErr == nil {
+		evalErr = e.validateEvaluatorResultsComplete(etec, result)
 	}
 
 	if evalErr != nil {
@@ -196,7 +202,10 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec *
 		clone.Status = entity.TurnRunState_Fail
 		clone.ErrMsg = errno.SerializeErr(evalErr)
 	} else {
-		clone.Status = gcond.If(result.AsyncAbort, clone.Status, entity.TurnRunState_Success)
+		if !result.AsyncAbort {
+			clone.Status = entity.TurnRunState_Success
+			clone.ErrMsg = ""
+		}
 	}
 
 	result.SetEvalErr(evalErr)
@@ -211,6 +220,35 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec *
 	return nil
 }
 
+func (e *ExptItemEvalCtxExecutor) validateEvaluatorResultsComplete(etec *entity.ExptTurnEvalCtx, result *entity.ExptTurnRunResult) error {
+	if etec == nil || etec.Expt == nil || result == nil || result.AsyncAbort {
+		return nil
+	}
+	if etec.Expt.EvalConf == nil || etec.Expt.EvalConf.ConnectorConf.EvaluatorsConf == nil || len(etec.Expt.Evaluators) == 0 {
+		return nil
+	}
+
+	missing := make([]string, 0)
+	for _, evaluator := range etec.Expt.Evaluators {
+		if evaluator == nil {
+			continue
+		}
+		evaluatorVersionID := evaluator.GetEvaluatorVersionID()
+		if evaluatorVersionID == 0 {
+			continue
+		}
+		record := result.GetEvaluatorRecord(evaluatorVersionID)
+		if record == nil || record.ID == 0 {
+			missing = append(missing, strconv.FormatInt(evaluatorVersionID, 10))
+		}
+	}
+	if len(missing) == 0 {
+		return nil
+	}
+
+	return errno.NewEvaluatorResultErr(fmt.Sprintf("evaluator result missing, evaluator_version_ids: %s", strings.Join(missing, ",")))
+}
+
 func (e *ExptItemEvalCtxExecutor) SetItemRunProcessing(ctx context.Context, exptID, exptRunID, itemID, spaceID int64, session *entity.Session) error {
 	return e.ItemResultRepo.UpdateItemRunLog(ctx, exptID, exptRunID, []int64{itemID}, map[string]any{"status": int32(entity.ItemRunState_Processing)}, spaceID)
 }

backend/modules/evaluation/domain/service/expt_run_item_impl_test.go

@@ -444,7 +444,7 @@ func Test_ExptItemEvalCtxExecutor_storeTurnRunResult(t *testing.T) {
 	})
 
 	t.Run("正常流程", func(t *testing.T) {
-		turnResultLog := &entity.ExptTurnResultRunLog{ID: 1, TurnID: 1}
+		turnResultLog := &entity.ExptTurnResultRunLog{ID: 1, TurnID: 1, ErrMsg: "old error"}
 		etec := &entity.ExptTurnEvalCtx{
 			Turn: &entity.Turn{ID: 1},
 			ExptItemEvalCtx: &entity.ExptItemEvalCtx{
@@ -458,9 +458,50 @@ func Test_ExptItemEvalCtxExecutor_storeTurnRunResult(t *testing.T) {
 			TargetResult:     &entity.EvalTargetRecord{ID: 10},
 			EvaluatorResults: map[int64]*entity.EvaluatorRecord{1: {ID: 100, EvaluatorVersionID: 1}},
 		}
-		mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil)
+		mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, logs []*entity.ExptTurnResultRunLog) error {
+			require.Len(t, logs, 1)
+			assert.Equal(t, entity.TurnRunState_Success, logs[0].Status)
+			assert.Empty(t, logs[0].ErrMsg)
+			return nil
+		})
+		err := executor.storeTurnRunResult(context.Background(), etec, result)
+		assert.NoError(t, err)
+	})
+
+	t.Run("缺少评估器结果时落失败状态", func(t *testing.T) {
+		turnResultLog := &entity.ExptTurnResultRunLog{ID: 1, TurnID: 1}
+		etec := &entity.ExptTurnEvalCtx{
+			Turn: &entity.Turn{ID: 1},
+			ExptItemEvalCtx: &entity.ExptItemEvalCtx{
+				Expt: &entity.Experiment{
+					ID:      1,
+					SpaceID: 2,
+					Evaluators: []*entity.Evaluator{
+						{EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: 1}},
+						{EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: 2}},
+					},
+					EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{EvaluatorsConf: &entity.EvaluatorsConf{}}},
+				},
+				Event:               &entity.ExptItemEvalEvent{ExptRunID: 3},
+				EvalSetItem:         &entity.EvaluationSetItem{ItemID: 2},
+				ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{1: turnResultLog}},
+			},
+		}
+		result := &entity.ExptTurnRunResult{
+			TargetResult:     &entity.EvalTargetRecord{ID: 10},
+			EvaluatorResults: map[int64]*entity.EvaluatorRecord{1: {ID: 100, EvaluatorVersionID: 1}},
+		}
+		mockConfiger.EXPECT().GetErrCtrl(gomock.Any()).Return(entity.DefaultExptErrCtrl())
+		mockTurnResultRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, logs []*entity.ExptTurnResultRunLog) error {
+			require.Len(t, logs, 1)
+			assert.Equal(t, entity.TurnRunState_Fail, logs[0].Status)
+			assert.Contains(t, logs[0].ErrMsg, "evaluator result missing")
+			return nil
+		})
+
 		err := executor.storeTurnRunResult(context.Background(), etec, result)
 		assert.NoError(t, err)
+		assert.Error(t, result.GetEvalErr())
 	})
 
 	t.Run("ctx取消后仍落turn失败状态", func(t *testing.T) {