Support task params for Schedules/Integations (#3159)

* feat(ui): refactor task form

* refactor(tasks): move ansible params to separate component

* feat(tasks): add task params table

* feat(schedule): support task params

* feat(integrations): save task settings

* feat(schedule): spaces

* fix(schedule): cron builder

* fix(schedule): sql query

* fix(tasks): override env via survay variables

* feat(schedule): use task params

* fix(integrations): empty task params

* test: fix test

Author: fiftin

api/integration.go

@@ -249,16 +249,13 @@ func MatchCompare(value any, method db.IntegrationMatchMethodType, expected stri
 	}
 }
 
-func RunIntegration(integration db.Integration, project db.Project, r *http.Request, payload []byte) {
-
-	log.Info(fmt.Sprintf("Running integration %d", integration.ID))
+func GetTaskDefinition(integration db.Integration, payload []byte, r *http.Request) (taskDefinition db.Task, err error) {
 
 	var envValues = make([]db.IntegrationExtractValue, 0)
 	var taskValues = make([]db.IntegrationExtractValue, 0)
 
-	extractValuesForExtractor, err := helpers.Store(r).GetIntegrationExtractValues(project.ID, db.RetrieveQueryParams{}, integration.ID)
+	extractValuesForExtractor, err := helpers.Store(r).GetIntegrationExtractValues(integration.ProjectID, db.RetrieveQueryParams{}, integration.ID)
 	if err != nil {
-		log.Error(err)
 		return
 	}
 
@@ -273,25 +270,45 @@ func RunIntegration(integration db.Integration, project db.Project, r *http.Requ
 
 	var extractedEnvResults = Extract(envValues, r, payload)
 
-	environmentJSONBytes, err := json.Marshal(extractedEnvResults)
+	taskDefinition = integration.TaskParams.CreateTask()
+	taskDefinition.TemplateID = integration.TemplateID
+
+	var env map[string]any
+	err = json.Unmarshal([]byte(taskDefinition.Environment), &env)
 	if err != nil {
-		log.Error(err)
 		return
 	}
 
-	var extractedTaskResults = ExtractAsAnyForTaskParams(taskValues, r, payload)
+	for k, v := range extractedEnvResults {
+		env[k] = v
+	}
 
-	var environmentJSONString = string(environmentJSONBytes)
-	var taskDefinition = db.Task{
-		TemplateID:    integration.TemplateID,
-		ProjectID:     integration.ProjectID,
-		Environment:   environmentJSONString,
-		IntegrationID: &integration.ID,
+	envStr, err := json.Marshal(env)
+	if err != nil {
+		return
 	}
 
-	// Only assign extractedTaskResults to Params if it's not empty
-	if len(extractedTaskResults) > 0 {
-		taskDefinition.Params = extractedTaskResults
+	taskDefinition.Environment = string(envStr)
+
+	extractedTaskResults := ExtractAsAnyForTaskParams(taskValues, r, payload)
+	for k, v := range extractedTaskResults {
+		taskDefinition.Params[k] = v
+	}
+
+	return
+}
+
+func RunIntegration(integration db.Integration, project db.Project, r *http.Request, payload []byte) {
+
+	log.Info(fmt.Sprintf("Running integration %d", integration.ID))
+
+	taskDefinition, err := GetTaskDefinition(integration, payload, r)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"context":        "integrations",
+			"integration_id": integration.ID,
+		}).Error("Failed to get task definition")
+		return
 	}
 
 	tpl, err := helpers.Store(r).GetTemplate(integration.ProjectID, integration.TemplateID)

db/Integration.go

@@ -98,7 +98,10 @@ type Integration struct {
 	AuthHeader   string                `db:"auth_header" json:"auth_header"`
 	AuthSecret   AccessKey             `db:"-" json:"-" backup:"-"`
 	Searchable   bool                  `db:"searchable" json:"searchable"`
-	TaskParams   MapStringAnyField     `db:"task_params" json:"task_params"`
+	//TaskParams   MapStringAnyField     `db:"task_params" json:"task_params"`
+
+	TaskParamsID *int        `db:"task_params_id" json:"-" backup:"-"`
+	TaskParams   *TaskParams `db:"-" json:"task_params,omitempty" backup:"task_params"`
 }
 
 func (alias IntegrationAlias) ToAlias() Alias {

db/Migration.go

@@ -31,6 +31,7 @@ func GetMigrations(dialect string) []Migration {
 			{Version: "2.15.4"},
 			{Version: "2.16.0"},
 			{Version: "2.16.1"},
+			{Version: "2.16.2"},
 		}
 	}
 
@@ -111,6 +112,7 @@ func GetMigrations(dialect string) []Migration {
 		{Version: "2.15.4"},
 		{Version: "2.16.0"},
 		{Version: "2.16.1"},
+		{Version: "2.16.2"},
 	}
 }
 

db/Schedule.go

@@ -10,6 +10,9 @@ type Schedule struct {
 
 	LastCommitHash *string `db:"last_commit_hash" json:"-" backup:"-"`
 	RepositoryID   *int    `db:"repository_id" json:"repository_id" backup:"-"`
+
+	TaskParamsID *int        `db:"task_params_id" json:"-" backup:"-"`
+	TaskParams   *TaskParams `db:"-" json:"task_params,omitempty" backup:"task_params"`
 }
 
 type ScheduleWithTpl struct {

db/Store.go

@@ -484,6 +484,13 @@ var IntegrationProps = ObjectProps{
 	DefaultSortingColumn:  "name",
 }
 
+var TaskParamsProps = ObjectProps{
+	TableName:             "project__task_params",
+	Type:                  reflect.TypeOf(TaskParams{}),
+	PrimaryColumnName:     "id",
+	ReferringColumnSuffix: "params_id",
+}
+
 var IntegrationExtractValueProps = ObjectProps{
 	TableName:            "project__integration_extract_value",
 	Type:                 reflect.TypeOf(IntegrationExtractValue{}),

db/TaskParams.go

@@ -0,0 +1,35 @@
+package db
+
+type TaskParams struct {
+	ID        int `db:"id" json:"-"`
+	ProjectID int `db:"project_id" json:"-"`
+
+	Environment string  `db:"environment" json:"environment,omitempty"`
+	Arguments   *string `db:"arguments" json:"arguments,omitempty"`
+	GitBranch   *string `db:"git_branch" json:"git_branch,omitempty"`
+
+	Message string `db:"message" json:"message,omitempty"`
+
+	// Version is a build version.
+	// This field available only for Build tasks.
+	Version *string `db:"version" json:"version,omitempty"`
+
+	InventoryID *int `db:"inventory_id" json:"inventory_id,omitempty"`
+
+	Params MapStringAnyField `db:"params" json:"params,omitempty"`
+}
+
+func (p TaskParams) CreateTask() (task Task) {
+	task = Task{
+		ProjectID:   p.ProjectID,
+		Environment: p.Environment,
+		Arguments:   p.Arguments,
+		GitBranch:   p.GitBranch,
+		Message:     p.Message,
+		Version:     p.Version,
+		InventoryID: p.InventoryID,
+		Params:      p.Params,
+	}
+
+	return
+}

db/sql/SqlDb.go

@@ -94,6 +94,7 @@ func (d *SqlDbConnection) Connect() {
 	d.sql.AddTableWithName(db.Template{}, "project__template").SetKeys(true, "id")
 	d.sql.AddTableWithName(db.User{}, "user").SetKeys(true, "id")
 	d.sql.AddTableWithName(db.Session{}, "session").SetKeys(true, "id")
+	d.sql.AddTableWithName(db.TaskParams{}, "project__task_params").SetKeys(true, "id")
 
 	//if d.GetDialect() == util.DbDriverSQLite {
 	//	_, err = d.exec("PRAGMA foreign_keys = ON")

db/sql/integration.go

@@ -12,18 +12,29 @@ func (d *SqlDb) CreateIntegration(integration db.Integration) (newIntegration db
 		return
 	}
 
+	if integration.TaskParams != nil {
+		params := *integration.TaskParams
+		params.ProjectID = integration.ProjectID
+		err = d.Sql().Insert(&params)
+		if err != nil {
+			return
+		}
+		integration.TaskParamsID = &params.ID
+	}
+
 	insertID, err := d.insert(
 		"id",
 		"insert into project__integration "+
-			"(project_id, name, template_id, auth_method, auth_secret_id, auth_header, searchable) values "+
-			"(?, ?, ?, ?, ?, ?, ?)",
+			"(project_id, name, template_id, auth_method, auth_secret_id, auth_header, searchable, task_params_id) values "+
+			"(?, ?, ?, ?, ?, ?, ?, ?)",
 		integration.ProjectID,
 		integration.Name,
 		integration.TemplateID,
 		integration.AuthMethod,
 		integration.AuthSecretID,
 		integration.AuthHeader,
-		integration.Searchable)
+		integration.Searchable,
+		integration.TaskParamsID)
 
 	if err != nil {
 		return
@@ -42,6 +53,20 @@ func (d *SqlDb) GetIntegrations(projectID int, params db.RetrieveQueryParams) (i
 
 func (d *SqlDb) GetIntegration(projectID int, integrationID int) (integration db.Integration, err error) {
 	err = d.getObject(projectID, db.IntegrationProps, integrationID, &integration)
+	if err != nil {
+		return
+	}
+
+	if integration.TaskParamsID != nil {
+		var taskParams db.TaskParams
+		err = d.getObject(projectID, db.TaskParamsProps, *integration.TaskParamsID, &taskParams)
+		if err != nil {
+			return
+		}
+
+		integration.TaskParams = &taskParams
+	}
+
 	return
 }
 
@@ -54,25 +79,72 @@ func (d *SqlDb) GetIntegrationRefs(projectID int, integrationID int) (referrers
 	return
 }
 
-func (d *SqlDb) DeleteIntegration(projectID int, integrationID int) error {
-	return d.deleteObject(projectID, db.IntegrationProps, integrationID)
+func (d *SqlDb) DeleteIntegration(projectID int, integrationID int) (err error) {
+	var integration db.Integration
+	err = d.getObject(projectID, db.IntegrationProps, integrationID, &integration)
+	if err != nil {
+		return
+	}
+
+	err = d.deleteObject(projectID, db.IntegrationProps, integrationID)
+	if err != nil {
+		return
+	}
+
+	if integration.TaskParamsID != nil {
+		err = d.deleteObject(projectID, db.TaskParamsProps, *integration.TaskParamsID)
+	}
+	return
 }
 
-func (d *SqlDb) UpdateIntegration(integration db.Integration) error {
-	err := integration.Validate()
+func (d *SqlDb) UpdateIntegration(integration db.Integration) (err error) {
 
-	if err != nil {
-		return err
+	if err = integration.Validate(); err != nil {
+		return
+	}
+
+	if integration.TaskParams != nil {
+		var curr db.Integration
+		err = d.getObject(integration.ProjectID, db.IntegrationProps, integration.ID, &curr)
+		if err != nil {
+			return
+		}
+
+		params := *integration.TaskParams
+		params.ProjectID = integration.ProjectID
+
+		if curr.TaskParamsID == nil {
+			err = d.Sql().Insert(&params)
+		} else {
+			params.ID = *curr.TaskParamsID
+			_, err = d.Sql().Update(&params)
+		}
+
+		if err != nil {
+			return
+		}
+
+		integration.TaskParamsID = &params.ID
 	}
 
 	_, err = d.exec(
-		"update project__integration set `name`=?, template_id=?, auth_method=?, auth_secret_id=?, auth_header=?, searchable=? where `id`=?",
+		"update project__integration set "+
+			"`name`=?, "+
+			"template_id=?, "+
+			"auth_method=?, "+
+			"auth_secret_id=?, "+
+			"auth_header=?, "+
+			"searchable=?, "+
+			"task_params_id=? "+
+			"where project_id=? AND `id`=?",
 		integration.Name,
 		integration.TemplateID,
 		integration.AuthMethod,
 		integration.AuthSecretID,
 		integration.AuthHeader,
 		integration.Searchable,
+		integration.TaskParamsID,
+		integration.ProjectID,
 		integration.ID)
 
 	return err

db/sql/migrations/v2.16.2.err.sql

@@ -0,0 +1,5 @@
+alter table project__schedule drop task_params_id;
+alter table project__integration drop task_params_id;
+alter table project__integration add task_params TEXT;
+
+drop table project__task_params;

db/sql/migrations/v2.16.2.sql

@@ -0,0 +1,20 @@
+create table project__task_params
+(
+    id           integer primary key autoincrement,
+
+    environment  TEXT,
+    project_id   int not null,
+    arguments    TEXT,
+    inventory_id int,
+    git_branch   varchar(255),
+    params       TEXT,
+    version      varchar(20),
+    message      varchar(250),
+
+    foreign key (`project_id`) references project (`id`) on delete cascade,
+    foreign key (`inventory_id`) references project__inventory (`id`) on delete cascade
+);
+
+alter table project__integration drop task_params;
+alter table project__schedule add task_params_id int references `project__task_params`(`id`);
+alter table project__integration add task_params_id int references `project__task_params`(`id`);