diff --git a/db/Task.go b/db/Task.go index 1511247ff..23fa15791 100644 --- a/db/Task.go +++ b/db/Task.go @@ -12,8 +12,7 @@ import ( "github.com/semaphoreui/semaphore/util" ) -type DefaultTaskParams struct { -} +type DefaultTaskParams struct{} type TerraformTaskParams struct { Plan bool `json:"plan"` @@ -27,6 +26,7 @@ type AnsibleTaskParams struct { Debug bool `json:"debug"` DryRun bool `json:"dry_run"` Diff bool `json:"diff"` + TaskID int } // Task is a model of a task which will be executed by the runner @@ -123,7 +123,6 @@ func (task *Task) GetIncomingVersion(d Store) *string { } buildTask, err := d.GetTask(task.ProjectID, *task.BuildTaskID) - if err != nil { return nil } @@ -150,7 +149,6 @@ func (task *Task) GetUrl() *string { } func (task *Task) ValidateNewTask(template Template) error { - var params interface{} switch template.App { case AppAnsible: diff --git a/db_lib/AnsibleApp.go b/db_lib/AnsibleApp.go index 93399434b..d94cf2bb6 100644 --- a/db_lib/AnsibleApp.go +++ b/db_lib/AnsibleApp.go @@ -2,6 +2,7 @@ package db_lib import ( "crypto/md5" + "errors" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "github.com/semaphoreui/semaphore/db" "github.com/semaphoreui/semaphore/pkg/task_logger" + "github.com/semaphoreui/semaphore/util" ) func getMD5Hash(filepath string) (string, error) { @@ -45,7 +47,7 @@ func writeMD5Hash(requirementsFile string, requirementsHashFile string) error { return err } - return os.WriteFile(requirementsHashFile, []byte(newFileMD5Hash), 0644) + return os.WriteFile(requirementsHashFile, []byte(newFileMD5Hash), 0o644) } type AnsibleApp struct { @@ -69,29 +71,76 @@ func (t *AnsibleApp) Log(msg string) { t.Logger.Log(msg) } -func (t *AnsibleApp) InstallRequirements(environmentVars []string, params interface{}) error { +func (t *AnsibleApp) getRepoPath() string { + repo := GitRepository{ + Logger: t.Logger, + TemplateID: t.Template.ID, + Repository: t.Repository, + Client: CreateDefaultGitClient(), + } + + return repo.GetFullPath() +} + +func (t *AnsibleApp) installRequirementsForPersonalTask(taskID int) error { + if err := t.installGalaxyRequirementsFileForPersonalTask(GalaxyRole, taskID); err != nil { + return err + } + if err := t.installGalaxyRequirementsFileForPersonalTask(GalaxyCollection, taskID); err != nil { + return err + } + + return nil +} + +func (t *AnsibleApp) installRequirements() error { if err := t.installCollectionsRequirements(); err != nil { return err } if err := t.installRolesRequirements(); err != nil { return err } + return nil } -func (t *AnsibleApp) getRepoPath() string { - repo := GitRepository{ - Logger: t.Logger, - TemplateID: t.Template.ID, - Repository: t.Repository, - Client: CreateDefaultGitClient(), +func (t *AnsibleApp) InstallRequirements(environmentVars []string, params interface{}) error { + if !util.Config.UsePersonalTaskRoles { + return t.installRequirements() } - return repo.GetFullPath() + ansibleParams, ok := params.(*db.AnsibleTaskParams) + if !ok { + return errors.New("undefined type task parameters") + } + + return t.installRequirementsForPersonalTask(ansibleParams.TaskID) } -func (t *AnsibleApp) installGalaxyRequirementsFile(requirementsType GalaxyRequirementsType, requirementsFilePath string) error { +func (t *AnsibleApp) installGalaxyRequirementsFileForPersonalTask(requirementsType GalaxyRequirementsType, taskID int) error { + requirementsFilePath := path.Join(t.GetPlaybookDir(), string(requirementsType)+"s", "requirements.yml") + + if _, err := os.Stat(requirementsFilePath); err != nil { + t.Log("No " + requirementsFilePath + " file found. Skip galaxy install process.\n") + return nil + } + if err := t.runGalaxy([]string{ + string(requirementsType), + "install", + "-r", + requirementsFilePath, + "--force", + "-p", + util.Config.FullPathToPersonalTaskRoles(taskID), + }); err != nil { + return err + } + + return nil +} + +func (t *AnsibleApp) installGalaxyRequirementsFile(requirementsType GalaxyRequirementsType, requirementsFilePath string) error { requirementsHashFilePath := fmt.Sprintf("%s.md5", requirementsFilePath) if _, err := os.Stat(requirementsFilePath); err != nil { diff --git a/services/tasks/LocalJob.go b/services/tasks/LocalJob.go index 490158a1a..958fc8911 100644 --- a/services/tasks/LocalJob.go +++ b/services/tasks/LocalJob.go @@ -59,7 +59,6 @@ func (t *LocalJob) SetCommit(hash, message string) { } func (t *LocalJob) getEnvironmentExtraVars(username string, incomingVersion *string) (extraVars map[string]interface{}, err error) { - extraVars = make(map[string]interface{}) if t.Environment.JSON != "" { @@ -179,7 +178,6 @@ func (t *LocalJob) getEnvironmentENV() (res []string, err error) { // nolint: gocyclo func (t *LocalJob) getShellArgs(username string, incomingVersion *string) (args []string, err error) { extraVars, err := t.getEnvironmentExtraVars(username, incomingVersion) - if err != nil { t.Log(err.Error()) t.Log("Error getting environment extra vars") @@ -220,11 +218,9 @@ func (t *LocalJob) getShellArgs(username string, incomingVersion *string) (args // nolint: gocyclo func (t *LocalJob) getTerraformArgs(username string, incomingVersion *string) (args []string, err error) { - args = []string{} extraVars, err := t.getEnvironmentExtraVars(username, incomingVersion) - if err != nil { t.Log(err.Error()) t.Log("Could not remove command environment, if existent it will be passed to --extra-vars. This is not fatal but be aware of side effects") @@ -269,7 +265,6 @@ func (t *LocalJob) getTerraformArgs(username string, incomingVersion *string) (a // nolint: gocyclo func (t *LocalJob) getPlaybookArgs(username string, incomingVersion *string) (args []string, inputs map[string]string, err error) { - inputMap := make(map[db.AccessKeyRole]string) inputs = make(map[string]string) @@ -431,7 +426,6 @@ func (t *LocalJob) getPlaybookArgs(username string, incomingVersion *string) (ar } func (t *LocalJob) getCLIArgs() (templateArgs []string, taskArgs []string, err error) { - if t.Template.Arguments != nil { err = json.Unmarshal([]byte(*t.Template.Arguments), &templateArgs) if err != nil { @@ -454,7 +448,9 @@ func (t *LocalJob) getCLIArgs() (templateArgs []string, taskArgs []string, err e func (t *LocalJob) getParams() (params interface{}, err error) { switch t.Template.App { case db.AppAnsible: - params = &db.AnsibleTaskParams{} + params = &db.AnsibleTaskParams{ + TaskID: t.Task.ID, + } case db.AppTerraform, db.AppTofu: params = &db.TerraformTaskParams{} default: @@ -462,7 +458,6 @@ func (t *LocalJob) getParams() (params interface{}, err error) { } err = t.Task.FillParams(params) - if err != nil { return } @@ -471,10 +466,13 @@ func (t *LocalJob) getParams() (params interface{}, err error) { } func (t *LocalJob) Run(username string, incomingVersion *string, alias string) (err error) { - defer func() { t.destroyKeys() t.destroyInventoryFile() + + if util.Config.UsePersonalTaskRoles { + t.destroyTaskRoles() + } }() t.SetStatus(task_logger.TaskRunningStatus) // It is required for local mode. Don't delete @@ -484,6 +482,13 @@ func (t *LocalJob) Run(username string, incomingVersion *string, alias string) ( return } + if util.Config.UsePersonalTaskRoles { + environmentVariables = append(environmentVariables, + fmt.Sprintf("ANSIBLE_ROLES_PATH=%s", util.Config.FullPathToPersonalTaskRoles(t.Task.ID)), + fmt.Sprintf("ANSIBLE_COLLECTIONS_PATH=%s", util.Config.FullPathToPersonalTaskRoles(t.Task.ID)), + ) + } + params, err := t.getParams() if err != nil { return @@ -544,11 +549,9 @@ func (t *LocalJob) Run(username string, incomingVersion *string, alias string) ( t.Process = p }, }) - } func (t *LocalJob) prepareRun(environmentVars []string, params interface{}) error { - t.Log("Preparing: " + strconv.Itoa(t.Task.ID)) if err := checkTmpDir(util.Config.TmpPath); err != nil { @@ -609,7 +612,6 @@ func (t *LocalJob) updateRepository() error { } err := repo.ValidateRepo() - if err != nil { if !os.IsNotExist(err) { err = os.RemoveAll(repo.GetFullPath()) @@ -636,7 +638,6 @@ func (t *LocalJob) updateRepository() error { } func (t *LocalJob) checkoutRepository() error { - repo := db_lib.GitRepository{ Logger: t.Logger, TemplateID: t.Template.ID, @@ -645,7 +646,6 @@ func (t *LocalJob) checkoutRepository() error { } err := repo.ValidateRepo() - if err != nil { return err } @@ -658,7 +658,6 @@ func (t *LocalJob) checkoutRepository() error { // store commit to TaskRunner table commitHash, err := repo.GetLastCommitHash() - if err != nil { return err } diff --git a/services/tasks/LocalJob_inventory.go b/services/tasks/LocalJob_inventory.go index a14769e73..09e1c42ff 100644 --- a/services/tasks/LocalJob_inventory.go +++ b/services/tasks/LocalJob_inventory.go @@ -84,7 +84,7 @@ func (t *LocalJob) installStaticInventory() error { fullPath := t.tmpInventoryFullPath() // create inventory file - return os.WriteFile(fullPath, []byte(t.Inventory.Inventory), 0664) + return os.WriteFile(fullPath, []byte(t.Inventory.Inventory), 0o664) } func (t *LocalJob) destroyInventoryFile() { @@ -94,6 +94,13 @@ func (t *LocalJob) destroyInventoryFile() { } } +func (t *LocalJob) destroyTaskRoles() { + fullPath := util.Config.FullPathToPersonalTaskRoles(t.Task.ID) + if err := os.RemoveAll(fullPath); err != nil { + log.Error(err) + } +} + func (t *LocalJob) destroyKeys() { err := t.sshKeyInstallation.Destroy() if err != nil { diff --git a/util/config.go b/util/config.go index 5395d2442..a0e26053d 100644 --- a/util/config.go +++ b/util/config.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "golang.org/x/crypto/bcrypt" "io" "net/url" "os" @@ -20,6 +19,8 @@ import ( "strconv" "strings" + "golang.org/x/crypto/bcrypt" + "github.com/google/go-github/github" "github.com/gorilla/securecookie" ) @@ -212,6 +213,9 @@ type ConfigType struct { // task concurrency MaxParallelTasks int `json:"max_parallel_tasks,omitempty" default:"10" rule:"^[0-9]{1,10}$" env:"SEMAPHORE_MAX_PARALLEL_TASKS"` + // When the flag is set, personal roles are created for each task. Fix https://github.com/semaphoreui/semaphore/pull/2316 + UsePersonalTaskRoles bool `json:"use_personal_task_roles,omitempty" env:"SEMAPHORE_USE_PERSONAL_TASK_ROLES"` + RunnerRegistrationToken string `json:"runner_registration_token,omitempty" env:"SEMAPHORE_RUNNER_REGISTRATION_TOKEN"` // feature switches @@ -301,8 +305,8 @@ func loadConfigFile(configPath string) (usedConfigPath *string) { configPath = os.Getenv("SEMAPHORE_CONFIG_PATH") } - //If the configPath option has been set try to load and decode it - //var usedPath string + // If the configPath option has been set try to load and decode it + // var usedPath string if configPath == "" { cwd, err := os.Getwd() @@ -339,8 +343,8 @@ func loadConfigFile(configPath string) (usedConfigPath *string) { } func loadDefaultsToObject(obj interface{}) error { - var t = reflect.TypeOf(obj) - var v = reflect.ValueOf(obj) + t := reflect.TypeOf(obj) + v := reflect.ValueOf(obj) if t.Kind() == reflect.Ptr { t = t.Elem() @@ -395,7 +399,6 @@ func loadDefaultsToObject(obj interface{}) error { } func loadConfigDefaults() { - err := loadDefaultsToObject(Config) if err != nil { panic(err) @@ -403,17 +406,14 @@ func loadConfigDefaults() { } func castStringToInt(value string) int { - valueInt, err := strconv.Atoi(value) if err != nil { panic(err) } return valueInt - } func castStringToBool(value string) bool { - var valueBool bool if value == "1" || strings.ToLower(value) == "true" || strings.ToLower(value) == "yes" { valueBool = true @@ -421,7 +421,6 @@ func castStringToBool(value string) bool { valueBool = false } return valueBool - } func CastValueToKind(value interface{}, kind reflect.Kind) (res interface{}, ok bool) { @@ -467,18 +466,15 @@ func CastValueToKind(value interface{}, kind reflect.Kind) (res interface{}, ok } func setConfigValue(attribute reflect.Value, value interface{}) { - if attribute.IsValid() { value, _ = CastValueToKind(value, attribute.Kind()) attribute.Set(reflect.ValueOf(value)) } else { panic(fmt.Errorf("got non-existent config attribute")) } - } func getConfigValue(path string) string { - attribute := reflect.ValueOf(Config) nested_path := strings.Split(path, ".") @@ -495,8 +491,8 @@ func getConfigValue(path string) string { } func validate(value interface{}) error { - var t = reflect.TypeOf(value) - var v = reflect.ValueOf(value) + t := reflect.TypeOf(value) + v := reflect.ValueOf(value) if t.Kind() == reflect.Ptr { t = t.Elem() @@ -544,17 +540,15 @@ func validate(value interface{}) error { } func validateConfig() { - err := validate(Config) - if err != nil { panic(err) } } func loadEnvironmentToObject(obj interface{}) error { - var t = reflect.TypeOf(obj) - var v = reflect.ValueOf(obj) + t := reflect.TypeOf(obj) + v := reflect.ValueOf(obj) if t.Kind() == reflect.Ptr { t = t.Elem() @@ -825,7 +819,6 @@ func (conf *ConfigType) GetDialect() (dialect string, err error) { func (conf *ConfigType) GetDBConfig() (dbConfig DbConfig, err error) { var dialect string dialect, err = conf.GetDialect() - if err != nil { return } @@ -857,6 +850,10 @@ func (conf *ConfigType) GenerateSecrets() { conf.AccessKeyEncryption = base64.StdEncoding.EncodeToString(accessKeyEncryption) } +func (conf *ConfigType) FullPathToPersonalTaskRoles(taskID int) string { + return path.Join(conf.TmpPath, "tasks", strconv.Itoa(taskID)) +} + var appCommands = map[string]string{ "ansible": "ansible-playbook", "terraform": "terraform", @@ -874,14 +871,12 @@ var appPriorities = map[string]int{ } func LookupDefaultApps() { - for appID, cmd := range appCommands { if _, ok := Config.Apps[appID]; ok { continue } _, err := exec.LookPath(cmd) - if err != nil { continue } @@ -920,7 +915,6 @@ func GetPublicHost() string { } return aliasURL - } func GetPublicAliasURL(scope string, alias string) string { @@ -936,7 +930,6 @@ func GetPublicAliasURL(scope string, alias string) string { } func GenerateRecoveryCode() (code string, hash string, err error) { - buf := make([]byte, 10) _, err = io.ReadFull(rand.Reader, buf) if err != nil {