From c82cd3d3d321668f9e069fb81a5c1f8b3990c630 Mon Sep 17 00:00:00 2001 From: Zverev Sergei Evgenevich Date: Wed, 5 Mar 2025 12:10:30 +0300 Subject: [PATCH 1/2] Add possibility dedicated folder for installed roles --- db/Task.go | 6 +-- db_lib/AnsibleApp.go | 69 ++++++++++++++++++++++++---- services/tasks/LocalJob.go | 28 +++++------ services/tasks/LocalJob_inventory.go | 9 +++- util/config.go | 41 +++++++---------- 5 files changed, 98 insertions(+), 55 deletions(-) diff --git a/db/Task.go b/db/Task.go index 1511247ff..23fa15791 100644 --- a/db/Task.go +++ b/db/Task.go @@ -12,8 +12,7 @@ import ( "github.com/semaphoreui/semaphore/util" ) -type DefaultTaskParams struct { -} +type DefaultTaskParams struct{} type TerraformTaskParams struct { Plan bool `json:"plan"` @@ -27,6 +26,7 @@ type AnsibleTaskParams struct { Debug bool `json:"debug"` DryRun bool `json:"dry_run"` Diff bool `json:"diff"` + TaskID int } // Task is a model of a task which will be executed by the runner @@ -123,7 +123,6 @@ func (task *Task) GetIncomingVersion(d Store) *string { } buildTask, err := d.GetTask(task.ProjectID, *task.BuildTaskID) - if err != nil { return nil } @@ -150,7 +149,6 @@ func (task *Task) GetUrl() *string { } func (task *Task) ValidateNewTask(template Template) error { - var params interface{} switch template.App { case AppAnsible: diff --git a/db_lib/AnsibleApp.go b/db_lib/AnsibleApp.go index 93399434b..d94cf2bb6 100644 --- a/db_lib/AnsibleApp.go +++ b/db_lib/AnsibleApp.go @@ -2,6 +2,7 @@ package db_lib import ( "crypto/md5" + "errors" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "github.com/semaphoreui/semaphore/db" "github.com/semaphoreui/semaphore/pkg/task_logger" + "github.com/semaphoreui/semaphore/util" ) func getMD5Hash(filepath string) (string, error) { @@ -45,7 +47,7 @@ func writeMD5Hash(requirementsFile string, requirementsHashFile string) error { return err } - return os.WriteFile(requirementsHashFile, []byte(newFileMD5Hash), 0644) + return os.WriteFile(requirementsHashFile, []byte(newFileMD5Hash), 0o644) } type AnsibleApp struct { @@ -69,29 +71,76 @@ func (t *AnsibleApp) Log(msg string) { t.Logger.Log(msg) } -func (t *AnsibleApp) InstallRequirements(environmentVars []string, params interface{}) error { +func (t *AnsibleApp) getRepoPath() string { + repo := GitRepository{ + Logger: t.Logger, + TemplateID: t.Template.ID, + Repository: t.Repository, + Client: CreateDefaultGitClient(), + } + + return repo.GetFullPath() +} + +func (t *AnsibleApp) installRequirementsForPersonalTask(taskID int) error { + if err := t.installGalaxyRequirementsFileForPersonalTask(GalaxyRole, taskID); err != nil { + return err + } + if err := t.installGalaxyRequirementsFileForPersonalTask(GalaxyCollection, taskID); err != nil { + return err + } + + return nil +} + +func (t *AnsibleApp) installRequirements() error { if err := t.installCollectionsRequirements(); err != nil { return err } if err := t.installRolesRequirements(); err != nil { return err } + return nil } -func (t *AnsibleApp) getRepoPath() string { - repo := GitRepository{ - Logger: t.Logger, - TemplateID: t.Template.ID, - Repository: t.Repository, - Client: CreateDefaultGitClient(), +func (t *AnsibleApp) InstallRequirements(environmentVars []string, params interface{}) error { + if !util.Config.UsePersonalTaskRoles { + return t.installRequirements() } - return repo.GetFullPath() + ansibleParams, ok := params.(*db.AnsibleTaskParams) + if !ok { + return errors.New("undefined type task parameters") + } + + return t.installRequirementsForPersonalTask(ansibleParams.TaskID) } -func (t *AnsibleApp) installGalaxyRequirementsFile(requirementsType GalaxyRequirementsType, requirementsFilePath string) error { +func (t *AnsibleApp) installGalaxyRequirementsFileForPersonalTask(requirementsType GalaxyRequirementsType, taskID int) error { + requirementsFilePath := path.Join(t.GetPlaybookDir(), string(requirementsType)+"s", "requirements.yml") + + if _, err := os.Stat(requirementsFilePath); err != nil { + t.Log("No " + requirementsFilePath + " file found. Skip galaxy install process.\n") + return nil + } + if err := t.runGalaxy([]string{ + string(requirementsType), + "install", + "-r", + requirementsFilePath, + "--force", + "-p", + util.Config.FullPathToPersonalTaskRoles(taskID), + }); err != nil { + return err + } + + return nil +} + +func (t *AnsibleApp) installGalaxyRequirementsFile(requirementsType GalaxyRequirementsType, requirementsFilePath string) error { requirementsHashFilePath := fmt.Sprintf("%s.md5", requirementsFilePath) if _, err := os.Stat(requirementsFilePath); err != nil { diff --git a/services/tasks/LocalJob.go b/services/tasks/LocalJob.go index 8f2f2b0bd..0e103e7e4 100644 --- a/services/tasks/LocalJob.go +++ b/services/tasks/LocalJob.go @@ -5,7 +5,6 @@ import ( "fmt" "maps" "os" - "path" "strconv" @@ -58,7 +57,6 @@ func (t *LocalJob) SetCommit(hash, message string) { } func (t *LocalJob) getEnvironmentExtraVars(username string, incomingVersion *string) (extraVars map[string]interface{}, err error) { - extraVars = make(map[string]interface{}) if t.Environment.JSON != "" { @@ -178,7 +176,6 @@ func (t *LocalJob) getEnvironmentENV() (res []string, err error) { // nolint: gocyclo func (t *LocalJob) getShellArgs(username string, incomingVersion *string) (args []string, err error) { extraVars, err := t.getEnvironmentExtraVars(username, incomingVersion) - if err != nil { t.Log(err.Error()) t.Log("Error getting environment extra vars") @@ -219,11 +216,9 @@ func (t *LocalJob) getShellArgs(username string, incomingVersion *string) (args // nolint: gocyclo func (t *LocalJob) getTerraformArgs(username string, incomingVersion *string) (args []string, err error) { - args = []string{} extraVars, err := t.getEnvironmentExtraVars(username, incomingVersion) - if err != nil { t.Log(err.Error()) t.Log("Could not remove command environment, if existent it will be passed to --extra-vars. This is not fatal but be aware of side effects") @@ -268,7 +263,6 @@ func (t *LocalJob) getTerraformArgs(username string, incomingVersion *string) (a // nolint: gocyclo func (t *LocalJob) getPlaybookArgs(username string, incomingVersion *string) (args []string, inputs map[string]string, err error) { - inputMap := make(map[db.AccessKeyRole]string) inputs = make(map[string]string) @@ -405,7 +399,6 @@ func (t *LocalJob) getPlaybookArgs(username string, incomingVersion *string) (ar } func (t *LocalJob) getCLIArgs() (templateArgs []string, taskArgs []string, err error) { - if t.Template.Arguments != nil { err = json.Unmarshal([]byte(*t.Template.Arguments), &templateArgs) if err != nil { @@ -428,7 +421,9 @@ func (t *LocalJob) getCLIArgs() (templateArgs []string, taskArgs []string, err e func (t *LocalJob) getParams() (params interface{}, err error) { switch t.Template.App { case db.AppAnsible: - params = &db.AnsibleTaskParams{} + params = &db.AnsibleTaskParams{ + TaskID: t.Task.ID, + } case db.AppTerraform, db.AppTofu: params = &db.TerraformTaskParams{} default: @@ -436,7 +431,6 @@ func (t *LocalJob) getParams() (params interface{}, err error) { } err = t.Task.FillParams(params) - if err != nil { return } @@ -445,10 +439,13 @@ func (t *LocalJob) getParams() (params interface{}, err error) { } func (t *LocalJob) Run(username string, incomingVersion *string, alias string) (err error) { - defer func() { t.destroyKeys() t.destroyInventoryFile() + + if util.Config.UsePersonalTaskRoles { + t.destroyTaskRoles() + } }() t.SetStatus(task_logger.TaskRunningStatus) // It is required for local mode. Don't delete @@ -458,6 +455,11 @@ func (t *LocalJob) Run(username string, incomingVersion *string, alias string) ( return } + if util.Config.UsePersonalTaskRoles { + environmentVariables = append(environmentVariables, + fmt.Sprintf("ANSIBLE_ROLES_PATH=%s", util.Config.FullPathToPersonalTaskRoles(t.Task.ID))) + } + params, err := t.getParams() if err != nil { return @@ -518,11 +520,9 @@ func (t *LocalJob) Run(username string, incomingVersion *string, alias string) ( t.Process = p }, }) - } func (t *LocalJob) prepareRun(environmentVars []string, params interface{}) error { - t.Log("Preparing: " + strconv.Itoa(t.Task.ID)) if err := checkTmpDir(util.Config.TmpPath); err != nil { @@ -583,7 +583,6 @@ func (t *LocalJob) updateRepository() error { } err := repo.ValidateRepo() - if err != nil { if !os.IsNotExist(err) { err = os.RemoveAll(repo.GetFullPath()) @@ -610,7 +609,6 @@ func (t *LocalJob) updateRepository() error { } func (t *LocalJob) checkoutRepository() error { - repo := db_lib.GitRepository{ Logger: t.Logger, TemplateID: t.Template.ID, @@ -619,7 +617,6 @@ func (t *LocalJob) checkoutRepository() error { } err := repo.ValidateRepo() - if err != nil { return err } @@ -632,7 +629,6 @@ func (t *LocalJob) checkoutRepository() error { // store commit to TaskRunner table commitHash, err := repo.GetLastCommitHash() - if err != nil { return err } diff --git a/services/tasks/LocalJob_inventory.go b/services/tasks/LocalJob_inventory.go index a14769e73..09e1c42ff 100644 --- a/services/tasks/LocalJob_inventory.go +++ b/services/tasks/LocalJob_inventory.go @@ -84,7 +84,7 @@ func (t *LocalJob) installStaticInventory() error { fullPath := t.tmpInventoryFullPath() // create inventory file - return os.WriteFile(fullPath, []byte(t.Inventory.Inventory), 0664) + return os.WriteFile(fullPath, []byte(t.Inventory.Inventory), 0o664) } func (t *LocalJob) destroyInventoryFile() { @@ -94,6 +94,13 @@ func (t *LocalJob) destroyInventoryFile() { } } +func (t *LocalJob) destroyTaskRoles() { + fullPath := util.Config.FullPathToPersonalTaskRoles(t.Task.ID) + if err := os.RemoveAll(fullPath); err != nil { + log.Error(err) + } +} + func (t *LocalJob) destroyKeys() { err := t.sshKeyInstallation.Destroy() if err != nil { diff --git a/util/config.go b/util/config.go index 5a32f45a2..38f42e5bd 100644 --- a/util/config.go +++ b/util/config.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "golang.org/x/crypto/bcrypt" "io" "net/url" "os" @@ -20,6 +19,8 @@ import ( "strconv" "strings" + "golang.org/x/crypto/bcrypt" + "github.com/google/go-github/github" "github.com/gorilla/securecookie" ) @@ -212,6 +213,9 @@ type ConfigType struct { // task concurrency MaxParallelTasks int `json:"max_parallel_tasks,omitempty" default:"10" rule:"^[0-9]{1,10}$" env:"SEMAPHORE_MAX_PARALLEL_TASKS"` + // When the flag is set, personal roles are created for each task. Fix https://github.com/semaphoreui/semaphore/pull/2316 + UsePersonalTaskRoles bool `json:"use_personal_task_roles,omitempty" env:"SEMAPHORE_USE_PERSONAL_TASK_ROLES"` + RunnerRegistrationToken string `json:"runner_registration_token,omitempty" env:"SEMAPHORE_RUNNER_REGISTRATION_TOKEN"` // feature switches @@ -290,8 +294,8 @@ func loadConfigFile(configPath string) (usedConfigPath *string) { configPath = os.Getenv("SEMAPHORE_CONFIG_PATH") } - //If the configPath option has been set try to load and decode it - //var usedPath string + // If the configPath option has been set try to load and decode it + // var usedPath string if configPath == "" { cwd, err := os.Getwd() @@ -328,8 +332,8 @@ func loadConfigFile(configPath string) (usedConfigPath *string) { } func loadDefaultsToObject(obj interface{}) error { - var t = reflect.TypeOf(obj) - var v = reflect.ValueOf(obj) + t := reflect.TypeOf(obj) + v := reflect.ValueOf(obj) if t.Kind() == reflect.Ptr { t = t.Elem() @@ -384,7 +388,6 @@ func loadDefaultsToObject(obj interface{}) error { } func loadConfigDefaults() { - err := loadDefaultsToObject(Config) if err != nil { panic(err) @@ -392,17 +395,14 @@ func loadConfigDefaults() { } func castStringToInt(value string) int { - valueInt, err := strconv.Atoi(value) if err != nil { panic(err) } return valueInt - } func castStringToBool(value string) bool { - var valueBool bool if value == "1" || strings.ToLower(value) == "true" || strings.ToLower(value) == "yes" { valueBool = true @@ -410,7 +410,6 @@ func castStringToBool(value string) bool { valueBool = false } return valueBool - } func CastValueToKind(value interface{}, kind reflect.Kind) (res interface{}, ok bool) { @@ -456,18 +455,15 @@ func CastValueToKind(value interface{}, kind reflect.Kind) (res interface{}, ok } func setConfigValue(attribute reflect.Value, value interface{}) { - if attribute.IsValid() { value, _ = CastValueToKind(value, attribute.Kind()) attribute.Set(reflect.ValueOf(value)) } else { panic(fmt.Errorf("got non-existent config attribute")) } - } func getConfigValue(path string) string { - attribute := reflect.ValueOf(Config) nested_path := strings.Split(path, ".") @@ -484,8 +480,8 @@ func getConfigValue(path string) string { } func validate(value interface{}) error { - var t = reflect.TypeOf(value) - var v = reflect.ValueOf(value) + t := reflect.TypeOf(value) + v := reflect.ValueOf(value) if t.Kind() == reflect.Ptr { t = t.Elem() @@ -533,17 +529,15 @@ func validate(value interface{}) error { } func validateConfig() { - err := validate(Config) - if err != nil { panic(err) } } func loadEnvironmentToObject(obj interface{}) error { - var t = reflect.TypeOf(obj) - var v = reflect.ValueOf(obj) + t := reflect.TypeOf(obj) + v := reflect.ValueOf(obj) if t.Kind() == reflect.Ptr { t = t.Elem() @@ -814,7 +808,6 @@ func (conf *ConfigType) GetDialect() (dialect string, err error) { func (conf *ConfigType) GetDBConfig() (dbConfig DbConfig, err error) { var dialect string dialect, err = conf.GetDialect() - if err != nil { return } @@ -846,6 +839,10 @@ func (conf *ConfigType) GenerateSecrets() { conf.AccessKeyEncryption = base64.StdEncoding.EncodeToString(accessKeyEncryption) } +func (conf *ConfigType) FullPathToPersonalTaskRoles(taskID int) string { + return path.Join(conf.TmpPath, "tasks", strconv.Itoa(taskID)) +} + var appCommands = map[string]string{ "ansible": "ansible-playbook", "terraform": "terraform", @@ -863,14 +860,12 @@ var appPriorities = map[string]int{ } func LookupDefaultApps() { - for appID, cmd := range appCommands { if _, ok := Config.Apps[appID]; ok { continue } _, err := exec.LookPath(cmd) - if err != nil { continue } @@ -909,7 +904,6 @@ func GetPublicHost() string { } return aliasURL - } func GetPublicAliasURL(scope string, alias string) string { @@ -925,7 +919,6 @@ func GetPublicAliasURL(scope string, alias string) string { } func GenerateRecoveryCode() (code string, hash string, err error) { - buf := make([]byte, 10) _, err = io.ReadFull(rand.Reader, buf) if err != nil { From c39db134d43263554a50b0d22735bbda678ba522 Mon Sep 17 00:00:00 2001 From: Bazhin Maksim Sergeevich Date: Wed, 26 Mar 2025 11:06:42 +0300 Subject: [PATCH 2/2] Added ANSIBLE_COLLECTIONS_PATH variable --- services/tasks/LocalJob.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/tasks/LocalJob.go b/services/tasks/LocalJob.go index de605790b..958fc8911 100644 --- a/services/tasks/LocalJob.go +++ b/services/tasks/LocalJob.go @@ -484,7 +484,9 @@ func (t *LocalJob) Run(username string, incomingVersion *string, alias string) ( if util.Config.UsePersonalTaskRoles { environmentVariables = append(environmentVariables, - fmt.Sprintf("ANSIBLE_ROLES_PATH=%s", util.Config.FullPathToPersonalTaskRoles(t.Task.ID))) + fmt.Sprintf("ANSIBLE_ROLES_PATH=%s", util.Config.FullPathToPersonalTaskRoles(t.Task.ID)), + fmt.Sprintf("ANSIBLE_COLLECTIONS_PATH=%s", util.Config.FullPathToPersonalTaskRoles(t.Task.ID)), + ) } params, err := t.getParams()