From 3b3418afd7425923edfcef3cc3804b78cb411e98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= <ofalvai@gmail.com>
Date: Tue, 27 Aug 2024 09:05:35 +0200
Subject: [PATCH 1/7] Activate step executable

---
 activator/steplib/activate.go                 | 99 ++++++-------------
 activator/steplib/activate_executable.go      | 84 ++++++++++++++++
 activator/steplib/activate_executable_test.go | 60 +++++++++++
 activator/steplib/activate_source.go          | 77 +++++++++++++++
 activator/steplib/testdata/file.txt           |  1 +
 models/models.go                              | 20 +++-
 6 files changed, 267 insertions(+), 74 deletions(-)
 create mode 100644 activator/steplib/activate_executable.go
 create mode 100644 activator/steplib/activate_executable_test.go
 create mode 100644 activator/steplib/activate_source.go
 create mode 100644 activator/steplib/testdata/file.txt

diff --git a/activator/steplib/activate.go b/activator/steplib/activate.go
index 0f192a5d..037ce940 100644
--- a/activator/steplib/activate.go
+++ b/activator/steplib/activate.go
@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"slices"
+	"time"
 
 	"github.com/bitrise-io/go-utils/command"
 	"github.com/bitrise-io/go-utils/pathutil"
@@ -12,7 +14,7 @@ import (
 	"github.com/bitrise-io/stepman/stepman"
 )
 
-var errStepNotAvailableOfflineMode error = fmt.Errorf("step not available in offline mode")
+const precompiledStepsEnv = "BITRISE_EXPERIMENT_PRECOMPILED_STEPS"
 
 func ActivateStep(stepLibURI, id, version, destination, destinationStepYML string, log stepman.Logger, isOfflineMode bool) error {
 	stepCollection, err := stepman.ReadStepSpec(stepLibURI)
@@ -20,41 +22,34 @@ func ActivateStep(stepLibURI, id, version, destination, destinationStepYML strin
 		return fmt.Errorf("failed to read %s steplib: %s", stepLibURI, err)
 	}
 
-	step, version, err := queryStep(stepCollection, stepLibURI, id, version)
+	step, version, err := queryStepMetadata(stepCollection, stepLibURI, id, version)
 	if err != nil {
 		return fmt.Errorf("failed to find step: %s", err)
 	}
 
-	srcFolder, err := activateStep(stepCollection, stepLibURI, id, version, step, log, isOfflineMode)
-	if err != nil {
-		if err == errStepNotAvailableOfflineMode {
-			availableVersions := ListCachedStepVersions(log, stepCollection, stepLibURI, id)
-			versionList := "Other versions available in the local cache:"
-			for _, version := range availableVersions {
-				versionList = versionList + fmt.Sprintf("\n- %s", version)
+	if os.Getenv(precompiledStepsEnv) == "true" {
+		platform := fmt.Sprintf("%s-%s", runtime.GOOS, runtime.GOARCH)
+		executableForPlatform, ok := step.Executables[platform]
+		if ok {
+			log.Debugf("Downloading executable for %s", platform)
+			downloadStart := time.Now()
+			err = activateStepExecutable(stepLibURI, id, version, executableForPlatform, destination, destinationStepYML)
+			if err != nil {
+				log.Warnf("Failed to download step executable, falling back to source build: %s", err)
+				return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
 			}
-
-			errMsg := fmt.Sprintf("version is not available in the local cache and $BITRISE_OFFLINE_MODE is set. %s", versionList)
-			return fmt.Errorf("failed to download step: %s", errMsg)
+			log.Debugf("Downloaded executable in %s", time.Since(downloadStart).Round(time.Millisecond))
+			return nil
+		} else {
+			log.Infof("No prebuilt executable found for %s, falling back to source build", platform)
+			return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
 		}
-
-		return fmt.Errorf("failed to download step: %s", err)
-	}
-
-	if err := copyStep(srcFolder, destination); err != nil {
-		return fmt.Errorf("copy step failed: %s", err)
+	} else {
+		return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
 	}
-
-	if destinationStepYML != "" {
-		if err := copyStepYML(stepLibURI, id, version, destinationStepYML); err != nil {
-			return fmt.Errorf("copy step.yml failed: %s", err)
-		}
-	}
-
-	return nil
 }
 
-func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, version string) (models.StepModel, string, error) {
+func queryStepMetadata(stepLib models.StepCollectionModel, stepLibURI string, id, version string) (models.StepModel, string, error) {
 	step, stepFound, versionFound := stepLib.GetStep(id, version)
 
 	if !stepFound {
@@ -75,46 +70,6 @@ func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, versio
 	return step, version, nil
 }
 
-func activateStep(stepLib models.StepCollectionModel, stepLibURI, id, version string, step models.StepModel, log stepman.Logger, isOfflineMode bool) (string, error) {
-	route, found := stepman.ReadRoute(stepLibURI)
-	if !found {
-		return "", fmt.Errorf("no route found for %s steplib", stepLibURI)
-	}
-
-	stepCacheDir := stepman.GetStepCacheDirPath(route, id, version)
-	if exist, err := pathutil.IsPathExists(stepCacheDir); err != nil {
-		return "", fmt.Errorf("failed to check if %s path exist: %s", stepCacheDir, err)
-	} else if exist {
-		return stepCacheDir, nil
-	}
-
-	// version specific source cache not exists
-	if isOfflineMode {
-		return "", errStepNotAvailableOfflineMode
-	}
-
-	if err := stepman.DownloadStep(stepLibURI, stepLib, id, version, step.Source.Commit, log); err != nil {
-		return "", fmt.Errorf("download failed: %s", err)
-	}
-
-	return stepCacheDir, nil
-}
-
-func copyStep(src, dst string) error {
-	if exist, err := pathutil.IsPathExists(dst); err != nil {
-		return fmt.Errorf("failed to check if %s path exist: %s", dst, err)
-	} else if !exist {
-		if err := os.MkdirAll(dst, 0777); err != nil {
-			return fmt.Errorf("failed to create dir for %s path: %s", dst, err)
-		}
-	}
-
-	if err := command.CopyDir(src+"/", dst, true); err != nil {
-		return fmt.Errorf("copy command failed: %s", err)
-	}
-	return nil
-}
-
 func copyStepYML(libraryURL, id, version, dest string) error {
 	route, found := stepman.ReadRoute(libraryURL)
 	if !found {
@@ -138,8 +93,14 @@ func copyStepYML(libraryURL, id, version, dest string) error {
 func ListCachedStepVersions(log stepman.Logger, stepLib models.StepCollectionModel, stepLibURI, stepID string) []string {
 	versions := []models.Semver{}
 
-	for version, step := range stepLib.Steps[stepID].Versions {
-		_, err := activateStep(stepLib, stepLibURI, stepID, version, step, log, true)
+	route, found := stepman.ReadRoute(stepLibURI)
+	if !found {
+		return nil
+	}
+
+	for version := range stepLib.Steps[stepID].Versions {
+		stepCacheDir := stepman.GetStepCacheDirPath(route, stepID, version)
+		_, err := os.Stat(stepCacheDir)
 		if err != nil {
 			continue
 		}
diff --git a/activator/steplib/activate_executable.go b/activator/steplib/activate_executable.go
new file mode 100644
index 00000000..7c0973b9
--- /dev/null
+++ b/activator/steplib/activate_executable.go
@@ -0,0 +1,84 @@
+package steplib
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/bitrise-io/stepman/models"
+	"github.com/hashicorp/go-retryablehttp"
+)
+
+func activateStepExecutable(
+	stepLibURI string,
+	stepID string,
+	version string,
+	executable models.Executable,
+	destination string,
+	destinationStepYML string,
+) error {
+	resp, err := retryablehttp.Get(executable.Url)
+	if err != nil {
+		return fmt.Errorf("fetch from %s: %w", executable.Url, err)
+	}
+	defer resp.Body.Close()
+
+	path := filepath.Join(destination, stepID)
+	file, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create file %s: %w", path, err)
+	}
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return fmt.Errorf("download %s to %s: %w", executable.Url, path, err)
+	}
+
+	err = validateHash(path, executable.Hash)
+	if err != nil {
+		return fmt.Errorf("validate hash: %s", err)
+	}
+
+	err = os.Chmod(path, 0755)
+	if err != nil {
+		return fmt.Errorf("set executable permission on file: %s", err)
+	}
+
+	if err := copyStepYML(stepLibURI, stepID, version, destinationStepYML); err != nil {
+		return fmt.Errorf("copy step.yml: %s", err)
+	}
+
+	return nil
+}
+
+func validateHash(filePath string, expectedHash string) error {
+	if expectedHash == "" {
+		return fmt.Errorf("hash is empty")
+	}
+
+	if !strings.HasPrefix(expectedHash, "sha256-") {
+		return fmt.Errorf("only SHA256 hashes supported at this time, make sure to prefix the hash with `sha256-`. Found hash value: %s", expectedHash)
+	}
+
+	expectedHash = strings.TrimPrefix(expectedHash, "sha256-")
+
+	reader, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+
+	h := sha256.New()
+	_, err = io.Copy(h, reader)
+	if err != nil {
+		return fmt.Errorf("calculate hash: %w", err)
+	}
+	actualHash := hex.EncodeToString(h.Sum(nil))
+	if actualHash != expectedHash {
+		return fmt.Errorf("hash mismatch: expected sha256-%s, got sha256-%s", expectedHash, actualHash)
+	}
+	return nil
+}
diff --git a/activator/steplib/activate_executable_test.go b/activator/steplib/activate_executable_test.go
new file mode 100644
index 00000000..a244b213
--- /dev/null
+++ b/activator/steplib/activate_executable_test.go
@@ -0,0 +1,60 @@
+package steplib
+
+import (
+	"fmt"
+	"testing"
+)
+
+
+func TestValidateHash(t *testing.T) {
+	tests := []struct {
+		name string
+		filePath     string
+		expectedHash string
+		expectedErr  error
+	}{
+		{
+			name: "Valid hash",
+			filePath:     "testdata/file.txt",
+			expectedHash: "sha256-f2040af3939f5033be8ca9b363055b3e53107c4688ba39b71d4529869a9cc9b2",
+			expectedErr:  nil,
+		},
+		{
+			name: "Hash mismatch",
+			filePath:     "testdata/file.txt",
+			expectedHash: "sha256-1234567890abcdef",
+			expectedErr:  fmt.Errorf("hash mismatch: expected sha256-1234567890abcdef, got sha256-f2040af3939f5033be8ca9b363055b3e53107c4688ba39b71d4529869a9cc9b2"),
+		},
+		{
+			name: "Nonexistent file",
+			filePath:     "testdata/nonexistent.txt",
+			expectedHash: "sha256-3b6b4f1e2e8b8a9e4f7a4b5e6c7d8e9f",
+			expectedErr:  fmt.Errorf("open testdata/nonexistent.txt: no such file or directory"),
+		},
+		{
+			name: "Empty hash",
+			filePath:     "testdata/file.txt",
+			expectedHash: "",
+			expectedErr:  fmt.Errorf("hash is empty"),
+		},
+		{
+			name: "Invalid hash type",
+			filePath:     "testdata/file.txt",
+			expectedHash: "md5-3b6b4f1e2e8b8a9e4f7a4b5e6c7d8e9f",
+			expectedErr:  fmt.Errorf("only SHA256 hashes supported at this time, make sure to prefix the hash with `sha256-`. Found hash value: md5-3b6b4f1e2e8b8a9e4f7a4b5e6c7d8e9f"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateHash(tt.filePath, tt.expectedHash)
+			if err != nil && tt.expectedErr == nil {
+				t.Errorf("unexpected error: %s", err)
+			} else if err == nil && tt.expectedErr != nil {
+				t.Errorf("expected error: %s, but got nil", tt.expectedErr)
+			} else if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() {
+				t.Errorf("expected error: %s, but got: %s", tt.expectedErr, err)
+			}
+		})
+	}
+}
diff --git a/activator/steplib/activate_source.go b/activator/steplib/activate_source.go
new file mode 100644
index 00000000..38975aea
--- /dev/null
+++ b/activator/steplib/activate_source.go
@@ -0,0 +1,77 @@
+package steplib
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/bitrise-io/go-utils/command"
+	"github.com/bitrise-io/go-utils/pathutil"
+	"github.com/bitrise-io/stepman/models"
+	"github.com/bitrise-io/stepman/stepman"
+)
+
+func activateStepSource(
+	stepLib models.StepCollectionModel,
+	stepLibURI, id, version string,
+	step models.StepModel,
+	destination string,
+	stepYMLDestination string,
+	log stepman.Logger,
+	isOfflineMode bool,
+) error {
+	route, found := stepman.ReadRoute(stepLibURI)
+	if !found {
+		return fmt.Errorf("no route found for %s steplib", stepLibURI)
+	}
+
+	stepCacheDir := stepman.GetStepCacheDirPath(route, id, version)
+	if exist, err := pathutil.IsPathExists(stepCacheDir); err != nil {
+		return fmt.Errorf("failed to check if %s path exist: %s", stepCacheDir, err)
+	} else if exist {
+		if err := copyStep(stepCacheDir, destination); err != nil {
+			return fmt.Errorf("copy step failed: %s", err)
+		}
+		return nil
+	}
+
+	// version specific source cache not exists
+	if isOfflineMode {
+		availableVersions := ListCachedStepVersions(log, stepLib, stepLibURI, id)
+		versionList := "Other versions available in the local cache:"
+		for _, version := range availableVersions {
+			versionList = versionList + fmt.Sprintf("\n- %s", version)
+		}
+
+		errMsg := fmt.Sprintf("version is not available in the local cache and $BITRISE_OFFLINE_MODE is set. %s", versionList)
+		return fmt.Errorf("download step: %s", errMsg)
+	}
+
+	if err := stepman.DownloadStep(stepLibURI, stepLib, id, version, step.Source.Commit, log); err != nil {
+		return fmt.Errorf("download failed: %s", err)
+	}
+
+	if err := copyStep(stepCacheDir, destination); err != nil {
+		return fmt.Errorf("copy step failed: %s", err)
+	}
+
+	if err := copyStepYML(stepLibURI, id, version, stepYMLDestination); err != nil {
+		return fmt.Errorf("copy step.yml failed: %s", err)
+	}
+
+	return nil
+}
+
+func copyStep(src, dst string) error {
+	if exist, err := pathutil.IsPathExists(dst); err != nil {
+		return fmt.Errorf("failed to check if %s path exist: %s", dst, err)
+	} else if !exist {
+		if err := os.MkdirAll(dst, 0777); err != nil {
+			return fmt.Errorf("failed to create dir for %s path: %s", dst, err)
+		}
+	}
+
+	if err := command.CopyDir(src+"/", dst, true); err != nil {
+		return fmt.Errorf("copy command failed: %s", err)
+	}
+	return nil
+}
diff --git a/activator/steplib/testdata/file.txt b/activator/steplib/testdata/file.txt
new file mode 100644
index 00000000..4335f02a
--- /dev/null
+++ b/activator/steplib/testdata/file.txt
@@ -0,0 +1 @@
+hash verification test
diff --git a/models/models.go b/models/models.go
index ec6a31c5..98d81be8 100644
--- a/models/models.go
+++ b/models/models.go
@@ -56,12 +56,11 @@ type SwiftStepToolkitModel struct {
 }
 
 type StepToolkitModel struct {
-	Bash   *BashStepToolkitModel   `json:"bash,omitempty" yaml:"bash,omitempty"`
-	Go     *GoStepToolkitModel     `json:"go,omitempty" yaml:"go,omitempty"`
-	Swift  *SwiftStepToolkitModel  `json:"swift,omitempty" yaml:"swift,omitempty"`
+	Bash  *BashStepToolkitModel  `json:"bash,omitempty" yaml:"bash,omitempty"`
+	Go    *GoStepToolkitModel    `json:"go,omitempty" yaml:"go,omitempty"`
+	Swift *SwiftStepToolkitModel `json:"swift,omitempty" yaml:"swift,omitempty"`
 }
 
-// StepModel ...
 type StepModel struct {
 	Title       *string `json:"title,omitempty" yaml:"title,omitempty"`
 	Summary     *string `json:"summary,omitempty" yaml:"summary,omitempty"`
@@ -70,10 +69,13 @@ type StepModel struct {
 	Website       *string `json:"website,omitempty" yaml:"website,omitempty"`
 	SourceCodeURL *string `json:"source_code_url,omitempty" yaml:"source_code_url,omitempty"`
 	SupportURL    *string `json:"support_url,omitempty" yaml:"support_url,omitempty"`
+
 	// auto-generated at share
 	PublishedAt *time.Time        `json:"published_at,omitempty" yaml:"published_at,omitempty"`
 	Source      *StepSourceModel  `json:"source,omitempty" yaml:"source,omitempty"`
+	Executables Executables       `json:"executables,omitempty" yaml:"executables,omitempty"`
 	AssetURLs   map[string]string `json:"asset_urls,omitempty" yaml:"asset_urls,omitempty"`
+
 	//
 	HostOsTags          []string          `json:"host_os_tags,omitempty" yaml:"host_os_tags,omitempty"`
 	ProjectTypeTags     []string          `json:"project_type_tags,omitempty" yaml:"project_type_tags,omitempty"`
@@ -120,6 +122,15 @@ type StepGroupModel struct {
 	Versions            map[string]StepModel `json:"versions,omitempty" yaml:"versions,omitempty"`
 }
 
+// Key: platform, as in runtime.GOOS + runtime.GOARCH
+// Examples: darwin-arm64, linux-amd64
+type Executables map[string]Executable
+
+type Executable struct {
+	Url string `json:"url,omitempty" yaml:"url,omitempty"`
+	Hash string `json:"hash,omitempty" yaml:"hash,omitempty"`
+}
+
 func (stepGroup StepGroupModel) LatestVersion() (StepModel, bool) {
 	step, found := stepGroup.Versions[stepGroup.LatestVersionNumber]
 	if !found {
@@ -174,4 +185,3 @@ type SteplibInfoModel struct {
 	URI      string `json:"uri,omitempty" yaml:"uri,omitempty"`
 	SpecPath string `json:"spec_path,omitempty" yaml:"spec_path,omitempty"`
 }
-

From 81ea523fde4d01fe881230139840fbbffca8d505 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= <ofalvai@gmail.com>
Date: Tue, 10 Sep 2024 11:05:25 +0200
Subject: [PATCH 2/7] Add missing MkdirAll

---
 activator/steplib/activate_executable.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/activator/steplib/activate_executable.go b/activator/steplib/activate_executable.go
index 7c0973b9..7c5d0f8f 100644
--- a/activator/steplib/activate_executable.go
+++ b/activator/steplib/activate_executable.go
@@ -27,6 +27,11 @@ func activateStepExecutable(
 	}
 	defer resp.Body.Close()
 
+	err = os.MkdirAll(destination, 0755)
+	if err != nil {
+		return fmt.Errorf("create directory %s: %w", destination, err)
+	}
+
 	path := filepath.Join(destination, stepID)
 	file, err := os.Create(path)
 	if err != nil {

From bdedd1d6ee3adc135f7ba398e9f55527c1d56a9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= <ofalvai@gmail.com>
Date: Tue, 10 Sep 2024 15:28:02 +0200
Subject: [PATCH 3/7] Expose executable path

---
 activator/activator.go                   |  2 ++
 activator/steplib/activate.go            | 19 +++++++++++--------
 activator/steplib/activate_executable.go | 18 +++++++++---------
 activator/steplib_ref.go                 |  3 ++-
 4 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/activator/activator.go b/activator/activator.go
index 47dbf732..f2365440 100644
--- a/activator/activator.go
+++ b/activator/activator.go
@@ -3,6 +3,8 @@ package activator
 type ActivatedStep struct {
 	StepYMLPath string
 
+	ExecutablePath string
+
 	// DidStepLibUpdate indicates that the local steplib cache was updated while resolving the exact step version.
 	DidStepLibUpdate bool
 }
diff --git a/activator/steplib/activate.go b/activator/steplib/activate.go
index 037ce940..e3a0d15c 100644
--- a/activator/steplib/activate.go
+++ b/activator/steplib/activate.go
@@ -16,15 +16,15 @@ import (
 
 const precompiledStepsEnv = "BITRISE_EXPERIMENT_PRECOMPILED_STEPS"
 
-func ActivateStep(stepLibURI, id, version, destination, destinationStepYML string, log stepman.Logger, isOfflineMode bool) error {
+func ActivateStep(stepLibURI, id, version, destination, destinationStepYML string, log stepman.Logger, isOfflineMode bool) (string, error) {
 	stepCollection, err := stepman.ReadStepSpec(stepLibURI)
 	if err != nil {
-		return fmt.Errorf("failed to read %s steplib: %s", stepLibURI, err)
+		return "", fmt.Errorf("failed to read %s steplib: %s", stepLibURI, err)
 	}
 
 	step, version, err := queryStepMetadata(stepCollection, stepLibURI, id, version)
 	if err != nil {
-		return fmt.Errorf("failed to find step: %s", err)
+		return "", fmt.Errorf("failed to find step: %s", err)
 	}
 
 	if os.Getenv(precompiledStepsEnv) == "true" {
@@ -33,19 +33,22 @@ func ActivateStep(stepLibURI, id, version, destination, destinationStepYML strin
 		if ok {
 			log.Debugf("Downloading executable for %s", platform)
 			downloadStart := time.Now()
-			err = activateStepExecutable(stepLibURI, id, version, executableForPlatform, destination, destinationStepYML)
+			execPath, err := activateStepExecutable(stepLibURI, id, version, executableForPlatform, destination, destinationStepYML)
 			if err != nil {
 				log.Warnf("Failed to download step executable, falling back to source build: %s", err)
-				return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
+				err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
+				return "", err
 			}
 			log.Debugf("Downloaded executable in %s", time.Since(downloadStart).Round(time.Millisecond))
-			return nil
+			return execPath, nil
 		} else {
 			log.Infof("No prebuilt executable found for %s, falling back to source build", platform)
-			return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
+			err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
+			return "", err
 		}
 	} else {
-		return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
+		err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
+		return "", err
 	}
 }
 
diff --git a/activator/steplib/activate_executable.go b/activator/steplib/activate_executable.go
index 7c5d0f8f..5b2b9507 100644
--- a/activator/steplib/activate_executable.go
+++ b/activator/steplib/activate_executable.go
@@ -20,44 +20,44 @@ func activateStepExecutable(
 	executable models.Executable,
 	destination string,
 	destinationStepYML string,
-) error {
+) (string, error) {
 	resp, err := retryablehttp.Get(executable.Url)
 	if err != nil {
-		return fmt.Errorf("fetch from %s: %w", executable.Url, err)
+		return "", fmt.Errorf("fetch from %s: %w", executable.Url, err)
 	}
 	defer resp.Body.Close()
 
 	err = os.MkdirAll(destination, 0755)
 	if err != nil {
-		return fmt.Errorf("create directory %s: %w", destination, err)
+		return "", fmt.Errorf("create directory %s: %w", destination, err)
 	}
 
 	path := filepath.Join(destination, stepID)
 	file, err := os.Create(path)
 	if err != nil {
-		return fmt.Errorf("create file %s: %w", path, err)
+		return "", fmt.Errorf("create file %s: %w", path, err)
 	}
 
 	_, err = io.Copy(file, resp.Body)
 	if err != nil {
-		return fmt.Errorf("download %s to %s: %w", executable.Url, path, err)
+		return "", fmt.Errorf("download %s to %s: %w", executable.Url, path, err)
 	}
 
 	err = validateHash(path, executable.Hash)
 	if err != nil {
-		return fmt.Errorf("validate hash: %s", err)
+		return "", fmt.Errorf("validate hash: %s", err)
 	}
 
 	err = os.Chmod(path, 0755)
 	if err != nil {
-		return fmt.Errorf("set executable permission on file: %s", err)
+		return "", fmt.Errorf("set executable permission on file: %s", err)
 	}
 
 	if err := copyStepYML(stepLibURI, stepID, version, destinationStepYML); err != nil {
-		return fmt.Errorf("copy step.yml: %s", err)
+		return "", fmt.Errorf("copy step.yml: %s", err)
 	}
 
-	return nil
+	return path, nil
 }
 
 func validateHash(filePath string, expectedHash string) error {
diff --git a/activator/steplib_ref.go b/activator/steplib_ref.go
index 30e1efda..b18d84e3 100644
--- a/activator/steplib_ref.go
+++ b/activator/steplib_ref.go
@@ -33,10 +33,11 @@ func ActivateSteplibRefStep(
 		return activationResult, err
 	}
 
-	err = steplib.ActivateStep(id.SteplibSource, id.IDorURI, stepInfo.Version, activatedStepDir, stepYMLPath, log, isOfflineMode)
+	execPath, err := steplib.ActivateStep(id.SteplibSource, id.IDorURI, stepInfo.Version, activatedStepDir, stepYMLPath, log, isOfflineMode)
 	if err != nil {
 		return activationResult, err
 	}
+	activationResult.ExecutablePath = execPath
 
 	// TODO: this is sketchy, we should clean this up, but this pointer originates in the CLI codebase
 	stepInfoPtr.ID = stepInfo.ID

From 9b1ab78ab2a1557914a4a84deea0c7f2ea0fe9d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= <ofalvai@gmail.com>
Date: Tue, 10 Sep 2024 16:39:49 +0200
Subject: [PATCH 4/7] Expose executable to toolkits

---
 toolkits/bash.go        |  4 ++--
 toolkits/golang.go      | 14 ++++++++++++--
 toolkits/golang_test.go |  2 +-
 toolkits/swift.go       |  4 ++--
 toolkits/toolkit.go     |  4 ++--
 5 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/toolkits/bash.go b/toolkits/bash.go
index c8d31042..10ede307 100644
--- a/toolkits/bash.go
+++ b/toolkits/bash.go
@@ -49,11 +49,11 @@ func (toolkit BashToolkit) ToolkitName() string {
 	return "bash"
 }
 
-func (toolkit BashToolkit) PrepareForStepRun(_ models.StepModel, _ stepid.CanonicalID, _ string) error {
+func (toolkit BashToolkit) PrepareForStepRun(_ models.StepModel, _ stepid.CanonicalID, _, _ string) error {
 	return nil
 }
 
-func (toolkit BashToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error) {
+func (toolkit BashToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, _ string) ([]string, error) {
 	entryFile := "step.sh"
 	if step.Toolkit != nil && step.Toolkit.Bash != nil && step.Toolkit.Bash.EntryFile != "" {
 		entryFile = step.Toolkit.Bash.EntryFile
diff --git a/toolkits/golang.go b/toolkits/golang.go
index b91316ad..f26b69ed 100644
--- a/toolkits/golang.go
+++ b/toolkits/golang.go
@@ -279,7 +279,7 @@ func stepBinaryCacheFullPath(sIDData stepid.CanonicalID) string {
 }
 
 // PrepareForStepRun ...
-func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) error {
+func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) error {
 	fullStepBinPath := stepBinaryCacheFullPath(sIDData)
 
 	// try to use cached binary, if possible
@@ -291,6 +291,11 @@ func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid
 		}
 	}
 
+	if stepExecutablePath != "" {
+		// It's a steplib ref step activated as a precompiled binary, there is nothing to prepare
+		return nil
+	}
+
 	// it's not cached, so compile it
 	if step.Toolkit == nil {
 		return errors.New("no toolkit information specified in step")
@@ -314,7 +319,12 @@ func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid
 // === Toolkit: Step Run ===
 
 // StepRunCommandArguments ...
-func (toolkit GoToolkit) StepRunCommandArguments(_ models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error) {
+func (toolkit GoToolkit) StepRunCommandArguments(_ models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) ([]string, error) {
+	if stepExecutablePath != "" {
+		// It's a steplib ref step activated as a precompiled binary, just run it
+		return []string{stepExecutablePath}, nil
+	}
+
 	fullStepBinPath := stepBinaryCacheFullPath(sIDData)
 	return []string{fullStepBinPath}, nil
 }
diff --git a/toolkits/golang_test.go b/toolkits/golang_test.go
index b5a99313..d1999ca5 100644
--- a/toolkits/golang_test.go
+++ b/toolkits/golang_test.go
@@ -219,7 +219,7 @@ func Benchmark_goBuildStep(b *testing.B) {
 		require.NoError(b, err)
 	}()
 
-	err = steplib.ActivateStep("https://github.com/bitrise-io/bitrise-steplib", "xcode-test", "5.1.1", stepDir, "", logger, false)
+	_, err = steplib.ActivateStep("https://github.com/bitrise-io/bitrise-steplib", "xcode-test", "5.1.1", stepDir, "", logger, false)
 	require.NoError(b, err)
 
 	packageName := "github.com/bitrise-steplib/steps-xcode-test"
diff --git a/toolkits/swift.go b/toolkits/swift.go
index 501aff1b..3f10bd97 100644
--- a/toolkits/swift.go
+++ b/toolkits/swift.go
@@ -37,7 +37,7 @@ func (toolkit SwiftToolkit) IsToolAvailableInPATH() bool {
 	return len(binPath) > 0
 }
 
-func (toolkit SwiftToolkit) PrepareForStepRun(step models.StepModel, _ stepid.CanonicalID, stepAbsDirPath string) error {
+func (toolkit SwiftToolkit) PrepareForStepRun(step models.StepModel, _ stepid.CanonicalID, stepAbsDirPath, _ string) error {
 	binaryLocation := step.Toolkit.Swift.BinaryLocation
 	if binaryLocation == "" {
 		return nil
@@ -58,7 +58,7 @@ func (toolkit SwiftToolkit) PrepareForStepRun(step models.StepModel, _ stepid.Ca
 	return nil
 }
 
-func (toolkit SwiftToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error) {
+func (toolkit SwiftToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, _ string) ([]string, error) {
 	binaryLocation := step.Toolkit.Swift.BinaryLocation
 	if binaryLocation == "" {
 		return []string{"swift", "run", "--package-path", stepAbsDirPath, "-c", "release"}, nil
diff --git a/toolkits/toolkit.go b/toolkits/toolkit.go
index ffbaa30b..b8fb2813 100644
--- a/toolkits/toolkit.go
+++ b/toolkits/toolkit.go
@@ -57,10 +57,10 @@ type Toolkit interface {
 	// the toolkit should/can be "enforced" here (e.g. during the compilation),
 	// BUT ONLY for this function! E.g. don't call `os.Setenv` or something similar
 	// which would affect other functions, just pass the required envs to the compilation command!
-	PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) error
+	PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) error
 
 	// StepRunCommandArguments ...
-	StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error)
+	StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) ([]string, error)
 }
 
 //

From 86573922681aa25f6262c21f0da3d8853fae5a8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= <ofalvai@gmail.com>
Date: Tue, 10 Sep 2024 17:07:29 +0200
Subject: [PATCH 5/7] Revert bad idea

---
 toolkits/bash.go    |  4 ++--
 toolkits/golang.go  | 14 ++------------
 toolkits/swift.go   |  4 ++--
 toolkits/toolkit.go |  4 ++--
 4 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/toolkits/bash.go b/toolkits/bash.go
index 10ede307..c8d31042 100644
--- a/toolkits/bash.go
+++ b/toolkits/bash.go
@@ -49,11 +49,11 @@ func (toolkit BashToolkit) ToolkitName() string {
 	return "bash"
 }
 
-func (toolkit BashToolkit) PrepareForStepRun(_ models.StepModel, _ stepid.CanonicalID, _, _ string) error {
+func (toolkit BashToolkit) PrepareForStepRun(_ models.StepModel, _ stepid.CanonicalID, _ string) error {
 	return nil
 }
 
-func (toolkit BashToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, _ string) ([]string, error) {
+func (toolkit BashToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error) {
 	entryFile := "step.sh"
 	if step.Toolkit != nil && step.Toolkit.Bash != nil && step.Toolkit.Bash.EntryFile != "" {
 		entryFile = step.Toolkit.Bash.EntryFile
diff --git a/toolkits/golang.go b/toolkits/golang.go
index f26b69ed..b91316ad 100644
--- a/toolkits/golang.go
+++ b/toolkits/golang.go
@@ -279,7 +279,7 @@ func stepBinaryCacheFullPath(sIDData stepid.CanonicalID) string {
 }
 
 // PrepareForStepRun ...
-func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) error {
+func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) error {
 	fullStepBinPath := stepBinaryCacheFullPath(sIDData)
 
 	// try to use cached binary, if possible
@@ -291,11 +291,6 @@ func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid
 		}
 	}
 
-	if stepExecutablePath != "" {
-		// It's a steplib ref step activated as a precompiled binary, there is nothing to prepare
-		return nil
-	}
-
 	// it's not cached, so compile it
 	if step.Toolkit == nil {
 		return errors.New("no toolkit information specified in step")
@@ -319,12 +314,7 @@ func (toolkit GoToolkit) PrepareForStepRun(step models.StepModel, sIDData stepid
 // === Toolkit: Step Run ===
 
 // StepRunCommandArguments ...
-func (toolkit GoToolkit) StepRunCommandArguments(_ models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) ([]string, error) {
-	if stepExecutablePath != "" {
-		// It's a steplib ref step activated as a precompiled binary, just run it
-		return []string{stepExecutablePath}, nil
-	}
-
+func (toolkit GoToolkit) StepRunCommandArguments(_ models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error) {
 	fullStepBinPath := stepBinaryCacheFullPath(sIDData)
 	return []string{fullStepBinPath}, nil
 }
diff --git a/toolkits/swift.go b/toolkits/swift.go
index 3f10bd97..501aff1b 100644
--- a/toolkits/swift.go
+++ b/toolkits/swift.go
@@ -37,7 +37,7 @@ func (toolkit SwiftToolkit) IsToolAvailableInPATH() bool {
 	return len(binPath) > 0
 }
 
-func (toolkit SwiftToolkit) PrepareForStepRun(step models.StepModel, _ stepid.CanonicalID, stepAbsDirPath, _ string) error {
+func (toolkit SwiftToolkit) PrepareForStepRun(step models.StepModel, _ stepid.CanonicalID, stepAbsDirPath string) error {
 	binaryLocation := step.Toolkit.Swift.BinaryLocation
 	if binaryLocation == "" {
 		return nil
@@ -58,7 +58,7 @@ func (toolkit SwiftToolkit) PrepareForStepRun(step models.StepModel, _ stepid.Ca
 	return nil
 }
 
-func (toolkit SwiftToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, _ string) ([]string, error) {
+func (toolkit SwiftToolkit) StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error) {
 	binaryLocation := step.Toolkit.Swift.BinaryLocation
 	if binaryLocation == "" {
 		return []string{"swift", "run", "--package-path", stepAbsDirPath, "-c", "release"}, nil
diff --git a/toolkits/toolkit.go b/toolkits/toolkit.go
index b8fb2813..ffbaa30b 100644
--- a/toolkits/toolkit.go
+++ b/toolkits/toolkit.go
@@ -57,10 +57,10 @@ type Toolkit interface {
 	// the toolkit should/can be "enforced" here (e.g. during the compilation),
 	// BUT ONLY for this function! E.g. don't call `os.Setenv` or something similar
 	// which would affect other functions, just pass the required envs to the compilation command!
-	PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) error
+	PrepareForStepRun(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) error
 
 	// StepRunCommandArguments ...
-	StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, stepExecutablePath string) ([]string, error)
+	StepRunCommandArguments(step models.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath string) ([]string, error)
 }
 
 //

From 8d861a90c5ada3c8ba05679d2c7eddcade659358 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= <ofalvai@gmail.com>
Date: Wed, 5 Feb 2025 12:33:41 +0100
Subject: [PATCH 6/7] Cleanups, minor tweaks

---
 activator/activator.go                   | 6 ++++++
 activator/steplib/activate.go            | 6 +++---
 activator/steplib/activate_executable.go | 4 ++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/activator/activator.go b/activator/activator.go
index f2365440..755cb177 100644
--- a/activator/activator.go
+++ b/activator/activator.go
@@ -3,6 +3,12 @@ package activator
 type ActivatedStep struct {
 	StepYMLPath string
 
+	// ExecutablePath is a local path to the main entrypoint of the step, ready for execution.
+	// This can be an empty string if:
+	// - step was activated from a git reference (we checked out the source dir directly)
+	// - step was activated from a local path (we copied the source dir directly)
+	// - step was activated from a steplib reference, but step.yml has no entry for pre-compiled binaries (we fallback to source checkout)
+	// - step was activated from a stpelib reference, but step.yml has no pre-compiled binary for the current OS+arch combo (we fallback to source checkout)
 	ExecutablePath string
 
 	// DidStepLibUpdate indicates that the local steplib cache was updated while resolving the exact step version.
diff --git a/activator/steplib/activate.go b/activator/steplib/activate.go
index e3a0d15c..e01ed09f 100644
--- a/activator/steplib/activate.go
+++ b/activator/steplib/activate.go
@@ -27,7 +27,7 @@ func ActivateStep(stepLibURI, id, version, destination, destinationStepYML strin
 		return "", fmt.Errorf("failed to find step: %s", err)
 	}
 
-	if os.Getenv(precompiledStepsEnv) == "true" {
+	if os.Getenv(precompiledStepsEnv) == "true" || os.Getenv(precompiledStepsEnv) == "1" {
 		platform := fmt.Sprintf("%s-%s", runtime.GOOS, runtime.GOARCH)
 		executableForPlatform, ok := step.Executables[platform]
 		if ok {
@@ -35,14 +35,14 @@ func ActivateStep(stepLibURI, id, version, destination, destinationStepYML strin
 			downloadStart := time.Now()
 			execPath, err := activateStepExecutable(stepLibURI, id, version, executableForPlatform, destination, destinationStepYML)
 			if err != nil {
-				log.Warnf("Failed to download step executable, falling back to source build: %s", err)
+				log.Warnf("Failed to download step executable, fallback to step source activation: %s", err)
 				err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
 				return "", err
 			}
 			log.Debugf("Downloaded executable in %s", time.Since(downloadStart).Round(time.Millisecond))
 			return execPath, nil
 		} else {
-			log.Infof("No prebuilt executable found for %s, falling back to source build", platform)
+			log.Infof("No prebuilt executable found for %s, fallback to step source activation", platform)
 			err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode)
 			return "", err
 		}
diff --git a/activator/steplib/activate_executable.go b/activator/steplib/activate_executable.go
index 5b2b9507..fbde4e5b 100644
--- a/activator/steplib/activate_executable.go
+++ b/activator/steplib/activate_executable.go
@@ -21,6 +21,10 @@ func activateStepExecutable(
 	destination string,
 	destinationStepYML string,
 ) (string, error) {
+	if strings.HasPrefix(executable.Url, "http://") {
+		return "", fmt.Errorf("http URL is unsupported, please use https: %s", executable.Url)
+	}
+
 	resp, err := retryablehttp.Get(executable.Url)
 	if err != nil {
 		return "", fmt.Errorf("fetch from %s: %w", executable.Url, err)

From 047076caedb37f77896ff5714bb2a1ec4956214e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= <ofalvai@gmail.com>
Date: Wed, 30 Apr 2025 15:50:08 +0200
Subject: [PATCH 7/7] *

---
 activator/activator.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/activator/activator.go b/activator/activator.go
index 755cb177..6f34b5f6 100644
--- a/activator/activator.go
+++ b/activator/activator.go
@@ -12,5 +12,7 @@ type ActivatedStep struct {
 	ExecutablePath string
 
 	// DidStepLibUpdate indicates that the local steplib cache was updated while resolving the exact step version.
+	// TODO: this is a leaky abstraction and we shouldn't signal this here, but it requires a bigger refactor.
+	// (stepman should keep track of this info in a file probably)
 	DidStepLibUpdate bool
 }