Convert SCP-style URLs (no explicit scheme) into proper SSH URLs

internal/exec/go_getter_utils.go

@@ -8,9 +8,11 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
+	log "github.com/charmbracelet/log"
 	"github.com/google/uuid"
 	"github.com/hashicorp/go-getter"
 
@@ -61,84 +63,215 @@ func IsValidScheme(scheme string) bool {
 	return validSchemes[scheme]
 }
 
-// CustomGitHubDetector intercepts GitHub URLs and transforms them
-// into something like git::https://<token>@github.com/... so we can
-// do a git-based clone with a token.
-type CustomGitHubDetector struct {
+// CustomGitDetector intercepts Git URLs (for GitHub, Bitbucket, GitLab, etc.)
+// and transforms them into a proper URL for cloning, optionally injecting tokens.
+type CustomGitDetector struct {
 	AtmosConfig schema.AtmosConfiguration
+	source      string
 }
 
 // Detect implements the getter.Detector interface for go-getter v1.
-func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) {
+func (d *CustomGitDetector) Detect(src, _ string) (string, bool, error) {
+	log.Debug("CustomGitDetector.Detect called")
+
 	if len(src) == 0 {
 		return "", false, nil
 	}
 
-	if !strings.Contains(src, "://") {
-		src = "https://" + src
-	}
+	// Ensure the URL has an explicit scheme.
+	src = d.ensureScheme(src)
 
+	// Parse the URL to extract the host and path.
 	parsedURL, err := url.Parse(src)
 	if err != nil {
-		u.LogDebug(fmt.Sprintf("Failed to parse URL %q: %v\n", src, err))
-		return "", false, fmt.Errorf("failed to parse URL %q: %w", src, err)
+		maskedSrc, _ := u.MaskBasicAuth(src)
+		log.Debug("Failed to parse URL", keyURL, maskedSrc, "error", err)
+		return "", false, fmt.Errorf("failed to parse URL %q: %w", maskedSrc, err)
 	}
 
-	if strings.ToLower(parsedURL.Host) != "github.com" {
-		u.LogDebug(fmt.Sprintf("Host is %q, not 'github.com', skipping token injection\n", parsedURL.Host))
-		return "", false, nil
-	}
+	// Normalize the path.
+	d.normalizePath(parsedURL)
 
-	parts := strings.SplitN(parsedURL.Path, "/", 4)
-	if len(parts) < 3 {
-		u.LogDebug(fmt.Sprintf("URL path %q doesn't look like /owner/repo\n", parsedURL.Path))
-		return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path)
+	// Adjust host check to support GitHub, Bitbucket, GitLab, etc.
+	host := strings.ToLower(parsedURL.Host)
+	if host != "github.com" && host != "bitbucket.org" && host != "gitlab.com" {
+		log.Debug("Skipping token injection for a unsupported host", "host", parsedURL.Host)
 	}
 
-	atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN")
-	gitHubToken := os.Getenv("GITHUB_TOKEN")
+	log.Debug("Reading config param", "InjectGithubToken", d.AtmosConfig.Settings.InjectGithubToken)
+	// Inject token if available.
+	d.injectToken(parsedURL, host)
 
-	var usedToken string
-	var tokenSource string
+	// Adjust subdirectory if needed.
+	d.adjustSubdir(parsedURL, d.source)
 
-	// 1. If ATMOS_GITHUB_TOKEN is set, always use that
-	if atmosGitHubToken != "" {
-		usedToken = atmosGitHubToken
-		tokenSource = "ATMOS_GITHUB_TOKEN"
-		u.LogDebug("ATMOS_GITHUB_TOKEN is set\n")
+	// Set "depth=1" for a shallow clone if not specified.
+	q := parsedURL.Query()
+	if _, exists := q["depth"]; !exists {
+		q.Set("depth", "1")
+	}
+	parsedURL.RawQuery = q.Encode()
+
+	finalURL := "git::" + parsedURL.String()
+	maskedFinal, err := u.MaskBasicAuth(strings.TrimPrefix(finalURL, "git::"))
+	if err != nil {
+		log.Debug("Masking failed", "error", err)
 	} else {
-		// 2. Otherwise, only inject GITHUB_TOKEN if cfg.Settings.InjectGithubToken == true
-		if d.AtmosConfig.Settings.InjectGithubToken && gitHubToken != "" {
-			usedToken = gitHubToken
-			tokenSource = "GITHUB_TOKEN"
-			u.LogTrace("InjectGithubToken=true and GITHUB_TOKEN is set, using it\n")
-		} else {
-			u.LogTrace("No ATMOS_GITHUB_TOKEN or GITHUB_TOKEN found\n")
+		log.Debug("Final URL", "final_url", "git::"+maskedFinal)
+	}
+
+	return finalURL, true, nil
+}
+
+const (
+	// Named constants for regex match indices.
+	matchIndexUser   = 1
+	matchIndexHost   = 3
+	matchIndexPath   = 4
+	matchIndexSuffix = 5
+	matchIndexExtra  = 6
+
+	// Key for logging repeated "url" field.
+	keyURL = "url"
+)
+
+// ensureScheme checks for an explicit scheme and rewrites SCP-style URLs if needed.
+// This version no longer returns an error since it never produces one.
+func (d *CustomGitDetector) ensureScheme(src string) string {
+	if !strings.Contains(src, "://") {
+		if newSrc, rewritten := rewriteSCPURL(src); rewritten {
+			maskedOld, _ := u.MaskBasicAuth(src)
+			maskedNew, _ := u.MaskBasicAuth(newSrc)
+			log.Debug("Rewriting SCP-style SSH URL", "old_url", maskedOld, "new_url", maskedNew)
+			return newSrc
+		}
+		src = "https://" + src
+		maskedSrc, _ := u.MaskBasicAuth(src)
+		log.Debug("Defaulting to https scheme", keyURL, maskedSrc)
+	}
+	return src
+}
+
+// rewriteSCPURL rewrites SCP-style URLs to a proper SSH URL if they match the expected pattern.
+// Returns the rewritten URL and a boolean indicating if rewriting occurred.
+func rewriteSCPURL(src string) (string, bool) {
+	scpPattern := regexp.MustCompile(`^(([\w.-]+)@)?([\w.-]+\.[\w.-]+):([\w./-]+)(\.git)?(.*)$`)
+	if scpPattern.MatchString(src) {
+		matches := scpPattern.FindStringSubmatch(src)
+		newSrc := "ssh://"
+		if matches[matchIndexUser] != "" {
+			newSrc += matches[matchIndexUser] // includes username and '@'
+		}
+		newSrc += matches[matchIndexHost] + "/" + matches[matchIndexPath]
+		if matches[matchIndexSuffix] != "" {
+			newSrc += matches[matchIndexSuffix]
 		}
+		if matches[matchIndexExtra] != "" {
+			newSrc += matches[matchIndexExtra]
+		}
+		return newSrc, true
+	}
+	return "", false
+}
+
+// normalizePath converts the URL path to use forward slashes.
+func (d *CustomGitDetector) normalizePath(parsedURL *url.URL) {
+	unescapedPath, err := url.PathUnescape(parsedURL.Path)
+	if err == nil {
+		parsedURL.Path = filepath.ToSlash(unescapedPath)
+	} else {
+		parsedURL.Path = filepath.ToSlash(parsedURL.Path)
+	}
+}
+
+// injectToken injects a token into the URL if available.
+func (d *CustomGitDetector) injectToken(parsedURL *url.URL, host string) {
+	token, tokenSource := d.resolveToken(host)
+	if token != "" {
+		defaultUsername := getDefaultUsername(host)
+		parsedURL.User = url.UserPassword(defaultUsername, token)
+		maskedURL, _ := u.MaskBasicAuth(parsedURL.String())
+		log.Debug("Injected token", "env", tokenSource, keyURL, maskedURL)
+	} else {
+		log.Debug("No token found for injection")
 	}
+}
 
-	if usedToken != "" {
-		user := parsedURL.User.Username()
-		pass, _ := parsedURL.User.Password()
-		if user == "" && pass == "" {
-			u.LogDebug(fmt.Sprintf("Injecting token from %s for %s\n", tokenSource, src))
-			parsedURL.User = url.UserPassword("x-access-token", usedToken)
+// resolveToken returns the token and its source based on the host.
+func (d *CustomGitDetector) resolveToken(host string) (string, string) {
+	var token, tokenSource string
+	switch host {
+	case "github.com":
+		if d.AtmosConfig.Settings.InjectGithubToken {
+			tokenSource = "ATMOS_GITHUB_TOKEN"
+			token = os.Getenv(tokenSource)
+			if token == "" {
+				tokenSource = "GITHUB_TOKEN"
+				token = os.Getenv(tokenSource)
+			}
 		} else {
-			u.LogDebug("Credentials found, skipping token injection\n")
+			tokenSource = "GITHUB_TOKEN"
+			token = os.Getenv(tokenSource)
+		}
+	case "bitbucket.org":
+		tokenSource = "BITBUCKET_TOKEN"
+		token = os.Getenv(tokenSource)
+		if token == "" {
+			tokenSource = "ATMOS_BITBUCKET_TOKEN"
+			token = os.Getenv(tokenSource)
+		}
+	case "gitlab.com":
+		tokenSource = "GITLAB_TOKEN"
+		token = os.Getenv(tokenSource)
+		if token == "" {
+			tokenSource = "ATMOS_GITLAB_TOKEN"
+			token = os.Getenv(tokenSource)
 		}
 	}
+	return token, tokenSource
+}
 
-	finalURL := "git::" + parsedURL.String()
+// getDefaultUsername returns the default username for token injection based on the host.
+func getDefaultUsername(host string) string {
+	switch host {
+	case "github.com":
+		return "x-access-token"
+	case "gitlab.com":
+		return "oauth2"
+	case "bitbucket.org":
+		defaultUsername := os.Getenv("ATMOS_BITBUCKET_USERNAME")
+		if defaultUsername == "" {
+			defaultUsername = os.Getenv("BITBUCKET_USERNAME")
+			if defaultUsername == "" {
+				return "x-token-auth"
+			}
+		}
+		log.Debug("Using Bitbucket username", "username", defaultUsername)
+		return defaultUsername
+	default:
+		return "x-access-token"
+	}
+}
 
-	return finalURL, true, nil
+// adjustSubdir appends "//." to the path if no subdirectory is specified.
+func (d *CustomGitDetector) adjustSubdir(parsedURL *url.URL, source string) {
+	normalizedSource := filepath.ToSlash(source)
+	if normalizedSource != "" && !strings.Contains(normalizedSource, "//") {
+		parts := strings.SplitN(parsedURL.Path, "/", 4)
+		if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 {
+			maskedSrc, _ := u.MaskBasicAuth(source)
+			log.Debug("Detected top-level repo with no subdir: appending '//.'", keyURL, maskedSrc)
+			parsedURL.Path += "//."
+		}
+	}
 }
 
 // RegisterCustomDetectors prepends the custom detector so it runs before
 // the built-in ones. Any code that calls go-getter should invoke this.
 func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) {
 	getter.Detectors = append(
 		[]getter.Detector{
-			&CustomGitHubDetector{AtmosConfig: atmosConfig},
+			&CustomGitDetector{AtmosConfig: atmosConfig},
 		},
 		getter.Detectors...,
 	)

internal/exec/vendor_utils.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/bmatcuk/doublestar/v4"
 	tea "github.com/charmbracelet/bubbletea"
+	log "github.com/charmbracelet/log"
 	cp "github.com/otiai10/copy"
 	"github.com/samber/lo"
 	"github.com/spf13/cobra"
@@ -634,7 +635,7 @@ func generateSkipFunction(atmosConfig schema.AtmosConfiguration, tempDir string,
 		}
 
 		// If 'included_paths' is not provided, include all files that were not excluded
-		u.LogTrace(fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src)))
+		log.Debug("Including ", "file", u.TrimBasePathFromPath(tempDir+"/", src))
 		return false, nil
 	}
 }

pkg/utils/url_utils.go

@@ -0,0 +1,20 @@
+package utils
+
+import (
+	"fmt"
+	"net/url"
+)
+
+// MaskBasicAuth replaces the username and password in a URL with "xxx" if present.
+func MaskBasicAuth(rawURL string) (string, error) {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse URL: %w", err)
+	}
+
+	if parsedURL.User != nil {
+		parsedURL.User = url.UserPassword("xxx", "xxx")
+	}
+
+	return parsedURL.String(), nil
+}

tests/fixtures/scenarios/vendor-pulls-ssh/atmos.yaml

@@ -0,0 +1,29 @@
+base_path: "./"
+settings:
+  inject_github_token: true
+
+
+components:
+  terraform:
+    base_path: "components/terraform"
+    apply_auto_approve: false
+    deploy_run_init: true
+    init_run_reconfigure: true
+    auto_generate_backend_file: false
+
+stacks:
+  base_path: "stacks"
+  included_paths:
+    - "deploy/**/*"
+  excluded_paths:
+    - "**/_defaults.yaml"
+  name_pattern: "{stage}"
+
+logs:
+  file: "/dev/stderr"
+  level: Info
+
+
+
+
+

tests/fixtures/scenarios/vendor-pulls-ssh/vendor.yaml

@@ -0,0 +1,35 @@
+apiVersion: atmos/v1
+kind: AtmosVendorConfig
+metadata:
+  name: demo-vendoring
+  description: Atmos vendoring manifest for Atmos demo component library
+spec:
+  imports: []
+
+  sources:
+    # Basic HTTPS default (token injection expected)
+    - component: "terraform-null-label-basic"
+      source: "github.com/cloudposse/terraform-null-label.git?ref={{ .Version }}"
+      version: "0.25.0"
+      targets:
+        - "library/basic/{{ .Component }}"
+      tags:
+        - demo
+
+    # Direct credentials provided in the URL (token injection should be skipped)
+    - component: "terraform-null-label-direct"
+      source: "https://myuser:[email protected]/cloudposse/terraform-null-label.git?ref={{ .Version }}"
+      version: "0.25.0"
+      targets:
+        - "library/direct/{{ .Component }}"
+      tags:
+        - demo
+
+    # HTTPS with pre-existing credentials (token injection skipped)
+    - component: "terraform-null-label-cred"
+      source: "https://[email protected]/cloudposse/terraform-null-label.git?ref={{ .Version }}"
+      version: "0.25.0"
+      targets:
+        - "library/cred/{{ .Component }}"
+      tags:
+        - demo