Merge pull request #76 from hashicorp/f-archive-deroot

Subdir supports glob patterns

README.md

@@ -119,6 +119,37 @@ The protocol-specific options are documented below the URL format
 section. But because they are part of the URL, we point it out here so
 you know they exist.
 
+### Subdirectories
+
+If you want to download only a specific subdirectory from a downloaded
+directory, you can specify a subdirectory after a double-slash `//`.
+go-getter will first download the URL specified _before_ the double-slash
+(as if you didn't specify a double-slash), but will then copy the
+path after the double slash into the target directory.
+
+For example, if you're downloading this GitHub repository, but you only
+want to download the `test-fixtures` directory, you can do the following:
+
+```
+https://github.com/hashicorp/go-getter.git//test-fixtures
+```
+
+If you downloaded this to the `/tmp` directory, then the file
+`/tmp/archive.gz` would exist. Notice that this file is in the `test-fixtures`
+directory in this repository, but because we specified a subdirectory,
+go-getter automatically copied only that directory contents.
+
+Subdirectory paths may contain may also use filesystem glob patterns.
+The path must match _exactly one_ entry or go-getter will return an error.
+This is useful if you're not sure the exact directory name but it follows
+a predictable naming structure.
+
+For example, the following URL would also work:
+
+```
+https://github.com/hashicorp/go-getter.git//test-*
+```
+
 ### Checksumming
 
 For file downloads of any protocol, go-getter can automatically verify

client.go

@@ -305,7 +305,13 @@ func (c *Client) Get() error {
 			return err
 		}
 
-		return copyDir(realDst, filepath.Join(dst, subDir), false)
+		// Process any globs
+		subDir, err := SubdirGlob(dst, subDir)
+		if err != nil {
+			return err
+		}
+
+		return copyDir(realDst, subDir, false)
 	}
 
 	return nil

get_http.go

@@ -98,7 +98,6 @@ func (g *HttpGetter) Get(dst string, u *url.URL) error {
 }
 
 func (g *HttpGetter) GetFile(dst string, u *url.URL) error {
-
 	if g.Netrc {
 		// Add auth from netrc if we can
 		if err := addAuthFromNetrc(u); err != nil {
@@ -140,13 +139,22 @@ func (g *HttpGetter) getSubdir(dst, source, subDir string) error {
 	}
 	defer os.RemoveAll(td)
 
+	// We have to create a subdirectory that doesn't exist for the file
+	// getter to work.
+	td = filepath.Join(td, "data")
+
 	// Download that into the given directory
 	if err := Get(td, source); err != nil {
 		return err
 	}
 
+	// Process any globbing
+	sourcePath, err := SubdirGlob(td, subDir)
+	if err != nil {
+		return err
+	}
+
 	// Make sure the subdir path actually exists
-	sourcePath := filepath.Join(td, subDir)
 	if _, err := os.Stat(sourcePath); err != nil {
 		return fmt.Errorf(
 			"Error downloading %s: %s", source, err)

get_http_test.go

@@ -86,6 +86,30 @@ func TestHttpGetter_metaSubdir(t *testing.T) {
 	}
 }
 
+func TestHttpGetter_metaSubdirGlob(t *testing.T) {
+	ln := testHttpServer(t)
+	defer ln.Close()
+
+	g := new(HttpGetter)
+	dst := tempDir(t)
+
+	var u url.URL
+	u.Scheme = "http"
+	u.Host = ln.Addr().String()
+	u.Path = "/meta-subdir-glob"
+
+	// Get it!
+	if err := g.Get(dst, &u); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the main file exists
+	mainPath := filepath.Join(dst, "sub.tf")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
 func TestHttpGetter_none(t *testing.T) {
 	ln := testHttpServer(t)
 	defer ln.Close()
@@ -194,6 +218,7 @@ func testHttpServer(t *testing.T) net.Listener {
 	mux.HandleFunc("/meta", testHttpHandlerMeta)
 	mux.HandleFunc("/meta-auth", testHttpHandlerMetaAuth)
 	mux.HandleFunc("/meta-subdir", testHttpHandlerMetaSubdir)
+	mux.HandleFunc("/meta-subdir-glob", testHttpHandlerMetaSubdirGlob)
 
 	var server http.Server
 	server.Handler = mux
@@ -234,6 +259,10 @@ func testHttpHandlerMetaSubdir(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic//subdir").String())))
 }
 
+func testHttpHandlerMetaSubdirGlob(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic//sub*").String())))
+}
+
 func testHttpHandlerNone(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(testHttpNoneStr))
 }

get_test.go

@@ -145,6 +145,44 @@ func TestGetAny_archive(t *testing.T) {
 	}
 }
 
+func TestGet_archiveRooted(t *testing.T) {
+	dst := tempDir(t)
+	u := testModule("archive-rooted/archive.tar.gz")
+	if err := Get(dst, u); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	mainPath := filepath.Join(dst, "root", "hello.txt")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestGet_archiveSubdirWild(t *testing.T) {
+	dst := tempDir(t)
+	u := testModule("archive-rooted/archive.tar.gz")
+	u += "//*"
+	if err := Get(dst, u); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	mainPath := filepath.Join(dst, "hello.txt")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestGet_archiveSubdirWildMultiMatch(t *testing.T) {
+	dst := tempDir(t)
+	u := testModule("archive-rooted-multi/archive.tar.gz")
+	u += "//*"
+	if err := Get(dst, u); err == nil {
+		t.Fatal("should error")
+	} else if !strings.Contains(err.Error(), "multiple") {
+		t.Fatalf("err: %s", err)
+	}
+}
+
 func TestGetAny_file(t *testing.T) {
 	dst := tempDir(t)
 	u := testModule("basic-file/foo.txt")

module_test.go

@@ -40,10 +40,15 @@ func testModule(n string) string {
 }
 
 func testModuleURL(n string) *url.URL {
+	n, subDir := SourceDirSubdir(n)
 	u, err := urlhelper.Parse(testModule(n))
 	if err != nil {
 		panic(err)
 	}
+	if subDir != "" {
+		u.Path += "//" + subDir
+		u.RawPath = u.Path
+	}
 
 	return u
 }

source.go

@@ -1,6 +1,8 @@
 package getter
 
 import (
+	"fmt"
+	"path/filepath"
 	"strings"
 )
 
@@ -34,3 +36,22 @@ func SourceDirSubdir(src string) (string, string) {
 
 	return src, subdir
 }
+
+// SubdirGlob returns the actual subdir with globbing processed.
+//
+// dst should be a destination directory that is already populated (the
+// download is complete) and subDir should be the set subDir. If subDir
+// is an empty string, this returns an empty string.
+//
+// The returned path is the full absolute path.
+func SubdirGlob(dst, subDir string) (string, error) {
+	matches, err := filepath.Glob(filepath.Join(dst, subDir))
+	if err != nil {
+		return "", err
+	}
+	if len(matches) > 1 {
+		return "", fmt.Errorf("subdir %q matches multiple paths", subDir)
+	}
+
+	return matches[0], nil
+}

source_test.go

@@ -21,6 +21,10 @@ func TestSourceDirSubdir(t *testing.T) {
 			"hashicorp.com//foo?bar=baz",
 			"hashicorp.com?bar=baz", "foo",
 		},
+		{
+			"https://hashicorp.com/path//*?archive=foo",
+			"https://hashicorp.com/path?archive=foo", "*",
+		},
 		{
 			"file://foo//bar",
 			"file://foo", "bar",