From 33529d6ed871a187d943964cd6c5633b8f91e332 Mon Sep 17 00:00:00 2001 From: lifangmoler Date: Fri, 8 Feb 2019 11:33:53 -0800 Subject: [PATCH] add gcs --- detect.go | 1 + detect_gcs.go | 37 +++++++++++ get.go | 1 + get_gcs.go | 170 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 detect_gcs.go create mode 100644 get_gcs.go diff --git a/detect.go b/detect.go index 1485aaa97..5bb750c9f 100644 --- a/detect.go +++ b/detect.go @@ -26,6 +26,7 @@ func init() { new(GitDetector), new(BitBucketDetector), new(S3Detector), + new(GCSDetector), new(FileDetector), } } diff --git a/detect_gcs.go b/detect_gcs.go new file mode 100644 index 000000000..66e2bc399 --- /dev/null +++ b/detect_gcs.go @@ -0,0 +1,37 @@ +package getter + +import ( + "fmt" + "net/url" + "strings" +) + +// GCSDetector implements Detector to detect GCS URLs and turn +// them into URLs that the GCSGetter can understand. +type GCSDetector struct{} + +func (d *GCSDetector) Detect(src, _ string) (string, bool, error) { + if len(src) == 0 { + return "", false, nil + } + + if strings.Contains(src, "googleapis.com/") { + return d.detectHTTP(fmt.Sprintf("https://www.googleapis.com/%s", + strings.SplitN(src, "googleapis.com/", 2)[1])) + } + + return "", false, nil +} + +func (d *GCSDetector) detectHTTP(src string) (string, bool, error) { + url, err := url.Parse(src) + if err != nil { + return "", false, fmt.Errorf("error parsing GCS URL: %s", err) + } + + pathParts := strings.SplitN(url.Path, "/", 5) + if len(pathParts) != 5 { + return "", false, fmt.Errorf("URL is not a valid GCS URL") + } + return "gcs::" + url.String(), true, nil +} diff --git a/get.go b/get.go index 9e79201a4..c233763c6 100644 --- a/get.go +++ b/get.go @@ -67,6 +67,7 @@ func init() { Getters = map[string]Getter{ "file": new(FileGetter), "git": new(GitGetter), + "gcs": new(GCSGetter), "hg": new(HgGetter), "s3": new(S3Getter), "http": httpGetter, diff --git a/get_gcs.go b/get_gcs.go new file mode 100644 index 000000000..479ffd2d5 --- /dev/null +++ b/get_gcs.go @@ -0,0 +1,170 @@ +package getter + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "cloud.google.com/go/storage" + "google.golang.org/api/iterator" +) + +// GCSGetter is a Getter implementation that will download a module from +// a GCS bucket. +type GCSGetter struct { + getter +} + +func (g *GCSGetter) ClientMode(u *url.URL) (ClientMode, error) { + ctx := g.Context() + + // Parse URL + bucket, object, err := g.parseURL(u) + if err != nil { + return 0, err + } + + sctx := context.Background() + client, err := storage.NewClient(sctx) + if err != nil { + return 0, err + } + iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object}) + count := 0 + for { + _, err := iter.Next() + if err != nil && err != iterator.Done { + return 0, err + } + count++ + if err == iterator.Done { + break + } + } + if count <= 1 { + // Return file if there are no matches as well. + // GetFile will fail in this case. + return ClientModeFile, nil + } else { + return ClientModeDir, nil + } +} + +func (g *GCSGetter) Get(dst string, u *url.URL) error { + ctx := g.Context() + + // Parse URL + bucket, object, err := g.parseURL(u) + if err != nil { + return err + } + + // Remove destination if it already exists + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // Remove the destination + if err := os.RemoveAll(dst); err != nil { + return err + } + } + + // Create all the parent directories + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + sctx := context.Background() + client, err := storage.NewClient(sctx) + if err != nil { + return err + } + + // Iterate through all matching objects. + iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object}) + for { + obj, err := iter.Next() + if err != nil && err != iterator.Done { + return err + } + if err == iterator.Done { + break + } + + // Get the object destination path + objDst, err := filepath.Rel(object, obj.Name) + if err != nil { + return err + } + objDst = filepath.Join(dst, objDst) + // Download the matching object. + err = g.getObject(ctx, client, objDst, bucket, obj.Name) + if err != nil { + return err + } + } + return nil +} + +func (g *GCSGetter) GetFile(dst string, u *url.URL) error { + ctx := g.Context() + + // Parse URL + bucket, object, err := g.parseURL(u) + if err != nil { + return err + } + + sctx := context.Background() + client, err := storage.NewClient(sctx) + if err != nil { + return err + } + return g.getObject(ctx, client, dst, bucket, object) +} + +func (g *GCSGetter) getObject(ctx context.Context, client *storage.Client, dst, bucket, object string) error { + rc, err := client.Bucket(bucket).Object(object).NewReader(ctx) + if err != nil { + return err + } + defer rc.Close() + + // Create all the parent directories + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + f, err := os.Create(dst) + if err != nil { + return err + } + defer f.Close() + + _, err = Copy(ctx, f, rc) + return err +} + +func (g *GCSGetter) parseURL(u *url.URL) (bucket, path string, err error) { + if strings.Contains(u.Host, "googleapis.com") { + hostParts := strings.Split(u.Host, ".") + if len(hostParts) != 3 { + err = fmt.Errorf("URL is not a valid GCS URL") + return + } + + pathParts := strings.SplitN(u.Path, "/", 5) + if len(pathParts) != 5 { + err = fmt.Errorf("URL is not a valid GCS URL") + return + } + bucket = pathParts[3] + path = pathParts[4] + } + return +}