spec: add image tags

This patch introduces the concept of image tags.

Since many ACEs (like rkt) wants to implement local caching I tried to
define Image Tags considering this as a primary requirement.

While an image is static since its labels are written in its manifest and
changing them will change the image id, image tags are dynamic.

In short words it adds another file that contains a map to convert a tag
to another tag (alias) and/or expand a tag to a set of labels.
It pratically adds a layer above current image discovery used to obtain
the final set of labels to use to calculte `ac-discovery` URLs.

Image Tags are defined to be per image name. This is needed to satisfy
the need to locally cache them (since they should be cached and retrieved
using a key and this key is the image name).

This will be useful to fix different issues and enhancements.

This patch:

* Introduce Image Tags format.

* Details the discovery process (since there's now an optional
additional step)

* Add a "tag" inside image dependencies in image manifest.

* Change the app string format parsing function. It now defines the
value after the ":" as a Tag instead of the version label.

* Add code for doing tag resolution and labels merging (that can be used
by an ACE). As a fallback, if no image tags data is provided, the
version label will be set from the tag value.

* The image tags fetching and verification logic is left to the ACE
(like done for an ACI).

-- Docker compatibility

Actually docker2aci converts a docker image (squashing it) to an ACI and
sets the version label to the image tag. This is a fast but not
consisten way to docker tags since they can be dynamically changed.

With this proposal an idea will be to convert docker tags to an
Image Tags file (using the docker APIs). Docker tags are per
repository and a repository is mapped to an ACI image name. So this
should match the proposal that dfines imagetags being per image.

Docker images don't have a version label but can have a label
representing the docker image ID. So an image tag can point to that
"dockerimageid" label.

-- rkt related work

For rkt, the changes to implement to satisfy image tags should be:

* Image Tags fetching and validation
* Labels Merging (calling imageTags.MergeTag providing the starting
* labels and tag value)
* Caching of Tags Data per app name
* Use of imageTags.MergeTag before calling GetACI.
* Removal of "latest" column from the store since it was a hack around
* current spec default "latest" version.

Some open points:

* Since there're an additional layer and an additional file there's the need to:
 * inspect Image Tags data (like the current `rkt image list` there
 should be something like `rkt imagetags list`)
 * fetch/update them (they are automatically fetched using `rkt
 --no-store fetch imagename`, but this will also fetch an image). To
 just fetch/update ImageTags something like `rkt imagetags --no-store fetch {IMAGENAME|file|URL}`
 should be added. Providing and image name will use discovery, while the other will
 just fetch it from the provided file/URL.

* Since rkt also accept fetching an image from a file or URL, doing this
won't carry Image Tags information. So a successive attemps to run an
image by it's image string can use a different image than the one that
will be run using an image discovery returning Image Tags data (due to
the fallback of setting the version label value to tag if no image tag
data is available). So a command like the above `rkt imagetags fetch` is
needed also for this reason.

Author: sgotti

actool/discover.go

@@ -15,12 +15,18 @@
 package main
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"net"
+	"net/http"
+	"net/url"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/appc/spec/discovery"
+	"github.com/appc/spec/schema"
 )
 
 var (
@@ -62,12 +68,37 @@ func runDiscover(args []string) (exit int) {
 		if transportFlags.Insecure {
 			insecure = discovery.InsecureTLS | discovery.InsecureHTTP
 		}
+		tagsEndpoints, attempts, err := discovery.DiscoverImageTags(*app, nil, insecure)
+		if err != nil {
+			stderr("error fetching endpoints for %s: %s", name, err)
+			return 1
+		}
+		for _, a := range attempts {
+			fmt.Printf("discover tags walk: prefix: %s error: %v\n", a.Prefix, a.Error)
+		}
+		if len(tagsEndpoints) != 0 {
+			tags, err := fetchImageTags(tagsEndpoints[0].ImageTags, insecure)
+			if err != nil {
+				stderr("error fetching tags info: %s", err)
+				return 1
+			}
+			// Merge tag labels
+			app, err = app.MergeTag(tags)
+			if err != nil {
+				stderr("error resolving tags to labels: %s", err)
+				return 1
+			}
+		} else {
+			fmt.Printf("no discover tags found")
+		}
+
 		eps, attempts, err := discovery.DiscoverACIEndpoints(*app, nil, insecure)
 		if err != nil {
 			stderr("error fetching endpoints for %s: %s", name, err)
 			return 1
 		}
 		for _, a := range attempts {
+
 			fmt.Printf("discover endpoints walk: prefix: %s error: %v\n", a.Prefix, a.Error)
 		}
 		publicKeys, attempts, err := discovery.DiscoverPublicKeys(*app, nil, insecure)
@@ -104,3 +135,61 @@ func runDiscover(args []string) (exit int) {
 
 	return
 }
+
+func fetchImageTags(urlStr string, insecure discovery.InsecureOption) (*schema.ImageTags, error) {
+	t := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		Dial: func(n, a string) (net.Conn, error) {
+			return net.DialTimeout(n, a, 5*time.Second)
+		},
+	}
+	if insecure&discovery.InsecureTLS != 0 {
+		t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+	client := &http.Client{
+		Transport: t,
+	}
+
+	fetch := func(scheme string) (res *http.Response, err error) {
+		u, err := url.Parse(urlStr)
+		if err != nil {
+			return nil, err
+		}
+		u.Scheme = scheme
+		urlStr := u.String()
+		req, err := http.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			return nil, err
+		}
+		res, err = client.Do(req)
+		return
+	}
+	closeBody := func(res *http.Response) {
+		if res != nil {
+			res.Body.Close()
+		}
+	}
+	res, err := fetch("https")
+	if err != nil || res.StatusCode != http.StatusOK {
+		if insecure&discovery.InsecureHTTP != 0 {
+			closeBody(res)
+			res, err = fetch("http")
+		}
+	}
+
+	if res != nil && res.StatusCode != http.StatusOK {
+		err = fmt.Errorf("expected a 200 OK got %d", res.StatusCode)
+	}
+
+	if err != nil {
+		closeBody(res)
+		return nil, err
+	}
+
+	var tags *schema.ImageTags
+	jd := json.NewDecoder(res.Body)
+	jd.Decode(&tags)
+	closeBody(res)
+
+	return tags, nil
+}

discovery/discovery.go

@@ -37,21 +37,25 @@ type ACIEndpoint struct {
 	ASC string
 }
 
+type ImageTagsEndpoint struct {
+	ImageTags string
+	ASC       string
+}
+
 // A struct containing both discovered endpoints and keys. Used to avoid
 // function duplication (one for endpoints and one for keys, so to avoid two
 // doDiscover, two DiscoverWalkFunc)
 type discoveryData struct {
-	ACIEndpoints []ACIEndpoint
-	PublicKeys   []string
+	ACIEndpoints       []ACIEndpoint
+	PublicKeys         []string
+	ImageTagsEndpoints []ImageTagsEndpoint
 }
 
 type ACIEndpoints []ACIEndpoint
 
 type PublicKeys []string
 
-const (
-	defaultVersion = "latest"
-)
+type ImageTagsEndpoints []ImageTagsEndpoint
 
 var (
 	templateExpression = regexp.MustCompile(`{.*?}`)
@@ -128,9 +132,6 @@ func createTemplateVars(app App) []string {
 
 func doDiscover(pre string, hostHeaders map[string]http.Header, app App, insecure InsecureOption) (*discoveryData, error) {
 	app = *app.Copy()
-	if app.Labels["version"] == "" {
-		app.Labels["version"] = defaultVersion
-	}
 
 	_, body, err := httpsOrHTTP(pre, hostHeaders, insecure)
 	if err != nil {
@@ -165,6 +166,20 @@ func doDiscover(pre string, hostHeaders map[string]http.Header, app App, insecur
 
 		case "ac-discovery-pubkeys":
 			dd.PublicKeys = append(dd.PublicKeys, m.uri)
+		case "ac-discovery-tags":
+			// Only name is used for tags discovery
+			tplVars := []string{"{name}", app.Name.String()}
+			// Ignore not handled variables as {ext} isn't already rendered.
+			uri, _ := renderTemplate(m.uri, tplVars...)
+			asc, ok := renderTemplate(uri, "{ext}", "aci.asc")
+			if !ok {
+				continue
+			}
+			tags, ok := renderTemplate(uri, "{ext}", "aci")
+			if !ok {
+				continue
+			}
+			dd.ImageTagsEndpoints = append(dd.ImageTagsEndpoints, ImageTagsEndpoint{ImageTags: tags, ASC: asc})
 		}
 	}
 
@@ -175,6 +190,7 @@ func doDiscover(pre string, hostHeaders map[string]http.Header, app App, insecur
 // optionally will use HTTP if insecure is set. hostHeaders specifies the
 // header to apply depending on the host (e.g. authentication). Based on the
 // response of the discoverFn it will continue to recurse up the tree.
+// If no discovery data can be found an empty discoveryData will be returned.
 func DiscoverWalk(app App, hostHeaders map[string]http.Header, insecure InsecureOption, discoverFn DiscoverWalkFunc) (dd *discoveryData, err error) {
 	parts := strings.Split(string(app.Name), "/")
 	for i := range parts {
@@ -187,7 +203,7 @@ func DiscoverWalk(app App, hostHeaders map[string]http.Header, insecure Insecure
 		}
 	}
 
-	return nil, fmt.Errorf("discovery failed")
+	return &discoveryData{}, nil
 }
 
 // DiscoverWalkFunc can stop a DiscoverWalk by returning non-nil error.
@@ -232,10 +248,13 @@ func DiscoverACIEndpoints(app App, hostHeaders map[string]http.Header, insecure
 		return nil, attempts, err
 	}
 
+	if len(dd.ACIEndpoints) == 0 {
+		return nil, attempts, fmt.Errorf("No ACI endpoints discovered")
+	}
 	return dd.ACIEndpoints, attempts, nil
 }
 
-// DiscoverPublicKey will make HTTPS requests to find the ac-public-keys meta
+// DiscoverPublicKeys will make HTTPS requests to find the ac-discovery-pubkeys meta
 // tags and optionally will use HTTP if insecure is set. hostHeaders
 // specifies the header to apply depending on the host (e.g. authentication).
 // It will not give up until it has exhausted the path or found an public key.
@@ -253,5 +272,29 @@ func DiscoverPublicKeys(app App, hostHeaders map[string]http.Header, insecure In
 		return nil, attempts, err
 	}
 
+	if len(dd.PublicKeys) == 0 {
+		return nil, attempts, fmt.Errorf("No public keys discovered")
+	}
 	return dd.PublicKeys, attempts, nil
 }
+
+// DiscoverImageTags will make HTTPS requests to find the ac-discovery-imagetags meta
+// tags and optionally will use HTTP if insecure is set. hostHeaders
+// specifies the header to apply depending on the host (e.g. authentication).
+// It will not give up until it has exhausted the path or found an imagetag.
+func DiscoverImageTags(app App, hostHeaders map[string]http.Header, insecure InsecureOption) (ImageTagsEndpoints, []FailedAttempt, error) {
+	testFn := func(pre string, dd *discoveryData, err error) error {
+		if len(dd.ImageTagsEndpoints) != 0 {
+			return errEnough
+		}
+		return nil
+	}
+
+	attempts := []FailedAttempt{}
+	dd, err := DiscoverWalk(app, hostHeaders, insecure, walker(&attempts, testFn))
+	if err != nil && err != errEnough {
+		return nil, attempts, err
+	}
+
+	return dd.ImageTagsEndpoints, attempts, nil
+}