Merge refs/heads/feature/project_deployment_versions

Documentation/SUMMARY.md

@@ -19,7 +19,6 @@
 
 * [About modules](stackhead-modules/stackhead-modules.md)
 * [List of modules](stackhead-modules/list-of-modules.md)
-* [Development](stackhead-modules/development/README.md)
 
 ## Technical Documentation
 

Documentation/development/basics.md

@@ -14,10 +14,10 @@ When developing for StackHead we encourage you to test with an actual remote ser
 
 We recommend a basic Ubuntu server on [Hetzner Cloud](https://hetzner.cloud/?ref=n7H3qhWcZ2QS).
 Right now the cheapest option comes in at 4,15€ per month (3,56€ Server + 0,60€ IPv4).
-However it is charged per-use. So you'll only paying the time the server is actually running.
+However, it is charged per-use. So you'll only paying the time the server is actually running.
 So you should be paying only a few cents (or even nothing) when running it for a few hours while testing.
 
-Make sure to setup the server with SSH key access, so you can connect to it from your local PC with root user.
+Make sure to set up the server with SSH key access, so you can connect to it from your local PC with root user.
 Verify you can connect to it via `ssh root@[IPv4 address]`.
 
 Then, set the A record of an actual domain or subdomain to the IP address.
@@ -27,3 +27,20 @@ Setup server:
 
 Deploy project:
 `./bin/stackhead-cli project deploy my_file.stackhead.yml [IPv4 address]`
+
+## Debugging
+
+### Debugging stackhead user
+
+If you want to connect to a server with the `stackhead` user, set its private key in your SSH config:
+
+```shell
+Host [IP_Address]
+    User stackhead
+    IdentityFile ~/.config/getstackhead/stackhead/ssh/remotes/[IP_Address]/private_key.pem
+```
+
+{% hint style="warning" %}
+Docker Registry credentials are only available during deployment. If you need to manually pull Docker images
+with the stackhead user, you'll need to authenticate against your Docker Registry again.
+{% endhint %}

Documentation/introduction/getting-started.md

@@ -16,7 +16,7 @@ If you wish to change the software used for proxy or containers, please [create
 
 ## Creating a project definition
 
-Create a new project definitions file at `./stackhead/example_app.yml` and the following content:
+Create a new project definitions file at `./stackhead/example_app.stackhead.yml` and the following content:
 
 ```yaml
 ---

Documentation/introduction/project-definition.md

@@ -72,7 +72,7 @@ container:
 
 ## Settings
 
-### `domains.\*.expose`
+### `domains.[*].expose`
 
 The web server will proxy all web traffic to the service and port specified in `expose` setting.
 
@@ -82,20 +82,20 @@ In the example above, the web server will proxy web requests to the "app" contai
 
 Name of the Container service to receive the web request.
 
-#### `internal\_port`
+#### `internal_port`
 
 Port of the given container service to receive the web request.
 
-#### `external\_port`
+#### `external_port`
 
 Port that Nginx listens to.
 
 {% hint style="danger" %}
-Setting _external\_port_ to 443 is not allowed, as HTTPS forwarding is automatically enabled for exposes with `external_port=80`.
+Setting _external_port_ to 443 is not allowed, as HTTPS forwarding is automatically enabled for exposes with `external_port=80`.
 {% endhint %}
 
 {% hint style="warning" %}
-Make sure to define the different _external\_port_ within one project definition, so that each port is only used once!
+Make sure to define the different _external_port_ within one project definition, so that each port is only used once!
 {% endhint %}
 
 #### `proxy_websocket_locations`
@@ -107,7 +107,7 @@ Please do not set `/` or `/.well-known/acme-challenge` as WebSocket locations.
 {% endhint %}
 
 
-### `domains.\*.dns`
+### `domains.[*].dns`
 
 Configure which DNS service to use for this domain.
 Make sure to install the required module and set the required configurations according to the DNS StackHead module's documentation.
@@ -116,7 +116,7 @@ Make sure to install the required module and set the required configurations acc
 
 Fully-qualified name of the provider to use (e.g. `getstackhead.stackhead_dns_cloudflare`).
 
-### `domains.\*.security`
+### `domains.[*].security`
 
 These options can be used to add further security to your projects.
 
@@ -205,7 +205,7 @@ Below you can see a comparison of the project definition \(left\) and the equiva
 
 {% tabs %}
 {% tab title="StackHead" %}
-{% code title="example\_project.yml" %}
+{% code title="example_project.stackhead.yml" %}
 ```yaml
 services:
   - name: nginx
@@ -240,9 +240,9 @@ services:
 {% endtab %}
 {% endtabs %}
 
-#### `volumes\_from`
+#### `volumes_from`
 
-See [docker-compose documentation on volumes\_from](https://docs.docker.com/compose/compose-file/compose-file-v2/#volumes_from).
+See [docker-compose documentation on volumes_from](https://docs.docker.com/compose/compose-file/compose-file-v2/#volumes_from).
 
 #### `environment`
 

commands/project/deploy.go

@@ -2,6 +2,7 @@ package project
 
 import (
 	"fmt"
+	xfs "github.com/saitho/golang-extended-fs/v2"
 
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/slices"
@@ -42,34 +43,52 @@ var DeployApplication = func() *cobra.Command {
 
 			err = taskRunner.RunTask(routines.PrepareProjectTask(projectDefinition))
 			if err != nil {
+				if system.Context.CurrentDeployment.Version > 0 {
+					_ = xfs.DeleteFolder("ssh://"+system.Context.CurrentDeployment.GetPath(), true)
+				}
 				return
 			}
 			err = taskRunner.RunTask(routines.CollectResourcesTask(projectDefinition))
 			if err != nil {
+				_ = xfs.DeleteFolder("ssh://"+system.Context.CurrentDeployment.GetPath(), true)
 				return
 			}
 
 			if autoConfirm {
-				_ = taskRunner.RunTask(routines.CreateResources)
+				err = taskRunner.RunTask(routines.CreateResources)
 			} else {
 				// Confirm resource creation
 				fmt.Println("\nStackHead will try to create or update the following resources:")
-				for _, resourceGroup := range system.Context.Resources {
+				for _, resourceGroup := range system.Context.CurrentDeployment.ResourceGroups {
 					for _, resource := range resourceGroup.Resources {
 						fmt.Println(fmt.Sprintf("- %s", resource.ToString(false)))
 					}
 				}
 				fmt.Println("")
 				fmt.Print("Please confirm with \"y\" or \"yes\": ")
 				if askForConfirmation() {
-					_ = taskRunner.RunTask(routines.CreateResources)
+					err = taskRunner.RunTask(routines.CreateResources)
+				} else {
+					// Abort deployment -> delete
+					if err := xfs.DeleteFolder("ssh://"+system.Context.CurrentDeployment.GetPath(), true); err != nil {
+						fmt.Print("Unable to remove deployment directory.")
+						return
+					}
+					fmt.Print("Deployment aborted.")
+					return
 				}
 			}
 
+			if err != nil {
+				// ensure rollback is performed due to errors caused by StackHead module Run scripts not resource creation
+				routines.RollbackResources.Disabled = false
+			}
+
 			if !noRollback {
 				// Rollback may be skipped if CreateResources does not trigger a rollback
 				_ = taskRunner.RunTask(routines.RollbackResources)
 			}
+			_ = taskRunner.RunTask(routines.FinalizeDeployment)
 		},
 	}
 	command.PersistentFlags().BoolVar(&autoConfirm, "autoconfirm", false, "Whether to auto-confirm resource changes")

commands/project/destroy.go

@@ -1,8 +1,6 @@
 package project
 
 import (
-	"fmt"
-
 	xfs "github.com/saitho/golang-extended-fs/v2"
 	"github.com/spf13/cobra"
 
@@ -12,6 +10,12 @@ import (
 	"github.com/getstackhead/stackhead/system"
 )
 
+func reverse[S ~[]E, E any](s S) {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+}
+
 // DestroyApplication is a command object for Cobra that provides the destroy command
 var DestroyApplication = &cobra.Command{
 	Use:     "destroy [path to project definition] [ipv4 address]",
@@ -26,22 +30,28 @@ var DestroyApplication = &cobra.Command{
 		}
 		commands.PrepareContext(args[1], system.ContextActionProjectDeploy, projectDefinition)
 
-		modules := system.Context.GetModulesInOrder()
-		for i, j := 0, len(modules)-1; i < j; i, j = i+1, j-1 { // reverse module list
-			modules[i], modules[j] = modules[j], modules[i]
+		latestDeployment, err := system.GetLatestDeployment(projectDefinition)
+		if err != nil {
+			panic("unable to load latest deployment" + err.Error())
 		}
+		system.Context.CurrentDeployment = *latestDeployment
 
-		// Init modules
+		modules := system.Context.GetModulesInOrder()
+		reverse(modules)
+
+		// Run modules destroy steps
 		for _, module := range modules {
 			moduleSettings := system.GetModuleSettings(module.GetConfig().Name)
 			module.Init(moduleSettings)
 		}
 		taskRunner := routines.TaskRunner{}
 
-		subTasks := []routines.Task{}
+		subTasks := []routines.Task{
+			// Remove resources from deployment
+			routines.RemoveResources(latestDeployment),
+		}
 
 		if hasProjectDir, _ := xfs.HasFolder("ssh://" + projectDefinition.GetDirectoryPath()); hasProjectDir {
-
 			// Run destroy scripts from plugins
 			for _, module := range modules {
 				moduleSettings := system.GetModuleSettings(module.GetConfig().Name)
@@ -65,13 +75,10 @@ var DestroyApplication = &cobra.Command{
 			})
 		}
 
-		_ = taskRunner.RunTask(routines.Task{
-			Name: fmt.Sprintf("Destroying project \"%s\" on server with IP \"%s\"", args[0], args[1]),
-			Run: func(r *routines.Task) error {
-				return nil
-			},
-			SubTasks: subTasks,
-			//RunAllSubTasksDespiteError: true,
-		})
+		for _, task := range subTasks {
+			if err = taskRunner.RunTask(task); err != nil {
+				panic(err)
+			}
+		}
 	},
 }

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/knadh/koanf v1.4.4
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/saitho/diff-docker-compose v1.1.3
-	github.com/saitho/golang-extended-fs/v2 v2.0.2
+	github.com/saitho/golang-extended-fs/v2 v2.1.0
 	github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0
 	github.com/sirupsen/logrus v1.9.2
 	github.com/spf13/cast v1.3.1

go.sum

@@ -862,8 +862,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
 github.com/saitho/diff-docker-compose v1.1.3 h1:ZKx+F7yMmoAKzI76BFpr7kY+Thd7quk7d/ETug0qNY8=
 github.com/saitho/diff-docker-compose v1.1.3/go.mod h1:O3V+mwGtlXQ7UERA4+yunV319kyNLwIKeNSw8PODyEU=
-github.com/saitho/golang-extended-fs/v2 v2.0.2 h1:QWGOIP4wFTaSODIU54qDCuH1IQXWcjL6irLezJiHqOA=
-github.com/saitho/golang-extended-fs/v2 v2.0.2/go.mod h1:aTmESz9Z7Rwbx80zxGWTeuBmW6MLdzYrBZMJOfXsJVw=
+github.com/saitho/golang-extended-fs/v2 v2.1.0 h1:j1X3hu5LQRUM0WQzis8Kyh6zZhqCjYQox7Gw1h/ruX0=
+github.com/saitho/golang-extended-fs/v2 v2.1.0/go.mod h1:aTmESz9Z7Rwbx80zxGWTeuBmW6MLdzYrBZMJOfXsJVw=
 github.com/saitho/jsonschema-validator v1.2.0 h1:bWSvTla54F5cziZsxLBR0mlN3gTw9hjrkFuZU55kO+E=
 github.com/saitho/jsonschema-validator v1.2.0/go.mod h1:W/1Q1xQ+vyYxANlTEpi6hQBEHJPEi4wJmCBkdIehFvQ=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=

modules/container/docker/README.md

@@ -20,6 +20,7 @@ Setting up the server with this plugin will install the following:
 * Docker Compose Plugin
 * containerd.io
 * Pass (for registry credential storage)
+  * **Note:** Right now defining multiple credentials for the same registry results in authentication issues. That is why the registry credentials are only available during deployment and not persisted on the target server.
 * golang-docker-credential-helpers
 
 ## Configuration

modules/container/docker/definitions/volume.go

@@ -3,24 +3,27 @@ package container_docker_definitions
 import (
 	"github.com/getstackhead/stackhead/project"
 	"github.com/getstackhead/stackhead/system"
+	"path"
 )
 
 type DockerPaths struct {
-	BaseDir string
+	DataDir       string
+	DeploymentDir string
 }
 
 func GetDockerPaths() DockerPaths {
 	return DockerPaths{
-		BaseDir: system.Context.Project.GetRuntimeDataDirectoryPath() + "/container",
+		DataDir:       path.Join(system.Context.Project.GetRuntimeDataDirectoryPath(), "container"),
+		DeploymentDir: path.Join(system.Context.CurrentDeployment.GetPath(), "container"),
 	}
 }
 
 func (p DockerPaths) GetHooksDir() string {
-	return p.BaseDir + "/hooks"
+	return path.Join(p.DeploymentDir, "hooks")
 }
 
 func (p DockerPaths) getDataDir() string {
-	return p.BaseDir + "/data"
+	return path.Join(p.DataDir, "data")
 }
 
 func (p DockerPaths) GetServiceDataDir(service project.ContainerService, volume project.ContainerServiceVolume) string {