use urls from zos

bootstrap/bootstrap/README.md

@@ -19,8 +19,8 @@ will do a multiple stage bootstrap. Currently this is only two stages:
 - Bootstrap is used by [0-initramfs](https://github.com/threefoldtech/0-initramfs/blob/development-zos-v3/packages/modules.sh) to basically add `internet` and `bootstrap` services to the base image
 - After internet service is fully started, bootstrap will start to download flists needed for zos node to work properly
 - As described above bootstrap run in two stages:
-  - The first stage is used to update bootstrap itself, and it is done like that to avoid re-building the image if we only changed the bootstrap code. this update is basically done from `tf-autobuilder` repo in the [hub/tf-autobuilder](https://hub.grid.tf/tf-autobuilder) and download the latest bootstrap flist
-  - For the second stage bootstrap will download the flists for that env. bootstrap cares about `runmode` argument that we pass during the start of the node. for example if we passed `runmode=dev` it will get the the tag `development` under [hub/tf-zos](https://hub.grid.tf/tf-zos) each tag is linked to a sub-directory where all flists for this env exists to be downloaded and installed on the node
+  - The first stage is used to update bootstrap itself, and it is done like that to avoid re-building the image if we only changed the bootstrap code. this update is basically done from `tf-autobuilder` repo in the [hub/tf-autobuilder](https://hub.threefold.me/tf-autobuilder) and download the latest bootstrap flist
+  - For the second stage bootstrap will download the flists for that env. bootstrap cares about `runmode` argument that we pass during the start of the node. for example if we passed `runmode=dev` it will get the the tag `development` under [hub/tf-zos](https://hub.threefold.me/tf-zos) each tag is linked to a sub-directory where all flists for this env exists to be downloaded and installed on the node
 
 ## Testing in Developer setup
 
@@ -40,6 +40,6 @@ after:
   - internet
 ```
 
-- remove overlay link under `qemu/overlay `
+- remove overlay link under `qemu/overlay`
 - create a new link pointing to overlay.custom under zos/qemu `ln -s overlay.custom overlay`
 - boot your vm as normal

bootstrap/bootstrap/src/hub.rs

@@ -6,7 +6,7 @@ use std::fs::{write, OpenOptions};
 use std::io::copy;
 use std::path::Path;
 
-const HUB: &str = "https://hub.grid.tf";
+const HUB: &str = "https://hub.threefold.me";
 
 pub struct Repo {
     name: String,
@@ -172,7 +172,7 @@ mod tests {
         let flist = repo.get("test.flist")?;
         assert_eq!(flist.name, "test.flist");
         assert_eq!(flist.kind, Kind::Regular);
-        assert_eq!(flist.url, "https://hub.grid.tf/azmy/test.flist");
+        assert_eq!(flist.url, "https://hub.threefold.me/azmy/test.flist");
 
         Ok(())
     }
@@ -227,7 +227,7 @@ mod tests {
 
         let found = found.unwrap();
         assert_eq!(found.name, "test.flist");
-        assert_eq!(found.url, "https://hub.grid.tf/azmy/test.flist");
+        assert_eq!(found.url, "https://hub.threefold.me/azmy/test.flist");
 
         Ok(())
     }

pkg/container/README.md

@@ -14,43 +14,43 @@ the container rootfs
 package main
 
 import (
-	"github.com/threefoldtech/zbus"
-	"github.com/threefoldtech/zosbase/pkg"
-	"github.com/threefoldtech/zosbase/pkg/stubs"
+ "github.com/threefoldtech/zbus"
+ "github.com/threefoldtech/zosbase/pkg"
+ "github.com/threefoldtech/zosbase/pkg/stubs"
 )
 
 func main() {
-	client, err := zbus.NewRedisClient("tcp://localhost:6379")
-	if err != nil {
-		panic(client)
-	}
+ client, err := zbus.NewRedisClient("tcp://localhost:6379")
+ if err != nil {
+  panic(client)
+ }
 
-	containerd := stubs.NewContainerModuleStub(client)
-	namespace := "example"
+ containerd := stubs.NewContainerModuleStub(client)
+ namespace := "example"
 
-	// make sure u have a network namespace ready using ip
-	// sudo ip netns add mynetns
+ // make sure u have a network namespace ready using ip
+ // sudo ip netns add mynetns
 
-	info := pkg.Container{
-		Name: "test",
-		FList: "https://hub.grid.tf/thabet/redis.flist",
-		Env: []string{},
-		Network: pkg.NetworkInfo{Namespace: "mynetns"},
-		Mounts: nil,
-		Entrypoint: "redis-server",
-	}
+ info := pkg.Container{
+  Name: "test",
+  FList: "https://hub.threefold.me/thabet/redis.flist",
+  Env: []string{},
+  Network: pkg.NetworkInfo{Namespace: "mynetns"},
+  Mounts: nil,
+  Entrypoint: "redis-server",
+ }
 
-	id, err := containerd.Run(namespace, info)
+ id, err := containerd.Run(namespace, info)
 
-	if err != nil {
-		panic(err)
-	}
+ if err != nil {
+  panic(err)
+ }
 
-	// DO WORK WITH CONTAINER ...
+ // DO WORK WITH CONTAINER ...
 
-	if err = containerd.Delete(namespace, id); err != nil {
-		panic(err)
-	}
+ if err = containerd.Delete(namespace, id); err != nil {
+  panic(err)
+ }
 
 }
 ```

pkg/environment/config.go

@@ -30,7 +30,15 @@ type Config struct {
 	RolloutUpgrade struct {
 		TestFarms []uint32 `json:"test_farms"`
 	} `json:"rollout_upgrade"`
-	RelaysURLs []string `json:"relays_urls"`
+	RelaysURLs    []string `json:"relays_urls"`
+	SubstrateURL  []string `json:"substrate_urls"`
+	ActivationURL []string `json:"activation_urls"`
+	GraphQL       []string `json:"graphql_urls"`
+	KycURL        string   `json:"kyc_url"`
+	RegistrarURL  string   `json:"registrar_url"`
+	FlistURL      string   `json:"flist_url"`
+	HubURL        string   `json:"hub_url"`
+	BinRepo       string   `json:"bin_repo"`
 }
 
 // Merge, updates current config with cfg merging and override config

pkg/environment/environment.go

@@ -1,6 +1,7 @@
 package environment
 
 import (
+	"net/http"
 	"os"
 	"slices"
 	"strconv"
@@ -16,6 +17,10 @@ import (
 
 const (
 	baseExtendedURL = "https://raw.githubusercontent.com/threefoldtech/zos-config/main/"
+	hubURL          = "https://hub.threefold.me"
+	v4HubURL        = "https://v4.hub.threefold.me"
+	flistURL        = "redis://hub.threefold.me:9900"
+	v4FlistURL      = "redis://v4.hub.threefold.me:9940"
 )
 
 // PubMac specify how the mac address of the public nic
@@ -53,6 +58,7 @@ type Environment struct {
 	GraphQL       []string
 	KycURL        string
 	RegistrarURL  string
+	HubURL        string
 
 	// private vlan to join
 	// if set, zos will use this as its priv vlan
@@ -121,7 +127,8 @@ var (
 			"https://activation.dev.grid.tf/activation/activate",
 			"https://activation.02.dev.grid.tf/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
+		FlistURL: flistURL,
+		HubURL:   hubURL,
 		BinRepo:  "tf-zos-v3-bins.dev",
 		GraphQL: []string{
 			"https://graphql.dev.grid.tf/graphql",
@@ -145,7 +152,8 @@ var (
 			"https://activation.test.grid.tf/activation/activate",
 			"https://activation.02.test.grid.tf/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
+		FlistURL: flistURL,
+		HubURL:   hubURL,
 		BinRepo:  "tf-zos-v3-bins.test",
 		GraphQL: []string{
 			"https://graphql.test.grid.tf/graphql",
@@ -169,7 +177,8 @@ var (
 			"https://activation.qa.grid.tf/activation/activate",
 			"https://activation.02.qa.grid.tf/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
+		FlistURL: flistURL,
+		HubURL:   hubURL,
 		BinRepo:  "tf-zos-v3-bins.qanet",
 		GraphQL: []string{
 			"https://graphql.qa.grid.tf/graphql",
@@ -193,17 +202,20 @@ var (
 			// "wss://relay.02.grid.tf",
 		},
 		ActivationURL: []string{
+			"https://activation.grid.threefold.me/activation/activate",
 			"https://activation.grid.tf/activation/activate",
 			"https://activation.02.grid.tf/activation/activate",
 		},
-		FlistURL: "redis://hub.grid.tf:9900",
+		HubURL:   hubURL,
+		FlistURL: flistURL,
 		BinRepo:  "tf-zos-v3-bins",
 		GraphQL: []string{
+			"https://graphql.grid.threefold.me/graphql",
 			"https://graphql.grid.tf/graphql",
 			"https://graphql.02.grid.tf/graphql",
 		},
-		KycURL:       "https://kyc.grid.tf",
-		RegistrarURL: "https://registrar.prod4.grid.tf",
+		KycURL:       "https://kyc.threefold.me",
+		RegistrarURL: "https://registrar.prod4.threefold.me",
 	}
 )
 
@@ -283,22 +295,59 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 		env = envProd
 	}
 
+	config, err := getConfig(env.RunningMode, baseExtendedURL, http.DefaultClient)
+	if err != nil {
+		// maybe the node can't reach the internet right now
+		// this will enforce node to skip config
+		config = Config{}
+	}
+
 	if substrate, ok := params.Get("substrate"); ok {
 		if len(substrate) > 0 {
 			env.SubstrateURL = substrate
 		}
+	} else if substrate := config.SubstrateURL; len(substrate) > 0 {
+		env.SubstrateURL = substrate
 	}
 
 	if relay, ok := params.Get("relay"); ok {
 		if len(relay) > 0 {
 			env.relaysURLs = relay
 		}
+	} else if relay := config.RelaysURLs; len(relay) > 0 {
+		env.relaysURLs = relay
 	}
 
 	if activation, ok := params.Get("activation"); ok {
 		if len(activation) > 0 {
 			env.ActivationURL = activation
 		}
+	} else if activation := config.ActivationURL; len(activation) > 0 {
+		env.ActivationURL = activation
+	}
+
+	if graphql := config.GraphQL; len(graphql) > 0 {
+		env.GraphQL = graphql
+	}
+
+	if flist := config.FlistURL; len(flist) > 0 {
+		env.FlistURL = flist
+	}
+
+	if hub := config.HubURL; len(hub) > 0 {
+		env.HubURL = hub
+	}
+
+	if bin := config.BinRepo; len(bin) > 0 {
+		env.BinRepo = bin
+	}
+
+	if kyc := config.KycURL; len(kyc) > 0 {
+		env.KycURL = kyc
+	}
+
+	if registrar := config.RegistrarURL; len(registrar) > 0 {
+		env.RegistrarURL = registrar
 	}
 
 	if farmSecret, ok := params.Get("secret"); ok {
@@ -378,9 +427,10 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 		env.BinRepo = e
 	}
 
-	// if the node running v4 chage flisturl to use v4.hub.grid.tf
+	// if the node running v4 chage flisturl to use v4.hub.threefold.me
 	if params.IsV4() {
-		env.FlistURL = "redis://v4.hub.grid.tf:9940"
+		env.FlistURL = v4FlistURL
+		env.HubURL = v4HubURL
 	}
 
 	return env, nil

pkg/flist/flist_test.go

@@ -3,6 +3,7 @@ package flist
 import (
 	"bytes"
 	"context"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -15,6 +16,7 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	"github.com/threefoldtech/zosbase/pkg"
+	"github.com/threefoldtech/zosbase/pkg/environment"
 	"github.com/threefoldtech/zosbase/pkg/gridtypes"
 )
 
@@ -159,7 +161,10 @@ func TestMountUnmount(t *testing.T) {
 
 	sys.On("Mount", "overlay", filepath.Join(root, "mountpoint", name), "overlay", uintptr(syscall.MS_NOATIME), mock.Anything).Return(nil)
 
-	mnt, err := flister.mountInNamespace(name, "https://hub.grid.tf/thabet/redis.flist", pkg.DefaultMountOptions, "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(t, err)
+
+	mnt, err := flister.mountInNamespace(name, redisFlist, pkg.DefaultMountOptions, "")
 	require.NoError(t, err)
 
 	// Trick flister into thinking that 0-fs has exited
@@ -186,7 +191,10 @@ func TestMountUnmountRO(t *testing.T) {
 	flist := mock.Anything
 	sys.On("Mount", flist, filepath.Join(root, "mountpoint", name), "bind", uintptr(syscall.MS_BIND), "").Return(nil)
 
-	mnt, err := flister.mountInNamespace(name, "https://hub.grid.tf/thabet/redis.flist", pkg.ReadOnlyMountOptions, "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(t, err)
+
+	mnt, err := flister.mountInNamespace(name, redisFlist, pkg.ReadOnlyMountOptions, "")
 	require.NoError(t, err)
 
 	// Trick flister into thinking that 0-fs has exited
@@ -223,11 +231,14 @@ func TestIsolation(t *testing.T) {
 	name2 := "test2"
 	sys.On("Mount", "overlay", filepath.Join(root, "mountpoint", name2), "overlay", uintptr(syscall.MS_NOATIME), mock.Anything).Return(nil)
 
-	path1, err := flister.mountInNamespace(name1, "https://hub.grid.tf/thabet/redis.flist", pkg.DefaultMountOptions, "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(err)
+
+	path1, err := flister.mountInNamespace(name1, redisFlist, pkg.DefaultMountOptions, "")
 	require.NoError(err)
 	args1 := cmder.m
 
-	path2, err := flister.mountInNamespace(name2, "https://hub.grid.tf/thabet/redis.flist", pkg.DefaultMountOptions, "")
+	path2, err := flister.mountInNamespace(name2, redisFlist, pkg.DefaultMountOptions, "")
 	require.NoError(err)
 	args2 := cmder.m
 
@@ -246,14 +257,17 @@ func TestDownloadFlist(t *testing.T) {
 
 	f := newFlister(root, strg, cmder, sys)
 
-	hash1, path1, err := f.downloadFlist("https://hub.grid.tf/thabet/redis.flist", "")
+	redisFlist, err := url.JoinPath(environment.MustGet().HubURL, "thabet", "redis.flist")
+	require.NoError(err)
+
+	hash1, path1, err := f.downloadFlist(redisFlist, "")
 	require.NoError(err)
 
 	// now corrupt the flist
 	err = os.Truncate(string(path1), 512)
 	require.NoError(err)
 
-	hash2, path2, err := f.downloadFlist("https://hub.grid.tf/thabet/redis.flist", "")
+	hash2, path2, err := f.downloadFlist(redisFlist, "")
 	require.NoError(err)
 
 	require.EqualValues(path1, path2)

pkg/gateway/flist.go

@@ -3,24 +3,29 @@ package gateway
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zbus"
 	"github.com/threefoldtech/zosbase/pkg"
+	"github.com/threefoldtech/zosbase/pkg/environment"
 	"github.com/threefoldtech/zosbase/pkg/stubs"
 )
 
-const (
-	flist = "https://hub.grid.tf/tf-autobuilder/traefik-2.9.9.flist"
-)
-
 // ensureTraefikBin makes sure traefik flist is mounted.
 // TODO: we need to "update" traefik and restart the service
 // if new version is available!
 func ensureTraefikBin(ctx context.Context, cl zbus.Client) (string, error) {
 	const bin = "traefik"
 	flistd := stubs.NewFlisterStub(cl)
+
+	env := environment.MustGet()
+	flist, err := url.JoinPath(env.HubURL, "tf-autobuilder", "traefik-2.9.9.flist")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to constract traefik flist url")
+	}
+
 	hash, err := flistd.FlistHash(ctx, flist)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to get traefik flist hash")