Upgrade test(LTS): use network area to federate cluster (#19934)

- Join areas
- wait for members alive and validate cross area service discovery

testing/deployer/sprawl/acl.go

@@ -171,7 +171,7 @@ node_prefix "" {
 	policy = "write"
 }
 
-operator = "read"
+operator = "write"
 `
 	policy, _, err := acl.PolicyCreate(
 		&api.ACLPolicy{

testing/deployer/sprawl/boot.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	retry "github.com/avast/retry-go"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/go-multierror"
 
@@ -146,6 +147,10 @@ func (s *Sprawl) launchType(firstTime bool, launchPhase LaunchPhase) (launchErr
 		return fmt.Errorf("waitForPeeringEstablishment: %w", err)
 	}
 
+	if err := s.waitForNetworkAreaEstablishment(); err != nil {
+		return fmt.Errorf("waitForNetworkAreaEstablishment: %w", err)
+	}
+
 	cleanupFuncs = nil // reset
 
 	return nil
@@ -198,7 +203,7 @@ func (s *Sprawl) assignIPAddresses() error {
 					return fmt.Errorf("unknown network %q", addr.Network)
 				}
 				addr.IPAddress = net.IPByIndex(node.Index)
-				s.logger.Info("assign addr", "node", node.Name, "addr", addr.IPAddress, "enabled", !node.Disabled)
+				s.logger.Info("assign addr", "node", node.Name, "addr", addr.IPAddress, "type", addr.Type, "enabled", !node.Disabled)
 			}
 		}
 	}
@@ -315,7 +320,17 @@ func (s *Sprawl) createFirstTime() error {
 		return fmt.Errorf("generator[agents]: %w", err)
 	}
 	for _, cluster := range s.topology.Clusters {
-		if err := s.waitForClientAntiEntropyOnce(cluster); err != nil {
+		err := retry.Do(
+			func() error {
+				if err := s.waitForClientAntiEntropyOnce(cluster); err != nil {
+					return fmt.Errorf("create first time - waitForClientAntiEntropyOnce[%s]: %w", cluster.Name, err)
+				}
+				return nil
+			},
+			retry.MaxDelay(5*time.Second),
+			retry.Attempts(15),
+		)
+		if err != nil {
 			return fmt.Errorf("create first time - waitForClientAntiEntropyOnce[%s]: %w", cluster.Name, err)
 		}
 	}
@@ -344,6 +359,10 @@ func (s *Sprawl) createFirstTime() error {
 	if err := s.initPeerings(); err != nil {
 		return fmt.Errorf("initPeerings: %w", err)
 	}
+
+	if err := s.initNetworkAreas(); err != nil {
+		return fmt.Errorf("initNetworkAreas: %w", err)
+	}
 	return nil
 }
 

testing/deployer/sprawl/internal/tfgen/agent.go

@@ -84,7 +84,9 @@ func (g *Generator) generateAgentHCL(node *topology.Node, enableV2, enableV2Tena
 		b.add("prometheus_retention_time", "168h")
 	})
 
-	b.add("encrypt", g.sec.ReadGeneric(node.Cluster, secrets.GossipKey))
+	if !cluster.DisableGossipEncryption {
+		b.add("encrypt", g.sec.ReadGeneric(node.Cluster, secrets.GossipKey))
+	}
 
 	{
 		var (

testing/deployer/sprawl/network_area_ce.go

@@ -0,0 +1,14 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !consulent
+
+package sprawl
+
+func (s *Sprawl) initNetworkAreas() error {
+	return nil
+}
+
+func (s *Sprawl) waitForNetworkAreaEstablishment() error {
+	return nil
+}

testing/deployer/topology/compile.go

@@ -694,11 +694,12 @@ func compile(logger hclog.Logger, raw *Config, prev *Topology) (*Topology, error
 	}
 
 	t := &Topology{
-		ID:       id,
-		Networks: networks,
-		Clusters: clusters,
-		Images:   images,
-		Peerings: raw.Peerings,
+		ID:           id,
+		Networks:     networks,
+		Clusters:     clusters,
+		Images:       images,
+		Peerings:     raw.Peerings,
+		NetworkAreas: raw.NetworkAreas,
 	}
 
 	if prev != nil {

testing/deployer/topology/topology.go

@@ -35,6 +35,10 @@ type Topology struct {
 	// Peerings defines the list of pairwise peerings that should be established
 	// between clusters.
 	Peerings []*Peering `json:",omitempty"`
+
+	// NetworkAreas defines the list of pairwise network area that should be established
+	// between clusters.
+	NetworkAreas []*NetworkArea `json:",omitempty"`
 }
 
 func (t *Topology) DigestExposedProxyPort(netName string, proxyPort int) (bool, error) {
@@ -100,6 +104,10 @@ type Config struct {
 	// Peerings defines the list of pairwise peerings that should be established
 	// between clusters.
 	Peerings []*Peering
+
+	// NetworkAreas defines the list of pairwise NetworkArea that should be established
+	// between clusters.
+	NetworkAreas []*NetworkArea
 }
 
 func (c *Config) Cluster(name string) *Cluster {
@@ -293,6 +301,10 @@ type Cluster struct {
 
 	// Segments is a map of network segment name and the ports
 	Segments map[string]int
+
+	// DisableGossipEncryption disables gossip encryption on the cluster
+	// Default is false to enable gossip encryption
+	DisableGossipEncryption bool `json:",omitempty"`
 }
 
 func (c *Cluster) inheritFromExisting(existing *Cluster) {
@@ -1055,6 +1067,13 @@ type Peering struct {
 	Accepting PeerCluster
 }
 
+// NetworkArea - a pair of clusters that are peered together
+// through network area. PeerCluster type is reused here.
+type NetworkArea struct {
+	Primary   PeerCluster
+	Secondary PeerCluster
+}
+
 type PeerCluster struct {
 	Name      string
 	Partition string