Fix e2e tests to run on cloudtop

Author: mattcary

test/e2e/tests/setup_e2e_test.go

@@ -43,6 +43,7 @@ var (
 	imageURL        = flag.String("image-url", "projects/debian-cloud/global/images/family/debian-11", "OS image url to get image from")
 	runInProw       = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
 	deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run")
+	cloudtopHost    = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP")
 
 	testContexts        = []*remote.TestContext{}
 	computeService      *compute.Service
@@ -116,11 +117,21 @@ var _ = AfterSuite(func() {
 	}
 })
 
+func getRemoteInstanceConfig() *remote.InstanceConfig {
+	return &remote.InstanceConfig{
+		Project:        *project,
+		Architecture:   *architecture,
+		MachineType:    *machineType,
+		ServiceAccount: *serviceAccount,
+		ImageURL:       *imageURL,
+		CloudtopHost:   *cloudtopHost}
+}
+
 func NewTestContext(zone string) *remote.TestContext {
 	nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)
 	klog.Infof("Setting up node %s", nodeID)
 
-	i, err := remote.SetupInstance(*project, *architecture, zone, nodeID, *machineType, *serviceAccount, *imageURL, computeService)
+	i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService)
 	if err != nil {
 		klog.Fatalf("Failed to setup instance %v: %v", nodeID, err)
 	}

test/e2e/tests/single_zone_e2e_test.go

@@ -1289,7 +1289,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
 
 		zone := "us-central1-c"
 		nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)
-		i, err := remote.SetupInstance(*project, *architecture, zone, nodeID, *machineType, *serviceAccount, *imageURL, computeService)
+		i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService)
 
 		if err != nil {
 			klog.Fatalf("Failed to setup instance %v: %v", nodeID, err)

test/remote/instance.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"os/exec"
 	"strings"
 	"time"
 
@@ -44,12 +45,25 @@ const (
 	timestampFormat = "20060102T150405"
 )
 
+// InstanceConfig is the common bundle of options used for instance creation.
+type InstanceConfig struct {
+	Project        string
+	Architecture   string
+	MachineType    string
+	ServiceAccount string
+	ImageURL       string
+	CloudtopHost   bool
+}
+
 type InstanceInfo struct {
-	project      string
-	architecture string
-	zone         string
-	name         string
-	machineType  string
+	project        string
+	architecture   string
+	zone           string
+	name           string
+	machineType    string
+	serviceAccount string
+	imageURL       string
+	cloudtopHost   bool
 
 	// External IP is filled in after instance creation
 	externalIP string
@@ -69,19 +83,22 @@ func (i *InstanceInfo) GetNodeID() string {
 	return common.CreateNodeID(i.project, i.zone, i.name)
 }
 
-func CreateInstanceInfo(project, instanceArchitecture, instanceZone, name, machineType string, cs *compute.Service) (*InstanceInfo, error) {
+func CreateInstanceInfo(config *InstanceConfig, zone, name string, cs *compute.Service) (*InstanceInfo, error) {
 	return &InstanceInfo{
-		project:        project,
-		architecture:   instanceArchitecture,
-		zone:           instanceZone,
+		project:        config.Project,
+		architecture:   config.Architecture,
+		zone:           zone,
 		name:           name,
-		machineType:    machineType,
+		machineType:    config.MachineType,
+		cloudtopHost:   config.CloudtopHost,
+		serviceAccount: config.ServiceAccount,
+		imageURL:       config.ImageURL,
 		computeService: cs,
 	}, nil
 }
 
 // Provision a gce instance using image
-func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) error {
+func (i *InstanceInfo) CreateOrGetInstance() error {
 	var err error
 	var instance *compute.Instance
 	klog.V(4).Infof("Creating instance: %v", i.name)
@@ -112,14 +129,14 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro
 				Type:       "PERSISTENT",
 				InitializeParams: &compute.AttachedDiskInitializeParams{
 					DiskName:    "my-root-pd-" + myuuid,
-					SourceImage: imageURL,
+					SourceImage: i.imageURL,
 				},
 			},
 		},
 	}
 
 	saObj := &compute.ServiceAccount{
-		Email:  serviceAccount,
+		Email:  i.serviceAccount,
 		Scopes: []string{"https://www.googleapis.com/auth/cloud-platform"},
 	}
 	newInst.ServiceAccounts = []*compute.ServiceAccount{saObj}
@@ -187,6 +204,15 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro
 			return false, nil
 		}
 
+		if i.cloudtopHost {
+			output, err := exec.Command("gcloud", "compute", "ssh", i.name, "--zone", i.zone, "--project", i.project, "--", "-o", "ProxyCommand=corp-ssh-helper %h %p", "--", "echo").CombinedOutput()
+			if err != nil {
+				klog.Errorf("Failed to bootstrap ssh (%v): %s", err, string(output))
+				return false, nil
+			}
+			klog.V(4).Infof("Bootstrapped cloudtop ssh for instance %v", i.name)
+		}
+
 		externalIP := getexternalIP(instance)
 		if len(externalIP) > 0 {
 			i.externalIP = externalIP

test/remote/setup-teardown.go

@@ -54,14 +54,14 @@ type processes struct {
 }
 
 // SetupInstance sets up the specified GCE Instance for E2E testing and returns a handle to the instance object for future use.
-func SetupInstance(instanceProject, instanceArchitecture, instanceZone, instanceName, instanceMachineType, instanceServiceAccount, instanceImageURL string, cs *compute.Service) (*InstanceInfo, error) {
+func SetupInstance(config *InstanceConfig, instanceZone, instanceName string, cs *compute.Service) (*InstanceInfo, error) {
 	// Create the instance in the requisite zone
-	instance, err := CreateInstanceInfo(instanceProject, instanceArchitecture, instanceZone, instanceName, instanceMachineType, cs)
+	instance, err := CreateInstanceInfo(config, instanceZone, instanceName, cs)
 	if err != nil {
 		return nil, err
 	}
 
-	err = instance.CreateOrGetInstance(instanceImageURL, instanceServiceAccount)
+	err = instance.CreateOrGetInstance()
 	if err != nil {
 		return nil, err
 	}

test/run-e2e-local.sh

@@ -11,4 +11,9 @@ readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 # This requires application default credentials to be set up, eg by
 # `gcloud auth application-default login`
 
-ginkgo --v "test/e2e/tests" -- --project "${PROJECT}" --service-account "${IAM_NAME}" --v=6 --logtostderr
+CLOUDTOP_HOST=
+if hostname | grep -q c.googlers.com ; then
+  CLOUDTOP_HOST=--cloudtop-host
+fi
+
+ginkgo --v "test/e2e/tests" -- --project "${PROJECT}" --service-account "${IAM_NAME}" "${CLOUDTOP_HOST}" --v=6 --logtostderr