feat: Add a locust-loandgen chaos fault

bin/experiment/experiment.go

@@ -61,6 +61,7 @@ import (
 	ec2TerminateByID "github.com/litmuschaos/litmus-go/experiments/kube-aws/ec2-terminate-by-id/experiment"
 	ec2TerminateByTag "github.com/litmuschaos/litmus-go/experiments/kube-aws/ec2-terminate-by-tag/experiment"
 	k6Loadgen "github.com/litmuschaos/litmus-go/experiments/load/k6-loadgen/experiment"
+	locustLoadgen "github.com/litmuschaos/litmus-go/experiments/load/locust-loadgen/experiment"
 	springBootFaults "github.com/litmuschaos/litmus-go/experiments/spring-boot/spring-boot-faults/experiment"
 	vmpoweroff "github.com/litmuschaos/litmus-go/experiments/vmware/vm-poweroff/experiment"
 	cli "github.com/litmuschaos/litmus-go/pkg/clients"
@@ -209,6 +210,8 @@ func main() {
 		springBootFaults.Experiment(ctx, clients, *experimentName)
 	case "k6-loadgen":
 		k6Loadgen.Experiment(ctx, clients)
+	case "locust-loadgen":
+		locustLoadgen.Experiment(ctx, clients)
 	default:
 		log.Errorf("Unsupported -name %v, please provide the correct value of -name args", *experimentName)
 		return

chaoslib/litmus/locust-loadgen/lib/locust-loadgen.go

@@ -0,0 +1,160 @@
+package lib
+
+import (
+	"context"
+	"fmt"
+	"github.com/litmuschaos/litmus-go/pkg/cerrors"
+	"github.com/litmuschaos/litmus-go/pkg/clients"
+	"github.com/litmuschaos/litmus-go/pkg/events"
+	experimentTypes "github.com/litmuschaos/litmus-go/pkg/load/locust-loadgen/types"
+	"github.com/litmuschaos/litmus-go/pkg/log"
+	"github.com/litmuschaos/litmus-go/pkg/probe"
+	"github.com/litmuschaos/litmus-go/pkg/status"
+	"github.com/litmuschaos/litmus-go/pkg/telemetry"
+	"github.com/litmuschaos/litmus-go/pkg/types"
+	"github.com/litmuschaos/litmus-go/pkg/utils/common"
+	"github.com/litmuschaos/litmus-go/pkg/utils/stringutils"
+	"github.com/palantir/stacktrace"
+	"go.opentelemetry.io/otel"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"strconv"
+)
+
+func experimentExecution(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
+	ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectLocustLoadGenFault")
+	defer span.End()
+
+	if experimentsDetails.EngineName != "" {
+		msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on target pod"
+		types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails)
+		events.GenerateEvents(eventsDetails, clients, chaosDetails, "ChaosEngine")
+	}
+
+	// run the probes during chaos
+	if len(resultDetails.ProbeDetails) != 0 {
+		if err := probe.RunProbes(ctx, chaosDetails, clients, resultDetails, "DuringChaos", eventsDetails); err != nil {
+			return err
+		}
+	}
+
+	runID := stringutils.GetRunID()
+
+	// creating the helper pod to perform locust-loadgen chaos
+	if err := createHelperPod(ctx, experimentsDetails, clients, chaosDetails, runID); err != nil {
+		return stacktrace.Propagate(err, "could not create helper pod")
+	}
+
+	appLabel := fmt.Sprintf("app=%s-helper-%s", experimentsDetails.ExperimentName, runID)
+
+	//checking the status of the helper pod, wait till the pod comes to running state else fail the experiment
+	log.Info("[Status]: Checking the status of the helper pod")
+	if err := status.CheckHelperStatus(experimentsDetails.ChaosNamespace, appLabel, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
+		common.DeleteAllHelperPodBasedOnJobCleanupPolicy(appLabel, chaosDetails, clients)
+		return stacktrace.Propagate(err, "could not check helper status")
+	}
+
+	// Wait till the completion of the helper pod
+	// set an upper limit for the waiting time
+	log.Info("[Wait]: waiting till the completion of the helper pod")
+	podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, appLabel, clients, experimentsDetails.ChaosDuration+experimentsDetails.Timeout, common.GetContainerNames(chaosDetails)...)
+	if err != nil || podStatus == "Failed" {
+		common.DeleteAllHelperPodBasedOnJobCleanupPolicy(appLabel, chaosDetails, clients)
+		return common.HelperFailedError(err, appLabel, experimentsDetails.ChaosNamespace, true)
+	}
+
+	//Deleting all the helper pod for locust-loadgen chaos
+	log.Info("[Cleanup]: Deleting all the helper pods")
+	if err = common.DeleteAllPod(appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
+		return stacktrace.Propagate(err, "could not delete helper pod(s)")
+	}
+
+	return nil
+}
+
+// PrepareChaos contains the preparation steps before chaos injection
+func PrepareChaos(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
+	ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareLocustLoadGenFault")
+	defer span.End()
+
+	//Waiting for the ramp time before chaos injection
+	if experimentsDetails.RampTime != 0 {
+		log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", experimentsDetails.RampTime)
+		common.WaitForDuration(experimentsDetails.RampTime)
+	}
+	//Starting the locust-loadgen experiment
+	if err := experimentExecution(ctx, experimentsDetails, clients, resultDetails, eventsDetails, chaosDetails); err != nil {
+		return stacktrace.Propagate(err, "could not execute chaos")
+	}
+	//Waiting for the ramp time after chaos injection
+	if experimentsDetails.RampTime != 0 {
+		log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime)
+		common.WaitForDuration(experimentsDetails.RampTime)
+	}
+	return nil
+}
+
+// createHelperPod derive the attributes for helper pod and create the helper pod
+func createHelperPod(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails, runID string) error {
+	ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "CreateLocustLoadGenFaultHelperPod")
+	defer span.End()
+
+	const volumeName = "script-volume"
+	const mountPath = "/mnt"
+
+	helperPod := &corev1.Pod{
+		ObjectMeta: v1.ObjectMeta{
+			GenerateName: experimentsDetails.ExperimentName + "-helper-",
+			Namespace:    experimentsDetails.ChaosNamespace,
+			Labels:       common.GetHelperLabels(chaosDetails.Labels, runID, experimentsDetails.ExperimentName),
+			Annotations:  chaosDetails.Annotations,
+		},
+		Spec: corev1.PodSpec{
+			RestartPolicy:    corev1.RestartPolicyNever,
+			ImagePullSecrets: chaosDetails.ImagePullSecrets,
+			Containers: []corev1.Container{
+				{
+					Name:            experimentsDetails.ExperimentName,
+					Image:           experimentsDetails.LIBImage,
+					ImagePullPolicy: corev1.PullPolicy(experimentsDetails.LIBImagePullPolicy),
+					Command: []string{
+						"locust",
+						"--headless",
+					},
+					Args: []string{
+						"--users", strconv.Itoa(experimentsDetails.Users),
+						"--spawn-rate", strconv.Itoa(experimentsDetails.SpawnRate),
+						"-H", experimentsDetails.Host,
+						"-t", strconv.Itoa(experimentsDetails.ChaosDuration) + "s",
+						"-f", mountPath + "/" + experimentsDetails.ConfigMapKey,
+					},
+					Resources: chaosDetails.Resources,
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      volumeName,
+							MountPath: mountPath,
+						},
+					},
+				},
+			},
+			Volumes: []corev1.Volume{
+				{
+					Name: volumeName,
+					VolumeSource: corev1.VolumeSource{
+						ConfigMap: &corev1.ConfigMapVolumeSource{
+							LocalObjectReference: corev1.LocalObjectReference{
+								Name: experimentsDetails.ConfigMapName,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	_, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).Create(context.Background(), helperPod, v1.CreateOptions{})
+	if err != nil {
+		return cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Reason: fmt.Sprintf("unable to create helper pod: %s", err.Error())}
+	}
+	return nil
+}

contribute/developer-guide/templates/chaoslib_helper.tmpl

@@ -20,6 +20,9 @@ import (
 )
 
 func experimentExecution(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
+    // @TODO: setup tracing
+    // ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "experimentExecution[name-your-chaos]Fault")
+    // defer span.End()
 
     // Get the target pod details for the chaos execution
 	// if the target pod is not defined it will derive the random target pod list using pod affected percentage
@@ -58,6 +61,10 @@ func experimentExecution(ctx context.Context, experimentsDetails *experimentType
 }
 
 func runChaos(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, targetPodList corev1.PodList, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
+    // @TODO: setup tracing
+    // ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "experimentExecution[name-your-chaos]Fault")
+    // defer span.End()
+
 	if experimentsDetails.EngineName != "" {
     		msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on target pod"
     		types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails)

experiments/load/locust-loadgen/README.md

@@ -0,0 +1,14 @@
+## Experiment Metadata
+
+<table>
+<tr>
+<th> Name </th>
+<th> Description </th>
+<th> Documentation Link </th>
+</tr>
+<tr>
+ <td> Locust Load Generator </td>
+ <td> Locust is an open-source load testing tool that makes performance testing easy and productive for engineering teams.  You can easily run load testing through a single Python script. Learn how to use locust <a href="https://docs.locust.io/en/stable/">here</a> </td>
+ <td> <a href="https://litmuschaos.github.io/litmus/experiments/categories/load/locust-loadgen/"> Here </a> </td>
+ </tr>
+ </table>

experiments/load/locust-loadgen/experiment/locust-loadgen.go

@@ -0,0 +1,145 @@
+package experiment
+
+import (
+	"context"
+	"os"
+
+	"github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1"
+	litmusLIB "github.com/litmuschaos/litmus-go/chaoslib/litmus/locust-loadgen/lib"
+	"github.com/litmuschaos/litmus-go/pkg/clients"
+	"github.com/litmuschaos/litmus-go/pkg/events"
+	experimentEnv "github.com/litmuschaos/litmus-go/pkg/load/locust-loadgen/environment"
+	experimentTypes "github.com/litmuschaos/litmus-go/pkg/load/locust-loadgen/types"
+	"github.com/litmuschaos/litmus-go/pkg/log"
+	"github.com/litmuschaos/litmus-go/pkg/probe"
+	"github.com/litmuschaos/litmus-go/pkg/result"
+	"github.com/litmuschaos/litmus-go/pkg/types"
+	"github.com/litmuschaos/litmus-go/pkg/utils/common"
+	"github.com/sirupsen/logrus"
+)
+
+// Experiment contains steps to inject chaos
+func Experiment(ctx context.Context, clients clients.ClientSets) {
+
+	experimentsDetails := experimentTypes.ExperimentDetails{}
+	resultDetails := types.ResultDetails{}
+	eventsDetails := types.EventDetails{}
+	chaosDetails := types.ChaosDetails{}
+
+	//Fetching all the ENV passed from the runner pod
+	log.Infof("[PreReq]: Getting the ENV for the %v experiment", os.Getenv("EXPERIMENT_NAME"))
+	experimentEnv.GetENV(&experimentsDetails)
+
+	// Initialize the chaos attributes
+	types.InitialiseChaosVariables(&chaosDetails)
+
+	// Initialize Chaos Result Parameters
+	types.SetResultAttributes(&resultDetails, chaosDetails)
+
+	if experimentsDetails.EngineName != "" {
+		// Get values from chaosengine. Bail out upon error, as we haven't entered exp business logic yet
+		if err := types.GetValuesFromChaosEngine(&chaosDetails, clients, &resultDetails); err != nil {
+			log.Errorf("Unable to initialize the probes, err: %v", err)
+			return
+		}
+	}
+
+	//Updating the chaos result in the beginning of experiment
+	log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName)
+	if err := result.ChaosResult(&chaosDetails, clients, &resultDetails, "SOT"); err != nil {
+		log.Errorf("Unable to Create the Chaos Result, err: %v", err)
+		result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
+		return
+	}
+
+	// Set the chaos result uid
+	result.SetResultUID(&resultDetails, clients, &chaosDetails)
+
+	// generating the event in chaosresult to marked the verdict as awaited
+	msg := "experiment: " + experimentsDetails.ExperimentName + ", Result: Awaited"
+	types.SetResultEventAttributes(&eventsDetails, types.AwaitedVerdict, msg, "Normal", &resultDetails)
+	events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResult")
+
+	//DISPLAY THE APP INFORMATION
+	log.InfoWithValues("[Info]: The application information is as follows", logrus.Fields{
+		"Chaos Duration": experimentsDetails.ChaosDuration,
+	})
+
+	// Calling AbortWatcher go routine, it will continuously watch for the abort signal and generate the required events and result
+	go common.AbortWatcher(experimentsDetails.ExperimentName, clients, &resultDetails, &chaosDetails, &eventsDetails)
+
+	if experimentsDetails.EngineName != "" {
+		// marking AUT as running, as we already checked the status of application under test
+		msg := "AUT: Running"
+
+		// run the probes in the pre-chaos check
+		if len(resultDetails.ProbeDetails) != 0 {
+
+			if err := probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PreChaos", &eventsDetails); err != nil {
+				log.Errorf("Probe Failed, err: %v", err)
+				msg := "AUT: Running, Probes: Unsuccessful"
+				types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Warning", &chaosDetails)
+				events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
+				result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
+				return
+			}
+			msg = "AUT: Running, Probes: Successful"
+		}
+		// generating the events for the pre-chaos check
+		types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Normal", &chaosDetails)
+		events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
+	}
+
+	chaosDetails.Phase = types.ChaosInjectPhase
+	if err := litmusLIB.PrepareChaos(ctx, &experimentsDetails, clients, &resultDetails, &eventsDetails, &chaosDetails); err != nil {
+		log.Errorf("Chaos injection failed, err: %v", err)
+		result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
+		return
+	}
+
+	log.Infof("[Confirmation]: %v chaos has been injected successfully", experimentsDetails.ExperimentName)
+	resultDetails.Verdict = v1alpha1.ResultVerdictPassed
+	chaosDetails.Phase = types.PostChaosPhase
+
+	if experimentsDetails.EngineName != "" {
+		// marking AUT as running, as we already checked the status of application under test
+		msg := "AUT: Running"
+
+		// run the probes in the post-chaos check
+		if len(resultDetails.ProbeDetails) != 0 {
+			if err := probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PostChaos", &eventsDetails); err != nil {
+				log.Errorf("Probes Failed, err: %v", err)
+				msg := "AUT: Running, Probes: Unsuccessful"
+				types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Warning", &chaosDetails)
+				events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
+				result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
+				return
+			}
+			msg = "AUT: Running, Probes: Successful"
+		}
+
+		// generating post chaos event
+		types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Normal", &chaosDetails)
+		events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
+	}
+
+	//Updating the chaosResult in the end of experiment
+	log.Infof("[The End]: Updating the chaos result of %v experiment (EOT)", experimentsDetails.ExperimentName)
+	if err := result.ChaosResult(&chaosDetails, clients, &resultDetails, "EOT"); err != nil {
+		log.Errorf("Unable to Update the Chaos Result, err: %v", err)
+		result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails)
+		return
+	}
+
+	// generating the event in chaosresult to mark the verdict as pass/fail
+	msg = "experiment: " + experimentsDetails.ExperimentName + ", Result: " + string(resultDetails.Verdict)
+	reason, eventType := types.GetChaosResultVerdictEvent(resultDetails.Verdict)
+	types.SetResultEventAttributes(&eventsDetails, reason, msg, eventType, &resultDetails)
+	events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResult")
+
+	if experimentsDetails.EngineName != "" {
+		msg := experimentsDetails.ExperimentName + " experiment has been " + string(resultDetails.Verdict) + "ed"
+		types.SetEngineEventAttributes(&eventsDetails, types.Summary, msg, "Normal", &chaosDetails)
+		events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
+	}
+}

experiments/load/locust-loadgen/rbac.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: locust-loadgen-sa
+  namespace: default
+  labels:
+    name: locust-loadgen-sa
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: locust-loadgen-sa
+  namespace: default
+  labels:
+    name: locust-loadgen-sa
+rules:
+- apiGroups: ["","litmuschaos.io","batch","apps"]
+  resources: ["pods","configmaps","jobs","pods/exec","pods/log","events","chaosengines","chaosexperiments","chaosresults"]
+  verbs: ["create","list","get","patch","update","delete","deletecollection"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: locust-loadgen-sa
+  namespace: default
+  labels:
+    name: locust-loadgen-sa
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: locust-loadgen-sa
+subjects:
+- kind: ServiceAccount
+  name: locust-loadgen-sa
+  namespace: default
+