change devbox to stop if exceeded quota (labring#5182)

HUAHUAI23 · Oct 31, 2024 · e6a2f8b · e6a2f8b
1 parent 0fe154f
commit e6a2f8b
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 3 deletions.
diff --git a/controllers/devbox/internal/controller/devbox_controller.go b/controllers/devbox/internal/controller/devbox_controller.go
@@ -261,7 +261,19 @@ func (r *DevboxReconciler) syncPod(ctx context.Context, devbox *devboxv1alpha1.D
 		case 0:
 			logger.Info("create pod")
 			logger.Info("next commit history", "commit", nextCommitHistory)
-			return r.createPod(ctx, devbox, expectPod, nextCommitHistory)
+			err := r.createPod(ctx, devbox, expectPod, nextCommitHistory)
+			if err != nil && helper.IsExceededQuotaError(err) {
+				logger.Info("devbox is exceeded quota, change devbox state to Stopped")
+				r.Recorder.Eventf(devbox, corev1.EventTypeWarning, "Devbox is exceeded quota", "Devbox is exceeded quota")
+				devbox.Spec.State = devboxv1alpha1.DevboxStateStopped
+				_ = r.Update(ctx, devbox)
+				return nil
+			}
+			if err != nil {
+				logger.Error(err, "create pod failed")
+				return err
+			}
+			return nil
 		case 1:
 			pod := &podList.Items[0]
 			// check pod container size, if it is 0, it means the pod is not running, return an error
@@ -419,11 +431,9 @@ func (r *DevboxReconciler) getRuntime(ctx context.Context, devbox *devboxv1alpha
 
 // create a new pod, add predicated status to nextCommitHistory
 func (r *DevboxReconciler) createPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, expectPod *corev1.Pod, nextCommitHistory *devboxv1alpha1.CommitHistory) error {
-	logger := log.FromContext(ctx)
 	nextCommitHistory.Status = devboxv1alpha1.CommitStatusPending
 	nextCommitHistory.PredicatedStatus = devboxv1alpha1.CommitStatusPending
 	if err := r.Create(ctx, expectPod); err != nil {
-		logger.Error(err, "create pod failed")
 		return err
 	}
 	devbox.Status.CommitHistory = append(devbox.Status.CommitHistory, nextCommitHistory)

diff --git a/controllers/devbox/internal/controller/helper/devbox.go b/controllers/devbox/internal/controller/helper/devbox.go
@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"log/slog"
 	"sort"
+	"strings"
 
 	"crypto/ed25519"
 	"crypto/rand"
@@ -397,6 +398,10 @@ func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, requestEphemera
 	}
 }
 
+func IsExceededQuotaError(err error) bool {
+	return strings.Contains(err.Error(), "exceeded quota")
+}
+
 func calculateResourceRequest(limit corev1.ResourceList) corev1.ResourceList {
 	if limit == nil {
 		return nil