feat: add ReclaimPolicy for GameServer (#115)

Signed-off-by: ChrisLiu <[email protected]>

Author: chrisliu1995

apis/v1alpha1/gameserverset_types.go

@@ -64,8 +64,22 @@ type GameServerTemplate struct {
 	// +kubebuilder:validation:Schemaless
 	corev1.PodTemplateSpec `json:",inline"`
 	VolumeClaimTemplates   []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
+	// ReclaimPolicy indicates the reclaim policy for GameServer.
+	// Default is Cascade.
+	ReclaimPolicy GameServerReclaimPolicy `json:"reclaimPolicy,omitempty"`
 }
 
+type GameServerReclaimPolicy string
+
+const (
+	// CascadeGameServerReclaimPolicy indicates that GameServer is deleted when the pod is deleted.
+	// The age of GameServer is exactly the same as that of the pod.
+	CascadeGameServerReclaimPolicy GameServerReclaimPolicy = "Cascade"
+	// DeleteGameServerReclaimPolicy indicates that GameServers will be deleted when replicas of GameServerSet decreases.
+	// The GameServer will not be deleted when the corresponding pod is deleted due to manual deletion, update, eviction, etc.
+	DeleteGameServerReclaimPolicy GameServerReclaimPolicy = "Delete"
+)
+
 type Network struct {
 	NetworkType string              `json:"networkType,omitempty"`
 	NetworkConf []NetworkConfParams `json:"networkConf,omitempty"`

config/crd/bases/game.kruise.io_gameserversets.yaml

@@ -70,6 +70,10 @@ spec:
                 description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
                   Important: Run "make" to regenerate code after modifying this file'
                 properties:
+                  reclaimPolicy:
+                    description: ReclaimPolicy indicates the reclaim policy for GameServer.
+                      Default is Cascade.
+                    type: string
                   volumeClaimTemplates:
                     items:
                       description: PersistentVolumeClaim is a user's request for and

docs/en/user_manuals/CRD_field_description.md

@@ -38,8 +38,24 @@ type GameServerTemplate struct {
 
     // Requests and claims for persistent volumes.
     VolumeClaimTemplates   []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
+
+    // ReclaimPolicy indicates the reclaim policy for GameServer.
+    // Default is Cascade.
+    ReclaimPolicy GameServerReclaimPolicy `json:"reclaimPolicy,omitempty"`
 }
 
+type GameServerReclaimPolicy string
+
+const (
+    // CascadeGameServerReclaimPolicy indicates that GameServer is deleted when the pod is deleted.
+    // The age of GameServer is exactly the same as that of the pod.
+    CascadeGameServerReclaimPolicy GameServerReclaimPolicy = "Cascade"
+    
+    // DeleteGameServerReclaimPolicy indicates that GameServers will be deleted when replicas of GameServerSet decreases.
+    // The GameServer will not be deleted when the corresponding pod is deleted due to manual deletion, update, eviction, etc.
+    DeleteGameServerReclaimPolicy GameServerReclaimPolicy = "Delete"
+)
+
 ```
 
 #### UpdateStrategy

docs/中文/用户手册/CRD字段说明.md

@@ -72,7 +72,23 @@ type GameServerTemplate struct {
 
     // 对持久卷的请求和声明
     VolumeClaimTemplates   []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
+    
+    // ReclaimPolicy 表明GameServer的回收策略
+    // 当前支持两种，Cascade与Delete。默认为Cascade
+    ReclaimPolicy GameServerReclaimPolicy `json:"reclaimPolicy,omitempty"`
 }
+
+type GameServerReclaimPolicy string
+
+const (
+    // Cascade 表明pod删除时GameServer一并删除。GameServer的生命周期与pod相同
+    CascadeGameServerReclaimPolicy GameServerReclaimPolicy = "Cascade"
+    
+    // Delete 表明 GameServers 只会在GameServerSet副本数目减少时被删除。
+    // 当对应的pod被手动删除、更新重建、被驱逐时，GameServer都不会被删除。
+    DeleteGameServerReclaimPolicy GameServerReclaimPolicy = "Delete"
+)
+
 ```
 
 ### UpdateStrategy

pkg/controllers/gameserver/gameserver_controller.go

@@ -26,7 +26,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
@@ -173,8 +172,15 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	if podFound && !gsFound {
-		err := r.initGameServer(pod)
-		if err != nil && !errors.IsAlreadyExists(err) && !errors.IsNotFound(err) {
+		gss, err := r.getGameServerSet(pod)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return reconcile.Result{}, nil
+			}
+			return reconcile.Result{}, err
+		}
+		err = r.initGameServerByPod(gss, pod)
+		if err != nil && !errors.IsAlreadyExists(err) {
 			klog.Errorf("failed to create GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error())
 			return reconcile.Result{}, err
 		}
@@ -237,57 +243,24 @@ func (r *GameServerReconciler) getGameServerSet(pod *corev1.Pod) (*gamekruiseiov
 	return gss, err
 }
 
-func (r *GameServerReconciler) initGameServer(pod *corev1.Pod) error {
-	gs := &gamekruiseiov1alpha1.GameServer{}
-	gs.Name = pod.GetName()
-	gs.Namespace = pod.GetNamespace()
-
-	// set owner reference
-	gss, err := r.getGameServerSet(pod)
-	if err != nil {
-		return err
-	}
-	ors := make([]metav1.OwnerReference, 0)
-	or := metav1.OwnerReference{
-		APIVersion:         gss.APIVersion,
-		Kind:               gss.Kind,
-		Name:               gss.GetName(),
-		UID:                gss.GetUID(),
-		Controller:         pointer.BoolPtr(true),
-		BlockOwnerDeletion: pointer.BoolPtr(true),
-	}
-	ors = append(ors, or)
-	gs.OwnerReferences = ors
-
-	// set Labels
-	gsLabels := gss.Spec.GameServerTemplate.GetLabels()
-	if gsLabels == nil {
-		gsLabels = make(map[string]string)
-	}
-	gsLabels[gamekruiseiov1alpha1.GameServerOwnerGssKey] = gss.GetName()
-	gs.SetLabels(gsLabels)
-
-	// set Annotations
-	gsAnnotations := gss.Spec.GameServerTemplate.GetAnnotations()
-	if gsAnnotations == nil {
-		gsAnnotations = make(map[string]string)
+func (r *GameServerReconciler) initGameServerByPod(gss *gamekruiseiov1alpha1.GameServerSet, pod *corev1.Pod) error {
+	// default fields
+	gs := util.InitGameServer(gss, pod.Name)
+
+	if gss.Spec.GameServerTemplate.ReclaimPolicy == gamekruiseiov1alpha1.CascadeGameServerReclaimPolicy || gss.Spec.GameServerTemplate.ReclaimPolicy == "" {
+		// rewrite ownerReferences
+		ors := make([]metav1.OwnerReference, 0)
+		or := metav1.OwnerReference{
+			APIVersion:         pod.APIVersion,
+			Kind:               pod.Kind,
+			Name:               pod.GetName(),
+			UID:                pod.GetUID(),
+			Controller:         pointer.BoolPtr(true),
+			BlockOwnerDeletion: pointer.BoolPtr(true),
+		}
+		ors = append(ors, or)
+		gs.OwnerReferences = ors
 	}
-	gsAnnotations[gamekruiseiov1alpha1.GsTemplateMetadataHashKey] = util.GetGsTemplateMetadataHash(gss)
-	gs.SetAnnotations(gsAnnotations)
-
-	// set NetWork
-	gs.Spec.NetworkDisabled = false
-
-	// set OpsState
-	gs.Spec.OpsState = gamekruiseiov1alpha1.None
-
-	// set UpdatePriority
-	updatePriority := intstr.FromInt(0)
-	gs.Spec.UpdatePriority = &updatePriority
-
-	// set deletionPriority
-	deletionPriority := intstr.FromInt(0)
-	gs.Spec.DeletionPriority = &deletionPriority
 
 	return r.Client.Create(context.Background(), gs)
 }

pkg/controllers/gameserverset/gameserverset_controller.go

@@ -252,12 +252,6 @@ func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return reconcile.Result{}, err
 	}
 
-	err = gsm.SyncGameServerReplicas()
-	if err != nil {
-		klog.Errorf("GameServerSet %s failed to adjust the replicas of GameServers to match that of Pods in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error())
-		return reconcile.Result{}, err
-	}
-
 	return ctrl.Result{}, nil
 }
 

pkg/controllers/gameserverset/gameserverset_manager.go

@@ -18,12 +18,6 @@ package gameserverset
 
 import (
 	"context"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/client-go/tools/record"
-	"sort"
-	"sync"
-	"time"
-
 	kruiseV1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1"
 	kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1"
 	corev1 "k8s.io/api/core/v1"
@@ -32,9 +26,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sort"
+	"strconv"
+	"sync"
 
 	gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
 	"github.com/openkruise/kruise-game/pkg/util"
@@ -47,7 +45,6 @@ type Control interface {
 	IsNeedToScale() bool
 	IsNeedToUpdateWorkload() bool
 	SyncPodProbeMarker() error
-	SyncGameServerReplicas() error
 	GetReplicasAfterKilling() *int32
 }
 
@@ -129,7 +126,15 @@ func (manager *GameServerSetManager) GameServerScale() error {
 	klog.Infof("GameServers %s/%s already has %d replicas, expect to have %d replicas.", gss.GetNamespace(), gss.GetName(), currentReplicas, expectedReplicas)
 	manager.eventRecorder.Eventf(gss, corev1.EventTypeNormal, ScaleReason, "scale from %d to %d", currentReplicas, expectedReplicas)
 
-	newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType)
+	newManageIds, newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, podList, gss.Spec.ScaleStrategy.ScaleDownStrategyType)
+
+	if gss.Spec.GameServerTemplate.ReclaimPolicy == gameKruiseV1alpha1.DeleteGameServerReclaimPolicy {
+		err := SyncGameServer(gss, c, newManageIds, util.GetIndexListFromPodList(podList))
+		if err != nil {
+			return err
+		}
+	}
+
 	asts.Spec.ReserveOrdinals = newReserveIds
 	asts.Spec.Replicas = gss.Spec.Replicas
 	asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{
@@ -157,7 +162,7 @@ func (manager *GameServerSetManager) GameServerScale() error {
 	return nil
 }
 
-func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) []int {
+func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) ([]int, []int) {
 	workloadManageIds := util.GetIndexListFromPodList(pods)
 
 	var toAdd []int
@@ -215,12 +220,13 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl
 		}
 	}
 
+	newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...)
+	newManageIds = util.GetSliceInANotInB(newManageIds, toDelete)
+
 	if scaleDownType == gameKruiseV1alpha1.ReserveIdsScaleDownStrategyType {
-		return append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...)
+		return newManageIds, append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...)
 	}
 
-	newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...)
-	newManageIds = util.GetSliceInANotInB(newManageIds, toDelete)
 	var newReserveIds []int
 	if len(newManageIds) != 0 {
 		sort.Ints(newManageIds)
@@ -231,63 +237,80 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl
 		}
 	}
 
-	return newReserveIds
+	return newManageIds, newReserveIds
 }
 
-func (manager *GameServerSetManager) SyncGameServerReplicas() error {
-	gss := manager.gameServerSet
-	gsList := &gameKruiseV1alpha1.GameServerList{}
-	err := manager.client.List(context.Background(), gsList, &client.ListOptions{
-		Namespace: gss.GetNamespace(),
-		LabelSelector: labels.SelectorFromSet(map[string]string{
-			gameKruiseV1alpha1.GameServerOwnerGssKey: gss.GetName(),
-		})})
-	if err != nil {
-		return err
-	}
-	podIds := util.GetIndexListFromPodList(manager.podList)
-
-	gsMap := make(map[int]int)
-	deleteIds := make([]int, 0)
-	for id, gs := range gsList.Items {
-		gsId := util.GetIndexFromGsName(gs.Name)
-		if !util.IsNumInList(gsId, podIds) {
-			gsMap[gsId] = id
-			deleteIds = append(deleteIds, gsId)
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+func SyncGameServer(gss *gameKruiseV1alpha1.GameServerSet, c client.Client, newManageIds, oldManageIds []int) error {
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	errch := make(chan error, len(deleteIds))
+	addIds := util.GetSliceInANotInB(newManageIds, oldManageIds)
+	deleteIds := util.GetSliceInANotInB(oldManageIds, newManageIds)
+
+	errch := make(chan error, len(addIds)+len(deleteIds))
 	var wg sync.WaitGroup
-	for _, gsId := range deleteIds {
+	for _, gsId := range append(addIds, deleteIds...) {
 		wg.Add(1)
 		id := gsId
 		go func(ctx context.Context) {
 			defer wg.Done()
 			defer ctx.Done()
 
-			gs := gsList.Items[gsMap[id]].DeepCopy()
-			gsLabels := make(map[string]string)
-			gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true"
-			patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}}
-			patchBytes, err := json.Marshal(patchGs)
+			gs := &gameKruiseV1alpha1.GameServer{}
+			gsName := gss.Name + "-" + strconv.Itoa(id)
+			err := c.Get(ctx, types.NamespacedName{
+				Name:      gsName,
+				Namespace: gss.Namespace,
+			}, gs)
 			if err != nil {
+				if errors.IsNotFound(err) {
+					return
+				}
 				errch <- err
+				return
 			}
-			err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, patchBytes))
-			if err != nil && !errors.IsNotFound(err) {
-				errch <- err
+
+			if util.IsNumInList(id, addIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] == "true" {
+				gsLabels := make(map[string]string)
+				gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "false"
+				patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}}
+				patchBytes, err := json.Marshal(patchGs)
+				if err != nil {
+					errch <- err
+					return
+				}
+				err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes))
+				if err != nil && !errors.IsNotFound(err) {
+					errch <- err
+					return
+				}
+				klog.Infof("GameServer %s/%s DeletingKey turn into false", gss.Namespace, gsName)
 			}
+
+			if util.IsNumInList(id, deleteIds) && gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] != "true" {
+				gsLabels := make(map[string]string)
+				gsLabels[gameKruiseV1alpha1.GameServerDeletingKey] = "true"
+				patchGs := map[string]interface{}{"metadata": map[string]map[string]string{"labels": gsLabels}}
+				patchBytes, err := json.Marshal(patchGs)
+				if err != nil {
+					errch <- err
+					return
+				}
+				err = c.Patch(ctx, gs, client.RawPatch(types.MergePatchType, patchBytes))
+				if err != nil && !errors.IsNotFound(err) {
+					errch <- err
+					return
+				}
+				klog.Infof("GameServer %s/%s DeletingKey turn into true, who will be deleted", gss.Namespace, gsName)
+			}
+
 		}(ctx)
 	}
+
 	wg.Wait()
 	close(errch)
-	err = <-errch
+	err := <-errch
 	if err != nil {
-		klog.Errorf("failed to delete GameServers %s in %s because of %s.", gss.GetName(), gss.GetNamespace(), err.Error())
 		return err
 	}