From c681e55d61aa275436860a90476f6049a1d80d36 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Mon, 14 Jul 2025 12:34:43 +0000 Subject: [PATCH] [Feature] Compact Action --- CHANGELOG.md | 3 +- docs/generated/actions.md | 2 + internal/actions.yaml | 5 + pkg/apis/deployment/v1/actions.generated.go | 12 ++ .../deployment/v2alpha1/actions.generated.go | 12 ++ pkg/deployment/client/client.go | 2 + pkg/deployment/client/compact.go | 60 +++++++++ pkg/deployment/reconcile/action.go | 4 +- .../reconcile/action.register.generated.go | 17 +++ .../action.register.generated_test.go | 10 ++ .../reconcile/action_backup_restore.go | 9 +- .../reconcile/action_compact_member.go | 123 ++++++++++++++++++ pkg/deployment/reconcile/plan_builder_test.go | 2 +- 13 files changed, 253 insertions(+), 8 deletions(-) create mode 100644 pkg/deployment/client/compact.go create mode 100644 pkg/deployment/reconcile/action_compact_member.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f217450c8..f09c9d46f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ - (Documentation) ManualUpgrade Docs - (Documentation) Add Required & Skip in Docs - (Feature) (Platform) ECS Storage -- (Bugfix) (Platform) Prevent NPE in case of missing Helm Release +- (Bugfix) (Platform) Prevent NPE in case of missing Helm Release +- (Feature) Compact Action ## [1.2.50](https://github.com/arangodb/kube-arangodb/tree/1.2.50) (2025-07-04) - (Feature) (Platform) MetaV1 Integration Service diff --git a/docs/generated/actions.md b/docs/generated/actions.md index bc9905583..3a1445a21 100644 --- a/docs/generated/actions.md +++ b/docs/generated/actions.md @@ -25,6 +25,7 @@ nav_order: 11 | CleanTLSCACertificate | no | 30m0s | no | Enterprise Only | Remove Certificate from CA TrustStore | | CleanTLSKeyfileCertificate | no | 30m0s | no | Enterprise Only | Remove old TLS certificate from server | | ClusterMemberCleanup | no | 10m0s | no | Community & Enterprise | Remove member from Cluster if it is gone already (Coordinators) | +| CompactMember | no | 8h0m0s | no | Community & Enterprise | Runs the Compact API on the Member | | Delay | no | 10m0s | yes | Community & Enterprise | Define delay operation | | ~~DisableClusterScaling~~ | no | 10m0s | no | Community & Enterprise | Disable Cluster Scaling integration | | DisableMaintenance | no | 10m0s | no | Community & Enterprise | Disable ArangoDB maintenance mode | @@ -123,6 +124,7 @@ spec: CleanTLSCACertificate: 30m0s CleanTLSKeyfileCertificate: 30m0s ClusterMemberCleanup: 10m0s + CompactMember: 8h0m0s Delay: 10m0s DisableClusterScaling: 10m0s DisableMaintenance: 10m0s diff --git a/internal/actions.yaml b/internal/actions.yaml index a5c03b3c9..24960c560 100644 --- a/internal/actions.yaml +++ b/internal/actions.yaml @@ -24,6 +24,11 @@ actions: RecreateMember: description: Recreate member with same ID and Data timeout: 15m + CompactMember: + description: Runs the Compact API on the Member + timeout: 8h + scopes: + - Normal CleanOutMember: description: Run the CleanOut job on member timeout: 48h diff --git a/pkg/apis/deployment/v1/actions.generated.go b/pkg/apis/deployment/v1/actions.generated.go index 58382a643..b37feb102 100644 --- a/pkg/apis/deployment/v1/actions.generated.go +++ b/pkg/apis/deployment/v1/actions.generated.go @@ -65,6 +65,9 @@ const ( // ActionClusterMemberCleanupDefaultTimeout define default timeout for action ActionClusterMemberCleanup ActionClusterMemberCleanupDefaultTimeout time.Duration = ActionsDefaultTimeout + // ActionCompactMemberDefaultTimeout define default timeout for action ActionCompactMember + ActionCompactMemberDefaultTimeout time.Duration = 28800 * time.Second // 8h0m0s + // ActionDelayDefaultTimeout define default timeout for action ActionDelay ActionDelayDefaultTimeout time.Duration = ActionsDefaultTimeout @@ -328,6 +331,9 @@ const ( // ActionTypeClusterMemberCleanup in scopes Normal. Remove member from Cluster if it is gone already (Coordinators) ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup" + // ActionTypeCompactMember in scopes Normal. Runs the Compact API on the Member + ActionTypeCompactMember ActionType = "CompactMember" + // ActionTypeDelay in scopes High and Normal. Define delay operation ActionTypeDelay ActionType = "Delay" @@ -589,6 +595,8 @@ func (a ActionType) DefaultTimeout() time.Duration { return ActionCleanTLSKeyfileCertificateDefaultTimeout case ActionTypeClusterMemberCleanup: return ActionClusterMemberCleanupDefaultTimeout + case ActionTypeCompactMember: + return ActionCompactMemberDefaultTimeout case ActionTypeDelay: return ActionDelayDefaultTimeout case ActionTypeDisableClusterScaling: @@ -771,6 +779,8 @@ func (a ActionType) Priority() ActionPriority { return ActionPriorityNormal case ActionTypeClusterMemberCleanup: return ActionPriorityNormal + case ActionTypeCompactMember: + return ActionPriorityNormal case ActionTypeDelay: return ActionPriorityHigh case ActionTypeDisableClusterScaling: @@ -979,6 +989,8 @@ func (a ActionType) Optional() bool { return false case ActionTypeClusterMemberCleanup: return false + case ActionTypeCompactMember: + return false case ActionTypeDelay: return true case ActionTypeDisableClusterScaling: diff --git a/pkg/apis/deployment/v2alpha1/actions.generated.go b/pkg/apis/deployment/v2alpha1/actions.generated.go index fa795e84d..854a3f8f3 100644 --- a/pkg/apis/deployment/v2alpha1/actions.generated.go +++ b/pkg/apis/deployment/v2alpha1/actions.generated.go @@ -65,6 +65,9 @@ const ( // ActionClusterMemberCleanupDefaultTimeout define default timeout for action ActionClusterMemberCleanup ActionClusterMemberCleanupDefaultTimeout time.Duration = ActionsDefaultTimeout + // ActionCompactMemberDefaultTimeout define default timeout for action ActionCompactMember + ActionCompactMemberDefaultTimeout time.Duration = 28800 * time.Second // 8h0m0s + // ActionDelayDefaultTimeout define default timeout for action ActionDelay ActionDelayDefaultTimeout time.Duration = ActionsDefaultTimeout @@ -328,6 +331,9 @@ const ( // ActionTypeClusterMemberCleanup in scopes Normal. Remove member from Cluster if it is gone already (Coordinators) ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup" + // ActionTypeCompactMember in scopes Normal. Runs the Compact API on the Member + ActionTypeCompactMember ActionType = "CompactMember" + // ActionTypeDelay in scopes High and Normal. Define delay operation ActionTypeDelay ActionType = "Delay" @@ -589,6 +595,8 @@ func (a ActionType) DefaultTimeout() time.Duration { return ActionCleanTLSKeyfileCertificateDefaultTimeout case ActionTypeClusterMemberCleanup: return ActionClusterMemberCleanupDefaultTimeout + case ActionTypeCompactMember: + return ActionCompactMemberDefaultTimeout case ActionTypeDelay: return ActionDelayDefaultTimeout case ActionTypeDisableClusterScaling: @@ -771,6 +779,8 @@ func (a ActionType) Priority() ActionPriority { return ActionPriorityNormal case ActionTypeClusterMemberCleanup: return ActionPriorityNormal + case ActionTypeCompactMember: + return ActionPriorityNormal case ActionTypeDelay: return ActionPriorityHigh case ActionTypeDisableClusterScaling: @@ -979,6 +989,8 @@ func (a ActionType) Optional() bool { return false case ActionTypeClusterMemberCleanup: return false + case ActionTypeCompactMember: + return false case ActionTypeDelay: return true case ActionTypeDisableClusterScaling: diff --git a/pkg/deployment/client/client.go b/pkg/deployment/client/client.go index dabcef42a..fd2a08b2f 100644 --- a/pkg/deployment/client/client.go +++ b/pkg/deployment/client/client.go @@ -55,6 +55,8 @@ type Client interface { RefreshJWT(ctx context.Context) (JWTDetails, error) DeleteExpiredJobs(ctx context.Context, timeout time.Duration) error + + Compact(ctx context.Context, request *CompactRequest) error } type client struct { diff --git a/pkg/deployment/client/compact.go b/pkg/deployment/client/compact.go new file mode 100644 index 000000000..314d2c1d7 --- /dev/null +++ b/pkg/deployment/client/compact.go @@ -0,0 +1,60 @@ +// +// DISCLAIMER +// +// Copyright 2025 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package client + +import ( + "context" + goHttp "net/http" +) + +type CompactRequest struct { + CompactBottomMostLevel *bool `json:"compactBottomMostLevel,omitempty"` + ChangeLevel *bool `json:"changeLevel,omitempty"` +} + +const CompactUrl = "/_admin/compact" + +func (c *client) Compact(ctx context.Context, request *CompactRequest) error { + req, err := c.c.NewRequest(goHttp.MethodPut, CompactUrl) + if err != nil { + return err + } + + if request == nil { + request = new(CompactRequest) + } + + req, err = req.SetBody(request) + if err != nil { + return err + } + + resp, err := c.c.Do(ctx, req) + if err != nil { + return err + } + + if err := resp.CheckStatus(goHttp.StatusOK); err != nil { + return err + } + + return nil +} diff --git a/pkg/deployment/reconcile/action.go b/pkg/deployment/reconcile/action.go index 8f4d41095..c98a8aba2 100644 --- a/pkg/deployment/reconcile/action.go +++ b/pkg/deployment/reconcile/action.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2025 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,6 +34,8 @@ import ( const ( DefaultStartFailureGracePeriod = 10 * time.Second + + LocalJobID api.PlanLocalKey = "jobID" ) func GetAllActions() []api.ActionType { diff --git a/pkg/deployment/reconcile/action.register.generated.go b/pkg/deployment/reconcile/action.register.generated.go index 8e4213231..6e7d99d6e 100644 --- a/pkg/deployment/reconcile/action.register.generated.go +++ b/pkg/deployment/reconcile/action.register.generated.go @@ -66,6 +66,9 @@ var ( _ Action = &actionClusterMemberCleanup{} _ actionFactory = newClusterMemberCleanupAction + _ Action = &actionCompactMember{} + _ actionFactory = newCompactMemberAction + _ Action = &actionDelay{} _ actionFactory = newDelayAction @@ -459,6 +462,20 @@ func init() { registerAction(action, function) } + // CompactMember + { + // Get Action type + action := api.ActionTypeCompactMember + + // Get Action defition + function := newCompactMemberAction + + // Wrap action main function + + // Register action + registerAction(action, function) + } + // Delay { // Get Action type diff --git a/pkg/deployment/reconcile/action.register.generated_test.go b/pkg/deployment/reconcile/action.register.generated_test.go index 01f41c55f..c607aa564 100644 --- a/pkg/deployment/reconcile/action.register.generated_test.go +++ b/pkg/deployment/reconcile/action.register.generated_test.go @@ -160,6 +160,16 @@ func Test_Actions(t *testing.T) { }) }) + t.Run("CompactMember", func(t *testing.T) { + ActionsExistence(t, api.ActionTypeCompactMember) + t.Run("Internal", func(t *testing.T) { + require.False(t, api.ActionTypeCompactMember.Internal()) + }) + t.Run("Optional", func(t *testing.T) { + require.False(t, api.ActionTypeCompactMember.Optional()) + }) + }) + t.Run("Delay", func(t *testing.T) { ActionsExistence(t, api.ActionTypeDelay) t.Run("Internal", func(t *testing.T) { diff --git a/pkg/deployment/reconcile/action_backup_restore.go b/pkg/deployment/reconcile/action_backup_restore.go index e8be1a5c2..037d54873 100644 --- a/pkg/deployment/reconcile/action_backup_restore.go +++ b/pkg/deployment/reconcile/action_backup_restore.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2025 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,7 +34,6 @@ import ( ) const ( - actionBackupRestoreLocalJobID api.PlanLocalKey = "jobID" actionBackupRestoreLocalBackupName api.PlanLocalKey = "backupName" ) @@ -114,7 +113,7 @@ func (a actionBackupRestore) restoreAsync(ctx context.Context, backup *backupApi if err := dbc.Backup().Restore(ctxChild, driver.BackupID(backup.Status.Backup.ID), nil); err != nil { if id, ok := conn.IsAsyncJobInProgress(err); ok { - a.actionCtx.Add(actionBackupRestoreLocalJobID, id, true) + a.actionCtx.Add(LocalJobID, id, true) a.actionCtx.Add(actionBackupRestoreLocalBackupName, backup.GetName(), true) // Async request has been sent @@ -169,9 +168,9 @@ func (a actionBackupRestore) CheckProgress(ctx context.Context) (bool, bool, err return false, false, errors.Errorf("Local Key is missing in action: %s", actionBackupRestoreLocalBackupName) } - job, ok := a.actionCtx.Get(a.action, actionBackupRestoreLocalJobID) + job, ok := a.actionCtx.Get(a.action, LocalJobID) if !ok { - return false, false, errors.Errorf("Local Key is missing in action: %s", actionBackupRestoreLocalJobID) + return false, false, errors.Errorf("Local Key is missing in action: %s", LocalJobID) } ctxChild, cancel := globals.GetGlobalTimeouts().ArangoD().WithTimeout(ctx) diff --git a/pkg/deployment/reconcile/action_compact_member.go b/pkg/deployment/reconcile/action_compact_member.go new file mode 100644 index 000000000..36f84d576 --- /dev/null +++ b/pkg/deployment/reconcile/action_compact_member.go @@ -0,0 +1,123 @@ +// +// DISCLAIMER +// +// Copyright 2025 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package reconcile + +import ( + "context" + + "github.com/arangodb-helper/go-helper/pkg/arangod/conn" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/client" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/errors" +) + +// newCompactMemberAction creates a new Action that implements the given +// planned CompactMember action. +func newCompactMemberAction(action api.Action, actionCtx ActionContext) Action { + a := &actionCompactMember{} + + a.actionImpl = newActionImplDefRef(action, actionCtx) + + return a +} + +// actionCompactMember implements an CompactMemberAction. +type actionCompactMember struct { + // actionImpl implement timeout and member id functions + actionImpl +} + +func (a *actionCompactMember) Start(ctx context.Context) (bool, error) { + m, g, ok := a.actionCtx.GetMemberStatusAndGroupByID(a.action.MemberID) + if !ok { + return false, errors.Errorf("expecting member to be present in list, but it is not") + } + + switch g { + case api.ServerGroupDBServers: + dbc, err := a.actionCtx.GetServerAsyncClient(m.ID) + if err != nil { + return false, errors.Wrapf(err, "Unable to create client") + } + + c := client.NewClient(dbc.Connection(), logger) + + if err := c.Compact(ctx, &client.CompactRequest{ + CompactBottomMostLevel: util.NewType(true), + ChangeLevel: util.NewType(true), + }); err != nil { + if id, ok := conn.IsAsyncJobInProgress(err); ok { + a.actionCtx.Add(LocalJobID, id, true) + + return false, nil + } + + return false, err + } + } + + return true, nil +} + +func (a actionCompactMember) CheckProgress(ctx context.Context) (bool, bool, error) { + m, g, ok := a.actionCtx.GetMemberStatusAndGroupByID(a.action.MemberID) + if !ok { + return false, true, errors.Errorf("expecting member to be present in list, but it is not") + } + + job, ok := a.actionCtx.Get(a.action, LocalJobID) + if !ok { + return false, true, errors.Errorf("Local Key is missing in action: %s", LocalJobID) + } + + switch g { + case api.ServerGroupDBServers: + dbc, err := a.actionCtx.GetServerAsyncClient(m.ID) + if err != nil { + return false, false, errors.Wrapf(err, "Unable to create client") + } + + c := client.NewClient(dbc.Connection(), logger) + + if err := c.Compact(conn.WithAsyncID(ctx, job), &client.CompactRequest{ + CompactBottomMostLevel: util.NewType(true), + ChangeLevel: util.NewType(true), + }); err != nil { + if _, ok := conn.IsAsyncJobInProgress(err); ok { + return false, false, nil + } + + if ok := conn.IsAsyncErrorNotFound(err); ok { + // Job not found + return false, true, err + } + + return false, false, err + } + + // Job Completed + return true, false, nil + } + + return true, false, nil +} diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index 9d7fbad83..422f2422a 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -134,7 +134,7 @@ func (c *testContext) GetDatabaseAsyncClient(ctx context.Context) (driver.Client panic("implement me") } -func (ac *testContext) GetServerAsyncClient(id string) (driver.Client, error) { +func (c *testContext) GetServerAsyncClient(id string) (driver.Client, error) { //TODO implement me panic("implement me") }