Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 7317d1a

Browse files
committedDec 25, 2023
support migarte etcd mds and snapshot
Signed-off-by: caoxianfei1 <caoxianfei@corp.netease.com>
1 parent b1a3a4e commit 7317d1a

17 files changed

+627
-11
lines changed
 

‎cli/command/migrate.go

+24-6
Original file line numberDiff line numberDiff line change
@@ -39,23 +39,31 @@ import (
3939

4040
var (
4141
MIGRATE_ETCD_STEPS = []int{
42-
playbook.STOP_SERVICE,
43-
playbook.CLEAN_SERVICE, // only container
42+
playbook.ADD_ETCD_MEMBER,
4443
playbook.PULL_IMAGE,
4544
playbook.CREATE_CONTAINER,
4645
playbook.SYNC_CONFIG,
46+
playbook.AMEND_ETCD_CONFIG,
4747
playbook.START_ETCD,
48+
playbook.AMEND_MDS_CONFIG, // add a etcd endpoint
49+
playbook.RESTART_SERVICE, // restart all mds then add a etcd endpoint in mds.conf
50+
playbook.REMOVE_ETCD_MEMBER,
51+
playbook.STOP_SERVICE,
52+
playbook.CLEAN_SERVICE, // only container
53+
// playbook.AMEND_MDS_CONFIG, // remove a etcd endpoint
54+
// playbook.RESTART_SERVICE, // restart all mds then remove a etcd endpoint in mds.conf
55+
// playbook.RELOAD_METASERVER
4856
playbook.UPDATE_TOPOLOGY,
4957
}
5058

5159
// mds
5260
MIGRATE_MDS_STEPS = []int{
53-
playbook.STOP_SERVICE,
54-
playbook.CLEAN_SERVICE, // only container
5561
playbook.PULL_IMAGE,
5662
playbook.CREATE_CONTAINER,
5763
playbook.SYNC_CONFIG,
5864
playbook.START_MDS,
65+
playbook.STOP_SERVICE,
66+
playbook.CLEAN_SERVICE, // only container
5967
playbook.UPDATE_TOPOLOGY,
6068
}
6169

@@ -157,7 +165,7 @@ func checkMigrateTopology(curveadm *cli.CurveAdm, data string) error {
157165
} else if len(dcs2add) < len(dcs2del) {
158166
return errno.ERR_DELETE_SERVICE_WHILE_MIGRATING_IS_DENIED
159167
}
160-
// len(dcs2add) == len(dcs2del)
168+
161169
if len(dcs2add) == 0 {
162170
return errno.ERR_NO_SERVICES_FOR_MIGRATING
163171
}
@@ -199,6 +207,7 @@ func genMigratePlaybook(curveadm *cli.CurveAdm,
199207
migrates := getMigrates(curveadm, data)
200208
role := migrates[0].From.GetRole()
201209
steps := MIGRATE_ROLE_STEPS[role]
210+
etcdDCs := curveadm.FilterDeployConfigByRole(dcs, topology.ROLE_ETCD)
202211

203212
// post clean
204213
if options.clean {
@@ -221,10 +230,14 @@ func genMigratePlaybook(curveadm *cli.CurveAdm,
221230
config := dcs2add
222231
switch step {
223232
case playbook.STOP_SERVICE,
224-
playbook.CLEAN_SERVICE:
233+
playbook.CLEAN_SERVICE,
234+
playbook.ADD_ETCD_MEMBER:
225235
config = dcs2del
226236
case playbook.BACKUP_ETCD_DATA:
227237
config = curveadm.FilterDeployConfigByRole(dcs, topology.ROLE_ETCD)
238+
case playbook.AMEND_MDS_CONFIG,
239+
playbook.RESTART_SERVICE:
240+
config = curveadm.FilterDeployConfigByRole(dcs, topology.ROLE_MDS)
228241
case
229242
playbook.CREATE_PHYSICAL_POOL,
230243
playbook.CREATE_LOGICAL_POOL,
@@ -251,6 +264,11 @@ func genMigratePlaybook(curveadm *cli.CurveAdm,
251264
optionsKV[comm.KEY_POOLSET] = poolset
252265
case playbook.UPDATE_TOPOLOGY:
253266
optionsKV[comm.KEY_NEW_TOPOLOGY_DATA] = data
267+
case playbook.ADD_ETCD_MEMBER,
268+
playbook.AMEND_ETCD_CONFIG,
269+
playbook.AMEND_MDS_CONFIG:
270+
optionsKV[comm.KEY_MIGRATE_SERVERS] = migrates
271+
optionsKV[comm.KEY_CLUSTER_DCS] = etcdDCs
254272
}
255273

256274
pb.AddStep(&playbook.PlaybookStep{

‎client.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
kind: curvefs
2+
s3.ak: curve
3+
s3.sk: Netease@2023
4+
s3.endpoint: 10.182.26.46:19005
5+
s3.bucket_name: curvefs
6+
container_image: quay.io/opencurve/curve/curvefs:v2.7.0-rc1_d8a1137
7+
mdsOpt.rpcRetryOpt.addrs: 10.182.26.46:6700,10.182.26.35:6700,10.182.26.36:6700
8+
log_dir: /home/caoxianfei/client/logs
9+
data_dir: /mnt/v27cache
10+
diskCache.fullRatio: 90
11+
diskCache.safeRatio: 70
12+
diskCache.trimRatio: 50
13+
diskCache.maxUsableSpaceBytes: 10737418240
14+
diskCache.maxFileNums: 1000000
15+
diskCache.trimCheckIntervalSec: 5
16+
client.loglevel: 9

‎curve

16.4 MB
Binary file not shown.

‎internal/common/common.go

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ const (
5757
// migrate
5858
KEY_MIGRATE_STATUS = "MIGRATE_STATUS"
5959
KEY_MIGRATE_COMMON_STATUS = "MIGRATE_COMMON_STATUS"
60+
KEY_CLUSTER_DCS = "CLUSTER_DCS"
6061

6162
// check
6263
KEY_CHECK_WITH_WEAK = "CHECK_WITH_WEAK"

‎internal/configure/topology/dc_get.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,11 @@ func (dc *DeployConfig) GetInstances() int { return dc.instanc
121121
func (dc *DeployConfig) GetHostSequence() int { return dc.hostSequence }
122122
func (dc *DeployConfig) GetInstancesSequence() int { return dc.instancesSequence }
123123
func (dc *DeployConfig) GetServiceConfig() map[string]string { return dc.serviceConfig }
124-
func (dc *DeployConfig) GetVariables() *variable.Variables { return dc.variables }
124+
func (dc *DeployConfig) SetServiceConfig(key, value string) {
125+
dc.serviceConfig[key] = value
126+
}
127+
128+
func (dc *DeployConfig) GetVariables() *variable.Variables { return dc.variables }
125129

126130
// (2): config item
127131
func (dc *DeployConfig) GetPrefix() string { return dc.getString(CONFIG_PREFIX) }

‎internal/configure/topology/variables.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,9 @@ var (
118118
{name: "cluster_mds_dummy_addr"},
119119
{name: "cluster_mds_dummy_port"},
120120
{name: "cluster_chunkserver_addr", kind: []string{KIND_CURVEBS}},
121-
{name: "cluster_snapshotclone_addr", kind: []string{KIND_CURVEBS}},
121+
{name: "cluster_snapshotclone_addr"},
122122
{name: "cluster_snapshotclone_proxy_addr", kind: []string{KIND_CURVEBS}},
123-
{name: "cluster_snapshotclone_dummy_port", kind: []string{KIND_CURVEBS}},
123+
{name: "cluster_snapshotclone_dummy_port"},
124124
{name: "cluster_snapshotclone_nginx_upstream", kind: []string{KIND_CURVEBS}},
125125
{name: "cluster_snapshot_addr"}, // tools-v2: compatible with some old version image
126126
{name: "cluster_snapshot_dummy_addr"}, // tools-v2

‎internal/errno/errno.go

+3
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,9 @@ var (
404404
ERR_GET_CHUNKSERVER_COPYSET = EC(410026, "failed to get chunkserver copyset")
405405
ERR_GET_MIGRATE_COPYSET = EC(410027, "migrate chunkserver copyset info must be 2")
406406
ERR_CONTAINER_NOT_REMOVED = EC(410027, "container not removed")
407+
ERR_GET_CLUSTER_ETCD_ADDR = EC(410028, "failed to get cluster_etcd_addr variable")
408+
ERR_ADD_ETCD_MEMEBER = EC(410029, "failed to add etcd member to existing etcd cluster")
409+
ERR_REMOVE_ETCD_MEMBER = EC(410030, "failed to remove etcd member from existing etcd cluster")
407410
// 420: common (curvebs client)
408411
ERR_VOLUME_ALREADY_MAPPED = EC(420000, "volume already mapped")
409412
ERR_VOLUME_CONTAINER_LOSED = EC(420001, "volume container is losed")

‎internal/playbook/factory.go

+12
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ const (
8484
INSTALL_CLIENT
8585
UNINSTALL_CLIENT
8686
ATTACH_LEADER_OR_RANDOM_CONTAINER
87+
ADD_ETCD_MEMBER
88+
AMEND_ETCD_CONFIG
89+
AMEND_MDS_CONFIG
90+
REMOVE_ETCD_MEMBER
8791

8892
// bs
8993
FORMAT_CHUNKFILE_POOL
@@ -251,6 +255,14 @@ func (p *Playbook) createTasks(step *PlaybookStep) (*tasks.Tasks, error) {
251255
t, err = comm.NewInstallClientTask(curveadm, config.GetCC(i))
252256
case UNINSTALL_CLIENT:
253257
t, err = comm.NewUninstallClientTask(curveadm, nil)
258+
case ADD_ETCD_MEMBER:
259+
t, err = comm.NewAddEtcdMemberTask(curveadm, config.GetDC(i))
260+
case AMEND_ETCD_CONFIG:
261+
t, err = comm.NewAmendEtcdConfigTask(curveadm, config.GetDC(i))
262+
case AMEND_MDS_CONFIG:
263+
t, err = comm.NewAmendMdsConfigTask(curveadm, config.GetDC(i))
264+
case REMOVE_ETCD_MEMBER:
265+
t, err = comm.NewRemoveEtcdMemberTask(curveadm, config.GetDC(i))
254266
// bs
255267
case FORMAT_CHUNKFILE_POOL:
256268
t, err = bs.NewFormatChunkfilePoolTask(curveadm, config.GetFC(i))

‎internal/task/scripts/enable_etcd_auth.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@
1212
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313
* See the License for the specific language governing permissions and
1414
* limitations under the License.
15-
*/
15+
*/
1616

1717
/*
1818
* Project: Curveadm
1919
* Created Date: 2023-08-02
2020
* Author: wanghai (SeanHai)
21-
*/
21+
*/
2222

2323
package scripts
2424

‎internal/task/scripts/script.go

+4
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ var (
3939
WAIT string
4040
//go:embed shell/report.sh
4141
REPORT string
42+
//go:embed shell/add_etcd.sh
43+
ADD_ETCD string
44+
//go:embed shell/remove_etcd.sh
45+
REMOVE_ETCD string
4246

4347
// CurveBS
4448

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/usr/bin/env bash
2+
3+
# Usage:
4+
# Example:
5+
# Created Date: 2023-12-15
6+
# Author: Caoxianfei
7+
8+
etcdctl=$1
9+
endpoints=$2
10+
old_name=$3
11+
new_name=$4
12+
new_peer_url=$5
13+
14+
${etcdctl} --endpoints=${endpoints} member add ${new_name} --peer-urls ${new_peer_url} > add_etcd.log 2>&1
15+
if [ $? -ne 0 ]; then
16+
if cat add_etcd.log | grep -q "Peer URLs already exists"; then
17+
exit 0
18+
else
19+
exit 1
20+
fi
21+
fi
22+
23+
# output=$(${etcdctl} --endpoints=${endpoints} member list)
24+
# if [ $? -ne 0 ]; then
25+
# echo "failed to list all etcd members"
26+
# exit 1
27+
# fi
28+
29+
# id=$(echo "$output" | awk -v name="$old_name" -F ', ' '$3 == name {print $1}')
30+
# if [ -z "${id}" ]; then
31+
# echo "failed to get id of member ${old_name}"
32+
# exit 1
33+
# fi
34+
35+
# ${etcdctl} --endpoints=${endpoints} member remove ${id}
36+
# if [ $? -ne 0 ]; then
37+
# echo "failed to remove member ${old_name}"
38+
# exit 1
39+
# fi
40+
41+
42+
43+
+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
3+
# Usage:
4+
# Example:
5+
# Created Date: 2023-12-15
6+
# Author: Caoxianfei
7+
8+
etcdctl=$1
9+
endpoints=$2
10+
old_name=$3
11+
12+
output=$(${etcdctl} --endpoints=${endpoints} member list)
13+
if [ $? -ne 0 ]; then
14+
echo "failed to list all etcd members"
15+
exit 1
16+
fi
17+
18+
id=$(echo "$output" | awk -v name="$old_name" -F ', ' '$3 == name {print $1}')
19+
if [ -z "${id}" ]; then
20+
echo "failed to get id of member ${old_name}"
21+
exit 1
22+
fi
23+
24+
${etcdctl} --endpoints=${endpoints} member remove ${id}
25+
if [ $? -ne 0 ]; then
26+
echo "failed to remove member ${old_name}"
27+
exit 1
28+
fi
29+
30+
31+
32+
+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright (c) 2023 NetEase Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*
18+
* Project: CurveAdm
19+
* Created Date: 2023-12-20
20+
* Author: Caoxianfei
21+
*/
22+
23+
package common
24+
25+
import (
26+
"fmt"
27+
"strconv"
28+
29+
"github.com/opencurve/curveadm/cli/cli"
30+
comm "github.com/opencurve/curveadm/internal/common"
31+
"github.com/opencurve/curveadm/internal/configure"
32+
"github.com/opencurve/curveadm/internal/configure/topology"
33+
"github.com/opencurve/curveadm/internal/errno"
34+
"github.com/opencurve/curveadm/internal/task/context"
35+
"github.com/opencurve/curveadm/internal/task/scripts"
36+
"github.com/opencurve/curveadm/internal/task/step"
37+
"github.com/opencurve/curveadm/internal/task/task"
38+
tui "github.com/opencurve/curveadm/internal/tui/common"
39+
)
40+
41+
func checkAddEtcdMemberStatus(success *bool, out *string) step.LambdaType {
42+
return func(ctx *context.Context) error {
43+
if !*success {
44+
return errno.ERR_ADD_ETCD_MEMEBER.S(*out)
45+
}
46+
return nil
47+
}
48+
}
49+
50+
func NewAddEtcdMemberTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (*task.Task, error) {
51+
serviceId := curveadm.GetServiceId(dc.GetId())
52+
containerId, err := curveadm.GetContainerId(serviceId)
53+
if curveadm.IsSkip(dc) {
54+
return nil, nil
55+
} else if err != nil {
56+
return nil, err
57+
}
58+
hc, err := curveadm.GetHost(dc.GetHost())
59+
if err != nil {
60+
return nil, err
61+
}
62+
63+
subname := fmt.Sprintf("host=%s role=%s containerId=%s",
64+
dc.GetHost(), dc.GetRole(), tui.TrimContainerId(containerId))
65+
t := task.NewTask("Backup Etcd Data", subname, hc.GetSSHConfig())
66+
67+
host, role := dc.GetHost(), dc.GetRole()
68+
script := scripts.ADD_ETCD
69+
layout := dc.GetProjectLayout()
70+
scriptPath := fmt.Sprintf("%s/add_etcd.sh", layout.ServiceBinDir)
71+
etcdctlPath := layout.ServiceBinDir + "/etcdctl"
72+
endpoints, err := dc.GetVariables().Get("cluster_etcd_addr")
73+
if err != nil {
74+
return nil, errno.ERR_GET_CLUSTER_ETCD_ADDR
75+
}
76+
oldName := fmt.Sprint("etcd", strconv.Itoa(dc.GetHostSequence()), strconv.Itoa(dc.GetInstancesSequence()))
77+
newName := fmt.Sprint("etcd", strconv.Itoa(dc.GetHostSequence()+3), strconv.Itoa(dc.GetInstancesSequence()))
78+
migrates := []*configure.MigrateServer{}
79+
if curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS) != nil {
80+
migrates = curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS).([]*configure.MigrateServer)
81+
}
82+
toService := migrates[0].To
83+
peerUrl := fmt.Sprint("http://", toService.GetListenIp(), ":", strconv.Itoa(toService.GetListenPort()))
84+
addEtcdCmd := fmt.Sprintf("/bin/bash %s %s %s %s %s %s", scriptPath, etcdctlPath, endpoints, oldName, newName, peerUrl)
85+
86+
var success bool
87+
var out string
88+
t.AddStep(&step.ListContainers{
89+
ShowAll: true,
90+
Format: `"{{.ID}}"`,
91+
Filter: fmt.Sprintf("id=%s", containerId),
92+
Out: &out,
93+
ExecOptions: curveadm.ExecOptions(),
94+
})
95+
t.AddStep(&step.Lambda{
96+
Lambda: CheckContainerExist(host, role, containerId, &out),
97+
})
98+
t.AddStep(&step.InstallFile{
99+
ContainerId: &containerId,
100+
ContainerDestPath: scriptPath,
101+
Content: &script,
102+
ExecOptions: curveadm.ExecOptions(),
103+
})
104+
t.AddStep(&step.ContainerExec{
105+
ContainerId: &containerId,
106+
Success: &success,
107+
Out: &out,
108+
Command: addEtcdCmd,
109+
ExecOptions: curveadm.ExecOptions(),
110+
})
111+
t.AddStep(&step.Lambda{
112+
Lambda: checkAddEtcdMemberStatus(&success, &out),
113+
})
114+
115+
return t, nil
116+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright (c) 2023 NetEase Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*
18+
* Project: CurveAdm
19+
* Created Date: 2023-12-20
20+
* Author: Caoxianfei
21+
*/
22+
23+
package common
24+
25+
import (
26+
"fmt"
27+
"strconv"
28+
"strings"
29+
30+
"github.com/opencurve/curveadm/cli/cli"
31+
comm "github.com/opencurve/curveadm/internal/common"
32+
"github.com/opencurve/curveadm/internal/configure"
33+
"github.com/opencurve/curveadm/internal/configure/topology"
34+
"github.com/opencurve/curveadm/internal/task/step"
35+
"github.com/opencurve/curveadm/internal/task/task"
36+
tui "github.com/opencurve/curveadm/internal/tui/common"
37+
)
38+
39+
const (
40+
AMEND_NAME = "name"
41+
AMEND_ENDPOINTS = "initial-cluster"
42+
AMEND_STATE = "initial-cluster-state"
43+
)
44+
45+
var options = make(map[string]interface{})
46+
47+
func mutateEtcdConf(dc *topology.DeployConfig, delimiter string, forceRender bool) step.Mutate {
48+
return func(in, key, value string) (out string, err error) {
49+
if len(key) == 0 {
50+
out = in
51+
return
52+
}
53+
if key == AMEND_NAME {
54+
value = options[AMEND_NAME].(string)
55+
} else if key == AMEND_ENDPOINTS {
56+
value = options[AMEND_ENDPOINTS].(string)
57+
} else if key == AMEND_STATE {
58+
value = "existing"
59+
}
60+
61+
out = fmt.Sprintf("%s%s%s", key, delimiter, value)
62+
return
63+
}
64+
}
65+
66+
func NewAmendEtcdConfigTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (*task.Task, error) {
67+
serviceId := curveadm.GetServiceId(dc.GetId())
68+
containerId, err := curveadm.GetContainerId(serviceId)
69+
if curveadm.IsSkip(dc) {
70+
return nil, nil
71+
} else if err != nil {
72+
return nil, err
73+
}
74+
hc, err := curveadm.GetHost(dc.GetHost())
75+
if err != nil {
76+
return nil, err
77+
}
78+
79+
subname := fmt.Sprintf("host=%s role=%s containerId=%s",
80+
dc.GetHost(), dc.GetRole(), tui.TrimContainerId(containerId))
81+
t := task.NewTask("Override Etcd configure", subname, hc.GetSSHConfig())
82+
83+
layout := dc.GetProjectLayout()
84+
migrates := []*configure.MigrateServer{}
85+
if curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS) != nil {
86+
migrates = curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS).([]*configure.MigrateServer)
87+
}
88+
dcs := []*topology.DeployConfig{}
89+
if curveadm.MemStorage().Get(comm.KEY_CLUSTER_DCS) != nil {
90+
dcs = curveadm.MemStorage().Get(comm.KEY_CLUSTER_DCS).([]*topology.DeployConfig)
91+
}
92+
endpoints := []string{}
93+
for _, dc := range dcs {
94+
ept := fmt.Sprint("etcd", strconv.Itoa(dc.GetHostSequence()), strconv.Itoa(dc.GetInstancesSequence()),
95+
"=", "http://", dc.GetListenIp(), ":", strconv.Itoa(dc.GetListenPort()))
96+
endpoints = append(endpoints, ept)
97+
}
98+
toService := migrates[0].To
99+
newName := fmt.Sprint("etcd", strconv.Itoa(toService.GetHostSequence()+3), strconv.Itoa(toService.GetInstancesSequence()))
100+
toSeriveEndpint := fmt.Sprint(newName, "=", "http://",
101+
toService.GetListenIp(), ":", strconv.Itoa(toService.GetListenPort()))
102+
endpoints = append(endpoints, toSeriveEndpint)
103+
endpointsStr := strings.Join(endpoints, ",")
104+
105+
options[AMEND_NAME] = newName
106+
options[AMEND_ENDPOINTS] = endpointsStr
107+
108+
t.AddStep(&step.SyncFile{ // sync etcd.conf config
109+
ContainerSrcId: &containerId,
110+
ContainerSrcPath: layout.ServiceConfPath,
111+
ContainerDestId: &containerId,
112+
ContainerDestPath: layout.ServiceConfPath,
113+
KVFieldSplit: ETCD_CONFIG_DELIMITER,
114+
Mutate: mutateEtcdConf(dc, ETCD_CONFIG_DELIMITER, false),
115+
ExecOptions: curveadm.ExecOptions(),
116+
})
117+
118+
return t, nil
119+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/*
2+
* Copyright (c) 2023 NetEase Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*
18+
* Project: CurveAdm
19+
* Created Date: 2023-12-20
20+
* Author: Caoxianfei
21+
*/
22+
23+
package common
24+
25+
import (
26+
"fmt"
27+
"strconv"
28+
29+
"github.com/opencurve/curveadm/cli/cli"
30+
comm "github.com/opencurve/curveadm/internal/common"
31+
"github.com/opencurve/curveadm/internal/configure"
32+
"github.com/opencurve/curveadm/internal/configure/topology"
33+
"github.com/opencurve/curveadm/internal/task/step"
34+
"github.com/opencurve/curveadm/internal/task/task"
35+
tui "github.com/opencurve/curveadm/internal/tui/common"
36+
)
37+
38+
const (
39+
ETCD_ENDPOINT = "etcd.endpoint"
40+
)
41+
42+
func mutateMDSConf(dc *topology.DeployConfig, delimiter string, forceRender bool) step.Mutate {
43+
return func(in, key, value string) (out string, err error) {
44+
if len(key) == 0 {
45+
out = in
46+
return
47+
}
48+
if key == ETCD_ENDPOINT {
49+
value = options[ETCD_ENDPOINT].(string)
50+
}
51+
52+
out = fmt.Sprintf("%s%s%s", key, delimiter, value)
53+
return
54+
}
55+
}
56+
57+
func NewAmendMdsConfigTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (*task.Task, error) {
58+
serviceId := curveadm.GetServiceId(dc.GetId())
59+
containerId, err := curveadm.GetContainerId(serviceId)
60+
if curveadm.IsSkip(dc) {
61+
return nil, nil
62+
} else if err != nil {
63+
return nil, err
64+
}
65+
hc, err := curveadm.GetHost(dc.GetHost())
66+
if err != nil {
67+
return nil, err
68+
}
69+
70+
subname := fmt.Sprintf("host=%s role=%s containerId=%s",
71+
dc.GetHost(), dc.GetRole(), tui.TrimContainerId(containerId))
72+
t := task.NewTask("Override Mds configure", subname, hc.GetSSHConfig())
73+
74+
layout := dc.GetProjectLayout()
75+
endpoints, err := dc.GetVariables().Get("cluster_etcd_addr")
76+
if err != nil {
77+
return nil, err
78+
}
79+
migrates := []*configure.MigrateServer{}
80+
if curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS) != nil {
81+
migrates = curveadm.MemStorage().Get(comm.KEY_MIGRATE_SERVERS).([]*configure.MigrateServer)
82+
}
83+
toService := migrates[0].To
84+
toSeriveEndpint := fmt.Sprint(toService.GetListenIp(), ":", strconv.Itoa(toService.GetListenClientPort()))
85+
endpoints = fmt.Sprint(endpoints, ",", toSeriveEndpint)
86+
options[ETCD_ENDPOINT] = endpoints
87+
88+
t.AddStep(&step.SyncFile{ // sync mds.conf config again
89+
ContainerSrcId: &containerId,
90+
ContainerSrcPath: layout.ServiceConfPath,
91+
ContainerDestId: &containerId,
92+
ContainerDestPath: layout.ServiceConfPath,
93+
KVFieldSplit: DEFAULT_CONFIG_DELIMITER,
94+
Mutate: mutateMDSConf(dc, DEFAULT_CONFIG_DELIMITER, false),
95+
ExecOptions: curveadm.ExecOptions(),
96+
})
97+
98+
return t, nil
99+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Copyright (c) 2023 NetEase Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*
18+
* Project: CurveAdm
19+
* Created Date: 2023-12-20
20+
* Author: Caoxianfei
21+
*/
22+
23+
package common
24+
25+
import (
26+
"fmt"
27+
"strconv"
28+
29+
"github.com/opencurve/curveadm/cli/cli"
30+
"github.com/opencurve/curveadm/internal/configure/topology"
31+
"github.com/opencurve/curveadm/internal/errno"
32+
"github.com/opencurve/curveadm/internal/task/context"
33+
"github.com/opencurve/curveadm/internal/task/scripts"
34+
"github.com/opencurve/curveadm/internal/task/step"
35+
"github.com/opencurve/curveadm/internal/task/task"
36+
tui "github.com/opencurve/curveadm/internal/tui/common"
37+
)
38+
39+
func checkRemoveEtcdMemberStatus(success *bool, out *string) step.LambdaType {
40+
return func(ctx *context.Context) error {
41+
if !*success {
42+
return errno.ERR_REMOVE_ETCD_MEMBER.S(*out)
43+
}
44+
return nil
45+
}
46+
}
47+
48+
func NewRemoveEtcdMemberTask(curveadm *cli.CurveAdm, dc *topology.DeployConfig) (*task.Task, error) {
49+
serviceId := curveadm.GetServiceId(dc.GetId())
50+
containerId, err := curveadm.GetContainerId(serviceId)
51+
if curveadm.IsSkip(dc) {
52+
return nil, nil
53+
} else if err != nil {
54+
return nil, err
55+
}
56+
hc, err := curveadm.GetHost(dc.GetHost())
57+
if err != nil {
58+
return nil, err
59+
}
60+
61+
subname := fmt.Sprintf("host=%s role=%s containerId=%s",
62+
dc.GetHost(), dc.GetRole(), tui.TrimContainerId(containerId))
63+
t := task.NewTask("Remove Old Etcd Member", subname, hc.GetSSHConfig())
64+
65+
host, role := dc.GetHost(), dc.GetRole()
66+
script := scripts.REMOVE_ETCD
67+
layout := dc.GetProjectLayout()
68+
scriptPath := fmt.Sprintf("%s/remove_etcd.sh", layout.ServiceBinDir)
69+
etcdctlPath := layout.ServiceBinDir + "/etcdctl"
70+
endpoints, err := dc.GetVariables().Get("cluster_etcd_addr")
71+
if err != nil {
72+
return nil, errno.ERR_GET_CLUSTER_ETCD_ADDR
73+
}
74+
oldName := fmt.Sprint("etcd", strconv.Itoa(dc.GetHostSequence()), strconv.Itoa(dc.GetInstancesSequence()))
75+
removeEtcdCmd := fmt.Sprintf("/bin/bash %s %s %s %s", scriptPath, etcdctlPath, endpoints, oldName)
76+
77+
var success bool
78+
var out string
79+
t.AddStep(&step.ListContainers{
80+
ShowAll: true,
81+
Format: `"{{.ID}}"`,
82+
Filter: fmt.Sprintf("id=%s", containerId),
83+
Out: &out,
84+
ExecOptions: curveadm.ExecOptions(),
85+
})
86+
t.AddStep(&step.Lambda{
87+
Lambda: CheckContainerExist(host, role, containerId, &out),
88+
})
89+
t.AddStep(&step.InstallFile{
90+
ContainerId: &containerId,
91+
ContainerDestPath: scriptPath,
92+
Content: &script,
93+
ExecOptions: curveadm.ExecOptions(),
94+
})
95+
t.AddStep(&step.ContainerExec{
96+
ContainerId: &containerId,
97+
Success: &success,
98+
Out: &out,
99+
Command: removeEtcdCmd,
100+
ExecOptions: curveadm.ExecOptions(),
101+
})
102+
t.AddStep(&step.Lambda{
103+
Lambda: checkRemoveEtcdMemberStatus(&success, &out),
104+
})
105+
106+
return t, nil
107+
}

‎topology.yaml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
kind: curvefs
2+
global:
3+
container_image: quay.io/opencurve/curve/curvefs:v2.7.0-rc1_d8a1137
4+
log_dir: ${home}/curvefs27/logs/${service_role}
5+
data_dir: ${home}/curvefs27/data/${service_role}
6+
variable:
7+
home: /data/cxf/chunkserver0
8+
machine1: curve7
9+
machine2: curve15
10+
machine3: curve16
11+
machine4: curve21
12+
13+
etcd_services:
14+
config:
15+
listen.ip: ${service_host}
16+
listen.port: 23800
17+
listen.client_port: 23790
18+
deploy:
19+
- host: ${machine1}
20+
- host: ${machine2}
21+
- host: ${machine3}
22+
mds_services:
23+
config:
24+
listen.ip: ${service_host}
25+
listen.port: 6700
26+
listen.dummy_port: 7700
27+
deploy:
28+
- host: ${machine1}
29+
- host: ${machine2}
30+
- host: ${machine3}
31+
32+
metaserver_services:
33+
config:
34+
listen.ip: ${service_host}
35+
listen.port: 16800
36+
listen.external_port: 17800
37+
trash.scanPeriodSec: 1
38+
trash.expiredAfterSec: 1
39+
deploy:
40+
- host: ${machine1}
41+
- host: ${machine2}
42+
- host: ${machine3}

0 commit comments

Comments
 (0)
Please sign in to comment.