Skip to content

Commit f313e60

Browse files
hasaketahasaketa
committed
Merge branch 'TSPROJ-7651'
2 parents 3966c69 + 048f989 commit f313e60

File tree

4 files changed

+267
-16
lines changed

4 files changed

+267
-16
lines changed

‎main.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,12 +119,20 @@ func main() {
119119
clustertopology.Dcrlog = &dcrlog
120120

121121
clustertopology.MongoshCapture.S = &cred
122+
123+
// discover all nodes of cluster
122124
err = clustertopology.GetAllNodes()
123125
if err != nil {
124126
dcrlog.Error(fmt.Sprintf("Error in Topology finding: %s", err.Error()))
125127
log.Fatal("Error in Topology finding cannot proceed aborting:", err)
126128
}
127129

130+
// dedup any mongo node entries because public/private hostnames point to same ip
131+
err = clustertopology.KeepUniqueNodes()
132+
if err != nil {
133+
dcrlog.Warn(fmt.Sprintf("Unable to filter for unique hostnames non-fatal: %s", err.Error()))
134+
}
135+
128136
for _, host := range clustertopology.Allnodes.Nodes {
129137

130138
dcrlog.Info(fmt.Sprintf("host: %s, port: %d", host.Hostname, host.Port))

‎topologyfinder/topologyfinder.go

Lines changed: 109 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ type ClusterNodes struct {
4040
// - Can find nodes that are part of ReplicaSet
4141
// - Can find nodes that are "not hidden", passives and arbiters
4242
// - Returns the hostname information that mongod has - could be PRIVATE hostnames as well!!!!
43+
// - If multiple hostnames point to same IP address only the unique IP is returned
4344

4445
type TopologyFinder struct {
4546
Allnodes ClusterNodes
@@ -87,14 +88,14 @@ func (tf *TopologyFinder) parseHelloOutput() error {
8788
var hostsArray []string
8889

8990
if err := json.Unmarshal(tf.GetHelloOutput.Bytes(), &hostsArray); err != nil {
90-
log.Fatalf("Error parsing hello output during topology discovery: %s", err)
91+
log.Fatalf("tftf - error parsing hello output during topology discovery: %s", err)
9192
}
9293

9394
for _, mongonodestring := range hostsArray {
9495

9596
mongonodeslice := strings.Split(mongonodestring, ":")
9697
if len(mongonodeslice) != 2 {
97-
log.Fatalf("Invalid mongo node string: %s", mongonodeslice)
98+
log.Fatalf("tftf - invalid mongo node string: %s", mongonodeslice)
9899
}
99100

100101
hostname := mongonodeslice[0]
@@ -103,7 +104,7 @@ func (tf *TopologyFinder) parseHelloOutput() error {
103104
port, err := strconv.Atoi(portStr)
104105
if err != nil {
105106
log.Fatalf(
106-
"In parseHelloOutput: Invalid port string format for node %s: %s ",
107+
"tftf - in parseHelloOutput: Invalid port string format for node %s: %s",
107108
mongonodestring,
108109
portStr,
109110
)
@@ -114,7 +115,7 @@ func (tf *TopologyFinder) parseHelloOutput() error {
114115
Port: port,
115116
}
116117

117-
tf.Dcrlog.Debug(fmt.Sprintf("appending node %s to allnodes list", mongonodestring))
118+
tf.Dcrlog.Debug(fmt.Sprintf("tftf - appending node %s to allnodes list", mongonodestring))
118119
tf.Allnodes.Nodes = append(tf.Allnodes.Nodes, mongonode)
119120

120121
}
@@ -124,38 +125,63 @@ func (tf *TopologyFinder) parseHelloOutput() error {
124125
func (tf *TopologyFinder) parseShardMapOutput() error {
125126
var shardMap map[string]interface{}
126127

127-
tf.Dcrlog.Debug("parsing shard cluster output")
128+
tf.Dcrlog.Debug("tftf - parsing shard cluster output")
128129
if err := json.Unmarshal(tf.GetShardMapOutput.Bytes(), &shardMap); err != nil {
129-
log.Fatalf("Error parsing shardmap output: %s", err)
130+
log.Fatalf("tftf - error parsing shardmap output: %s", err)
130131
}
131132

132-
tf.Dcrlog.Debug("extract hosts from shard cluster output")
133+
tf.Dcrlog.Debug("tftf - extract hosts from shard cluster output")
134+
135+
// hosts document is of format {'hostname1:port1' : 'shardn/config'... }
133136
allhosts, ok := shardMap["hosts"].(map[string]interface{})
134137
if !ok {
135-
log.Fatalf("error reading sharmap hosts document")
138+
log.Fatalf("tftf - error reading sharmap hosts document")
136139
}
137140

141+
tf.Dcrlog.Debug(fmt.Sprintf("tftf - parsing shard output allhosts is: %s", allhosts))
142+
// hosts document is of format {'hostname1:port1' : 'shardn/config'... }
143+
// ignore the values part only need the keys which are 'hostname1:port1' ...
138144
for mongonodestring := range allhosts {
139145

140146
mongonodeslice := strings.Split(mongonodestring, ":")
147+
tf.Dcrlog.Debug(
148+
fmt.Sprintf("tftf - parsing shard output mongonodestring is: %s", mongonodestring),
149+
)
150+
tf.Dcrlog.Debug(
151+
fmt.Sprintf("tftf - parsing shard output mongonodeslice is: %s", mongonodeslice),
152+
)
153+
tf.Dcrlog.Debug(
154+
fmt.Sprintf(
155+
"tftf - parsing shard output length of mongonodeslice is: %d",
156+
len(mongonodeslice),
157+
),
158+
)
141159
if len(mongonodeslice) != 2 {
142-
log.Fatalf("invalid mongo node string: %s", mongonodeslice)
160+
log.Fatalf("tftf - invalid mongo node string: %s", mongonodeslice)
143161
}
144162

163+
// hosts document is of format {'hostname1:port1' : 'shardn/config'... }
145164
hostname := mongonodeslice[0]
165+
tf.Dcrlog.Debug(fmt.Sprintf("tftf - parsing shard output hostname is: %s", hostname))
166+
146167
portStr := mongonodeslice[1]
168+
tf.Dcrlog.Debug(fmt.Sprintf("tftf - parsing shard output portStr is: %s", portStr))
147169

148170
port, err := strconv.Atoi(portStr)
149171
if err != nil {
150-
log.Fatalf("invalid port string format for node %s: %s ", mongonodestring, portStr)
172+
log.Fatalf(
173+
"tftf - invalid port string format for node %s: %s",
174+
mongonodestring,
175+
portStr,
176+
)
151177
}
152178

153179
mongonode := ClusterNode{
154180
Hostname: hostname,
155181
Port: port,
156182
}
157183

158-
tf.Dcrlog.Debug(fmt.Sprintf("appending node %s to allnodes list", mongonodestring))
184+
tf.Dcrlog.Debug(fmt.Sprintf("tftf - appending node %s to allnodes list", mongonodestring))
159185
tf.Allnodes.Nodes = append(tf.Allnodes.Nodes, mongonode)
160186

161187
}
@@ -168,17 +194,17 @@ func (tf *TopologyFinder) addSeedMongosNode() error {
168194
seedhostname := tf.MongoshCapture.S.Seedmongodhost
169195
isSeedHostinList := false
170196

171-
tf.Dcrlog.Debug("looking up seed node in the allnodes list")
197+
tf.Dcrlog.Debug("tftf - looking up seed node in the allnodes list")
172198
for _, host := range tf.Allnodes.Nodes {
173199
if seedhostname == string(host.Hostname) &&
174200
seedport == strconv.Itoa(host.Port) {
175201
isSeedHostinList = true
176-
tf.Dcrlog.Debug("found seed node in the allnodes list")
202+
tf.Dcrlog.Debug("tftf - found seed node in the allnodes list")
177203
}
178204
}
179205

180206
if !isSeedHostinList {
181-
tf.Dcrlog.Debug("seed node not in the list, adding seed node in the allnodes list")
207+
tf.Dcrlog.Debug("tftf - seed node not in the list, adding seed node in the allnodes list")
182208
err := tf.addSeedNode()
183209
if err != nil {
184210
return err
@@ -188,7 +214,7 @@ func (tf *TopologyFinder) addSeedMongosNode() error {
188214
}
189215

190216
func (tf *TopologyFinder) GetAllNodes() error {
191-
tf.Dcrlog.Debug("building allnodes list for data collection")
217+
tf.Dcrlog.Debug("tftf - building allnodes list for data collection")
192218
err := tf.runShardMapDBCommand()
193219
if err != nil {
194220
return err
@@ -199,7 +225,7 @@ func (tf *TopologyFinder) GetAllNodes() error {
199225
if tf.isShardMap() {
200226

201227
tf.Dcrlog.Debug(
202-
"we are connected to sharded cluster proceeding with extracting mongo hostnames",
228+
"tftf - we are connected to sharded cluster proceeding with extracting mongo hostnames",
203229
)
204230
err := tf.parseShardMapOutput()
205231
if err != nil {
@@ -221,6 +247,73 @@ func (tf *TopologyFinder) GetAllNodes() error {
221247
return nil
222248
}
223249

250+
// find and retain only unique nodes
251+
func (tf *TopologyFinder) KeepUniqueNodes() error {
252+
tf.Dcrlog.Debug("tftf - will attempt to find if multiple hostnames mapped to single IP")
253+
uniqueipfinder := UniqueIPfinder{}
254+
uniqueipfinder.Dcrlog = tf.Dcrlog
255+
uniqueipfinder.AllNodes = tf.Allnodes
256+
257+
// build list of strings with hostnames and port from the obtained Allnodes
258+
hostportList := make([]string, 0)
259+
for _, node := range tf.Allnodes.Nodes {
260+
hostportList = append(hostportList, node.Hostname+":"+fmt.Sprintf("%d", node.Port))
261+
}
262+
263+
// generate a set of unique IP address to possible multiple hostnames
264+
ipportTohostportset, err := uniqueipfinder.IpportTohostportMap(hostportList)
265+
if err != nil {
266+
// TODO: add a help message here
267+
// Allnodes is intact at this point
268+
return err
269+
}
270+
271+
allNodesNew := make([]ClusterNode, 0)
272+
tf.Allnodes.Nodes = allNodesNew
273+
274+
// make split the ip port string and build Allnodes again
275+
// if there are more than one hostname mapped to single ip keep the hostname which is not internal
276+
// Allnodes is rebuilt with the hostnames removing duplicates
277+
for ipportKey, hostportList := range ipportTohostportset {
278+
if len(hostportList) == 0 {
279+
tf.Dcrlog.Warn(fmt.Sprintf(
280+
"tftf - warn - the ipport %s has no corresponding hostname match hence skipping",
281+
ipportKey,
282+
))
283+
// TODO: fill uniqueHostname as the IP
284+
continue
285+
}
286+
287+
// of the possible multiple hostnames from the set keep only the first
288+
uniqueHostname, uniqueListenPort, err := splitHostPort(hostportList[0], tf.Dcrlog)
289+
if err != nil {
290+
tf.Dcrlog.Warn(
291+
fmt.Sprintf(
292+
"tftf - warn - unable to properly split hosport string: %s",
293+
hostportList[0],
294+
),
295+
)
296+
// TODO: fill uniqueHostname as the IP
297+
continue
298+
}
299+
300+
// add to the Allnodes
301+
mongonode := ClusterNode{
302+
Hostname: uniqueHostname,
303+
Port: uniqueListenPort,
304+
}
305+
306+
tf.Dcrlog.Debug(
307+
fmt.Sprintf("tftf - appending node %s to allnodes list", hostportList[0]),
308+
)
309+
allNodesNew = append(allNodesNew, mongonode)
310+
}
311+
312+
// replace existing Allnodes
313+
tf.Allnodes.Nodes = allNodesNew
314+
return nil
315+
}
316+
224317
func (tf *TopologyFinder) addSeedNode() error {
225318
seedport, err := strconv.Atoi(tf.MongoshCapture.S.Seedmongodport)
226319
if err != nil {

‎topologyfinder/uniqueIPfinder.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// Copyright 2023 MongoDB Inc
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package topologyfinder
16+
17+
import (
18+
"fmt"
19+
"net"
20+
"strconv"
21+
22+
"dcrcli/dcrlogger"
23+
)
24+
25+
type UniqueIPfinder struct {
26+
AllNodes ClusterNodes
27+
Dcrlog *dcrlogger.DCRLogger
28+
}
29+
30+
func (uf *UniqueIPfinder) netLookupIP(host string) ([]net.IP, error) {
31+
uf.Dcrlog.Debug(fmt.Sprintf("tfuf - looking up IPs for host: %s", host))
32+
addrs, err := net.LookupIP(host)
33+
if err != nil {
34+
uf.Dcrlog.Error(fmt.Sprintf("tfuf - lookup %s: %v", host, err))
35+
return nil, fmt.Errorf("tfuf - lookup %s: %v", host, err)
36+
}
37+
validAddrs := make([]net.IP, 0, len(addrs))
38+
for _, addr := range addrs {
39+
if addr.To4() != nil {
40+
uf.Dcrlog.Debug(
41+
fmt.Sprintf("tfuf - found ipv4 addr %s for host %s", addr.String(), host),
42+
)
43+
validAddrs = append(validAddrs, addr)
44+
}
45+
}
46+
return validAddrs, nil
47+
}
48+
49+
func (uf *UniqueIPfinder) IpportTohostportMap(hostportList []string) (map[string][]string, error) {
50+
// create an empty set
51+
// key : unique IP address
52+
// values : [hostname1, hostname2]
53+
ipportToHostPortSet := make(map[string][]string)
54+
55+
uf.Dcrlog.Debug(fmt.Sprintf("tfuf - hostPortList is : %s", hostportList))
56+
57+
for _, hostPort := range hostportList {
58+
uf.Dcrlog.Debug(fmt.Sprintf("tfuf - hostPort is : %s", hostPort))
59+
60+
hostname, listenPort, err := splitHostPort(hostPort, uf.Dcrlog)
61+
if err != nil {
62+
uf.Dcrlog.Warn(
63+
fmt.Sprintf(
64+
"tfuf - unable to properly split hostPort string: %s with err: %v",
65+
hostPort,
66+
err,
67+
),
68+
)
69+
// skip empty elements if any
70+
continue
71+
}
72+
73+
ipAddrs, err := uf.netLookupIP(hostname)
74+
// if one lookup fails skip that
75+
if err != nil {
76+
uf.Dcrlog.Warn(
77+
fmt.Sprintf("tfuf - lookup for hostname %s failed with err: %v", hostname,
78+
err),
79+
)
80+
continue
81+
}
82+
83+
for _, ipAddr := range ipAddrs {
84+
// join ip address and port to form a unique key
85+
ipPortKey := fmt.Sprintf("%s:%d", ipAddr.String(), listenPort)
86+
// create a set with ip+port as the key with possible multiple hostnames
87+
ipportToHostPortSet[ipPortKey] = append(
88+
ipportToHostPortSet[ipPortKey],
89+
hostname+":"+strconv.Itoa(listenPort),
90+
)
91+
}
92+
}
93+
94+
return ipportToHostPortSet, nil
95+
}

‎topologyfinder/utils.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright 2023 MongoDB Inc
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package topologyfinder
16+
17+
import (
18+
"fmt"
19+
"strconv"
20+
"strings"
21+
22+
"dcrcli/dcrlogger"
23+
)
24+
25+
func splitHostPort(hostPort string, logger *dcrlogger.DCRLogger) (string, int, error) {
26+
colonPos := strings.IndexByte(hostPort, ':')
27+
28+
logger.Debug(fmt.Sprintf("tf - sph - splitter colonPos: %d", colonPos))
29+
30+
if colonPos == -1 {
31+
logger.Debug(fmt.Sprintf("tf - sph - error in splitting string on colon: %d", colonPos))
32+
return "", -1, fmt.Errorf("tf - sph - failed to parse hostport string")
33+
}
34+
35+
hostname := hostPort[:colonPos]
36+
listenPort := hostPort[colonPos+1:]
37+
38+
port, err := strconv.Atoi(listenPort)
39+
if err != nil {
40+
logger.Debug(
41+
fmt.Sprintf(
42+
"tf - sph - error in converting listenPort string to int (host, port): (%s, %s)",
43+
hostname,
44+
listenPort,
45+
),
46+
)
47+
return "", -1, fmt.Errorf(
48+
"tf - sph - invalid port string format for node %s: %s ",
49+
hostname,
50+
listenPort,
51+
)
52+
}
53+
54+
return hostname, port, nil
55+
}

0 commit comments

Comments
 (0)