Skip to content

Commit c1c7cff

Browse files
AlekSiAlekSi
authored
Merge pull request #85 from percona/PMM-1764-small-improvements
PMM-1764 Various small improvements
2 parents a65811a + f58598a commit c1c7cff

16 files changed

+658
-67
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ env:
1414
- MONGODB_IMAGE=mongo:3.4
1515
- MONGODB_IMAGE=mongo:3.6
1616
- MONGODB_IMAGE=percona/percona-server-mongodb:3.4
17+
- MONGODB_IMAGE=perconalab/percona-server-mongodb:3.6
1718

1819
services:
1920
- docker

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
## v0.4.0 (not released yet)
44

5+
* New flags `-collect.database` and `-collect.collection` can be used to enable collection of database and collection
6+
metrics. They are disabled by default.
7+
* MongoDB connections are now kept between the scrapes. New flag `-mongodb.max-connections` (with the default value `1`)
8+
controls the maximum number of established connections.
9+
* Add standard metrics:
10+
* `mongodb_scrape_errors_total`
11+
* `mongodb_up`
12+
* Some queries now contain [cursor comments](https://www.percona.com/blog/2017/06/21/tracing-mongodb-queries-to-code-with-cursor-comments/)
13+
with source code locations.
514
* Go vendoring switched to [dep](https://github.com/golang/dep).
615

716
## v0.3.1 (2017-09-08)

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ db.getSiblingDB("admin").createUser({
6464
export MONGODB_URL=mongodb://mongodb_exporter:s3cr3tpassw0rd@localhost:27017
6565
```
6666

67+
If you use [x.509 Certificates to Authenticate Clients](https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/), pass in username and `authMechanism` via [connection options](https://docs.mongodb.com/manual/reference/connection-string/#connections-connection-options) to the MongoDB uri. Eg:
68+
69+
```
70+
mongodb://CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry@localhost:27017/?authMechanism=MONGODB-X509
71+
```
72+
6773
## Note about how this works
6874

6975
Point the process to any mongo port and it will detect if it is a mongos, replicaset member, or stand alone mongod and return the appropriate metrics for that type of node. This was done to preent the need to an exporter per type of process.

‎collector/mongod/collections_status.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package collector_mongod
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"github.com/prometheus/common/log"
6+
"gopkg.in/mgo.v2"
7+
"gopkg.in/mgo.v2/bson"
8+
)
9+
10+
var (
11+
collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
12+
Namespace: Namespace,
13+
Subsystem: "db_coll",
14+
Name: "size",
15+
Help: "The total size in memory of all records in a collection",
16+
}, []string{"db", "coll"})
17+
collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
18+
Namespace: Namespace,
19+
Subsystem: "db_coll",
20+
Name: "count",
21+
Help: "The number of objects or documents in this collection",
22+
}, []string{"db", "coll"})
23+
collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
24+
Namespace: Namespace,
25+
Subsystem: "db_coll",
26+
Name: "avgobjsize",
27+
Help: "The average size of an object in the collection (plus any padding)",
28+
}, []string{"db", "coll"})
29+
collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
30+
Namespace: Namespace,
31+
Subsystem: "db_coll",
32+
Name: "storage_size",
33+
Help: "The total amount of storage allocated to this collection for document storage",
34+
}, []string{"db", "coll"})
35+
collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
36+
Namespace: Namespace,
37+
Subsystem: "db_coll",
38+
Name: "indexes",
39+
Help: "The number of indexes on the collection",
40+
}, []string{"db", "coll"})
41+
collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
42+
Namespace: Namespace,
43+
Subsystem: "db_coll",
44+
Name: "indexes_size",
45+
Help: "The total size of all indexes",
46+
}, []string{"db", "coll"})
47+
)
48+
49+
// CollectionStatList contains stats from all collections
50+
type CollectionStatList struct {
51+
Members []CollectionStatus
52+
}
53+
54+
// CollectionStatus represents stats about a collection in database (mongod and raw from mongos)
55+
type CollectionStatus struct {
56+
Database string
57+
Name string
58+
Size int `bson:"size,omitempty"`
59+
Count int `bson:"count,omitempty"`
60+
AvgObjSize int `bson:"avgObjSize,omitempty"`
61+
StorageSize int `bson:"storageSize,omitempty"`
62+
Indexes int `bson:"indexSizes,omitempty"`
63+
IndexesSize int `bson:"totalIndexSize,omitempty"`
64+
}
65+
66+
// Export exports database stats to prometheus
67+
func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) {
68+
for _, member := range collStatList.Members {
69+
ls := prometheus.Labels{
70+
"db": member.Database,
71+
"coll": member.Name,
72+
}
73+
collectionSize.With(ls).Set(float64(member.Size))
74+
collectionObjectCount.With(ls).Set(float64(member.Count))
75+
collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize))
76+
collectionStorageSize.With(ls).Set(float64(member.StorageSize))
77+
collectionIndexes.With(ls).Set(float64(member.Indexes))
78+
collectionIndexesSize.With(ls).Set(float64(member.IndexesSize))
79+
}
80+
collectionSize.Collect(ch)
81+
collectionObjectCount.Collect(ch)
82+
collectionAvgObjSize.Collect(ch)
83+
collectionStorageSize.Collect(ch)
84+
collectionIndexes.Collect(ch)
85+
collectionIndexesSize.Collect(ch)
86+
}
87+
88+
// Describe describes database stats for prometheus
89+
func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) {
90+
collectionSize.Describe(ch)
91+
collectionObjectCount.Describe(ch)
92+
collectionAvgObjSize.Describe(ch)
93+
collectionStorageSize.Describe(ch)
94+
collectionIndexes.Describe(ch)
95+
collectionIndexesSize.Describe(ch)
96+
}
97+
98+
// GetDatabaseStatus returns stats for a given database
99+
func GetCollectionStatList(session *mgo.Session) *CollectionStatList {
100+
collectionStatList := &CollectionStatList{}
101+
database_names, err := session.DatabaseNames()
102+
if err != nil {
103+
log.Error("Failed to get database names")
104+
return nil
105+
}
106+
for _, db := range database_names {
107+
collection_names, err := session.DB(db).CollectionNames()
108+
if err != nil {
109+
log.Error("Failed to get collection names for db=" + db)
110+
return nil
111+
}
112+
for _, collection_name := range collection_names {
113+
collStatus := CollectionStatus{}
114+
err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus)
115+
collStatus.Database = db
116+
collStatus.Name = collection_name
117+
if err != nil {
118+
log.Error("Failed to get collection status.")
119+
return nil
120+
}
121+
collectionStatList.Members = append(collectionStatList.Members, collStatus)
122+
}
123+
}
124+
125+
return collectionStatList
126+
}

‎collector/mongod/database_status.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package collector_mongod
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"github.com/prometheus/common/log"
6+
"gopkg.in/mgo.v2"
7+
"gopkg.in/mgo.v2/bson"
8+
)
9+
10+
var (
11+
indexSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
12+
Namespace: Namespace,
13+
Subsystem: "db",
14+
Name: "index_size_bytes",
15+
Help: "The total size in bytes of all indexes created on this database",
16+
}, []string{"db"})
17+
dataSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
18+
Namespace: Namespace,
19+
Subsystem: "db",
20+
Name: "data_size_bytes",
21+
Help: "The total size in bytes of the uncompressed data held in this database",
22+
}, []string{"db"})
23+
collectionsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
24+
Namespace: Namespace,
25+
Subsystem: "db",
26+
Name: "collections_total",
27+
Help: "Contains a count of the number of collections in that database",
28+
}, []string{"db"})
29+
indexesTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
30+
Namespace: Namespace,
31+
Subsystem: "db",
32+
Name: "indexes_total",
33+
Help: "Contains a count of the total number of indexes across all collections in the database",
34+
}, []string{"db"})
35+
objectsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
36+
Namespace: Namespace,
37+
Subsystem: "db",
38+
Name: "objects_total",
39+
Help: "Contains a count of the number of objects (i.e. documents) in the database across all collections",
40+
}, []string{"db"})
41+
)
42+
43+
// DatabaseStatList contains stats from all databases
44+
type DatabaseStatList struct {
45+
Members []DatabaseStatus
46+
}
47+
48+
// DatabaseStatus represents stats about a database (mongod and raw from mongos)
49+
type DatabaseStatus struct {
50+
Name string `bson:"db,omitempty"`
51+
IndexSize int `bson:"indexSize,omitempty"`
52+
DataSize int `bson:"dataSize,omitempty"`
53+
Collections int `bson:"collections,omitempty"`
54+
Objects int `bson:"objects,omitempty"`
55+
Indexes int `bson:"indexes,omitempty"`
56+
}
57+
58+
// Export exports database stats to prometheus
59+
func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) {
60+
for _, member := range dbStatList.Members {
61+
ls := prometheus.Labels{"db": member.Name}
62+
indexSize.With(ls).Set(float64(member.IndexSize))
63+
dataSize.With(ls).Set(float64(member.DataSize))
64+
collectionsTotal.With(ls).Set(float64(member.Collections))
65+
indexesTotal.With(ls).Set(float64(member.Indexes))
66+
objectsTotal.With(ls).Set(float64(member.Objects))
67+
}
68+
indexSize.Collect(ch)
69+
dataSize.Collect(ch)
70+
collectionsTotal.Collect(ch)
71+
indexesTotal.Collect(ch)
72+
objectsTotal.Collect(ch)
73+
74+
}
75+
76+
// Describe describes database stats for prometheus
77+
func (dbStatList *DatabaseStatList) Describe(ch chan<- *prometheus.Desc) {
78+
indexSize.Describe(ch)
79+
dataSize.Describe(ch)
80+
collectionsTotal.Describe(ch)
81+
indexesTotal.Describe(ch)
82+
objectsTotal.Describe(ch)
83+
}
84+
85+
// GetDatabaseStatList returns stats for all databases
86+
func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList {
87+
dbStatList := &DatabaseStatList{}
88+
database_names, err := session.DatabaseNames()
89+
if err != nil {
90+
log.Error("Failed to get database names")
91+
return nil
92+
}
93+
for _, db := range database_names {
94+
dbStatus := DatabaseStatus{}
95+
err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus)
96+
if err != nil {
97+
log.Error("Failed to get database status.")
98+
return nil
99+
}
100+
dbStatList.Members = append(dbStatList.Members, dbStatus)
101+
}
102+
103+
return dbStatList
104+
}

‎collector/mongod/global_lock.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,6 @@ var (
3131
Name: "total",
3232
Help: "The value of totalTime represents the time, in microseconds, since the database last started and creation of the globalLock. This is roughly equivalent to total server uptime",
3333
})
34-
globalLockLockTotal = prometheus.NewCounter(prometheus.CounterOpts{
35-
Namespace: Namespace,
36-
Subsystem: "global_lock",
37-
Name: "lock_total",
38-
Help: "The value of lockTime represents the time, in microseconds, since the database last started, that the globalLock has been held",
39-
})
4034
)
4135
var (
4236
globalLockCurrentQueue = prometheus.NewGaugeVec(prometheus.GaugeOpts{

‎collector/mongod/oplog_status.go‎

Lines changed: 25 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,13 @@ import (
1919
"github.com/prometheus/common/log"
2020
"gopkg.in/mgo.v2"
2121
"gopkg.in/mgo.v2/bson"
22+
23+
"github.com/percona/mongodb_exporter/shared"
2224
)
2325

2426
var (
27+
oplogDb = "local"
28+
oplogCollection = "oplog.rs"
2529
oplogStatusCount = prometheus.NewGauge(prometheus.GaugeOpts{
2630
Namespace: Namespace,
2731
Subsystem: "replset_oplog",
@@ -69,44 +73,34 @@ func BsonMongoTimestampToUnix(timestamp bson.MongoTimestamp) float64 {
6973
return float64(timestamp >> 32)
7074
}
7175

72-
func GetOplogTimestamps(session *mgo.Session) (*OplogTimestamps, error) {
73-
oplogTimestamps := &OplogTimestamps{}
74-
var err error
75-
76-
// retry once if there is an error
77-
var tries int64 = 0
78-
var head_result struct {
76+
func getOplogTailOrHeadTimestamp(session *mgo.Session, returnHead bool) (float64, error) {
77+
var result struct {
7978
Timestamp bson.MongoTimestamp `bson:"ts"`
8079
}
81-
for tries < 2 {
82-
err = session.DB("local").C("oplog.rs").Find(nil).Sort("-$natural").Limit(1).One(&head_result)
83-
if err == nil {
84-
break
85-
}
86-
tries += 1
87-
}
88-
if err != nil {
89-
return oplogTimestamps, err
90-
}
9180

92-
// retry once if there is an error
93-
tries = 0
94-
var tail_result struct {
95-
Timestamp bson.MongoTimestamp `bson:"ts"`
81+
var sortCond string = "$natural"
82+
if returnHead {
83+
sortCond = "-$natural"
9684
}
97-
for tries < 2 {
98-
err = session.DB("local").C("oplog.rs").Find(nil).Sort("$natural").Limit(1).One(&tail_result)
99-
if err == nil {
100-
break
101-
}
102-
tries += 1
85+
86+
findQuery := session.DB(oplogDb).C(oplogCollection).Find(nil).Sort(sortCond).Limit(1)
87+
err := shared.AddCodeCommentToQuery(findQuery).One(&result)
88+
return BsonMongoTimestampToUnix(result.Timestamp), err
89+
}
90+
91+
func GetOplogTimestamps(session *mgo.Session) (*OplogTimestamps, error) {
92+
headTs, err := getOplogTailOrHeadTimestamp(session, true)
93+
if err != nil {
94+
return nil, err
10395
}
96+
tailTs, err := getOplogTailOrHeadTimestamp(session, false)
10497
if err != nil {
105-
return oplogTimestamps, err
98+
return nil, err
99+
}
100+
oplogTimestamps := &OplogTimestamps{
101+
Head: headTs,
102+
Tail: tailTs,
106103
}
107-
108-
oplogTimestamps.Tail = BsonMongoTimestampToUnix(tail_result.Timestamp)
109-
oplogTimestamps.Head = BsonMongoTimestampToUnix(head_result.Timestamp)
110104
return oplogTimestamps, err
111105
}
112106

‎collector/mongod/replset_status.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,6 @@ var (
115115
Name: "member_config_version",
116116
Help: "The configVersion value is the replica set configuration version.",
117117
}, []string{"set", "name", "state"})
118-
memberOptime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
119-
Namespace: Namespace,
120-
Subsystem: subsystem,
121-
Name: "member_optime",
122-
Help: "Information regarding the last operation from the operation log that this member has applied.",
123-
}, []string{"set", "name", "state"})
124118
)
125119

126120
// ReplSetStatus keeps the data returned by the GetReplSetStatus method

0 commit comments

Comments
 (0)