Merge pull request #89 from percona/develop

Version 0.4.0

Author: AlekSi

.travis.yml

@@ -3,7 +3,6 @@ sudo: required
 language: go
 
 go:
-  - 1.8.x
   - 1.9.x
   - master
 
@@ -13,7 +12,9 @@ matrix:
 
 env:
   - MONGODB_IMAGE=mongo:3.4
+  - MONGODB_IMAGE=mongo:3.6
   - MONGODB_IMAGE=percona/percona-server-mongodb:3.4
+  - MONGODB_IMAGE=perconalab/percona-server-mongodb:3.6
 
 services:
   - docker

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v0.4.0 (2017-01-17)
+
+* New flags `-collect.database` and `-collect.collection` can be used to enable collection of database and collection
+  metrics. They are disabled by default.
+* MongoDB connections are now kept between the scrapes. New flag `-mongodb.max-connections` (with the default value `1`)
+  controls the maximum number of established connections.
+* Add standard metrics:
+  * `mongodb_scrape_errors_total`
+  * `mongodb_up`
+* Some queries now contain [cursor comments](https://www.percona.com/blog/2017/06/21/tracing-mongodb-queries-to-code-with-cursor-comments/)
+  with source code locations.
+* Go vendoring switched to [dep](https://github.com/golang/dep).
+
 ## v0.3.1 (2017-09-08)
 
 * Better logging for scrape errors.

CONTRIBUTING.md

@@ -16,5 +16,5 @@ export MONGODB_URL='mongodb://localhost:27017'
 
 ## Vendoring
 
-We use [Glide](https://glide.sh) to vendor dependencies. Please use released version of Glide (i.e. do not `go get`
-from `master` branch). Also please use `--strip-vendor` flag.
+We use [dep](https://github.com/golang/dep) to vendor dependencies.
+Please use released version of dep (i.e. do not `go get` from `master` branch).

Gopkg.lock

@@ -0,0 +1,20 @@
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"

Gopkg.toml

@@ -64,6 +64,12 @@ db.getSiblingDB("admin").createUser({
 export MONGODB_URL=mongodb://mongodb_exporter:s3cr3tpassw0rd@localhost:27017
 ```
 
+If you use [x.509 Certificates to Authenticate Clients](https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/), pass in username and `authMechanism` via [connection options](https://docs.mongodb.com/manual/reference/connection-string/#connections-connection-options) to the MongoDB uri. Eg:
+
+```
+mongodb://CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry@localhost:27017/?authMechanism=MONGODB-X509
+```
+
 ## Note about how this works
 
 Point the process to any mongo port and it will detect if it is a mongos, replicaset member, or stand alone mongod and return the appropriate metrics for that type of node. This was done to preent the need to an exporter per type of process.

README.md

@@ -1 +1 @@
-0.3.2
+0.4.0

VERSION

@@ -0,0 +1,126 @@
+package collector_mongod
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+)
+
+var (
+	collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "size",
+		Help:      "The total size in memory of all records in a collection",
+	}, []string{"db", "coll"})
+	collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "count",
+		Help:      "The number of objects or documents in this collection",
+	}, []string{"db", "coll"})
+	collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "avgobjsize",
+		Help:      "The average size of an object in the collection (plus any padding)",
+	}, []string{"db", "coll"})
+	collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "storage_size",
+		Help:      "The total amount of storage allocated to this collection for document storage",
+	}, []string{"db", "coll"})
+	collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "indexes",
+		Help:      "The number of indexes on the collection",
+	}, []string{"db", "coll"})
+	collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "indexes_size",
+		Help:      "The total size of all indexes",
+	}, []string{"db", "coll"})
+)
+
+// CollectionStatList contains stats from all collections
+type CollectionStatList struct {
+	Members []CollectionStatus
+}
+
+// CollectionStatus represents stats about a collection in database (mongod and raw from mongos)
+type CollectionStatus struct {
+	Database    string
+	Name        string
+	Size        int `bson:"size,omitempty"`
+	Count       int `bson:"count,omitempty"`
+	AvgObjSize  int `bson:"avgObjSize,omitempty"`
+	StorageSize int `bson:"storageSize,omitempty"`
+	Indexes     int `bson:"indexSizes,omitempty"`
+	IndexesSize int `bson:"totalIndexSize,omitempty"`
+}
+
+// Export exports database stats to prometheus
+func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) {
+	for _, member := range collStatList.Members {
+		ls := prometheus.Labels{
+			"db":   member.Database,
+			"coll": member.Name,
+		}
+		collectionSize.With(ls).Set(float64(member.Size))
+		collectionObjectCount.With(ls).Set(float64(member.Count))
+		collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize))
+		collectionStorageSize.With(ls).Set(float64(member.StorageSize))
+		collectionIndexes.With(ls).Set(float64(member.Indexes))
+		collectionIndexesSize.With(ls).Set(float64(member.IndexesSize))
+	}
+	collectionSize.Collect(ch)
+	collectionObjectCount.Collect(ch)
+	collectionAvgObjSize.Collect(ch)
+	collectionStorageSize.Collect(ch)
+	collectionIndexes.Collect(ch)
+	collectionIndexesSize.Collect(ch)
+}
+
+// Describe describes database stats for prometheus
+func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) {
+	collectionSize.Describe(ch)
+	collectionObjectCount.Describe(ch)
+	collectionAvgObjSize.Describe(ch)
+	collectionStorageSize.Describe(ch)
+	collectionIndexes.Describe(ch)
+	collectionIndexesSize.Describe(ch)
+}
+
+// GetDatabaseStatus returns stats for a given database
+func GetCollectionStatList(session *mgo.Session) *CollectionStatList {
+	collectionStatList := &CollectionStatList{}
+	database_names, err := session.DatabaseNames()
+	if err != nil {
+		log.Error("Failed to get database names")
+		return nil
+	}
+	for _, db := range database_names {
+		collection_names, err := session.DB(db).CollectionNames()
+		if err != nil {
+			log.Error("Failed to get collection names for db=" + db)
+			return nil
+		}
+		for _, collection_name := range collection_names {
+			collStatus := CollectionStatus{}
+			err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus)
+			collStatus.Database = db
+			collStatus.Name = collection_name
+			if err != nil {
+				log.Error("Failed to get collection status.")
+				return nil
+			}
+			collectionStatList.Members = append(collectionStatList.Members, collStatus)
+		}
+	}
+
+	return collectionStatList
+}