Splunkenterprisereceiver add health metric (#36695) (#36695)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

Adds `splunk.health` metric that uses the introspection endpoint at
`/services/server/health/splunkd/details`. The metric features 2
attributes:

`splunk.feature`: Describes the part of the server's feature
`splunk.feature.health`: Describes (in red, yellow, green) the health
status of the given feature

The metric will remain a value of 1 unless the feature's health becomes
'red.'

<!--Describe what testing was performed and which tests were added.-->
#### Testing

Built a custom version of splunk-otel-collector that referenced this
code. Then ran it on a Splunk stack, and received metrics out. Also
created generated tests.

<!--Describe the documentation added.-->
#### Documentation

Added an entry to `documentation.md` describing the new metric.

---------

Co-authored-by: Sean Marciniak <[email protected]>

.chloggen/changes.yaml

@@ -0,0 +1,22 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'enhancement'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: 'splunkenterprisereceiver'
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Added a new `splunk.health` metric."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36695]
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

receiver/splunkenterprisereceiver/documentation.md

@@ -390,6 +390,21 @@ Size in bytes on disk of this index *Note:** Must be pointed at specific indexer
 | ---- | ----------- | ------ |
 | splunk.index.name | The name of the index reporting a specific KPI | Any Str |
 
+### splunk.health
+
+The status ('red', 'yellow', or 'green') of the Splunk server. Health of 'red' produces a 0 while all other colors produce a 1.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| {status} | Gauge | Int |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| splunk.feature | The Feature name from the Splunk Health Introspection Endpoint | Any Str |
+| splunk.feature.health | The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint | Any Str |
+
 ### splunk.indexer.throughput
 
 Gauge tracking average bytes per second throughput of indexer. *Note:** Must be pointed at specific indexer `endpoint` and gathers metrics from only that indexer.

receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml

@@ -19,6 +19,8 @@ all_set:
       enabled: true
     splunk.data.indexes.extended.total.size:
       enabled: true
+    splunk.health:
+      enabled: true
     splunk.indexer.avg.rate:
       enabled: true
     splunk.indexer.cpu.time:
@@ -101,6 +103,8 @@ none_set:
       enabled: false
     splunk.data.indexes.extended.total.size:
       enabled: false
+    splunk.health:
+      enabled: false
     splunk.indexer.avg.rate:
       enabled: false
     splunk.indexer.cpu.time:

receiver/splunkenterprisereceiver/metadata.yaml

@@ -39,6 +39,12 @@ attributes:
   splunk.searchartifacts.cache.type:
     description: The search artifacts cache type
     type: string
+  splunk.feature:
+    description: The Feature name from the Splunk Health Introspection Endpoint
+    type: string
+  splunk.feature.health:
+    description: The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint
+    type: string
 
 metrics:
   splunk.license.index.usage:
@@ -345,6 +351,14 @@ metrics:
       aggregation_temporality: cumulative
       value_type: int
     attributes: [splunk.host]
+  #`services/server/health/splunkd/details`
+  splunk.health:
+    enabled: false
+    description: The status ('red', 'yellow', or 'green') of the Splunk server. Health of 'red' produces a 0 while all other colors produce a 1.
+    unit: "{status}"
+    gauge:
+      value_type: int
+    attributes: [splunk.feature, splunk.feature.health]
 
 tests:
   config:

receiver/splunkenterprisereceiver/scraper.go

@@ -101,6 +101,7 @@ func (s *splunkScraper) scrape(ctx context.Context) (pmetric.Metrics, error) {
 		s.scrapeIndexerAvgRate,
 		s.scrapeKVStoreStatus,
 		s.scrapeSearchArtifacts,
+		s.scrapeHealth,
 	}
 	errChan := make(chan error, len(metricScrapes))
 
@@ -1075,12 +1076,12 @@ func unmarshallSearchReq(res *http.Response, sr *searchResponse) error {
 
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("Failed to read response: %w", err)
+		return fmt.Errorf("failed to read response: %w", err)
 	}
 
 	err = xml.Unmarshal(body, &sr)
 	if err != nil {
-		return fmt.Errorf("Failed to unmarshall response: %w", err)
+		return fmt.Errorf("failed to unmarshall response: %w", err)
 	}
 
 	return nil
@@ -1733,3 +1734,55 @@ func (s *splunkScraper) scrapeSearchArtifacts(ctx context.Context, now pcommon.T
 		}
 	}
 }
+
+// Scrape Health Introspection Endpoint
+func (s *splunkScraper) scrapeHealth(ctx context.Context, now pcommon.Timestamp, errs chan error) {
+	if !s.conf.MetricsBuilderConfig.Metrics.SplunkHealth.Enabled {
+		return
+	}
+
+	ctx = context.WithValue(ctx, endpointType("type"), typeCm)
+
+	ept := apiDict[`SplunkHealth`]
+	var ha healthArtifacts
+
+	req, err := s.splunkClient.createAPIRequest(ctx, ept)
+	if err != nil {
+		errs <- err
+		return
+	}
+
+	res, err := s.splunkClient.makeRequest(req)
+	if err != nil {
+		errs <- err
+		return
+	}
+	defer res.Body.Close()
+
+	if err := json.NewDecoder(res.Body).Decode(&ha); err != nil {
+		errs <- err
+		return
+	}
+
+	s.settings.Logger.Debug(fmt.Sprintf("Features: %s", ha.Entries))
+	for _, details := range ha.Entries {
+		s.traverseHealthDetailFeatures(details.Content, now)
+	}
+}
+
+func (s *splunkScraper) traverseHealthDetailFeatures(details healthDetails, now pcommon.Timestamp) {
+	if details.Features == nil {
+		return
+	}
+
+	for k, feature := range details.Features {
+		if feature.Health != "red" {
+			s.settings.Logger.Debug(feature.Health)
+			s.mb.RecordSplunkHealthDataPoint(now, 1, k, feature.Health)
+		} else {
+			s.settings.Logger.Debug(feature.Health)
+			s.mb.RecordSplunkHealthDataPoint(now, 0, k, feature.Health)
+		}
+		s.traverseHealthDetailFeatures(feature, now)
+	}
+}

receiver/splunkenterprisereceiver/search_result.go

@@ -25,6 +25,7 @@ var apiDict = map[string]string{
 	`SplunkIntrospectionQueues`: `/services/server/introspection/queues?output_mode=json&count=-1`,
 	`SplunkKVStoreStatus`:       `/services/kvstore/status?output_mode=json`,
 	`SplunkDispatchArtifacts`:   `/services/server/status/dispatch-artifacts?output_mode=json&count=-1`,
+	`SplunkHealth`:              `/services/server/health/splunkd/details?output_mode=json`,
 }
 
 type searchResponse struct {
@@ -156,3 +157,17 @@ type DispatchArtifactContent struct {
 	StatusCacheSize    string `json:"cached_job_status_status_csv_size_mb"`
 	CacheTotalEntries  string `json:"cached_job_status_total_entries"`
 }
+
+// '/services/server/health/splunkd/details
+type healthArtifacts struct {
+	Entries []healthArtifactEntry `json:"entry"`
+}
+
+type healthArtifactEntry struct {
+	Content healthDetails `json:"content"`
+}
+
+type healthDetails struct {
+	Health   string                   `json:"health"`
+	Features map[string]healthDetails `json:"features,omitempty"`
+}