Improve Prombench query behaviour (#618)

* Prombench: scrape cAdvisor metrics So that queries for container_memory_rss return some data. Signed-off-by: Bryan Boreham <[email protected]> * Prombench: adjust load-gen queries In some cases reduce the amount of data returned; in others reduce the frequency so there is a gap between one batch and the next. node_cpu has been renamed to node_cpu_seconds_total. Signed-off-by: Bryan Boreham <[email protected]> --------- Signed-off-by: Bryan Boreham <[email protected]>
prometheus · Nov 23, 2023 · ce97176 · ce97176
1 parent 2198f38
commit ce97176
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 9 deletions.
diff --git a/prombench/manifests/prombench/benchmark/3a_prometheus-test_configmap.yaml b/prombench/manifests/prombench/benchmark/3a_prometheus-test_configmap.yaml
@@ -30,6 +30,27 @@ data:
         target_label: __metrics_path__
         replacement: /api/v1/nodes/${1}/proxy/metrics
 
+    - job_name: cadvisor
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+      kubernetes_sd_configs:
+      - role: node
+      relabel_configs:
+      - action: keep
+        source_labels: [__meta_kubernetes_node_label_cloud_google_com_gke_nodepool]
+        regex: prometheus-{{ .PR_NUMBER }}|nodes-{{ .PR_NUMBER }}
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+
     - job_name: node-exporters
       kubernetes_sd_configs:
       - role: endpoints

diff --git a/prombench/manifests/prombench/benchmark/6_loadgen.yaml b/prombench/manifests/prombench/benchmark/6_loadgen.yaml
@@ -8,40 +8,40 @@ data:
     querier:
       groups:
       - name: simple_range
-        interval: 2s
+        interval: 10s
         type: range
         start: 2h
         end: 1h
         step: 15s
         queries:
         - expr: go_goroutines
         - expr: container_memory_rss
-        - expr: kube_pod_container_info
+        - expr: kubelet_running_pods
         - expr: codelab_api_http_requests_in_progress
-        - expr: codelab_api_requests_total
+        - expr: 'codelab_api_requests_total{method="GET",path="/api/bar",status="200"}'
       - name: aggr_instant
         interval: 5s
         type: instant
         queries:
         - expr: sum by(image) (container_memory_rss)
-        - expr: sum by(instance) (rate(node_cpu{mode!="idle"}[5m]))
-        - expr: sum by(instance) (rate(node_cpu[5m]))
+        - expr: sum by(instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))
+        - expr: sum by(instance) (rate(node_cpu_seconds_total[5m]))
         - expr: sum by(instance) (rate(codelab_api_requests_total[5m]))
         - expr: sum by(instance) (rate(codelab_api_requests_total{method=~"GET|POST"}[5m]))
       - name: aggr_range
-        interval: 10s
+        interval: 30s
         type: range
         start: 1h
         end: 0h
         step: 15s
         queries:
         - expr: sum by(image) (container_memory_rss)
-        - expr: sum by(instance) (rate(node_cpu{mode!="idle"}[5m]))
-        - expr: sum by(instance) (rate(node_cpu[5m]))
+        - expr: sum by(instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))
+        - expr: sum by(instance) (rate(node_cpu_seconds_total[5m]))
         - expr: sum by(instance) (rate(codelab_api_requests_total[5m]))
         - expr: sum by(instance) (rate(codelab_api_requests_total{method=~"GET|POST"}[5m]))
       - name: heavy_instant
-        interval: 10s
+        interval: 60s
         queries:
         - expr: rate(codelab_api_requests_total{method=~"GET|POST"}[5m])
         - expr: sum without(instance) (rate(codelab_api_requests_total{method=~"GET|POST"}[5m]))