fixing thanos querier dedup issue causing incorrect/high values when …

pkg/dedup/iter.go

@@ -5,6 +5,7 @@
 
 import (
 	"math"
+	"strings"
 
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
@@ -83,7 +84,27 @@
 		currMinTime := chunks[i].MinTime
 		for ri := range o.replicas {
 			if len(o.replicas[ri]) == 0 || o.replicas[ri][len(o.replicas[ri])-1].MaxTime < currMinTime {
-				o.replicas[ri] = append(o.replicas[ri], chunks[i])
+				//Approach 1, Just comment below line to not pick sample causing issue
+				//o.replicas[ri] = append(o.replicas[ri], chunks[i])
+
+				//Approach 2, whenever lower timestamps has higher values don't pick it if it is a counter metric, not filtering it is causing incorrect/
+				//higher values over rate/increase functions which is causing false alerts
+				if len(o.replicas[ri]) != 0 && (strings.HasSuffix(o.currLabels[0].Value, "count") ||
+					strings.HasSuffix(o.currLabels[0].Value, "sum") ||
+					strings.HasSuffix(o.currLabels[0].Value, "total")) {
+
+					chk, _ := chunkenc.FromData(chunkenc.EncXOR, o.replicas[ri][len(o.replicas[ri])-1].Raw.Data)
+					chk2, _ := chunkenc.FromData(chunkenc.EncXOR, chunks[i].Raw.Data)
+					samples := expandChunk(chk.Iterator(nil))
+					samples2 := expandChunk(chk2.Iterator(nil))
+
+					if samples[len(samples)-1].t < samples2[len(samples2)-1].t && samples[len(samples)-1].v < samples2[len(samples2)-1].v {
+						o.replicas[ri] = append(o.replicas[ri], chunks[i])
+					}
+				} else {
+					o.replicas[ri] = append(o.replicas[ri], chunks[i])
+				}
+
 				continue chunksLoop
 			}
 		}
@@ -92,6 +113,19 @@
 	return true
 }
 
+type dupSample struct {
+	t int64
+	v float64
+}
+
+func expandChunk(cit chunkenc.Iterator) (res []dupSample) {
+	for cit.Next() != chunkenc.ValNone {
+		t, v := cit.At()
+		res = append(res, dupSample{t, v})
+	}
+	return res
+}
+
 func (o *overlapSplitSet) At() (labels.Labels, []storepb.AggrChunk) {
 	return o.currLabels, o.replicas[o.currI]
 }