-
Notifications
You must be signed in to change notification settings - Fork 2.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[elasticsearchexporter] [chore] hasher interface
Extract a dataPointHasher interface from the mappingModel interface, so we can make the latter purely about encoding, and separate concerns.
- Loading branch information
Showing
4 changed files
with
134 additions
and
111 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package elasticsearchexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter" | ||
|
|
||
| import ( | ||
| "encoding/binary" | ||
| "hash" | ||
| "hash/fnv" | ||
| "math" | ||
| "slices" | ||
|
|
||
| "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter/internal/datapoints" | ||
| "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter/internal/elasticsearch" | ||
| "go.opentelemetry.io/collector/pdata/pcommon" | ||
| ) | ||
|
|
||
| // dataPointHasher is an interface for hashing data points by their identity, | ||
| // for grouping into a single document. | ||
| type dataPointHasher interface { | ||
| hashDataPoint(datapoints.DataPoint) uint32 | ||
| } | ||
|
|
||
| func newDataPointHasher(mode MappingMode) dataPointHasher { | ||
| switch mode { | ||
| case MappingOTel: | ||
| return otelDataPointHasher{} | ||
| default: | ||
| // // Defaults to ECS for backward compatibility | ||
| return ecsDataPointHasher{} | ||
| } | ||
| } | ||
|
|
||
| // TODO use https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/internal/exp/metrics/identity | ||
|
|
||
| type ecsDataPointHasher struct{} | ||
| type otelDataPointHasher struct{} | ||
|
|
||
| func (h ecsDataPointHasher) hashDataPoint(dp datapoints.DataPoint) uint32 { | ||
| hasher := fnv.New32a() | ||
|
|
||
| timestampBuf := make([]byte, 8) | ||
| binary.LittleEndian.PutUint64(timestampBuf, uint64(dp.Timestamp())) | ||
| hasher.Write(timestampBuf) | ||
|
|
||
| mapHashExcludeReservedAttrs(hasher, dp.Attributes()) | ||
|
|
||
| return hasher.Sum32() | ||
| } | ||
|
|
||
| func (h otelDataPointHasher) hashDataPoint(dp datapoints.DataPoint) uint32 { | ||
| hasher := fnv.New32a() | ||
|
|
||
| timestampBuf := make([]byte, 8) | ||
| binary.LittleEndian.PutUint64(timestampBuf, uint64(dp.Timestamp())) | ||
| hasher.Write(timestampBuf) | ||
|
|
||
| binary.LittleEndian.PutUint64(timestampBuf, uint64(dp.StartTimestamp())) | ||
| hasher.Write(timestampBuf) | ||
|
|
||
| hasher.Write([]byte(dp.Metric().Unit())) | ||
|
|
||
| mapHashExcludeReservedAttrs(hasher, dp.Attributes(), elasticsearch.MappingHintsAttrKey) | ||
|
|
||
| return hasher.Sum32() | ||
| } | ||
|
|
||
| // mapHashExcludeReservedAttrs is mapHash but ignoring some reserved attributes. | ||
| // e.g. index is already considered during routing and DS attributes do not need to be considered in hashing | ||
| func mapHashExcludeReservedAttrs(hasher hash.Hash, m pcommon.Map, extra ...string) { | ||
| m.Range(func(k string, v pcommon.Value) bool { | ||
| switch k { | ||
| case elasticsearch.DataStreamType, elasticsearch.DataStreamDataset, elasticsearch.DataStreamNamespace: | ||
| return true | ||
| } | ||
| if slices.Contains(extra, k) { | ||
| return true | ||
| } | ||
| hasher.Write([]byte(k)) | ||
| valueHash(hasher, v) | ||
|
|
||
| return true | ||
| }) | ||
| } | ||
|
|
||
| func mapHash(hasher hash.Hash, m pcommon.Map) { | ||
| m.Range(func(k string, v pcommon.Value) bool { | ||
| hasher.Write([]byte(k)) | ||
| valueHash(hasher, v) | ||
|
|
||
| return true | ||
| }) | ||
| } | ||
|
|
||
| func valueHash(h hash.Hash, v pcommon.Value) { | ||
| switch v.Type() { | ||
| case pcommon.ValueTypeEmpty: | ||
| h.Write([]byte{0}) | ||
| case pcommon.ValueTypeStr: | ||
| h.Write([]byte(v.Str())) | ||
| case pcommon.ValueTypeBool: | ||
| if v.Bool() { | ||
| h.Write([]byte{1}) | ||
| } else { | ||
| h.Write([]byte{0}) | ||
| } | ||
| case pcommon.ValueTypeDouble: | ||
| buf := make([]byte, 8) | ||
| binary.LittleEndian.PutUint64(buf, math.Float64bits(v.Double())) | ||
| h.Write(buf) | ||
| case pcommon.ValueTypeInt: | ||
| buf := make([]byte, 8) | ||
| binary.LittleEndian.PutUint64(buf, uint64(v.Int())) | ||
| h.Write(buf) | ||
| case pcommon.ValueTypeBytes: | ||
| h.Write(v.Bytes().AsRaw()) | ||
| case pcommon.ValueTypeMap: | ||
| mapHash(h, v.Map()) | ||
| case pcommon.ValueTypeSlice: | ||
| sliceHash(h, v.Slice()) | ||
| } | ||
| } | ||
|
|
||
| func sliceHash(h hash.Hash, s pcommon.Slice) { | ||
| for i := 0; i < s.Len(); i++ { | ||
| valueHash(h, s.At(i)) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters