Fix: geo_points can be imported from ECS (#476)

Author: mtojek

Diff for: internal/fields/validate.go

@@ -265,8 +265,9 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool {
 	k = strings.ReplaceAll(k, "*", "[^.]+")
 
 	// Workaround for potential geo_point, as "lon" and "lat" fields are not present in field definitions.
-	if def.Type == "geo_point" {
-		k += "\\.(lon|lat)"
+	// Unfortunately we have to assume that imported field could be a geo_point (nasty workaround).
+	if def.Type == "geo_point" || def.External != "" {
+		k += "(\\.lon|\\.lat|)"
 	}
 
 	k = fmt.Sprintf("^%s$", k)

Diff for: internal/fields/validate_test.go

@@ -223,3 +223,14 @@ func readSampleEvent(t *testing.T, path string) json.RawMessage {
 	require.NoError(t, err)
 	return c
 }
+
+func TestValidate_geo_point(t *testing.T) {
+	validator, err := CreateValidatorForDataStream("../../test/packages/fields_tests/data_stream/first")
+
+	require.NoError(t, err)
+	require.NotNil(t, validator)
+
+	e := readSampleEvent(t, "../../test/packages/fields_tests/data_stream/first/sample_event.json")
+	errs := validator.ValidateDocumentBody(e)
+	require.Empty(t, errs)
+}

Diff for: test/packages/fields_tests/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

Diff for: test/packages/fields_tests/_dev/build/docs/README.md

@@ -0,0 +1,5 @@
+# Fields Tests
+
+{{event "first"}}
+
+{{fields "first"}}

Diff for: test/packages/fields_tests/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/0 # FIXME Replace with the real PR link

Diff for: test/packages/fields_tests/data_stream/first/agent/stream/stream.yml.hbs

@@ -0,0 +1,7 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~

Diff for: test/packages/fields_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,10 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- set:
+    field: sample_field
+    value: "1"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

Diff for: test/packages/fields_tests/data_stream/first/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

Diff for: test/packages/fields_tests/data_stream/first/fields/geo-fields.yml

@@ -0,0 +1,6 @@
+- name: destination.geo.location
+  external: ecs
+- name: geo.location
+  external: ecs
+- name: source.geo.location
+  external: ecs

Diff for: test/packages/fields_tests/data_stream/first/manifest.yml

@@ -0,0 +1,13 @@
+title: "First"
+type: logs
+streams:
+  - input: logfile
+    title: Sample logs
+    description: Collect sample logs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        default:
+          - /var/log/*.log

Diff for: test/packages/fields_tests/data_stream/first/sample_event.json

@@ -0,0 +1,8 @@
+{
+    "source.geo.location": {
+        "lat": 1.0,
+        "lon": "2.0"
+    },
+    "geo.location.lat": 3.0,
+    "geo.location.lon": 4.0
+}

Diff for: test/packages/fields_tests/docs/README.md

@@ -0,0 +1,26 @@
+# Fields Tests
+
+An example event for `first` looks as following:
+
+```json
+{
+    "source.geo.location": {
+        "lat": 1.0,
+        "lon": "2.0"
+    },
+    "geo.location.lat": 3.0,
+    "geo.location.lon": 4.0
+}
+```
+
+**Exported fields**
+
+| Field | Description | Type |
+|---|---|---|
+| @timestamp | Event timestamp. | date |
+| data_stream.dataset | Data stream dataset. | constant_keyword |
+| data_stream.namespace | Data stream namespace. | constant_keyword |
+| data_stream.type | Data stream type. | constant_keyword |
+| destination.geo.location | Longitude and latitude. | geo_point |
+| geo.location | Longitude and latitude. | geo_point |
+| source.geo.location | Longitude and latitude. | geo_point |

Diff for: test/packages/fields_tests/manifest.yml

@@ -0,0 +1,22 @@
+format_version: 1.0.0
+name: fields_tests
+title: "Fields Tests"
+version: 0.0.1
+license: basic
+description: "These are tests of field validation."
+type: integration
+categories:
+  - custom
+release: experimental
+conditions:
+  kibana.version: "^7.15.0"
+policy_templates:
+  - name: sample
+    title: Sample logs
+    description: Collect sample logs
+    inputs:
+      - type: logfile
+        title: Collect sample logs from instances
+        description: Collecting sample logs
+owner:
+  github: elastic/integrations