From 6b885bf05765032d594f3e9317661cbf977023cd Mon Sep 17 00:00:00 2001 From: Yipeng Zhao Date: Fri, 13 Dec 2019 14:48:54 +0800 Subject: [PATCH 1/3] minor: update spec file for newly provided lite mmdb file from geolite.maxmind.com --- spec/filters/geoip_spec.rb | 6 +++--- vendor.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/filters/geoip_spec.rb b/spec/filters/geoip_spec.rb index c5b9202..93eb70f 100644 --- a/spec/filters/geoip_spec.rb +++ b/spec/filters/geoip_spec.rb @@ -93,8 +93,8 @@ it "should set other subfields of 'target' properly" do expect(event.get("target").to_hash.keys.sort).to eq(["city_name", "ip", "region_name"]) - expect(event.get("[target][city_name]")).to eq("Nashua") - expect(event.get("[target][region_name]")).to eq("New Hampshire") + expect(event.get("[target][city_name]")).to eq("Brighton") + expect(event.get("[target][region_name]")).to eq("Massachusetts") end end @@ -248,7 +248,7 @@ expect(event.get("geoip")).not_to be_empty expect(event.get("geoip")["ip"]).to eq("2607:f0d0:1002:51:0:0:0:4") expect(event.get("geoip").to_hash.keys.sort).to eq( - ["continent_code", "country_code2", "country_code3", "country_name", "ip", "latitude", "location", "longitude", "timezone"] + ["city_name", "continent_code", "country_code2", "country_code3", "country_name", "dma_code", "ip", "latitude", "location", "longitude", "postal_code", "region_code", "region_name", "timezone"] ) end end diff --git a/vendor.json b/vendor.json index 492af5a..574fe1e 100644 --- a/vendor.json +++ b/vendor.json @@ -1,7 +1,7 @@ [ { "url": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz", - "sha1": "8632d2abf2672fb33aec3845d0dcbb82986d1b4a" + "sha1": "f3c9d9c213de73c9b89ab2a965e022b2d4ce38a7" }, { "url": "https://s3.amazonaws.com/download.elasticsearch.org/logstash/maxmind/GeoLite2-ASN.mmdb", From 256917829f6e93bbdfdafd4f8c17e1790eb525a7 Mon Sep 17 00:00:00 2001 From: Yipeng Zhao Date: Fri, 13 Dec 2019 14:59:30 +0800 Subject: [PATCH 2/3] add support locales as arguments passed into plugin --- gradle/wrapper/gradle-wrapper.properties | 6 ++-- lib/logstash/filters/geoip.rb | 12 ++++++-- spec/filters/geoip_spec.rb | 23 +++++++++++++++ .../org/logstash/filters/GeoIPFilter.java | 29 ++++++++++++++----- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index bae7962..e5b897f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Mon Oct 31 23:15:11 PDT 2016 +#Fri Dec 13 10:58:02 CST 2019 +distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip +zipStoreBase=GRADLE_USER_HOME diff --git a/lib/logstash/filters/geoip.rb b/lib/logstash/filters/geoip.rb index 6fdfad9..e902ee6 100644 --- a/lib/logstash/filters/geoip.rb +++ b/lib/logstash/filters/geoip.rb @@ -56,6 +56,13 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base # `dma_code`, `ip`, `latitude`, `longitude`, `postal_code`, `region_name` and `timezone`. config :fields, :validate => :array + # An array of geoip locales to be used in the event. + # + # locales are gracefully resolved from beginning, with default as 'en' + # + # it matches locales support in mmdb files + config :locales, :validate => :array, :default => ['en'] + # Specify the field into which Logstash should store the geoip data. # This can be useful, for example, if you have `src_ip` and `dst_ip` fields and # would like the GeoIP information of both IPs. @@ -99,8 +106,9 @@ def register end @logger.info("Using geoip database", :path => @database) - - @geoipfilter = org.logstash.filters.GeoIPFilter.new(@source, @target, @fields, @database, @cache_size) + + locales = (@locales + ['en']).uniq + @geoipfilter = org.logstash.filters.GeoIPFilter.new(@source, @target, locales, @fields, @database, @cache_size) end # def register public diff --git a/spec/filters/geoip_spec.rb b/spec/filters/geoip_spec.rb index 93eb70f..fe957ec 100644 --- a/spec/filters/geoip_spec.rb +++ b/spec/filters/geoip_spec.rb @@ -290,8 +290,31 @@ expect(subject.get("geoip")["asn"]).to eq(15169) expect(subject.get("geoip")["as_org"]).to eq("Google Inc.") end + end + + describe "support locales" do + config <<-CONFIG + filter { + geoip { + source => "ip" + locales => ["zh-CN", "pt-BR"] + } + } + CONFIG + sample("ip" => "106.38.102.106") do + expect(subject.get("geoip")).not_to be_empty + expect(subject.get("geoip")["ip"]).to eq("106.38.102.106") + expect(subject.get("geoip")["city_name"]).to eq("北京") + expect(subject.get("geoip")["country_name"]).to eq("中国") + end + sample("ip" => "189.2.0.0") do + expect(subject.get("geoip")).not_to be_empty + expect(subject.get("geoip")["ip"]).to eq("189.2.0.0") + expect(subject.get("geoip")["city_name"]).to eq("São Paulo") + expect(subject.get("geoip")["country_name"]).to eq("巴西") + end end end diff --git a/src/main/java/org/logstash/filters/GeoIPFilter.java b/src/main/java/org/logstash/filters/GeoIPFilter.java index f23b9ad..82b6107 100644 --- a/src/main/java/org/logstash/filters/GeoIPFilter.java +++ b/src/main/java/org/logstash/filters/GeoIPFilter.java @@ -59,12 +59,14 @@ public class GeoIPFilter { private final String sourceField; private final String targetField; + private final List desiredLocales; private final Set desiredFields; private final DatabaseReader databaseReader; - public GeoIPFilter(String sourceField, String targetField, List fields, String databasePath, int cacheSize) { + public GeoIPFilter(String sourceField, String targetField, List locales, List fields, String databasePath, int cacheSize) { this.sourceField = sourceField; this.targetField = targetField; + this.desiredLocales = locales; final File database = new File(databasePath); try { this.databaseReader = new DatabaseReader.Builder(database).withCache(new CHMCache(cacheSize)).build(); @@ -206,7 +208,7 @@ private Map retrieveCityGeoData(InetAddress ipAddress) throws Geo for (Fields desiredField : this.desiredFields) { switch (desiredField) { case CITY_NAME: - String cityName = city.getName(); + String cityName = getNameAccordingLocales(city.getNames(), this.desiredLocales); if (cityName != null) { geoData.put(Fields.CITY_NAME.fieldName(), cityName); } @@ -218,13 +220,13 @@ private Map retrieveCityGeoData(InetAddress ipAddress) throws Geo } break; case CONTINENT_NAME: - String continentName = continent.getName(); + String continentName = getNameAccordingLocales(continent.getNames(), this.desiredLocales); if (continentName != null) { geoData.put(Fields.CONTINENT_NAME.fieldName(), continentName); } break; case COUNTRY_NAME: - String countryName = country.getName(); + String countryName = getNameAccordingLocales(country.getNames(), this.desiredLocales); if (countryName != null) { geoData.put(Fields.COUNTRY_NAME.fieldName(), countryName); } @@ -257,7 +259,7 @@ private Map retrieveCityGeoData(InetAddress ipAddress) throws Geo } break; case REGION_NAME: - String subdivisionName = subdivision.getName(); + String subdivisionName = getNameAccordingLocales(subdivision.getNames(), this.desiredLocales); if (subdivisionName != null) { geoData.put(Fields.REGION_NAME.fieldName(), subdivisionName); } @@ -320,13 +322,13 @@ private Map retrieveCountryGeoData(InetAddress ipAddress) throws } break; case COUNTRY_NAME: - String countryName = country.getName(); + String countryName = getNameAccordingLocales(country.getNames(), this.desiredLocales); if (countryName != null) { geoData.put(Fields.COUNTRY_NAME.fieldName(), countryName); } break; case CONTINENT_NAME: - String continentName = continent.getName(); + String continentName = getNameAccordingLocales(continent.getNames(), this.desiredLocales); if (continentName != null) { geoData.put(Fields.CONTINENT_NAME.fieldName(), continentName); } @@ -401,4 +403,17 @@ private Map retrieveAsnGeoData(InetAddress ipAddress) throws Geo return geoData; } + + private String getNameAccordingLocales(Map names, List locales) { + int size = locales.size(); + if (size <= 0) { + return null; + } + + String first = locales.get(0); + if (names.containsKey(first)) { + return names.get(first); + } + return this.getNameAccordingLocales(names, locales.subList(1, size)); + } } From 6401b68887b3100ec86d5979327a2d4a3a7664bd Mon Sep 17 00:00:00 2001 From: Yipeng Zhao Date: Fri, 13 Dec 2019 22:11:30 +0800 Subject: [PATCH 3/3] bump to v6.0.4 --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 12 ++++++++++++ logstash-filter-geoip.gemspec | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b4bc7c..185baf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 6.0.4 + - Add locales support for output fields, such as `country_name` and `city_name` [#160](https://github.com/logstash-plugins/logstash-filter-geoip/pull/160) + ## 6.0.3 - Fixed docs for missing region_code [#158](https://github.com/logstash-plugins/logstash-filter-geoip/pull/158) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 1f17ec2..b29961c 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -74,6 +74,7 @@ This plugin supports the following configuration options plus the <> |a valid filesystem path|No | <> |`City` or `ASN`|No | <> |<>|No +| <> |<>|No | <> |<>|Yes | <> |<>|No | <> |<>|No @@ -143,6 +144,17 @@ For the built-in GeoLite2 City database, the following are available: `dma_code`, `ip`, `latitude`, `longitude`, `postal_code`, `region_code`, `region_name` and `timezone`. +[id="plugins-{type}s-{plugin}-locales"] +===== `locales` + + * Value type is <> + * There is no default value for this setting. + +An array of geoip locales to be included in the event. + +It displays the output of city_name or country_name in preferred locale. The one appear +in the front of locales would resolve first, with 'en' as default. + [id="plugins-{type}s-{plugin}-source"] ===== `source` diff --git a/logstash-filter-geoip.gemspec b/logstash-filter-geoip.gemspec index 5e00019..73b7049 100644 --- a/logstash-filter-geoip.gemspec +++ b/logstash-filter-geoip.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-geoip' - s.version = '6.0.3' + s.version = '6.0.4' s.licenses = ['Apache License (2.0)'] s.summary = "Adds geographical information about an IP address" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"