diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c77326..1cddce3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,12 @@ +## 7.2.13 + - Added support for DB-IP Location+ISP (Enterprise) databases, [#174](https://github.com/logstash-plugins/logstash-filter-geoip/issues/174) and . It is recommended to explicitly configure fields in case of enabled ECS. + ## 7.2.12 - - [DOC] Add `http_proxy` environment variable for GeoIP service endpoint. The feature is included in 8.1.0, and was back-ported to 7.17.2 [#207](https://github.com/logstash-plugins/logstash-filter-geoip/pull/207) + - [DOC] Add `http_proxy` environment variable for GeoIP service endpoint. The feature is included in 8.1.0, and was back-ported to 7.17.2 [#207](https://github.com/logstash-plugins/logstash-filter-geoip/pull/207) ## 7.2.11 - Improved compatibility with the Elastic Common Schema [#206](https://github.com/logstash-plugins/logstash-filter-geoip/pull/206) - - Added support for ECS's composite `region_iso_code` (`US-WA`), which _replaces_ the non-ECS `region_code` (`WA`) as a default field with City databases. To get the stand-alone `region_code` in ECS mode, you must include it in the `fields` directive. + - Added support for ECS's composite `region_iso_code` (`US-WA`), which _replaces_ the non-ECS `region_code` (`WA`) as a default field with City databases. To get the stand-alone `region_code` in ECS mode, you must include it in the `fields` directive. - [DOC] Improve ECS-related documentation ## 7.2.10 @@ -77,14 +80,14 @@ - Update of GeoLite2 DB [#157](https://github.com/logstash-plugins/logstash-filter-geoip/pull/157) ## 6.0.1 - - Fixed deeplink to Elasticsearch Reference + - Fixed deeplink to Elasticsearch Reference [#151](https://github.com/logstash-plugins/logstash-filter-geoip/pull/151) ## 6.0.0 - Removed obsolete lru_cache_size field ## 5.0.3 - - Skip lookup operation if source field contains an empty string + - Skip lookup operation if source field contains an empty string - Update of the GeoIP2 DB ## 5.0.2 @@ -109,7 +112,7 @@ - Add ASN data support via GeoIP2-ISP database. ## 4.1.0 - - Removed from RubyGems.org since it was missing the default GeoIP2 database. + - Removed from RubyGems.org since it was missing the default GeoIP2 database. ## 4.0.6 - Docs: Remove patch classes from the main plugin file diff --git a/src/main/java/org/logstash/filters/geoip/Fields.java b/src/main/java/org/logstash/filters/geoip/Fields.java index 87d9e0f..41699f2 100644 --- a/src/main/java/org/logstash/filters/geoip/Fields.java +++ b/src/main/java/org/logstash/filters/geoip/Fields.java @@ -44,7 +44,8 @@ enum Fields { LOCATION("geo.location", "location"), LATITUDE("geo.location.lat", "latitude"), LONGITUDE("geo.location.lon", "longitude"), - ORGANIZATION("mmdb.organization", "organization"); + ORGANIZATION("mmdb.organization", "organization"), + CONNECTION_TYPE("mmdb.connection_type", "connection_type"); private final String fieldName; private final String ecsFieldName; diff --git a/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java b/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java index 68b00f5..aa1ef11 100644 --- a/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java +++ b/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java @@ -24,9 +24,11 @@ import com.maxmind.geoip2.exception.GeoIp2Exception; import com.maxmind.geoip2.model.AsnResponse; import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.EnterpriseResponse; import com.maxmind.geoip2.model.CountryResponse; import com.maxmind.geoip2.model.DomainResponse; import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.geoip2.model.ConnectionTypeResponse.ConnectionType; import com.maxmind.geoip2.record.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -58,6 +60,7 @@ public class GeoIPFilter { private static final String CITY_SOUTH_AMERICA_DB_TYPE = "GeoIP2-City-South-America"; private static final String COUNTRY_DB_TYPE = "GeoIP2-Country"; private static final String ISP_DB_TYPE = "GeoIP2-ISP"; + private static final String DBIP_LOCATION_ISP_TYPE = "DBIP-Location-ISP (compat=Enterprise)"; private static final String DOMAIN_DB_TYPE = "GeoIP2-Domain"; private final String sourceField; @@ -132,6 +135,11 @@ private Set createDesiredFields(List fields, final boolean ecsCo break; case DOMAIN_DB_TYPE: desiredFields = Fields.DEFAULT_DOMAIN_FIELDS; + case DBIP_LOCATION_ISP_TYPE: + desiredFields = Fields.ALL_FIELDS; + break; + default: + throw new IllegalStateException("Unsupported database type '" + databaseReader.getMetadata().getDatabaseType() + "'"); } } else { for (String fieldName : fields) { @@ -189,8 +197,11 @@ public boolean handleEvent(RubyEvent rubyEvent) { case DOMAIN_DB_TYPE: geoData = retrieveDomainGeoData(ipAddress); break; + case DBIP_LOCATION_ISP_TYPE: + geoData = retrieveEnterpriseGeoData(ipAddress); + break; default: - throw new IllegalStateException("Unsupported database type " + databaseReader.getMetadata().getDatabaseType() + ""); + throw new IllegalStateException("Unsupported database type '" + databaseReader.getMetadata().getDatabaseType() + "'"); } } catch (UnknownHostException e) { logger.debug("IP Field contained invalid IP address or hostname. exception={}, field={}, event={}", e, sourceField, event); @@ -352,6 +363,160 @@ private Map retrieveCityGeoData(InetAddress ipAddress) throws Geo return geoData; } + private Map retrieveEnterpriseGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + EnterpriseResponse response = databaseReader.enterprise(ipAddress); + Country country = response.getCountry(); + City city = response.getCity(); + + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Postal postal = response.getPostal(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + Traits traits = response.getTraits(); + + Map geoData = new HashMap<>(); + + // if location is empty, there is no point populating geo data + // and most likely all other fields are empty as well + if (location.getLatitude() == null && location.getLongitude() == null) { + return geoData; + } + + for (Fields desiredField : this.desiredFields) { + switch (desiredField) { + case CITY_NAME: + String cityName = city.getName(); + if (cityName != null) { + geoData.put(Fields.CITY_NAME, cityName); + } + break; + case CONTINENT_CODE: + String continentCode = continent.getCode(); + if (continentCode != null) { + geoData.put(Fields.CONTINENT_CODE, continentCode); + } + break; + case CONTINENT_NAME: + String continentName = continent.getName(); + if (continentName != null) { + geoData.put(Fields.CONTINENT_NAME, continentName); + } + break; + case COUNTRY_NAME: + String countryName = country.getName(); + if (countryName != null) { + geoData.put(Fields.COUNTRY_NAME, countryName); + } + break; + case COUNTRY_CODE2: + String countryCode2 = country.getIsoCode(); + if (countryCode2 != null) { + geoData.put(Fields.COUNTRY_CODE2, countryCode2); + } + break; + case COUNTRY_CODE3: + String countryCode3 = country.getIsoCode(); + if (countryCode3 != null) { + geoData.put(Fields.COUNTRY_CODE3, countryCode3); + } + break; + case IP: + geoData.put(Fields.IP, ipAddress.getHostAddress()); + break; + case POSTAL_CODE: + String postalCode = postal.getCode(); + if (postalCode != null) { + geoData.put(Fields.POSTAL_CODE, postalCode); + } + break; + case DMA_CODE: + Integer dmaCode = location.getMetroCode(); + if (dmaCode != null) { + geoData.put(Fields.DMA_CODE, dmaCode); + } + break; + case REGION_NAME: + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + geoData.put(Fields.REGION_NAME, subdivisionName); + } + break; + case REGION_CODE: + String subdivisionCode = subdivision.getIsoCode(); + if (subdivisionCode != null) { + geoData.put(Fields.REGION_CODE, subdivisionCode); + } + break; + case REGION_ISO_CODE: + String countryCodeForRegion = country.getIsoCode(); + String regionCode2 = subdivision.getIsoCode(); + if (countryCodeForRegion != null && regionCode2 != null) { + geoData.put(Fields.REGION_ISO_CODE, String.format("%s-%s", countryCodeForRegion, regionCode2)); + } + break; + case TIMEZONE: + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + geoData.put(Fields.TIMEZONE, locationTimeZone); + } + break; + case LOCATION: + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + geoData.put(Fields.LOCATION, locationObject); + } + break; + case LATITUDE: + Double lat = location.getLatitude(); + if (lat != null) { + geoData.put(Fields.LATITUDE, lat); + } + break; + case LONGITUDE: + Double lon = location.getLongitude(); + if (lon != null) { + geoData.put(Fields.LONGITUDE, lon); + } + break; + case AUTONOMOUS_SYSTEM_NUMBER: + Integer asn = traits.getAutonomousSystemNumber(); + if (asn != null) { + geoData.put(Fields.AUTONOMOUS_SYSTEM_NUMBER, asn); + } + break; + case AUTONOMOUS_SYSTEM_ORGANIZATION: + String aso = traits.getAutonomousSystemOrganization(); + if (aso != null) { + geoData.put(Fields.AUTONOMOUS_SYSTEM_ORGANIZATION, aso); + } + break; + case ISP: + String isp = traits.getIsp(); + if (isp != null) { + geoData.put(Fields.ISP, isp); + } + break; + case ORGANIZATION: + String org = traits.getOrganization(); + if (org != null) { + geoData.put(Fields.ORGANIZATION, org); + } + break; + case CONNECTION_TYPE: + ConnectionType type = traits.getConnectionType(); + if (type != null) { + geoData.put(Fields.CONNECTION_TYPE, type.toString()); + } + break; + } + } + return geoData; + } + private Map retrieveCountryGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { CountryResponse response = databaseReader.country(ipAddress); Country country = response.getCountry();