Add new Warpstream integration

.codecov.yml

@@ -315,6 +315,10 @@ coverage:
         target: 75
         flags:
         - speedtest
+      warpstream:
+        target: 75
+        flags:
+        - warpstream
       wayfinder:
         target: 75
         flags:
@@ -690,6 +694,11 @@ flags:
     paths:
     - vespa/datadog_checks/vespa
     - vespa/tests
+  warpstream:
+    carryforward: true
+    paths:
+    - warpstream/datadog_checks/warpstream
+    - warpstream/tests
   wayfinder:
     carryforward: true
     paths:

.github/workflows/test-all.yml

@@ -1474,6 +1474,25 @@ jobs:
       test-py3: ${{ inputs.test-py3 }}
       setup-env-vars: "${{ inputs.setup-env-vars }}"
     secrets: inherit
+  je48051a:
+    uses: DataDog/integrations-core/.github/workflows/test-target.yml@master
+    with:
+      job-name: warpstream
+      target: warpstream
+      platform: linux
+      runner: '["ubuntu-22.04"]'
+      repo: "${{ inputs.repo }}"
+      python-version: "${{ inputs.python-version }}"
+      standard: ${{ inputs.standard }}
+      latest: ${{ inputs.latest }}
+      agent-image: "${{ inputs.agent-image }}"
+      agent-image-py2: "${{ inputs.agent-image-py2 }}"
+      agent-image-windows: "${{ inputs.agent-image-windows }}"
+      agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}"
+      test-py2: ${{ inputs.test-py2 }}
+      test-py3: ${{ inputs.test-py3 }}
+      setup-env-vars: "${{ inputs.setup-env-vars }}"
+    secrets: inherit
   jf287d93:
     uses: DataDog/integrations-core/.github/workflows/test-target.yml@master
     with:

warpstream/assets/configuration/spec.yaml

@@ -0,0 +1,15 @@
+name: warpstream
+files:
+- name: warpstream.yaml
+  options:
+  - template: init_config
+    options:
+    - template: init_config/default
+  - template: instances
+    options:
+    - name: url
+      required: true
+      description: The URL used to connect to the Warpstream agent instance (use the Agent REST API endpoint).
+      value:
+        type: string
+    - template: instances/default

warpstream/assets/service_checks.json

@@ -0,0 +1 @@
+[]

warpstream/datadog_checks/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore

warpstream/datadog_checks/warpstream/__about__.py

@@ -0,0 +1 @@
+__version__ = '1.0.0'

warpstream/datadog_checks/warpstream/__init__.py

@@ -0,0 +1,4 @@
+from .__about__ import __version__
+from .check import WarpstreamCheck
+
+__all__ = ['__version__', 'WarpstreamCheck']

warpstream/datadog_checks/warpstream/check.py

@@ -0,0 +1,66 @@
+from requests.exceptions import ConnectionError, HTTPError, InvalidURL, Timeout
+
+from datadog_checks.base import AgentCheck, ConfigurationError
+
+
+class WarpstreamCheck(AgentCheck):
+
+    # This will be the prefix of every metric and service check the integration sends
+    __NAMESPACE__ = 'warpstream'
+
+    def __init__(self, name, init_config, instances):
+        super(WarpstreamCheck, self).__init__(name, init_config, instances)
+
+        self._url = self.instance.get('url', '')
+        self._tags = self.instance.get('tags', [])
+        # The Agent only makes one attempt to instantiate each AgentCheck so any errors occurring
+        # in `__init__` are logged just once, making it difficult to spot. Therefore, we emit
+        # potential configuration errors as part of the check run phase.
+        # The configuration is only parsed once if it succeed, otherwise it's retried.
+        self.check_initializations.append(self._parse_config)
+
+    def _parse_config(self):
+        if not self._url:
+            raise ConfigurationError('Missing configuration: url')
+
+    def check(self, _):
+        tags = ['url:{}'.format(self._url)] + self._tags
+
+        url_warpstream_agent = self._url + "/api/v1/status"
+
+        # Perform HTTP Requests with our HTTP wrapper.
+        # More info at https://datadoghq.dev/integrations-core/base/http/
+        try:
+            response = self.http.get(url_warpstream_agent)
+            response.raise_for_status()
+
+        except Timeout as e:
+            self.gauge('can_connect', 0, tags=tags)
+            self.service_check(
+                "can_connect",
+                AgentCheck.CRITICAL,
+                message="Request timeout: {}, {}".format(url_warpstream_agent, e),
+            )
+            raise
+
+        except (HTTPError, InvalidURL, ConnectionError) as e:
+            self.gauge('can_connect', 0, tags=tags)
+            self.service_check(
+                "can_connect",
+                AgentCheck.CRITICAL,
+                message="Request failed: {}, {}".format(url_warpstream_agent, e),
+            )
+            raise
+
+        except ValueError as e:
+            self.gauge('can_connect', 0, tags=tags)
+            self.service_check("can_connect", AgentCheck.CRITICAL, message=str(e))
+            raise
+
+        else:
+            if response.status_code == 200:
+                self.service_check('can_connect', AgentCheck.OK, tags=tags)
+            else:
+                self.service_check('can_connect', AgentCheck.WARNING, tags=tags)
+
+        self.gauge('can_connect', 1, tags=tags)

warpstream/datadog_checks/warpstream/config_models/__init__.py

@@ -0,0 +1,21 @@
+# This file is autogenerated.
+# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
+#     ddev -x validate config -s <INTEGRATION_NAME>
+#     ddev -x validate models -s <INTEGRATION_NAME>
+
+
+from .instance import InstanceConfig
+from .shared import SharedConfig
+
+
+class ConfigMixin:
+    _config_model_instance: InstanceConfig
+    _config_model_shared: SharedConfig
+
+    @property
+    def config(self) -> InstanceConfig:
+        return self._config_model_instance
+
+    @property
+    def shared_config(self) -> SharedConfig:
+        return self._config_model_shared

warpstream/datadog_checks/warpstream/config_models/defaults.py

@@ -0,0 +1,12 @@
+# This file is autogenerated.
+# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
+#     ddev -x validate config -s <INTEGRATION_NAME>
+#     ddev -x validate models -s <INTEGRATION_NAME>
+
+
+def instance_empty_default_hostname():
+    return False
+
+
+def instance_min_collection_interval():
+    return 15

warpstream/datadog_checks/warpstream/config_models/instance.py

@@ -0,0 +1,47 @@
+# This file is autogenerated.
+# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
+#     ddev -x validate config -s <INTEGRATION_NAME>
+#     ddev -x validate models -s <INTEGRATION_NAME>
+
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+
+from datadog_checks.base.utils.functions import identity
+from datadog_checks.base.utils.models import validation
+
+from . import defaults, validators
+
+
+class InstanceConfig(BaseModel):
+    model_config = ConfigDict(
+        validate_default=True,
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    empty_default_hostname: Optional[bool] = None
+    min_collection_interval: Optional[float] = None
+    service: Optional[str] = None
+    tags: Optional[tuple[str, ...]] = None
+
+    @model_validator(mode='before')
+    def _initial_validation(cls, values):
+        return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values))
+
+    @field_validator('*', mode='before')
+    def _validate(cls, value, info):
+        field = cls.model_fields[info.field_name]
+        field_name = field.alias or info.field_name
+        if field_name in info.context['configured_fields']:
+            value = getattr(validators, f'instance_{info.field_name}', identity)(value, field=field)
+        else:
+            value = getattr(defaults, f'instance_{info.field_name}', lambda: value)()
+
+        return validation.utils.make_immutable(value)
+
+    @model_validator(mode='after')
+    def _final_validation(cls, model):
+        return validation.core.check_model(getattr(validators, 'check_instance', identity)(model))

warpstream/datadog_checks/warpstream/config_models/shared.py

@@ -0,0 +1,44 @@
+# This file is autogenerated.
+# To change this file you should edit assets/configuration/spec.yaml and then run the following commands:
+#     ddev -x validate config -s <INTEGRATION_NAME>
+#     ddev -x validate models -s <INTEGRATION_NAME>
+
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+
+from datadog_checks.base.utils.functions import identity
+from datadog_checks.base.utils.models import validation
+
+from . import defaults, validators
+
+
+class SharedConfig(BaseModel):
+    model_config = ConfigDict(
+        validate_default=True,
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    service: Optional[str] = None
+
+    @model_validator(mode='before')
+    def _initial_validation(cls, values):
+        return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values))
+
+    @field_validator('*', mode='before')
+    def _validate(cls, value, info):
+        field = cls.model_fields[info.field_name]
+        field_name = field.alias or info.field_name
+        if field_name in info.context['configured_fields']:
+            value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field)
+        else:
+            value = getattr(defaults, f'shared_{info.field_name}', lambda: value)()
+
+        return validation.utils.make_immutable(value)
+
+    @model_validator(mode='after')
+    def _final_validation(cls, model):
+        return validation.core.check_model(getattr(validators, 'check_shared', identity)(model))

warpstream/datadog_checks/warpstream/config_models/validators.py

@@ -0,0 +1,9 @@
+# Here you can include additional config validators or transformers
+#
+# def initialize_instance(values, **kwargs):
+#     if 'my_option' not in values and 'my_legacy_option' in values:
+#         values['my_option'] = values['my_legacy_option']
+#     if values.get('my_number') > 10:
+#         raise ValueError('my_number max value is 10, got %s' % str(values.get('my_number')))
+#
+#     return values

warpstream/datadog_checks/warpstream/data/conf.yaml.example

@@ -0,0 +1,48 @@
+## All options defined here are available to all instances.
+#
+init_config:
+
+    ## @param service - string - optional
+    ## Attach the tag `service:<SERVICE>` to every metric, event, and service check emitted by this integration.
+    ##
+    ## Additionally, this sets the default `service` for every log source.
+    #
+    # service: <SERVICE>
+
+## Every instance is scheduled independently of the others.
+#
+instances:
+
+    ## @param url - string - required
+    ## The URL used to connect to the Warpstream agent instance (use the Agent REST API endpoint).
+    #
+  - url: <URL>
+
+    ## @param tags - list of strings - optional
+    ## A list of tags to attach to every metric and service check emitted by this instance.
+    ##
+    ## Learn more about tagging at https://docs.datadoghq.com/tagging
+    #
+    # tags:
+    #   - <KEY_1>:<VALUE_1>
+    #   - <KEY_2>:<VALUE_2>
+
+    ## @param service - string - optional
+    ## Attach the tag `service:<SERVICE>` to every metric, event, and service check emitted by this integration.
+    ##
+    ## Overrides any `service` defined in the `init_config` section.
+    #
+    # service: <SERVICE>
+
+    ## @param min_collection_interval - integer - optional - default: 15
+    ## This changes the collection interval of the check. For more information, see:
+    ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval
+    #
+    # min_collection_interval: 15
+
+    ## @param empty_default_hostname - boolean - optional - default: false
+    ## This forces the check to send metrics with no hostname.
+    ##
+    ## This is useful for cluster-level checks.
+    #
+    # empty_default_hostname: false

warpstream/hatch.toml

@@ -0,0 +1,4 @@
+[env.collectors.datadog-checks]
+
+[[envs.default.matrix]]
+python = ["3.12"]