Merge pull request #63 from mithrandi/59-libcloud-dns

Implement libcloud-based dns-01 challenge responder.

setup.py

@@ -44,15 +44,19 @@ def read(*parts):
         ],
     install_requires=[
         'Twisted[tls]>=15.5.0',
-        'acme>=0.4.0',
+        'acme>=0.9.0',
         'attrs',
         'eliot>=0.8.0',
         'pem>=16.1.0',
         'treq>=15.1.0',
         'txsni',
         ],
     extras_require={
+        'libcloud': [
+            'apache-libcloud',
+        ],
         'test': [
+            'apache-libcloud',
             'fixtures>=1.4.0',
             'hypothesis>=3.1.0,<4.0.0',
             'testrepository>=0.0.20',

src/integration/test_client.py

@@ -22,7 +22,7 @@
 from txsni.snimap import SNIMap
 from txsni.tlsendpoint import TLSEndpoint
 
-from txacme.challenges import TLSSNI01Responder
+from txacme.challenges import LibcloudDNSResponder, TLSSNI01Responder
 from txacme.client import (
     answer_challenge, Client, fqdn_identifier, LETSENCRYPT_STAGING_DIRECTORY,
     poll_until_valid)
@@ -31,9 +31,6 @@
 from txacme.util import csr_for_names, generate_private_key, tap
 
 
-HOST = u'acme-testing.mithrandi.net'
-
-
 class ClientTestsMixin(object):
     """
     Integration tests for the ACME client.
@@ -145,7 +142,7 @@ def _test_registration(self):
             .addCallback(self._test_chain)
             .addActionFinish())
 
-    def test_registration(self):
+    def test_issuing(self):
         action = start_action(action_type=u'integration')
         with action.context():
             return self._test_registration()
@@ -165,22 +162,21 @@ def _getenv(name, default=None):
     return value
 
 
-class LetsEncryptStagingTests(ClientTestsMixin, TestCase):
+class LetsEncryptStagingTLSSNI01Tests(ClientTestsMixin, TestCase):
     """
     Tests using the real ACME client against the Let's Encrypt staging
-    environment.
+    environment, and the tls-sni-01 challenge.
 
     You must set $ACME_HOST to a hostname that will, when connected to on port
     443, reach a listening socket opened by the tests on $ACME_ENDPOINT.
     """
     HOST = _getenv(u'ACME_HOST')
-    ENDPOINT = _getenv(u'ACME_ENDPOINT', u'tcp:443')
-    if not _PY3:
+    ENDPOINT = _getenv(u'ACME_ENDPOINT')
+    if None in [HOST, ENDPOINT]:
+        skip = 'Must provide $ACME_HOST and $ACME_ENDPOINT'
+    elif not _PY3:
         ENDPOINT = ENDPOINT.encode('utf-8')
 
-    if HOST is None:
-        skip = 'Must provide $ACME_HOST'
-
     def _create_client(self, key):
         return (
             Client.from_url(reactor, LETSENCRYPT_STAGING_DIRECTORY, key=key)
@@ -205,6 +201,43 @@ def _create_responder(self):
                 .addActionFinish())
 
 
+class LetsEncryptStagingLibcloudTests(ClientTestsMixin, TestCase):
+    """
+    Tests using the real ACME client against the Let's Encrypt staging
+    environment, and the dns-01 challenge.
+
+    You must set $ACME_HOST to a hostname that will be used for the challenge,
+    and $LIBCLOUD_PROVIDER, $LIBCLOUD_USERNAME, $LIBCLOUD_PASSWORD, and
+    $LIBCLOUD_ZONE to the appropriate values for the DNS provider to complete
+    the challenge with.
+    """
+    HOST = _getenv(u'ACME_HOST')
+    PROVIDER = _getenv(u'LIBCLOUD_PROVIDER')
+    USERNAME = _getenv(u'LIBCLOUD_USERNAME')
+    PASSWORD = _getenv(u'LIBCLOUD_PASSWORD')
+    ZONE = _getenv(u'LIBCLOUD_ZONE')
+
+    if None in (HOST, PROVIDER, USERNAME, PASSWORD):
+        skip = 'Must provide $ACME_HOST and $LIBCLOUD_*'
+
+    def _create_client(self, key):
+        return (
+            Client.from_url(reactor, LETSENCRYPT_STAGING_DIRECTORY, key=key)
+            .addCallback(tap(
+                lambda client: self.addCleanup(
+                    client._client._treq._agent._pool.closeCachedConnections)))
+            )
+
+    def _create_responder(self):
+        with start_action(action_type=u'integration:create_responder'):
+            return LibcloudDNSResponder.create(
+                reactor,
+                self.PROVIDER,
+                self.USERNAME,
+                self.PASSWORD,
+                self.ZONE)
+
+
 class FakeClientTests(ClientTestsMixin, TestCase):
     """
     Tests against our verified fake.
@@ -218,4 +251,4 @@ def _create_responder(self):
         return succeed(NullResponder(u'tls-sni-01'))
 
 
-__all__ = ['LetsEncryptStagingTests', 'FakeClientTests']
+__all__ = ['LetsEncryptStagingTLSSNI01Tests', 'FakeClientTests']

src/txacme/challenges/__init__.py

@@ -1,4 +1,6 @@
 from ._http import HTTP01Responder
+from ._libcloud import LibcloudDNSResponder
 from ._tls import TLSSNI01Responder
 
-__all__ = ['HTTP01Responder', 'TLSSNI01Responder']
+
+__all__ = ['HTTP01Responder', 'LibcloudDNSResponder', 'TLSSNI01Responder']

src/txacme/challenges/_libcloud.py

@@ -0,0 +1,183 @@
+import hashlib
+import time
+from threading import Thread
+
+import attr
+from acme import jose
+from libcloud.dns.providers import get_driver
+from twisted._threads import pool
+from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
+from zope.interface import implementer
+
+from txacme.errors import NotInZone, ZoneNotFound
+from txacme.interfaces import IResponder
+from txacme.util import const
+
+
+def _daemon_thread(*a, **kw):
+    """
+    Create a `threading.Thread`, but always set ``daemon``.
+    """
+    thread = Thread(*a, **kw)
+    thread.daemon = True
+    return thread
+
+
+def _defer_to_worker(deliver, worker, work, *args, **kwargs):
+    """
+    Run a task in a worker, delivering the result as a ``Deferred`` in the
+    reactor thread.
+    """
+    deferred = Deferred()
+
+    def wrapped_work():
+        try:
+            result = work(*args, **kwargs)
+        except:
+            f = Failure()
+            deliver(lambda: deferred.errback(f))
+        else:
+            deliver(lambda: deferred.callback(result))
+    worker.do(wrapped_work)
+    return deferred
+
+
+def _split_zone(server_name, zone_name):
+    """
+    Split the zone portion off from a DNS label.
+
+    :param str server_name: The full DNS label.
+    :param str zone_name: The zone name suffix.
+    """
+    if not (server_name == zone_name or
+            server_name.endswith(u'.' + zone_name)):
+        raise NotInZone(server_name=server_name, zone_name=zone_name)
+    return server_name[:-len(zone_name)].rstrip(u'.')
+
+
+def _get_existing(driver, zone_name, server_name, validation):
+    """
+    Get existing validation records.
+    """
+    if zone_name is None:
+        zones = sorted(
+            (z for z
+             in driver.list_zones()
+             if server_name.endswith(u'.' + z.domain)),
+            key=lambda z: len(z.domain),
+            reverse=True)
+        if len(zones) == 0:
+            raise NotInZone(server_name=server_name, zone_name=None)
+    else:
+        zones = [
+            z for z
+            in driver.list_zones()
+            if z.domain == zone_name]
+        if len(zones) == 0:
+            raise ZoneNotFound(zone_name=zone_name)
+    zone = zones[0]
+    subdomain = _split_zone(server_name, zone.domain)
+    existing = [
+        record for record
+        in zone.list_records()
+        if record.name == subdomain and
+        record.type == 'TXT' and
+        record.data == validation]
+    return zone, existing, subdomain
+
+
+def _validation(response):
+    """
+    Get the validation value for a challenge response.
+    """
+    h = hashlib.sha256(response.key_authorization.encode("utf-8"))
+    return jose.b64encode(h.digest()).decode()
+
+
+@attr.s(hash=False)
+@implementer(IResponder)
+class LibcloudDNSResponder(object):
+    """
+    A ``dns-01`` challenge responder using libcloud.
+
+    ..  warning:: Some libcloud backends are broken with regard to TXT records
+        at the time of writing; the Route 53 backend, for example. This makes
+        them unusable with this responder.
+
+    ..  note:: This implementation relies on invoking libcloud in a thread, so
+        may not be entirely production quality.
+    """
+    challenge_type = u'dns-01'
+
+    _reactor = attr.ib()
+    _thread_pool = attr.ib()
+    _driver = attr.ib()
+    zone_name = attr.ib()
+    settle_delay = attr.ib()
+
+    @classmethod
+    def create(cls, reactor, driver_name, username, password, zone_name=None,
+               settle_delay=60.0):
+        """
+        Create a responder.
+
+        :param reactor: The Twisted reactor to use for threading support.
+        :param str driver_name: The name of the libcloud DNS driver to use.
+        :param str username: The username to authenticate with (the meaning of
+            this is driver-specific).
+        :param str password: The username to authenticate with (the meaning of
+            this is driver-specific).
+        :param str zone_name: The zone name to respond in, or ``None`` to
+            automatically detect zones.  Usually auto-detection should be fine,
+            unless restricting responses to a single specific zone is desired.
+        :param float settle_delay: The time, in seconds, to allow for the DNS
+            provider to propagate record changes.
+        """
+        return cls(
+            reactor=reactor,
+            thread_pool=pool(const(1), threadFactory=_daemon_thread),
+            driver=get_driver(driver_name)(username, password),
+            zone_name=zone_name,
+            settle_delay=settle_delay)
+
+    def _defer(self, f):
+        """
+        Run a function in our private thread pool.
+        """
+        return _defer_to_worker(
+            self._reactor.callFromThread, self.thread_pool, f)
+
+    def start_responding(self, server_name, challenge, response):
+        """
+        Install a TXT challenge response record.
+        """
+        validation = _validation(response)
+        full_name = challenge.validation_domain_name(server_name)
+        _driver = self._driver
+
+        def _go():
+            zone, existing, subdomain = _get_existing(
+                _driver, self.zone_name, full_name, validation)
+            if len(existing) == 0:
+                zone.create_record(name=subdomain, type='TXT', data=validation)
+                time.sleep(self.settle_delay)
+        return self._defer(_go)
+
+    def stop_responding(self, server_name, challenge, response):
+        """
+        Remove a TXT challenge response record.
+        """
+        validation = _validation(response)
+        full_name = challenge.validation_domain_name(server_name)
+        _driver = self._driver
+
+        def _go():
+            zone, existing, subdomain = _get_existing(
+                _driver, self.zone_name, full_name, validation)
+            for record in existing:
+                record.delete()
+        return self._defer(_go)
+
+
+__all__ = ['LibcloudDNSResponder']

src/txacme/errors.py

@@ -0,0 +1,24 @@
+"""
+Exception types for txacme.
+"""
+import attr
+
+
+@attr.s
+class NotInZone(ValueError):
+    """
+    The given domain name is not in the configured zone.
+    """
+    server_name = attr.ib()
+    zone_name = attr.ib()
+
+
+@attr.s
+class ZoneNotFound(ValueError):
+    """
+    The configured zone was not found in the zones at the configured provider.
+    """
+    zone_name = attr.ib()
+
+
+__all__ = ['NotInZone', 'ZoneNotFound']

src/txacme/newsfragments/59.feature

@@ -0,0 +1 @@
+txacme.challenges.LibcloudDNSResponder implements a dns-01 challenge responder using libcloud. Installing txacme[libcloud] is necessary to pull in the dependencies for this.

src/txacme/test/doubles.py

@@ -0,0 +1,15 @@
+"""
+Test doubles.
+"""
+from twisted.internet.interfaces import IReactorFromThreads
+from zope.interface import implementer
+
+
+@implementer(IReactorFromThreads)
+class SynchronousReactorThreads(object):
+    """
+    An implementation of ``IReactorFromThreads`` that calls things
+    synchronously in the same thread.
+    """
+    def callFromThread(self, f, *args, **kwargs):  # noqa
+        f(*args, **kwargs)