Automating Let’s Encrypt Certificates on VMware Avi Load Balancer with Cloudflare DNS

'''
###
# Name: letsencrypt_mgmt_profile_cloudflare_dns.py
# Version: 0.1.2-cloudflare
# License: MIT
#
# Description -
#     Certificate Management Profile script for VMware Avi Load Balancer that requests
#     and renews certificates from Let's Encrypt using the DNS-01 challenge against a
#     Cloudflare-hosted zone. Because it uses DNS-01 it can issue wildcard certificates
#     (*.domain.tld) and needs no inbound connectivity and no Virtual Service.
#
#     Adapted from the official Infoblox DNS-01 script:
#     https://github.com/avinetworks/devops/blob/master/certificate_mgmt/letsencrypt_mgmt_profile_infoblox_dns.py
#     Changes from that script:
#       - Infoblox calls replaced with Cloudflare API v4 calls.
#       - Virtual Service lookup removed (it is only needed for HTTP-01 and it fails for
#         wildcard names), so no Avi user/password is required.
#       - Waits for DNS propagation before asking Let's Encrypt to validate.
#
# Setup -
#     1. Templates > Security > Certificate Management > Create, paste this script.
#     2. Add the script params listed below (mark cf_api_token as sensitive).
#     3. Templates > Security > SSL/TLS Certificates > Create > Application Certificate.
#     4. Type: CSR. Common Name: *.domain.tld. Add domain.tld as a SAN if you want the
#        apex covered as well. Select this Certificate Management profile.
#     5. Save. Issuance takes roughly a minute.
#
# Parameters -
#     cf_api_token        - Cloudflare API token with Zone / DNS / Edit on the zone. (required, sensitive)
#     cf_zone_id          - Cloudflare Zone ID. (recommended; if omitted the script looks the
#                           zone up by name, which needs Zone / Zone / Read on the token)
#     cf_propagation_wait - Seconds to wait after creating the TXT records. (Default: 30)
#     cf_ttl              - TTL of the TXT record in seconds, minimum 60. (Default: 60)
#     dryrun              - True/False. If True Let's Encrypt's staging server is used. (Default: False)
#                           Use this while testing so you do not hit rate limits.
#     contact             - E-mail address sent to Let's Encrypt for the account. (Default: None)
#     directory_url       - Use a different ACME server. (Default: Let's Encrypt Production)
#     letsencrypt_key     - Existing Let's Encrypt account key in PEM. (Default: None, one is generated)
#     debug               - True/False. Extra logging. (Default: False)
#
# Authors/Credits -
#     acme-tiny <https://github.com/diafygi/acme-tiny>
#     acme-dns-tiny <https://github.com/Trim/acme-dns-tiny>
#     Nikhil Kumar Yadav, Patrik Kernstock, Priya Koshta (VMware) for the original Avi scripts
###
'''

import base64, binascii, hashlib, os, json, re, ssl, subprocess, time
from urllib.request import urlopen, Request
from tempfile import NamedTemporaryFile
import requests

VERSION = "0.1.2-cloudflare"

DEFAULT_DIRECTORY_URL = "https://acme-v02.api.letsencrypt.org/directory"
DEFAULT_STAGING_DIRECTORY_URL = "https://acme-staging-v02.api.letsencrypt.org/directory"
ACCOUNT_KEY_PATH = "/tmp/letsencrypt.key"

CF_API = "https://api.cloudflare.com/client/v4"
CF_TIMEOUT = 30

# record ids created during this run, keyed by (record name, record value)
CF_RECORD_IDS = {}
# zone ids resolved during this run, keyed by record name
CF_ZONE_CACHE = {}


def _cf_request(method, path, kwargs, params=None, payload=None):
    token = kwargs.get('cf_api_token', None)
    if not token:
        raise Exception("Script param cf_api_token is required.")
    headers = {"Authorization": "Bearer {}".format(token.strip()), "Content-Type": "application/json"}
    r = requests.request(method, CF_API + path, headers=headers, params=params, json=payload, timeout=CF_TIMEOUT)
    try:
        body = r.json()
    except ValueError:
        raise Exception("Cloudflare API returned non-JSON response (HTTP {}): {}".format(r.status_code, r.text[:300]))
    return r.status_code, body


def _cf_errors(body):
    return "; ".join("{} {}".format(e.get('code'), e.get('message')) for e in (body.get('errors') or [])) or str(body)


def _cf_zone_id(record_name, kwargs):
    zone_id = kwargs.get('cf_zone_id', None)
    if zone_id:
        return zone_id.strip()
    if record_name in CF_ZONE_CACHE:
        return CF_ZONE_CACHE[record_name]
    # walk up the name until Cloudflare knows a zone: a.b.domain.tld -> b.domain.tld -> domain.tld
    labels = record_name.split(".")
    for i in range(1, len(labels) - 1):
        candidate = ".".join(labels[i:])
        code, body = _cf_request("GET", "/zones", kwargs, params={"name": candidate})
        if code == 200 and body.get('success') and body.get('result'):
            CF_ZONE_CACHE[record_name] = body['result'][0]['id']
            return CF_ZONE_CACHE[record_name]
        if code in (401, 403):
            raise Exception("Cloudflare rejected the API token during zone lookup ({}). "
                            "Set cf_zone_id or add Zone / Zone / Read to the token.".format(_cf_errors(body)))
    raise Exception("Could not find a Cloudflare zone for {}. Set the cf_zone_id script param.".format(record_name))


def _cf_find_record_ids(zone_id, record_name, value, kwargs):
    code, body = _cf_request("GET", "/zones/{}/dns_records".format(zone_id), kwargs,
                             params={"type": "TXT", "name": record_name, "per_page": 100})
    if code != 200 or not body.get('success'):
        raise Exception("Cloudflare record lookup failed: {}".format(_cf_errors(body)))
    # Cloudflare may return TXT content wrapped in double quotes
    return [rec['id'] for rec in body.get('result', []) if rec.get('content', '').strip('"') == value]


def add_dns_text_record(key_digest_64, txt_record_name, kwargs):
    # Create dns txt record with fqdn txt_record_name and value key_digest_64
    name = txt_record_name.rstrip(".")
    try:
        ttl = max(60, int(kwargs.get('cf_ttl', 60)))
    except ValueError:
        ttl = 60
    zone_id = _cf_zone_id(name, kwargs)
    payload = {"type": "TXT", "name": name, "content": key_digest_64, "ttl": ttl}
    code, body = _cf_request("POST", "/zones/{}/dns_records".format(zone_id), kwargs, payload=payload)
    if code in (200, 201) and body.get('success'):
        CF_RECORD_IDS[(name, key_digest_64)] = body['result']['id']
        print("Added DNS TXT record {}".format(name))
        return
    # 81057 / 81058: an identical record is already there, e.g. left over from an aborted run
    if any(e.get('code') in (81057, 81058) for e in (body.get('errors') or [])):
        print("DNS TXT record {} already exists, reusing it".format(name))
        return
    raise Exception("Error adding DNS TXT record {} (HTTP {}): {}".format(name, code, _cf_errors(body)))


def remove_dns_text_record(key_digest_64, txt_record_name, kwargs):
    # Remove the dns txt record with fqdn txt_record_name and value key_digest_64.
    # Never raises: a cleanup problem must not hide the real result of the challenge.
    name = txt_record_name.rstrip(".")
    try:
        zone_id = _cf_zone_id(name, kwargs)
        record_id = CF_RECORD_IDS.pop((name, key_digest_64), None)
        record_ids = [record_id] if record_id else _cf_find_record_ids(zone_id, name, key_digest_64, kwargs)
        for rid in record_ids:
            code, body = _cf_request("DELETE", "/zones/{}/dns_records/{}".format(zone_id, rid), kwargs)
            if code == 200 and body.get('success'):
                print("Deleted DNS TXT record {}".format(name))
            else:
                print("Warning: could not delete DNS TXT record {} ({}): {}".format(name, rid, _cf_errors(body)))
    except Exception as e:
        print("Warning: cleanup of DNS TXT record {} failed: {}".format(name, e))


def get_crt(csr, directory_url=DEFAULT_DIRECTORY_URL, contact=None, debug=False, kwargs=None):
    directory, acct_headers, alg, jwk = None, None, None, None # global variables

    # helper functions - base64 encode for jose spec
    def _b64(b):
        return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "")

    # helper function - run external commands
    def _cmd(cmd_list, stdin=None, cmd_input=None, err_msg="Command Line Error"):
        proc = subprocess.Popen(cmd_list, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = proc.communicate(cmd_input)
        if proc.returncode != 0:
            raise IOError("{0}\n{1}".format(err_msg, err))
        return out

    # helper function - make request and automatically parse json response
    def _do_request(url, data=None, err_msg="Error", depth=0):
        try:
            ctx = ssl.create_default_context()
            resp = urlopen(Request(url, data=data, headers={"Content-Type": "application/jose+json", "User-Agent": "acme-tiny"}), context=ctx)
            resp_data, code, headers = resp.read().decode("utf8"), resp.getcode(), resp.headers
        except IOError as e:
            resp_data = e.read().decode("utf8") if hasattr(e, "read") else str(e)
            code, headers = getattr(e, "code", None), {}
        try:
            resp_data = json.loads(resp_data) # try to parse json results
        except ValueError:
            pass # ignore json parsing errors
        if depth < 100 and code == 400 and isinstance(resp_data, dict) and resp_data.get('type') == "urn:ietf:params:acme:error:badNonce":
            raise IndexError(resp_data) # allow 100 retrys for bad nonces
        if code not in [200, 201, 204]:
            raise ValueError("{0}:\nUrl: {1}\nData: {2}\nResponse Code: {3}\nResponse: {4}".format(err_msg, url, data, code, resp_data))
        return resp_data, code, headers

    # helper function - make signed requests
    def _send_signed_request(url, payload, err_msg, depth=0):
        payload64 = "" if payload is None else _b64(json.dumps(payload).encode('utf8'))
        new_nonce = _do_request(directory['newNonce'])[2]['Replay-Nonce']
        protected = {"url": url, "alg": alg, "nonce": new_nonce}
        protected.update({"jwk": jwk} if acct_headers is None else {"kid": acct_headers['Location']})
        protected64 = _b64(json.dumps(protected).encode('utf8'))
        protected_input = "{0}.{1}".format(protected64, payload64).encode('utf8')
        out = _cmd(["openssl", "dgst", "-sha256", "-sign", ACCOUNT_KEY_PATH], stdin=subprocess.PIPE, cmd_input=protected_input, err_msg="OpenSSL Error")
        data = json.dumps({"protected": protected64, "payload": payload64, "signature": _b64(out)})
        try:
            return _do_request(url, data=data.encode('utf8'), err_msg=err_msg, depth=depth)
        except IndexError: # retry bad nonces (they raise IndexError)
            return _send_signed_request(url, payload, err_msg, depth=(depth + 1))

    # helper function - poll until complete
    def _poll_until_not(url, pending_statuses, err_msg):
        result, t0 = None, time.time()
        while result is None or result['status'] in pending_statuses:
            assert (time.time() - t0 < 600), "Polling timeout" # 10 minute timeout
            time.sleep(0 if result is None else 2)
            result, _, _ = _send_signed_request(url, None, err_msg)
        return result

    try:
        propagation_wait = max(0, int(kwargs.get('cf_propagation_wait', 30)))
    except ValueError:
        propagation_wait = 30

    if os.path.exists(ACCOUNT_KEY_PATH):
        if debug:
            print ("DEBUG: Reusing account key.")
    else:
        print ("Account key not found. Generating account key...")
        out = _cmd(["openssl", "genrsa", "4096"], err_msg="OpenSSL Error")
        with open(ACCOUNT_KEY_PATH, 'w') as f:
            f.write(out.decode("utf-8"))
        os.chmod(ACCOUNT_KEY_PATH, 0o600)

    # parse account key to get public key
    print ("Parsing account key...")
    out = _cmd(["openssl", "rsa", "-in", ACCOUNT_KEY_PATH, "-noout", "-text"], err_msg="OpenSSL Error")
    pub_pattern = r"modulus:[\s]+?00:([a-f0-9\:\s]+?)\npublicExponent: ([0-9]+)"
    pub_hex, pub_exp = re.search(pub_pattern, out.decode('utf8'), re.MULTILINE|re.DOTALL).groups()
    pub_exp = "{0:x}".format(int(pub_exp))
    pub_exp = "0{0}".format(pub_exp) if len(pub_exp) % 2 else pub_exp
    alg = "RS256"
    jwk = {
        "e": _b64(binascii.unhexlify(pub_exp.encode("utf-8"))),
        "kty": "RSA",
        "n": _b64(binascii.unhexlify(re.sub(r"(\s|:)", "", pub_hex).encode("utf-8"))),
    }
    accountkey_json = json.dumps(jwk, sort_keys=True, separators=(',', ':'))
    thumbprint = _b64(hashlib.sha256(accountkey_json.encode('utf8')).digest())

    # find domains
    print ("Parsing CSR...")
    out = _cmd(["openssl", "req", "-in", csr, "-noout", "-text"], err_msg="Error loading {0}".format(csr))
    domains = set([])
    common_name = re.search(r"Subject:.*? CN\s?=\s?([^\s,;/]+)", out.decode('utf8'))
    if common_name is not None:
        domains.add(common_name.group(1))
    subject_alt_names = re.search(r"X509v3 Subject Alternative Name: (?:critical)?\n +([^\n]+)\n", out.decode('utf8'), re.MULTILINE|re.DOTALL)
    if subject_alt_names is not None:
        for san in subject_alt_names.group(1).split(", "):
            if san.startswith("DNS:"):
                domains.add(san[4:])
    if not domains:
        raise ValueError("No domain names found in the CSR.")
    print ("Found domains: {0}".format(", ".join(sorted(domains))))

    # get the ACME directory of urls
    print ("Getting directory...")
    directory, _, _ = _do_request(directory_url, err_msg="Error getting directory")
    print ("Directory found!")

    # create account, update contact details (if any), and set the global key identifier
    print ("Registering account...")
    reg_payload = {"termsOfServiceAgreed": True}
    account, code, acct_headers = _send_signed_request(directory['newAccount'], reg_payload, "Error registering")
    print ("Registered!" if code == 201 else "Already registered!")
    if contact is not None:
        account, _, _ = _send_signed_request(acct_headers['Location'], {"contact": contact}, "Error updating contact details")
        # Let's Encrypt no longer stores or returns contact addresses, so the key may be absent
        returned = account.get('contact') if isinstance(account, dict) else None
        print ("Contact details sent. Server returned: {0}".format(", ".join(returned) if returned else "none (not stored by this CA)"))

    # create a new order
    print ("Creating new order...")
    order_payload = {"identifiers": [{"type": "dns", "value": d} for d in sorted(domains)]}
    order, _, order_headers = _send_signed_request(directory['newOrder'], order_payload, "Error creating new order")
    print ("Order created!")

    # Complete the authorizations. *.domain.tld and domain.tld both validate through
    # _acme-challenge.domain.tld, each with its own TXT value. All TXT records are
    # published BEFORE any challenge is triggered: Let's Encrypt's resolvers cache the
    # TXT answer for a short time, so adding the second value after the first validation
    # makes the second one fail against the cached (stale) answer.
    pending = []
    try:
        for auth_url in order['authorizations']:
            if debug:
                print ("DEBUG: Authorization URL is: {}".format(auth_url))

            authorization, _, _ = _send_signed_request(auth_url, None, "Error getting challenges")
            domain = authorization['identifier']['value']
            label = "*.{}".format(domain) if authorization.get('wildcard') else domain

            if authorization['status'] == "valid":
                print ("{0} is already authorized, skipping challenge.".format(label))
                continue

            # find the dns-01 challenge and generate the txt record name and value
            challenges = [c for c in authorization['challenges'] if c['type'] == "dns-01"]
            if not challenges:
                raise ValueError("No dns-01 challenge offered for {0}: {1}".format(label, authorization))
            challenge = challenges[0]
            token = re.sub(r"[^A-Za-z0-9_\-]", "_", challenge['token'])
            keyauthorization = "{0}.{1}".format(token, thumbprint)
            keydigest64 = _b64(hashlib.sha256(keyauthorization.encode("utf8")).digest())
            txt_record_name = "_acme-challenge.{0}.".format(domain)
            if debug:
                print ("DEBUG: Validation record for {} is: {}".format(label, txt_record_name))

            # register for cleanup before adding, so a half-created record is still removed
            pending.append((label, auth_url, challenge, keydigest64, txt_record_name))
            add_dns_text_record(keydigest64, txt_record_name, kwargs)

        if pending:
            print ("Waiting {} seconds for DNS propagation...".format(propagation_wait))
            time.sleep(propagation_wait)

        for label, auth_url, challenge, keydigest64, txt_record_name in pending:
            print ("Verifying {0}...".format(label))
            _send_signed_request(challenge['url'], {}, "Error submitting challenges: {0}".format(label))
            authorization = _poll_until_not(auth_url, ["pending", "processing"], "Error checking challenge status for {0}".format(label))
            if authorization['status'] != "valid":
                raise ValueError("Challenge did not pass for {0}: {1}".format(label, authorization))
            print ("Challenge passed for {0}".format(label))

    finally:
        if pending:
            print ("Cleaning up...")
        for label, auth_url, challenge, keydigest64, txt_record_name in pending:
            remove_dns_text_record(keydigest64, txt_record_name, kwargs)

    print ("All domains verified!")

    # finalize the order with the csr
    print ("Signing certificate...")
    csr_der = _cmd(["openssl", "req", "-in", csr, "-outform", "DER"], err_msg="DER Export Error")
    _send_signed_request(order['finalize'], {"csr": _b64(csr_der)}, "Error finalizing order")

    # poll the order to monitor when it's done
    order = _poll_until_not(order_headers['Location'], ["pending", "processing", "ready"], "Error checking order status")
    if order['status'] != "valid":
        raise ValueError("Order failed: {0}".format(order))

    # download the certificate
    certificate_pem, _, _ = _send_signed_request(order['certificate'], None, "Certificate download failed")
    print ("Certificate signed!")

    return certificate_pem


def certificate_request(csr, common_name, kwargs):
    dry_run = kwargs.get('dryrun', "false")
    contact = kwargs.get('contact', None)
    debug = kwargs.get('debug', "false")
    directory_url = kwargs.get('directory_url', None)
    letsencrypt_key = kwargs.get('letsencrypt_key', None)

    print ("Running version {}".format(VERSION))

    debug = str(debug).lower() == "true"
    if debug:
        print ("Debug enabled.")

    dry_run = str(dry_run).lower() == "true"
    print ("dry_run is: {}".format(str(dry_run)))

    if not directory_url:
        directory_url = DEFAULT_STAGING_DIRECTORY_URL if dry_run else DEFAULT_DIRECTORY_URL
    print ("directory_url is {}".format(directory_url))

    if contact and "@" in contact:
        contact = [ "mailto:{}".format(contact) ] # contact must be array as of ACME RFC
        print ("Contact set to: {}".format(contact))
    else:
        contact = None

    if letsencrypt_key:
        with open(ACCOUNT_KEY_PATH, 'w') as f:
            f.write(letsencrypt_key)
        os.chmod(ACCOUNT_KEY_PATH, 0o600)

    # Create CSR temp file.
    csr_temp_file = NamedTemporaryFile(mode='w', delete=False)
    csr_temp_file.close()

    with open(csr_temp_file.name, 'w') as f:
        f.write(csr)

    signed_crt = None
    try:
        signed_crt = get_crt(csr_temp_file.name, directory_url=directory_url,
                             contact=contact, debug=debug, kwargs=kwargs)
    finally:
        os.remove(csr_temp_file.name)

    print (signed_crt)
    return signed_crt