Merge branch 'master' of github.com:research/chocolate

server-ca/daemon_common.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+# functions common to the various kinds of daemon
+
+# TODO: define a log function that sends a pubsub message to the
+# logger daemon
+
+import time, binascii
+from Crypto import Random
+
+def signal_handler(a, b):
+    global clean_shutdown
+    clean_shutdown = True
+    r.publish("exit", "clean-exit")
+    r.lpush("exit", "clean-exit")
+
+def short(session):
+    """Return the first 12 bytes of a session ID, or, for a
+    challenge ID, the challenge ID with the session ID truncated."""
+    tmp = session.partition(":")
+    return tmp[0][:12] + "..." + tmp[1] + tmp[2]
+
+def random():
+    """Return 64 hex digits representing a new 32-byte random number."""
+    return binascii.hexlify(Random.get_random_bytes(32))
+
+def random_raw():
+    """Return 32 random bytes."""
+    return Random.get_random_bytes(32)

server-ca/issue-daemon.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+# This daemon runs on the CA side to look for requests in
+# the database that are waiting for a cert to be issued.
+
+import redis, redis_lock, CSR, sys, signal
+from sni_challenge.verify import verify_challenge
+from Crypto import Random
+
+r = redis.Redis()
+ps = r.pubsub()
+issue_lock = redis_lock.redis_lock(r, "issue_lock")
+# This lock guards the ability to issue certificates with "openssl ca",
+# which has no locking of its own.  We don't need locking for the updates
+# that the daemon performs on the sessions in the database because the
+# queues pending-makechallenge, pending-testchallenge, and pending-issue
+# are updated atomically and the daemon only ever acts on sessions that it
+# has removed from a queue.
+
+debug = "debug" in sys.argv
+clean_shutdown = False
+
+from daemon_common import signal_handler, short, random, random_raw
+
+signal.signal(signal.SIGTERM, signal_handler)
+signal.signal(signal.SIGINT, signal_handler)
+
+def issue(session):
+    if r.hget(session, "live") != "True":
+        # This session has died due to some other reason, like an
+        # illegal request or timeout, since it entered testchallenge
+        # state.  Consequently, we're not allowed to advance its
+        # state any further, and it should be removed from the
+        # pending-requests queue and not pushed into any other queue.
+        # We don't have to remove it from pending-testchallenge
+        # because the caller has already done so.
+        #
+        # Having a session in pending-issue die is a very weird case
+        # that probably suggests that timeouts are set incorrectly
+        # or that the client is misbehaving very badly.  This means
+        # that a request passed all of its challenges but the
+        # session nonetheless died for some reason unrelated to failing
+        # challenges before the cert could be issued.  Normally, this
+        # should never happen.
+        if debug: print "removing expired (issue-state!?) session", short(session)
+        r.lrem("pending-requests", session)
+        return
+    if r.hget(session, "state") != "issue":
+        return
+    csr = r.hget(session, "csr")
+    names = r.lrange("%s:names" % session, 0, -1)
+    with issue_lock:
+        cert = CSR.issue(csr, names)
+    r.hset(session, "cert", cert)
+    if cert:   # once issuing cert succeeded
+        if debug: print "%s: issued certificate for names: %s" % (short(session), ", ".join(names))
+        r.hset(session, "state", "done")
+        # r.lpush("pending-done", session)
+    else:       # should not be reached in deployed version
+        if debug: print "issuing for", short(session), "failed"
+        r.lpush("pending-issue", session)
+
+while True:
+    (where, what) = r.brpop(["exit", "pending-issue"])
+    if where == "exit":
+        r.lpush("exit", "exit")
+        break
+    elif where == "pending-issue":
+        issue(what)
+    if clean_shutdown:
+        print "daemon exiting cleanly"
+        break

server-ca/logging-daemon.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+# This daemon runs on the CA side to handle logging.
+
+import redis, signal, sys
+
+r = redis.Redis()
+ps = r.pubsub()
+
+debug = "debug" in sys.argv
+clean_shutdown = False
+
+from daemon_common import signal_handler
+
+signal.signal(signal.SIGTERM, signal_handler)
+signal.signal(signal.SIGINT, signal_handler)
+
+ps.subscribe(["logs", "exit"])
+for message in ps.listen():
+    if message["type"] != "message":
+        continue
+    if message["channel"] == "logs":
+        if debug: print message["data"]
+        continue
+    if message["channel"] == "exit":
+        break
+    if clean_shutdown:
+        print "daemon exiting cleanly"
+        break

server-ca/makechallenge-daemon.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+# This daemon runs on the CA side to look for requests in
+# the database that are waiting for challenges to be issued.
+
+import redis, time, sys, signal
+
+r = redis.Redis()
+ps = r.pubsub()
+
+debug = "debug" in sys.argv
+clean_shutdown = False
+
+from daemon_common import signal_handler, short, random, random_raw
+
+signal.signal(signal.SIGTERM, signal_handler)
+signal.signal(signal.SIGINT, signal_handler)
+
+def makechallenge(session):
+    if r.hget(session, "live") != "True":
+        # This session has died due to some other reason, like an
+        # illegal request or timeout, since it entered makechallenge
+        # state.  Consequently, we're not allowed to advance its
+        # state any further, and it should be removed from the
+        # pending-requests queue and not pushed into any other queue.
+        # We don't have to remove it from pending-makechallenge
+        # because the caller has already done so.
+        if debug: print "removing expired session", short(session)
+        r.lrem("pending-requests", session)
+        return
+    # Currently only makes challenges of type 0 (DomainValidateSNI)
+    # This challenge type has three internal data parameters:
+    #     dvsni:nonce,  dvsni:r,  dvsni:ext
+    # This challenge type sends three data parameters to the client:
+    #     nonce,  y = E(r),  ext
+    #
+    # Make one challenge for each name.  (This one-to-one relationship
+    # is not an inherent protocol requirement!)
+    names = r.lrange("%s:names" % session, 0, -1)
+    if debug: print "%s: new valid request" % session
+    if debug: print "%s: from requesting client at %s" % (short(session), r.hget(session, "client-addr"))
+    if debug: print "%s: for %d names: %s" % (short(session), len(names), ", ".join(names))
+    for i, name in enumerate(names):
+        challenge = "%s:%d" % (session, i)
+        r.hset(challenge, "challtime", int(time.time()))
+        r.hset(challenge, "type", 0)   # DomainValidateSNI
+        r.hset(challenge, "name", name)
+        r.hset(challenge, "satisfied", False)
+        r.hset(challenge, "failed", False)
+        r.hset(challenge, "dvsni:nonce", random())
+        r.hset(challenge, "dvsni:r", random_raw())
+        r.hset(challenge, "dvsni:ext", "1.3.3.7")
+        # Keep accurate count of how many challenges exist in this session.
+        r.hincrby(session, "challenges", 1)
+        if debug: print "created new challenge", short(challenge)
+    if True:  # challenges have been created
+        r.hset(session, "state", "testchallenge")
+    else:
+        r.lpush("pending-makechallenge", session)
+
+while True:
+    (where, what) = r.brpop(["exit", "pending-makechallenge"])
+    if where == "exit":
+        r.lpush("exit", "exit")
+        break
+    elif where == "pending-makechallenge":
+        makechallenge(what)
+    if clean_shutdown:
+        print "daemon exiting cleanly"
+        break

server-ca/payment-daemon.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+# Wait for news about payments received for sesssions and
+# then mark the sessions to show that that payment was received.
+
+# This daemon uses a different scheduling model from the
+# testchallenge daemon so ONLY ONE COPY OF THIS DAEMON SHOULD
+# BE RUN AT ONCE.  Since this daemon takes a minimal, discrete
+# action in response to a pubsub message, there should never be
+# a significant backlog associated with this daemon.
+
+import redis, signal, sys
+
+r = redis.Redis()
+ps = r.pubsub()
+
+debug = "debug" in sys.argv
+clean_shutdown = False
+
+from daemon_common import signal_handler
+
+signal.signal(signal.SIGTERM, signal_handler)
+signal.signal(signal.SIGINT, signal_handler)
+
+ps.subscribe(["payments", "exit"])
+for message in ps.listen():
+    if message["type"] != "message":
+        continue
+    if message["channel"] == "payments":
+        if debug: print message["data"]
+        # TODO: Actually process the payment here :-)
+        continue
+    if message["channel"] == "exit":
+        break
+    if clean_shutdown:
+        print "daemon exiting cleanly"
+        break

server-ca/policy.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+# This file should contain functions that set CA-side policies (that
+# could change over time or differ from CA to CA) on whether individual
+# aspects of a session are legitimate or appropriate.
+
+# Functions here can access Redis if necessary to examine details of
+# a session.
+
+# Examples: session expiry times
+
+import redis
+
+r = redis.Redis()
+
+def payment_required(session):
+    """Does this session require a payment?"""
+    return False
+
+def expire_session(session, state):
+    """Should this session be expired?"""
+    # Different maximum age policies apply to sessions that are waiting
+    # for a payment, and, in general, to sessions at different stages
+    # of their lifecycle.
+    # """Given that this session is in the specified named state,
+    # decide whether the daemon should forcibly expire it for being too
+    # old, even if no client request has caused the serve to mark the
+    # session as expired.  This is most relevant to truly abandoned
+    # sessions that no client ever asks about."""
+    age = int(time.time()) - int(r.hget(session, "created"))
+    if state == "makechallenge" and age > 120:
+        if debug: print "considered", short(session), "ancient"
+        return True
+    if state == "testchallenge" and age > 600:
+        if debug: print "considered", short(session), "ancient"
+        return True
+    if state == "testpayment" and age > 5000:
+        if debug: print "considered", short(session), "ancient"
+        return True
+    return False

server-ca/sni_challenge/exit.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+import exit_geography, random, socks
+
+# This file is currently unused.  It demonstrates how to make a
+# connection using exit_geography and a Tor exit node in a chosen
+# country.  This can be used to implement multipath probing to
+# perform SNI challenges from the vantage point of specified
+# countries.
+
+# NOTE: This requires a modification to your torrc: AllowDotExit 1
+
+node = random.choice(exit_geography.by_country["DE"])
+socksocket = socks.socksocket()
+socksocket.setproxy(socks.PROXY_TYPE_SOCKS4, "localhost", 9050)
+print node
+socksocket.connect(("theobroma.info.%s.exit" % node, 80))

server-ca/testchallenge-daemon.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+# This daemon runs on the CA side to look for requests in
+# the database that are waiting for the CA to test whether
+# challenges have been met, and to perform this test.
+
+import redis, time, sys, signal
+from redis_lock import redis_lock
+from sni_challenge.verify import verify_challenge
+
+r = redis.Redis()
+ps = r.pubsub()
+
+debug = "debug" in sys.argv
+clean_shutdown = False
+
+from daemon_common import signal_handler, short, random, random_raw
+
+def signal_handler(a, b):
+    global clean_shutdown
+    clean_shutdown = True
+    r.publish("exit", "clean-exit")
+
+signal.signal(signal.SIGTERM, signal_handler)
+signal.signal(signal.SIGINT, signal_handler)
+
+def testchallenge(session):
+    if r.hget(session, "live") != "True":
+        # This session has died due to some other reason, like an
+        # illegal request or timeout, since it entered testchallenge
+        # state.  Consequently, we're not allowed to advance its
+        # state any further, and it should be removed from the
+        # pending-requests queue and not pushed into any other queue.
+        # We don't have to remove it from pending-testchallenge
+        # because the caller has already done so.
+        if debug: print "removing expired session", short(session)
+        r.lrem("pending-requests", session)
+        return
+    if r.hget(session, "state") != "testchallenge":
+        return
+    if int(r.hincrby(session, "times-tested", 1)) > 3:
+        # This session has already been unsuccessfully tested three
+        # times.  Clearly, something has gone wrong or the client is
+        # just trying to annoy us.  Do not allow it to be tested again.
+        r.hset(session, "live", False)
+        r.lrem("pending-requests", session)
+        return
+    all_satisfied = True
+    for i, name in enumerate(r.lrange("%s:names" % session, 0, -1)):
+        challenge = "%s:%d" % (session, i)
+        if debug: print "testing challenge", short(challenge)
+        challtime = int(r.hget(challenge, "challtime"))
+        challtype = int(r.hget(challenge, "type"))
+        name = r.hget(challenge, "name")
+        satisfied = r.hget(challenge, "satisfied") == "True"
+        failed = r.hget(challenge, "failed") == "True"
+        # TODO: check whether this challenge is too old
+        if not satisfied and not failed:
+            # if debug: print "challenge", short(challenge), "being tested"
+            if challtype == 0:  # DomainValidateSNI
+                if debug: print "\tbeginning dvsni test to %s" % name
+                dvsni_nonce = r.hget(challenge, "dvsni:nonce")
+                dvsni_r = r.hget(challenge, "dvsni:r")
+                dvsni_ext = r.hget(challenge, "dvsni:ext")
+                direct_result, direct_reason = verify_challenge(name, dvsni_r, dvsni_nonce, False)
+                proxy_result, proxy_reason = verify_challenge(name, dvsni_r, dvsni_nonce, True)
+                if debug:
+                    print "\t...direct probe: %s (%s)" % (direct_result, direct_reason)
+                    print "\tTor proxy probe: %s (%s)" % (proxy_result, proxy_reason)
+                if direct_result and proxy_result:
+                    r.hset(challenge, "satisfied", True)
+                else: 
+                    all_satisfied = False
+                # TODO: distinguish permanent and temporarily failures
+                # can cause a permanent failure under some conditions, causing
+                # the session to become dead.  TODO: need to articulate what
+                # those conditions are
+            else:
+                # Don't know how to handle this challenge type
+                all_satisfied = False
+        elif not satisfied:
+             if debug: print "\tchallenge was not attempted"
+             all_satisfied = False
+    if all_satisfied:
+        # Challenges all succeeded, so we should prepare to issue
+        # the requested cert.
+        # TODO: double-check that there were > 0 challenges,
+        # so that we don't somehow mistakenly issue a cert in
+        # response to an empty list of challenges (even though
+        # the daemon that put this session on the queue should
+        # also have implicitly guaranteed this).
+        if debug: print "\t** All challenges satisfied; request %s GRANTED" % short(session)
+        r.hset(session, "state", "issue")
+        r.lpush("pending-issue", session)
+    else:
+        # Some challenges were not verified.  In the current
+        # design of this daemon, the client must contact
+        # us again to request that the session be placed back
+        # in pending-testchallenge!
+        pass
+
+while True:
+    (where, what) = r.brpop(["exit", "pending-testchallenge"])
+    if where == "exit":
+        r.lpush("exit", "exit")
+        break
+    elif where == "pending-testchallenge":
+        with redis_lock(r, "lock-" + what):
+            testchallenge(what)
+    if clean_shutdown:
+        print "daemon exiting cleanly"
+        break