From ee0c6b9bd53bed312376014de1db5484e09430e0 Mon Sep 17 00:00:00 2001 From: Brian Muller Date: Sat, 4 Jan 2014 14:27:45 -0500 Subject: [PATCH] added travis file and some more tests --- .travis.yml | 6 +++ Makefile | 6 +-- README.markdown | 37 +++++++++++-- example/server.py | 8 --- kademlia/network.py | 11 ++-- kademlia/protocol.py | 6 +-- kademlia/routing.py | 6 ++- kademlia/tests/test_node.py | 29 +++++++++++ kademlia/tests/test_routing.py | 52 +++++++++++++++++++ kademlia/tests/utils.py | 94 ++++++++++++++++++++++++++++++++++ server.tac | 9 ++++ setup.py | 2 +- 12 files changed, 242 insertions(+), 24 deletions(-) create mode 100644 .travis.yml delete mode 100644 example/server.py create mode 100644 kademlia/tests/test_routing.py create mode 100644 kademlia/tests/utils.py create mode 100644 server.tac diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..fff05b3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: python +python: + - "2.7" + - "pypy" +install: pip install . --use-mirrors +script: trial kademlia diff --git a/Makefile b/Makefile index 4c253c6..019571a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ PYDOCTOR=pydoctor +test: + trial kademlia + docs: $(PYDOCTOR) --make-html --html-output apidoc --add-package kademlia --project-name=kademlia --project-url=http://github.com/bmuller/kademlia --html-use-sorttable --html-use-splitlinks --html-shorten-lists @@ -9,6 +12,3 @@ lint: install: python setup.py install - -test: - trial kademlia diff --git a/README.markdown b/README.markdown index 515d054..9a677c9 100644 --- a/README.markdown +++ b/README.markdown @@ -1,18 +1,49 @@ # [Kademlia](http://en.wikipedia.org/wiki/Kademlia) in Python +[![Build Status](https://secure.travis-ci.org/bmuller/kademlia.png?branch=master)](https://travis-ci.org/bmuller/kademlia) ## Installation ``` -easy_install kademlia +pip install kademlia ``` ## Usage *This assumes you have a working familiarity with Twisted.* - +Assuming you want to connect to an existing network (run the standalone server example below if you don't have a network): ```python from twisted.internet import reactor +from twisted.python import log +from kademlia.network import Server +import sys -... +# log to std out +log.startLogging(sys.stdout) + +def quit(result): + print "Key result:", result + reactor.stop() + +def get(result, server): + reactor.stop() + #return server.get("a key").addCallback(quit) + +def done(found, server): + log.msg("Found nodes: %s" % found) + return server.set("a key", "a value").addCallback(get, server) + +server = Server() +# next line, or use reactor.listenUDP(5678, server) +server.listen(5678) +server.bootstrap([('127.0.0.1', 1234)]).addCallback(done, two) + +reactor.run() +``` + +## Stand-alone Server +If all you want to do is run a local server, just start the example server: + +``` +twistd -noy server.tac ``` diff --git a/example/server.py b/example/server.py deleted file mode 100644 index 3bed7c5..0000000 --- a/example/server.py +++ /dev/null @@ -1,8 +0,0 @@ -from twisted.internet import reactor -from twisted.python import log -from kademlia.network import Server -import sys - -log.startLogging(sys.stdout) -one = Server(1234) -reactor.run() diff --git a/kademlia/network.py b/kademlia/network.py index b21951e..6099336 100644 --- a/kademlia/network.py +++ b/kademlia/network.py @@ -4,7 +4,7 @@ Package for interacting on the network at a high level. import random from twisted.internet.task import LoopingCall -from twisted.internet import defer +from twisted.internet import defer, reactor from kademlia.log import Logger from kademlia.protocol import KademliaProtocol @@ -113,7 +113,7 @@ class Server(object): to start listening as an active node on the network. """ - def __init__(self, port, ksize=20, alpha=3): + def __init__(self, ksize=20, alpha=3): """ Create a server instance. This will start listening on the given port. @@ -125,10 +125,13 @@ class Server(object): self.alpha = alpha self.log = Logger(system=self) storage = ForgetfulStorage() - self.node = Node('127.0.0.1', port, digest(random.getrandbits(255))) - self.protocol = KademliaProtocol(self.node, storage, ksize) + self.node = Node(None, None, digest(random.getrandbits(255))) + self.protocol = KademliaProtocol(self.node.id, storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) + def listen(self, port): + return reactor.listenUDP(port, self.protocol) + def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour diff --git a/kademlia/protocol.py b/kademlia/protocol.py index 26bce7c..4d10c79 100644 --- a/kademlia/protocol.py +++ b/kademlia/protocol.py @@ -7,11 +7,11 @@ from kademlia.log import Logger class KademliaProtocol(RPCProtocol): - def __init__(self, node, storage, ksize): - RPCProtocol.__init__(self, node.port) + def __init__(self, sourceID, storage, ksize): + RPCProtocol.__init__(self) self.router = RoutingTable(self, ksize) self.storage = storage - self.sourceID = node.id + self.sourceID = sourceID self.log = Logger(system=self) def getRefreshIDs(self): diff --git a/kademlia/routing.py b/kademlia/routing.py index 5cd9799..1be310e 100644 --- a/kademlia/routing.py +++ b/kademlia/routing.py @@ -93,13 +93,15 @@ class RoutingTable(object): def __init__(self, protocol, ksize): self.protocol = protocol self.ksize = ksize - self.buckets = [KBucket(0, 2 ** 160, ksize)] + self.flush() + + def flush(self): + self.buckets = [KBucket(0, 2 ** 160, self.ksize)] def splitBucket(self, index): one, two = self.buckets[index].split() self.buckets[index] = one self.buckets.insert(index + 1, two) - # todo split one/two if needed based on section 4.2 def getLonelyBuckets(self): diff --git a/kademlia/tests/test_node.py b/kademlia/tests/test_node.py index 90cb3d2..921e046 100644 --- a/kademlia/tests/test_node.py +++ b/kademlia/tests/test_node.py @@ -3,7 +3,9 @@ import hashlib from twisted.trial import unittest +from kademlia.utils import digest from kademlia.node import Node, NodeHeap +from kademlia.tests.utils import mknode class NodeTest(unittest.TestCase): @@ -26,3 +28,30 @@ class NodeHeapTest(unittest.TestCase): def test_maxSize(self): n = NodeHeap(3) self.assertEqual(0, len(n)) + + for d in range(10): + n.push(d, mknode()) + self.assertEqual(3, len(n)) + + self.assertEqual(3, len(list(n))) + + def test_iteration(self): + heap = NodeHeap(5) + nodes = [mknode(id=digest(x)) for x in range(10)] + for index, node in enumerate(nodes): + heap.push(index, node) + for index, node in enumerate(heap): + self.assertEqual(digest(index), node.id) + self.assertTrue(index < 5) + + def test_remove(self): + heap = NodeHeap(5) + nodes = [mknode(id=digest(x)) for x in range(10)] + for index, node in enumerate(nodes): + heap.push(index, node) + + heap.remove([nodes[0].id, nodes[1].id]) + self.assertEqual(len(list(heap)), 5) + for index, node in enumerate(heap): + self.assertEqual(digest(index + 2), node.id) + self.assertTrue(index < 5) diff --git a/kademlia/tests/test_routing.py b/kademlia/tests/test_routing.py new file mode 100644 index 0000000..ecaf1ae --- /dev/null +++ b/kademlia/tests/test_routing.py @@ -0,0 +1,52 @@ +from twisted.trial import unittest + +from kademlia.routing import KBucket, RoutingTable +from kademlia.protocol import KademliaProtocol +from kademlia.tests.utils import mknode, FakeProtocol + + +class KBucketTest(unittest.TestCase): + def test_split(self): + bucket = KBucket(0, 10, 5) + bucket.addNode(mknode(intid=5)) + bucket.addNode(mknode(intid=6)) + one, two = bucket.split() + self.assertEqual(len(one), 1) + self.assertEqual(one.range, (0, 5)) + self.assertEqual(len(two), 1) + self.assertEqual(two.range, (6, 10)) + + def test_addNode(self): + # when full, return false + bucket = KBucket(0, 10, 2) + self.assertTrue(bucket.addNode(mknode())) + self.assertTrue(bucket.addNode(mknode())) + self.assertFalse(bucket.addNode(mknode())) + self.assertEqual(len(bucket), 2) + + # make sure when a node is double added it's put at the end + bucket = KBucket(0, 10, 3) + nodes = [mknode(), mknode(), mknode()] + for node in nodes: + bucket.addNode(node) + for index, node in enumerate(bucket.getNodes()): + self.assertEqual(node, nodes[index]) + + def test_inRange(self): + bucket = KBucket(0, 10, 10) + self.assertTrue(bucket.hasInRange(mknode(intid=5))) + self.assertFalse(bucket.hasInRange(mknode(intid=11))) + self.assertTrue(bucket.hasInRange(mknode(intid=10))) + self.assertTrue(bucket.hasInRange(mknode(intid=0))) + + +class RoutingTableTest(unittest.TestCase): + def setUp(self): + self.id = mknode().id + self.protocol = FakeProtocol(self.id) + self.router = self.protocol.router + + def test_addContact(self): + self.router.addContact(mknode()) + self.assertTrue(len(self.router.buckets), 1) + self.assertTrue(len(self.router.buckets[0].nodes), 1) diff --git a/kademlia/tests/utils.py b/kademlia/tests/utils.py new file mode 100644 index 0000000..4c8878d --- /dev/null +++ b/kademlia/tests/utils.py @@ -0,0 +1,94 @@ +""" +Utility functions for tests. +""" +import random +import hashlib +from struct import pack + +from kademlia.node import Node +from kademlia.routing import RoutingTable + + +def mknode(ip=None, port=None, id=None, intid=None): + """ + Make a node. Created a random id if not specified. + """ + if intid is not None: + id = pack('>l', intid) + id = id or hashlib.sha1(str(random.getrandbits(255))).digest() + return Node(ip, port, id) + + +class FakeProtocol(object): + def __init__(self, sourceID, ksize=20): + self.router = RoutingTable(self, ksize) + self.storage = {} + self.sourceID = sourceID + + def getRefreshIDs(self): + """ + Get ids to search for to keep old buckets up to date. + """ + ids = [] + for bucket in self.router.getLonelyBuckets(): + ids.append(random.randint(*bucket.range)) + return ids + + def rpc_ping(self, sender, nodeid): + source = Node(sender[0], sender[1], nodeid) + self.router.addContact(source) + return self.sourceID + + def rpc_store(self, sender, nodeid, key, value): + source = Node(sender[0], sender[1], nodeid) + self.router.addContact(source) + self.log.debug("got a store request from %s, storing value" % str(sender)) + self.storage[key] = value + + def rpc_find_node(self, sender, nodeid, key): + self.log.info("finding neighbors of %i in local table" % long(nodeid.encode('hex'), 16)) + source = Node(sender[0], sender[1], nodeid) + self.router.addContact(source) + node = Node(None, None, key) + return map(tuple, self.router.findNeighbors(node, exclude=source)) + + def rpc_find_value(self, sender, nodeid, key): + source = Node(sender[0], sender[1], nodeid) + self.router.addContact(source) + value = self.storage.get(key, None) + if value is None: + return self.rpc_find_node(sender, nodeid, key) + return { 'value': value } + + def callFindNode(self, nodeToAsk, nodeToFind): + address = (nodeToAsk.ip, nodeToAsk.port) + d = self.find_node(address, self.sourceID, nodeToFind.id) + return d.addCallback(self.handleCallResponse, nodeToAsk) + + def callFindValue(self, nodeToAsk, nodeToFind): + address = (nodeToAsk.ip, nodeToAsk.port) + d = self.find_value(address, self.sourceID, nodeToFind.id) + return d.addCallback(self.handleCallResponse, nodeToAsk) + + def callPing(self, nodeToAsk): + address = (nodeToAsk.ip, nodeToAsk.port) + d = self.ping(address, self.sourceID) + return d.addCallback(self.handleCallResponse, nodeToAsk) + + def callStore(self, nodeToAsk, key, value): + address = (nodeToAsk.ip, nodeToAsk.port) + d = self.store(address, self.sourceID, key, value) + return d.addCallback(self.handleCallResponse, nodeToAsk) + + def handleCallResponse(self, result, node): + """ + If we get a response, add the node to the routing table. If + we get no response, make sure it's removed from the routing table. + """ + if result[0]: + self.log.info("got response from %s, adding to router" % node) + self.router.addContact(node) + else: + self.log.debug("no response from %s, removing from router" % node) + self.router.removeContact(node) + return result diff --git a/server.tac b/server.tac new file mode 100644 index 0000000..f6d7abb --- /dev/null +++ b/server.tac @@ -0,0 +1,9 @@ +from twisted.application import service, internet +import sys, os +sys.path.append(os.path.dirname(__file__)) +from kademlia.network import Server + +application = service.Application("kademlia") +kserver = Server() +server = internet.UDPServer(1234, kserver.protocol) +server.setServiceParent(application) diff --git a/setup.py b/setup.py index b780d14..aab5be7 100755 --- a/setup.py +++ b/setup.py @@ -12,5 +12,5 @@ setup( url="http://github.com/bmuller/kademlia", packages=find_packages(), requires=["twisted", "rpcudp"], - install_requires=['twisted>=12.0', "rpcudp>=0.3"] + install_requires=['twisted>=13.0', "rpcudp>=1.0"] )