diff --git a/kademlia/network.py b/kademlia/network.py index 717d2d9..7b15bec 100644 --- a/kademlia/network.py +++ b/kademlia/network.py @@ -33,9 +33,9 @@ class Server(object): self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) - storage = ForgetfulStorage() + self.storage = ForgetfulStorage() self.node = Node(id or digest(random.getrandbits(255))) - self.protocol = KademliaProtocol(self.node, storage, ksize) + self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port): @@ -58,7 +58,15 @@ class Server(object): nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest) ds.append(spider.find()) - return defer.gatherResults(ds) + + def republishKeys(_): + ds = [] + # Republish keys older than one hour + for key, value in self.storage.iteritemsOlderThan(3600): + ds.append(self.set(key, value)) + return defer.gatherResults(ds) + + return defer.gatherResults(ds).addCallback(republishKeys) def bootstrappableNeighbors(self): """ diff --git a/kademlia/storage.py b/kademlia/storage.py index a97c343..78b9892 100644 --- a/kademlia/storage.py +++ b/kademlia/storage.py @@ -1,11 +1,15 @@ import time +from itertools import izip +from itertools import imap +from itertools import takewhile +import operator from collections import OrderedDict class ForgetfulStorage(object): - def __init__(self, ttl=7200): + def __init__(self, ttl=604800): """ - By default, max age is 2 hours + By default, max age is a week. """ self.data = OrderedDict() self.ttl = ttl @@ -13,22 +17,17 @@ class ForgetfulStorage(object): def __setitem__(self, key, value): if key in self.data: del self.data[key] - self.data[key] = (time.time() + self.ttl, value) + self.data[key] = (time.time(), value) self.cull() def cull(self): - pop = 0 - for value in self.data.itervalues(): - if value[0] >= time.time(): - break - pop += 1 - for _ in xrange(pop): + for k, v in self.iteritemsOlderThan(self.ttl): self.data.popitem(first=True) def get(self, key, default=None): self.cull() if key in self.data: - return self.data[key][1] + return self[key] return default def __getitem__(self, key): @@ -42,3 +41,21 @@ class ForgetfulStorage(object): def __repr__(self): self.cull() return repr(self.data) + + def iteritemsOlderThan(self, secondsOld): + minBirthday = time.time() - secondsOld + zipped = self._tripleIterable() + matches = takewhile(lambda r: minBirthday >= r[1], zipped) + return imap(operator.itemgetter(0, 2), matches) + + def _tripleIterable(self): + ikeys = self.data.iterkeys() + ibirthday = imap(operator.itemgetter(0), self.data.itervalues()) + ivalues = imap(operator.itemgetter(1), self.data.itervalues()) + return izip(ikeys, ibirthday, ivalues) + + def iteritems(self): + self.cull() + ikeys = self.data.iterkeys() + ivalues = imap(operator.itemgetter(1), self.data.itervalues()) + return izip(ikeys, ivalues)