now republishing keys once an hour

kademlia/network.py

@@ -33,9 +33,9 @@ class Server(object):
         self.ksize = ksize
         self.alpha = alpha
         self.log = Logger(system=self)
-        storage = ForgetfulStorage()
+        self.storage = ForgetfulStorage()
         self.node = Node(id or digest(random.getrandbits(255)))
-        self.protocol = KademliaProtocol(self.node, storage, ksize)
+        self.protocol = KademliaProtocol(self.node, self.storage, ksize)
         self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
 
     def listen(self, port):
@@ -58,8 +58,16 @@ class Server(object):
             nearest = self.protocol.router.findNeighbors(node, self.alpha)
             spider = NodeSpiderCrawl(self.protocol, node, nearest)
             ds.append(spider.find())
+
+        def republishKeys(_):
+            ds = []
+            # Republish keys older than one hour
+            for key, value in self.storage.iteritemsOlderThan(3600):
+                ds.append(self.set(key, value))
             return defer.gatherResults(ds)
 
+        return defer.gatherResults(ds).addCallback(republishKeys)
+
     def bootstrappableNeighbors(self):
         """
         Get a C{list} of (ip, port) C{tuple}s suitable for use as an argument

kademlia/storage.py

@@ -1,11 +1,15 @@
 import time
+from itertools import izip
+from itertools import imap
+from itertools import takewhile
+import operator
 from collections import OrderedDict
 
 
 class ForgetfulStorage(object):
-    def __init__(self, ttl=7200):
+    def __init__(self, ttl=604800):
         """
-        By default, max age is 2 hours
+        By default, max age is a week.
         """
         self.data = OrderedDict()
         self.ttl = ttl
@@ -13,22 +17,17 @@ class ForgetfulStorage(object):
     def __setitem__(self, key, value):
         if key in self.data:
             del self.data[key]
-        self.data[key] = (time.time() + self.ttl, value)
+        self.data[key] = (time.time(), value)
         self.cull()
 
     def cull(self):
-        pop = 0
-        for value in self.data.itervalues():
-            if value[0] >= time.time():
-                break
-            pop += 1
-        for _ in xrange(pop):
+        for k, v in self.iteritemsOlderThan(self.ttl):
             self.data.popitem(first=True)
 
     def get(self, key, default=None):
         self.cull()
         if key in self.data:
-            return self.data[key][1]
+            return self[key]
         return default
 
     def __getitem__(self, key):
@@ -42,3 +41,21 @@ class ForgetfulStorage(object):
     def __repr__(self):
         self.cull()
         return repr(self.data)
+
+    def iteritemsOlderThan(self, secondsOld):
+        minBirthday = time.time() - secondsOld
+        zipped = self._tripleIterable()
+        matches = takewhile(lambda r: minBirthday >= r[1], zipped)
+        return imap(operator.itemgetter(0, 2), matches)
+
+    def _tripleIterable(self):
+        ikeys = self.data.iterkeys()
+        ibirthday = imap(operator.itemgetter(0), self.data.itervalues())
+        ivalues = imap(operator.itemgetter(1), self.data.itervalues())
+        return izip(ikeys, ibirthday, ivalues)
+
+    def iteritems(self):
+        self.cull()
+        ikeys = self.data.iterkeys()
+        ivalues = imap(operator.itemgetter(1), self.data.itervalues())
+        return izip(ikeys, ivalues)