1
2
3
4
5
6 import sqlite3
7 import bsddb
8 import unittest
9
11 """Base class for stores."""
12 - def put(self, key, value):
13 raise NotImplementedError
14
16 raise NotImplementedError
17
19 raise NotImplementedError
20
22 """
23 Does not need to return an actual iterator,
24 just something that L{iterate} can recognize.
25 """
26 raise NotImplementedError
27
28 - def iterate(self, iterator, threshold):
29 """
30 Iterates over keys/values starting at iterator, until threshold bytes are accumulated.
31
32 @param iterator: Something that can describe the current position
33 @param threshold: How many bytes to accumulate before returning
34 """
35 raise NotImplementedError
36
38 for k, v in kvlist:
39 try:
40 v_curr = self.get(k)
41 v = resolver(v, v_curr)
42 except KeyError:
43
44 pass
45
46 self.put(k, v)
47
48
50 """Basic in-memory store."""
53
54 - def put(self, key, value):
55 self._store[key] = value
56
58 return self._store[key]
59
62
64 return self._store.iteritems()
65
66 - def iterate(self, iterator, threshold):
67 tot = 0
68 ret = []
69 try:
70 while tot < threshold:
71 k, v = iterator.next()
72 tot = tot + len(k) + len(v)
73 ret.append((k, v))
74 return ret, iterator
75 except StopIteration:
76 return ret, iterator
77
79 """Store using BerkeleyDB, specifically the B-Tree version"""
81 self._store = bsddb.btopen(filename)
82
83 - def put(self, key, value):
84 self._store[key] = value
85 self._store.sync()
86
88 return self._store[key]
89
91 del self._store[key]
92 self._store.sync()
93
95 try:
96 k, v = self._store.first()
97 return k
98 except bsddb.error:
99 return None
100
101 - def iterate(self, iterator, threshold):
102 if iterator is None:
103 return [], None
104 iterator, v = self._store.set_location(iterator)
105 tot = 0
106 ret = [(iterator, v)]
107 try:
108 while tot < threshold:
109 iterator, v = self._store.next()
110 tot = tot + len(iterator) + len(v)
111 ret.append((iterator, v))
112 return ret, iterator
113 except bsddb.error:
114 return ret, None
115
116
118 """Store that uses SQLite for storage."""
120 self._db = filename
121 self.conn = sqlite3.connect(self._db)
122 c = self.conn.cursor()
123
124 c.execute("CREATE TABLE IF NOT EXISTS blobkey(k BLOB PRIMARY KEY, v BLOB)")
125 self.conn.commit()
126 c.close()
127
128 - def put(self, key, value):
129 c = self.conn.cursor()
130 c.execute("INSERT OR REPLACE INTO blobkey(k, v) VALUES(?, ?)", (key, sqlite3.Binary(value)))
131 self.conn.commit()
132 c.close()
133
134 - def get(self, key):
135 c = self.conn.cursor()
136 c.execute("SELECT v FROM blobkey WHERE k = ?", (key,))
137 value = c.fetchone()
138 c.close()
139 if value is None:
140 raise KeyError(key)
141 return value[0]
142
144 c = self.conn.cursor()
145 c.execute("DELETE FROM blobkey WHERE k = ?", (key,))
146 self.conn.commit()
147 rows = c.rowcount
148 c.close()
149 if rows == 0:
150 raise KeyError
151
153 c = self.conn.cursor()
154 c.execute("SELECT k, v FROM blobkey")
155 return c
156
157 - def iterate(self, iterator, threshold):
158 tot = 0
159 ret = []
160 try:
161 while tot < threshold:
162 k, v = iterator.next()
163 tot = tot + len(k) + len(v)
164 ret.append((k, v))
165 return ret, iterator
166 except StopIteration:
167 return ret, iterator
168
171 contents = [("Key_%d"%i, "Val_%d"%i) for i in range(100)]
172 for k, v in contents:
173 d.put(k, v)
174 iterator = d.get_iterator()
175 kvlist = []
176 while True:
177 kv, iterator = d.iterate(iterator, 100)
178 if not kv:
179 break
180 kvlist.extend(kv)
181 self.assertEqual(set(contents), set([(str(k), str(v)) for k, v in kvlist]))
182
186
190
194
195 if __name__=="__main__":
196 unittest.main()
197