Package vinzclortho :: Module store
[hide private]
[frames] | no frames]

Source Code for Module vinzclortho.store

  1  # -*- coding: utf-8 -*- 
  2  # 
  3  # Copyright (c) 2001-2010 Pär Bohrarper. 
  4  # See LICENSE for details. 
  5   
  6  import sqlite3 
  7  import bsddb 
  8  import unittest 
  9   
10 -class Store(object):
11 """Base class for stores."""
12 - def put(self, key, value):
13 raise NotImplementedError
14
15 - def get(self, key):
16 raise NotImplementedError
17
18 - def delete(self, key):
19 raise NotImplementedError
20
21 - def get_iterator(self):
22 """ 23 Does not need to return an actual iterator, 24 just something that L{iterate} can recognize. 25 """ 26 raise NotImplementedError
27
28 - def iterate(self, iterator, threshold):
29 """ 30 Iterates over keys/values starting at iterator, until threshold bytes are accumulated. 31 32 @param iterator: Something that can describe the current position 33 @param threshold: How many bytes to accumulate before returning 34 """ 35 raise NotImplementedError
36
37 - def multi_put(self, kvlist, resolver):
38 for k, v in kvlist: 39 try: 40 v_curr = self.get(k) 41 v = resolver(v, v_curr) 42 except KeyError: 43 # This store doesn't have the key, no need to resolve 44 pass 45 # TODO: probably should check if the value was changed... 46 self.put(k, v)
47 48
49 -class DictStore(Store):
50 """Basic in-memory store."""
51 - def __init__(self):
52 self._store = {}
53
54 - def put(self, key, value):
55 self._store[key] = value
56
57 - def get(self, key):
58 return self._store[key]
59
60 - def delete(self, key):
61 del self._store[key]
62
63 - def get_iterator(self):
64 return self._store.iteritems()
65
66 - def iterate(self, iterator, threshold):
67 tot = 0 68 ret = [] 69 try: 70 while tot < threshold: 71 k, v = iterator.next() 72 tot = tot + len(k) + len(v) 73 ret.append((k, v)) 74 return ret, iterator 75 except StopIteration: 76 return ret, iterator
77
78 -class BerkeleyDBStore(Store):
79 """Store using BerkeleyDB, specifically the B-Tree version"""
80 - def __init__(self, filename):
81 self._store = bsddb.btopen(filename)
82
83 - def put(self, key, value):
84 self._store[key] = value 85 self._store.sync()
86
87 - def get(self, key):
88 return self._store[key]
89
90 - def delete(self, key):
91 del self._store[key] 92 self._store.sync()
93
94 - def get_iterator(self):
95 try: 96 k, v = self._store.first() 97 return k 98 except bsddb.error: 99 return None
100
101 - def iterate(self, iterator, threshold):
102 if iterator is None: 103 return [], None 104 iterator, v = self._store.set_location(iterator) 105 tot = 0 106 ret = [(iterator, v)] 107 try: 108 while tot < threshold: 109 iterator, v = self._store.next() 110 tot = tot + len(iterator) + len(v) 111 ret.append((iterator, v)) 112 return ret, iterator 113 except bsddb.error: 114 return ret, None
115 116
117 -class SQLiteStore(Store):
118 """Store that uses SQLite for storage."""
119 - def __init__(self, filename):
120 self._db = filename 121 self.conn = sqlite3.connect(self._db) 122 c = self.conn.cursor() 123 # Create table 124 c.execute("CREATE TABLE IF NOT EXISTS blobkey(k BLOB PRIMARY KEY, v BLOB)") 125 self.conn.commit() 126 c.close()
127
128 - def put(self, key, value):
129 c = self.conn.cursor() 130 c.execute("INSERT OR REPLACE INTO blobkey(k, v) VALUES(?, ?)", (key, sqlite3.Binary(value))) 131 self.conn.commit() 132 c.close()
133
134 - def get(self, key):
135 c = self.conn.cursor() 136 c.execute("SELECT v FROM blobkey WHERE k = ?", (key,)) 137 value = c.fetchone() 138 c.close() 139 if value is None: 140 raise KeyError(key) 141 return value[0]
142
143 - def delete(self, key):
144 c = self.conn.cursor() 145 c.execute("DELETE FROM blobkey WHERE k = ?", (key,)) 146 self.conn.commit() 147 rows = c.rowcount 148 c.close() 149 if rows == 0: 150 raise KeyError
151
152 - def get_iterator(self):
153 c = self.conn.cursor() 154 c.execute("SELECT k, v FROM blobkey") 155 return c
156
157 - def iterate(self, iterator, threshold):
158 tot = 0 159 ret = [] 160 try: 161 while tot < threshold: 162 k, v = iterator.next() 163 tot = tot + len(k) + len(v) 164 ret.append((k, v)) 165 return ret, iterator 166 except StopIteration: 167 return ret, iterator
168
169 -class TestStores(unittest.TestCase):
170 - def _test_iterate(self, d):
171 contents = [("Key_%d"%i, "Val_%d"%i) for i in range(100)] 172 for k, v in contents: 173 d.put(k, v) 174 iterator = d.get_iterator() 175 kvlist = [] 176 while True: 177 kv, iterator = d.iterate(iterator, 100) 178 if not kv: 179 break 180 kvlist.extend(kv) 181 self.assertEqual(set(contents), set([(str(k), str(v)) for k, v in kvlist]))
182
183 - def test_iterate_dict(self):
184 d = DictStore() 185 self._test_iterate(d)
186
187 - def test_iterate_bdb(self):
188 d = BerkeleyDBStore("bdb") 189 self._test_iterate(d)
190
191 - def test_iterate_sqlite(self):
192 d = SQLiteStore("sqlite") 193 self._test_iterate(d)
194 195 if __name__=="__main__": 196 unittest.main() 197