7JmruGTC

1#!/usr/bin/env python3
2
3import sqlite3
4import zlib
5import functools
6
7
8class Set(object):
9    def __init__(self, base=0x0F, name="set-dbs"):
10        self.base = base
11        self._dbs = [sqlite3.connect("{}/{:02x}".format(name, i))
12                     for i in range(base)]
13        self._cursors = [db.cursor() for db in self._dbs]
14        self._counter = [0] * base
15        for db in self._dbs:
16            db.execute("PRAGMA synchronous = off")
17            db.execute("""
18            CREATE TABLE IF NOT EXISTS t (
19                k TEXT PRIMARY KEY
20            )
21            """)
22            
23    def __del__(self):
24        self.close()
25
26    def close(self):
27        for db in self._dbs:
28            db.close()
29            
30    @functools.lru_cache(maxsize=10000)
31    def index(self, key):
32        return zlib.adler32(key.encode()) % self.base
33        
34    def __contains__(self, key):
35        idx = self.index(key)
36        c = self._cursors[idx]
37        c.execute("SELECT k FROM t WHERE k = ? LIMIT 1", (key,))
38        return c.fetchone()
39        
40    def _batched(method):
41        """
42        Notice for method who use this decorator: self._counter is important
43        """
44        @functools.wraps(method)
45        def wrapper(self, *keys):
46            for k in keys:
47                method(self, k)
48            for idx, n in enumerate(self._counter):
49                if n:
50                    self._dbs[idx].commit()
51                    self._counter[idx] = 0
52            assert not any(self._counter)
53        return wrapper
54
55    @_batched
56    def add(self, key):
57        idx = self.index(key)
58        c = self._cursors[idx]
59        c.execute("INSERT OR IGNORE INTO t(k) VALUES(?)", (key,))
60        self._counter[idx] += c.rowcount
61        
62    @_batched
63    def remove(self, key):
64        idx = self.index(key)
65        c = self._cursors[idx]
66        c.execute("DELETE FROM t WHERE k = ?", (key,))
67        self._counter[idx] += c.rowcount
68
69    def _batched(sql):
70        """
71        Another implementation
72        """
73        def method(self, *keys):
74            for key in keys:
75                idx = self.index(key)
76                c = self._cursors[idx]
77                c.execute(sql, (key,))
78                self._counter[idx] += c.rowcount
79            for idx, n in enumerate(self._counter):
80                if n:
81                    #print(idx, n)
82                    self._dbs[idx].commit()
83                    self._counter[idx] = 0
84            assert not any(self._counter)
85        return method
86
87    add = _batched("INSERT OR IGNORE INTO t(k) VALUES(?)")
88    remove = _batched("DELETE FROM t WHERE k = ?")
89    discard = remove
90    del _batched
91
92
93def main():
94    s = Set(256)
95    l = []
96    while True:
97        try:
98            x = input()
99        except EOFError:
100            break
101        l.append(x)
102        #assert x in s
103        #s.add(x)
104        #s.remove(x)
105
106    s.remove(*l)
107    del s
108
109
110if __name__ == "__main__":
111    main()