· 8 years ago · Jun 11, 2018, 06:14 AM
1import sqlite3
2import re
3
4def createtables(bot_db, bot_dbc):
5 bot_dbc.executescript("""
6 CREATE TABLE IF NOT EXISTS xsentence_links (
7 id_from INTEGER,
8 id_to INTEGER,
9 count INTEGER
10 );
11 CREATE TABLE IF NOT EXISTS isentence_links (
12 id_from INTEGER,
13 id_to INTEGER,
14 count INTEGER
15 );
16 CREATE TABLE IF NOT EXISTS fragments (
17 id INTEGER PRIMARY KEY,
18 count INTEGER,
19 text TEXT,
20 word1 TEXT,
21 word2 TEXT,
22 word3 TEXT
23 );
24 """)
25 bot_db.commit()
26
27 return bot_dbc
28
29bot_db = sqlite3.connect("bot.db")
30bot_dbc = bot_db.cursor()
31bash_db = sqlite3.connect("bash.db")
32bash_dbc = bash_db.cursor()
33
34bot_dbc = createtables(bot_db, bot_dbc)
35
36QUOTES = 2000#20589
37markov = 2
38
39def makemarkov(text):
40 words = text.split(" ")
41 lenwords = len(words)
42 fragments = []
43
44 #for i in range(0-markov,lenwords): #Not sure which to use
45 for i in range(0,lenwords,markov+1):
46 fragment = []
47 for j in range(0,markov+1):
48 if i+j < 0 or i+j >= lenwords:
49 fragment.append("")
50 else:
51 fragment.append(words[i+j])
52
53 fragments.append(fragment)
54
55 return fragments
56
57removenameregex = re.compile("<(.*)> (.*)")
58for k in range(1,QUOTES+1):
59 print(k)
60 bash_dbc = bash_dbc.execute("SELECT * FROM quotes WHERE quoteid=?", (k,))
61 quote = bash_dbc.fetchall() #quote is now a list of tuples containing the quote
62 quotelen = len(quote)
63
64 for i in range(0,quotelen):
65 quotetext = quote[i][1]
66 if quotetext == "": continue
67 if quotetext[0] != "<": continue #ignore actions and joins and stuff for now
68
69 #parse out the name
70 result = removenameregex.match(quotetext)
71 if result is None: continue
72 quotetext = result.group(2)
73 if quotetext == "": continue
74
75 """
76 make inter-sentence links
77 """
78
79 fragments = makemarkov(quotetext)
80 fragmentslen = len(fragments)
81 for j in range(0,fragmentslen):
82 fragment1 = bot_dbc.execute("SELECT * FROM fragments WHERE text=?",(str(fragments[j]),))
83 f1exists = fragment1.fetchone()
84 if f1exists is not None:
85 bot_dbc.execute("UPDATE fragments SET count=count+1 WHERE id=?",(f1exists[0],))
86 else:
87 bot_dbc.execute(
88 "INSERT INTO fragments (text, word1, word2, word3, count) VALUES (?, ?, ?, ?, 1)",
89 (str(fragments[j]), fragments[j][0], fragments[j][1], fragments[j][2],)
90 )
91
92 fragment1 = bot_dbc.execute("SELECT * FROM fragments WHERE text=?",(str(fragments[j]),))
93 f1exists = fragment1.fetchone()
94
95 #if we have another fragment
96 if j+1 < fragmentslen:
97 fragment2 = bot_dbc.execute("SELECT * FROM fragments WHERE text=?",(str(fragments[j+1]),))
98 f2exists = fragment2.fetchone()
99 if f2exists is not None:
100 bot_dbc.execute("UPDATE fragments SET count=count+1 WHERE id=?",(f2exists[0],))
101 else:
102 bot_dbc.execute(
103 "INSERT INTO fragments (text, word1, word2, word3, count) VALUES (?, ?, ?, ?, 1)",
104 (str(fragments[j+1]), fragments[j+1][0], fragments[j+1][1], fragments[j+1][2],)
105 )
106
107 fragment2 = bot_dbc.execute("SELECT * FROM fragments WHERE text=?",(str(fragments[j+1]),))
108 f2exists = fragment2.fetchone()
109
110 #fragment inter-sentence linking part
111 link = bot_dbc.execute("SELECT * FROM isentence_links WHERE id_from=? AND id_to=?",(f1exists[0], f2exists[0],))
112 linkexists = link.fetchone()
113 if linkexists is not None:
114 bot_dbc.execute(
115 "UPDATE isentence_links SET count=count+1 WHERE id_from=? AND id_to=?",
116 (linkexists[0],linkexists[1],)
117 )
118 else:
119 bot_dbc.execute(
120 "INSERT INTO isentence_links (id_from, id_to, count) VALUES (?, ?, 1)",
121 (f1exists[0], f2exists[0],)
122 )
123
124bot_db.commit()