· 7 years ago · Jan 25, 2019, 01:26 AM
1import sqlite3
2import argparse
3import json
4import glob
5
6
7conn = sqlite3.connect('galleries.db')
8c = conn.cursor()
9
10
11def update_db():
12 c.execute('''
13 CREATE TABLE IF NOT EXISTS galleries
14 (
15 id INTEGER PRIMARY KEY,
16 type TEXT,
17 artists TEXT,
18 groups TEXT,
19 parodys TEXT,
20 tags TEXT,
21 characters TEXT,
22 language TEXT,
23 galleryname TEXT,
24 UNIQUE (id)
25 )
26 ''')
27 db_keys = ['id', 'type', 'a', 'g', 'p', 't', 'c', 'l', 'n']
28 files = glob.glob('galleries*.json')
29 for i, filename in enumerate(files):
30 with open(filename, 'r', encoding='utf8') as f:
31 data = json.load(f)
32 print('({}/{}) reading file {} with {} entries'.format(i + 1, len(files), filename, len(data)))
33 for entry in data:
34 current_obj = [entry[key] if key in entry else '' for key in db_keys]
35 current_obj = ['|'.join(elem) if type(elem) is list else elem for elem in current_obj]
36 c.execute("INSERT OR REPLACE INTO galleries VALUES (?,?,?,?,?,?,?,?,?)", current_obj)
37 conn.commit()
38 c.execute('SELECT Count(*) FROM galleries')
39 num, = c.fetchone()
40 print('DB has {} entries'.format(num))
41
42
43def find(q):
44 q = 'SELECT id, type, artists, groups, parodys, tags, characters, language, galleryname FROM galleries WHERE {0}'.format(q)
45 print(q)
46 c.execute(q)
47 return c
48
49
50def print_info(title, arr):
51 if arr:
52 arr = arr.split('|')
53 print('{}:'.format(title), ', '.join(arr))
54
55
56def print_gallery(gallery_data):
57 id, type, artists, groups, parodys, tags, characters, language, galleryname = gallery_data
58 print('{} https://hitomi.la/galleries/{}.html'.format(type, id))
59 print_info('Title', galleryname)
60 print_info('Artists', artists)
61 print_info('Groups', groups)
62 print_info('Parodys', parodys)
63 print_info('Tags', tags)
64 print_info('Characters', characters)
65 print_info('Language', language)
66
67
68def fetch(q):
69 Ñ = find(q)
70 print('=' * 80)
71 n = 10
72 data = c.fetchmany(n)
73 while len(data):
74 for result in data:
75 print_gallery(result)
76 print('=' * 80)
77 data = c.fetchmany(n)
78 if len(data) and input('Press Enter for more, type "q" to exit\n') is 'q':
79 break
80 print('=' * 35, 'END', '=' * 40)
81
82
83def format_query(field, vals=[]):
84 if vals and len(vals):
85 return ['{} NOT LIKE "%{}%"'.format(field, v[1:]) if v[0] == '!' else '{} LIKE "%{}%"'.format(field, v) for v in vals]
86 return []
87
88
89def search_galleries(args):
90 if args.id:
91 fetch('id={}'.format(args.id))
92 else:
93 q = []
94
95 if args.type != 'all':
96 q.append('type="{}"'.format(args.type))
97
98 if args.language != 'all':
99 q.append('language="{}"'.format(args.language))
100
101 if args.name:
102 q.append('galleryname LIKE "%{}%"'.format(args.name))
103
104 q.extend(format_query('artists', args.artists))
105 q.extend(format_query('groups', args.groups))
106 q.extend(format_query('tags', args.tags))
107 q.extend(format_query('characters', args.characters))
108
109 if len(q):
110 fetch(' AND '.join(q))
111 return True
112 else:
113 return False
114
115
116if __name__ == '__main__':
117 parser = argparse.ArgumentParser(description='''Search for galleries in local copy of hitomi.la database.
118To use this script, first you have to manually download galleries*.json files from hitomi.la at urls:
119https://ltn.hitomi.la/galleries0.json,
120https://ltn.hitomi.la/galleries1.json,
121https://ltn.hitomi.la/galleries2.json,
122...,
123https://ltn.hitomi.la/galleries19.json, etc.
124(there are currently 20 files, but there can be more in the future)
125Save then in the same folder as this script.
126Then, call "python hitomy.py update" to create SQLite database. After that, you can use "python hitomy.py search ..." for quick and simple search.
127To update galleries with new entries, re-download last galleriesXX.json (and add new JSONs if there are any) file and run "python hitomy.py update" again.
128You can delete all JSON files after update is done.
129''',
130 epilog='Use "!" in search queries for exclusion, i.e.: "search -t shota !yaoi"')
131
132 parser.add_argument('action', type=str, choices=['search', 'update'],
133 help='update reads all galleries*.json files in directory and populates database, search will return results from query to database (see other options)')
134
135 parser.add_argument('-id', type=int,
136 help='search by ID (other options are ignored)')
137
138 parser.add_argument('--tags', '-t', metavar='T', type=str, nargs='+',
139 help='search tags')
140
141 parser.add_argument('--artists', '-a', metavar='A', type=str, nargs='+',
142 help='search for artists')
143
144 parser.add_argument('--groups', '-g', metavar='G', type=str, nargs='+',
145 help='search gallery groups')
146
147 parser.add_argument('--parodys', '-p', metavar='P', type=str, nargs='+',
148 help='search parodys')
149
150 parser.add_argument('--characters', '-c', metavar='C', type=str, nargs='+',
151 help='search for characters')
152
153 parser.add_argument('--language', '-l', metavar='L', default='all', choices=['all', 'czech', 'italian', 'persian', 'polish', 'thai', 'esperanto', 'tagalog', 'ukrainian', 'danish', 'french', 'arabic', 'portuguese', 'romanian', 'english', 'korean', 'russian', 'vietnamese', 'slovak', 'hungarian', 'turkish', 'dutch', 'latin', 'spanish', 'greek', 'german', 'indonesian', 'norwegian', 'japanese', 'hebrew', 'catalan', 'finnish', 'albanian', 'estonian', 'chinese', 'swedish'],
154 help='search media language')
155
156 parser.add_argument('--name', '-n', metavar='N', type=str,
157 help='search for title')
158
159 parser.add_argument('--type', '-T', default='all', choices=['all', 'doujinshi', 'manga', 'artistcg', 'gamecg', 'anime'],
160 help='search media type')
161
162 args = parser.parse_args()
163
164 if args.action == 'search':
165 r = search_galleries(args)
166 if not r:
167 parser.print_help()
168 elif args.action == 'update':
169 update_db()
170 else:
171 parser.print_help()
172
173 conn.close()