· 8 years ago · Jun 09, 2018, 02:28 PM
1#!/usr/local/bin/python
2
3import pgdb, httplib, sys, os
4from misc import *
5from mods import *
6
7VERSION="4"
8
9def scrape_board( db, scrape_id, ( id, name, sname, host, parser ), mods ):
10 # Fetch thread list
11 hc = httplib.HTTPConnection( host )
12 p_mod = __import__( parser )
13
14 all_threads = []
15
16 for _ in xrange( 0, 3 ):
17 try:
18 all_threads = p_mod.fetch_thread_list( hc, sname )
19 break
20 except FetchErrorNeedsReconnect:
21 hc = httplib.HTTPConnection( host )
22
23 if not all_threads:
24 return
25
26 # Cull un-updated threads
27 #db.execute( '''SELECT thread_id, post_id FROM s_threads WHERE board_id=%s AND thread_id IN ( %s )''' % ( id, ','.join( map( lambda x: str( x[0] ), all_threads ) ) ) )
28 db.execute( '''SELECT thread_id, MAX(post_no) FROM s_posts WHERE board_Id=%s AND thread_id IN ( %s ) GROUP BY thread_id''' % ( id, ','.join( map( lambda x: str( x[0] ), all_threads ) ) ) )
29 old_threads, threads = dict( db.fetchall() ), []
30
31 for tid, pid in all_threads:
32 if tid not in old_threads:
33 # Throw this into the database.
34 db.execute( '''INSERT INTO s_threads( thread_id, board_id, post_id ) VALUES( %s, %s, 0 )''', ( tid, id ) )
35 threads.append( ( tid, 0 ) )
36 elif pid > old_threads[tid]:
37 threads.append( ( tid, old_threads[tid] ) )
38
39 if not threads:
40 print( "No new posts to fetch." )
41 return
42
43 # Fetch threads
44 print( "Fetching and processing threads..." )
45
46 threads_done = 0
47 for tid, old in threads:
48 threads_done += 1
49
50 for _ in xrange( 0, 3 ):
51 try:
52 thread = p_mod.fetch_thread( hc, sname, tid )
53 break
54 except FetchErrorNeedsReconnect:
55 hc = httplib.HTTPConnection( host )
56 except ParseError, e:
57 sys.stderr.write( "ERROR: %s (#%s)\n" % ( e, tid ) )
58 thread = None
59 break
60
61 if not thread:
62 sys.stderr.write( "ERROR: Thread not fetched properly -- skipping.\n" )
63 continue
64
65 for mod in mods:
66 if 'process_thread' in mod.__dict__:
67 mod.process_thread( db, thread )
68
69 _, latest, comments = thread
70
71 # Insert new comments
72 comments_done = 0
73 for c in comments:
74 comments_done += 1
75 write_status_2( ( threads_done, len( threads ) ), ( comments_done, len( comments ) ) )
76
77 for mod in mods:
78 if 'process_post' in mod.__dict__:
79 mod.process_post( db, c )
80
81 if c[0] <= old:
82 continue
83
84 # Try to get the image
85 if c[7]:
86 for _ in xrange( 0, 3 ):
87 try:
88 img_id = process_image( hc, db, scrape_id, c[7], c[8], mods )
89 break
90
91 except FetchErrorNeedsReconnect:
92 hc = httplib.HTTPConnection( host )
93
94 except FetchError, e:
95 sys.stderr.write( "ERROR: %s\n" % e )
96 img_id = 0
97 break
98 else:
99 raise RuntimeError( "Unable to fetch image. Something is broken.\n%s" % c )
100 else:
101 img_id = 0
102
103 # Put into the database
104 try:
105 u = lambda x: unicode(x, 'utf-8')
106 db.execute( '''INSERT INTO s_posts( thread_id, board_id, img_id, post_no, post_name, post_email, post_trip, post_subject, post_date, post_comment, post_img, post_origimg, scrape_id, post_index ) VALUES( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )''', ( tid, id, img_id, c[0], u(c[1]), u(c[3]), u(c[2]), u(c[4]), c[6], u(c[5]), u(c[7]), u(c[9]), scrape_id, c[4] + ' ' + c[5] + ' ' + c[9] ) )
107 except UnicodeError, e:
108 print(e)
109
110 # Update the latest thread.
111 db.execute( '''UPDATE s_threads SET post_id=%s WHERE board_id=%s AND thread_id=%s''', ( latest, id, tid ) )
112
113def add_indexes(c, table):
114 indexes = ['id', 'hash', 'aspect', 'h', 'nsfw', 'w']
115 for i in indexes:
116 c.execute('''CREATE INDEX idx_%s_%s ON %s(img_%s)''' % (i, table, table, i))
117
118def regenerate_cache(db, c, cache_table, board_ids):
119 c.execute( '''DROP TABLE IF EXISTS %s''' % ( cache_table, ) )
120 c.execute( '''CREATE TABLE %s AS SELECT DISTINCT ON (i.img_id) i.img_id, i.img_hash, i.img_path, i.img_thumb, i.scrape_id, i.img_w, i.img_h, i.img_aspect, i.img_nsfw, i.img_nsfw_by, i.img_animated FROM s_images i JOIN s_posts p ON p.img_id = i.img_id WHERE p.img_id > 0 AND p.board_id IN (%s)''' % ( cache_table, ','.join( map( lambda x: str(x), board_ids ) ) ) )
121 add_indexes(c, cache_table)
122 db.commit()
123
124if __name__ == '__main__':
125 db = pgdb.connect( LOL="LOL" )
126 c = db.cursor()
127
128 mods = load_mods( c )
129
130 # Get the scrape ID
131 c.execute( '''INSERT INTO s_scrapes( scrape_date, scrape_version ) VALUES( NOW(), %s )''', ( VERSION, ) )
132 c.execute( '''SELECT currval('seq_scrape_id') FROM s_scrapes LIMIT 1''' )
133 scrape_id = c.fetchone()[0]
134
135 # Get the boards to scrape
136 c.execute( '''SELECT board_id, board_name, board_sname, board_host, board_parser FROM s_boards WHERE board_scrape=true''' )
137 boards = c.fetchall()
138
139 for b in boards:
140 print( "Scraping %s..." % b[1] )
141 scrape_board( c, scrape_id, b, mods )
142 db.commit()
143
144 print( "Flushing search cache..." )
145 c.execute( '''DELETE FROM s_search_cache''' )
146 c.execute( '''DELETE FROM s_psearch_cache''' )
147
148 print( "Decaying popularity..." )
149 c.execute( '''SELECT i.img_id FROM s_images i JOIN s_image_hits ih ON i.img_id=ih.img_id WHERE i.img_nsfw=0 ORDER BY ih.img_hits DESC LIMIT 20''' )
150 pop_sfw = map( lambda x: str( x[0] ), c.fetchall() )
151
152 c.execute( '''SELECT i.img_id FROM s_images i JOIN s_image_hits ih ON i.img_id=ih.img_id WHERE i.img_nsfw!=0 ORDER BY ih.img_hits DESC LIMIT 20''' )
153 pop_nsfw = map( lambda x: str( x[0] ), c.fetchall() )
154
155 if (pop_sfw + pop_nsfw):
156 c.execute( '''UPDATE s_image_hits SET img_hits=img_hits*0.95 WHERE img_id IN ( %s )''' % ','.join( pop_sfw + pop_nsfw ) );
157
158 print( "Updating board counts..." )
159 c.execute( '''UPDATE s_boards b SET board_imgs=(SELECT COUNT(*) FROM s_posts p WHERE p.board_id=b.board_id AND p.img_id>0)''' )
160 c.execute( '''UPDATE s_boards b SET board_posts=(SELECT COUNT(*) FROM s_posts p WHERE p.board_id=b.board_id)''' )
161
162 print( "Updating random views..." )
163 c.execute( '''SELECT board_id FROM s_boards''' )
164 boards = map( lambda x: x[0], c.fetchall() )
165
166 db.commit()
167
168 for board_id in boards:
169 regenerate_cache(db, c, 'v_random_%s' % (board_id,), [board_id])
170
171 regenerate_cache(db, c, 'v_random_w', [1,2])
172
173 c.execute('''SELECT COUNT(*) FROM s_posts WHERE scrape_id=%s''', (scrape_id,))
174 new_posts = c.fetchone()[0]
175
176 c.execute('''SELECT COUNT(*) FROM s_images WHERE scrape_id=%s''', (scrape_id,))
177 new_images = c.fetchone()[0]
178
179 os.system('./twit.sh 4scrape:mypasswordisawesome "%s new images in %s posts scraped."' % (str(new_images), str(new_posts)))
180
181 c.close()
182 db.commit()
183 db.close()
184 print( "All done." )