0me7vnaU

1import urllib
2import json
3import twurl
4import sqlite3
5
6TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
7
8conn = sqlite3.connect('spider.sqlite3')
9cur = conn.cursor()
10
11cur.execute ('CREATE TABLE IF NOT EXISTS People (id INTEGER PRIMARY KEY, name TEXT UNIQUE, retrieved INTEGER)')
12cur.execute ('CREATE TABLE IF NOT EXISTS Follows (from_id INTEGER, to_id INTEGER, UNIQUE(from_id, to_id))')
13while True:
14	acct = raw_input('Enter a Twitter account, or quit: ')
15	if acct =='quit':break
16	if len(acct)<1:
17		cur.execute('SELECT id, name FROM People WHERE retrieved = 0 LIMIT 1')
18		try:
19			(id, acct) = cur.fetchone()
20		except:
21			print 'No unretrieved Twitter accounts found'
22			continue
23	else:
24		cur.execute('SELECT id FROM People WHERE name = ? LIMIT 1', (acct, ))
25		try:
26			id = cur.fetchone()[0]
27		except:
28			cur.execute('INSERT OR IGNORE INTO People (name, retrieved) VALUES (?, 0)', (acct, ))
29			conn.commit()
30			if cur.rowcount !=1:
31				print 'Error inserting account:', acct
32				continue
33			id = cur.lastrowid
34			
35	url = twurl.augment(TWITTER_URL, {'screen_name': acct, 'count': '20'})
36	print 'Retrieving', url
37	connection = urllib.urlopen(url)
38	data = connection.read()
39	headers = connection.info().dict
40	
41	print 'Remaining', headers['x-rate-limit-remaining']
42	js = json.loads(data)
43	
44	cur.execute('UPDATE People SET retrieved = 1 WHERE name = ?', (acct, ))
45	countnew = 0
46	countold = 0
47	
48	for u in js['users']:
49		friend = u['screen_name']
50		print friend
51		cur.execute('SELECT id FROM People WHERE name = ? LIMIT 1', (friend, ))
52		try:
53			friend_id = cur.fetchone()[0]
54			countold = countold +1
55		except:
56			cur.execute('INSERT OR IGNORE INTO People(name, retrieved) VALUES (?, 0)', (friend, ))
57			conn.commit()
58			if cur.rowcount !=1:
59				print 'Error inserting account:', friend
60				continue
61			friend_id = cur.lastrowid
62			countnew = countnew +1
63	cur.execute('INSERT OR IGNORE INTO Follows (from_id, to_id) VALUES (?,?)', (id, friend_id))
64		
65	print 'New accounts=', countnew, 'revisited=', countold
66	conn.commit()
67cur.close()