G2mL4qan

1from urllib.request import urlopen
2import urllib.error
3import twitter as twurl
4import json
5import sqlite3
6import ssl
7
8TWITTER_URL = "https://api.twitter.com/1.1/friends/list.json"
9
10conn = sqlite3.connect("spider.sqlite")
11cur = conn.cursor()
12
13cur.execute('''
14            CREATE TABLE IF NOT EXISTS Twitter
15            (name TEXT, retrieved INTEGER, friends INTEGER)''')
16
17ctx = ssl.create_default_context()
18ctx.check_hostname = False
19ctx.verify_mode = ssl.CERT_NONE
20
21while True:
22    acct = input("Enter a Twitter account, or quit: ")
23    if acct == "quit":
24        break
25    if len(acct) < 1:
26        cur.execute("SELECT name FROM Twitter WHERE retrieved = 0 LIMIT 1")
27        try:
28            acct = cur.fetchone()[0]
29        except:
30            print("No unretrieved Twitter accounts found")
31            continue
32
33    url = twurl.augment(TWITTER_URL, {"screen_name": acct, "count": "5"})
34    print("retrieving", url)
35    connection = urlopen(url, context = ctx)
36    data = connection.read().decode()
37    headers = dict(connection.getheaders())
38
39    print("Remaining", headers["x-rate-limit-remaining"])
40    js = json.loads(data)
41
42    cur.execute("UPDATE Twitter SET retrieved=1 WHERE name = ?", (acct, ))
43
44    countnew = 0
45    countold = 0
46    for u in js["users"]:
47        friend = u["screen_name"]
48        print(friend)
49        cur.execute("SELECT friends FROM Twitter WHERE name = ? LIMIT 1", (friend, ))
50
51        try:
52            count = cur.fetchone()[0]
53            cur.execute("UPDATE Twitter SET friends = ? WHERE name = ?", (count+1, friend))
54            countold +=1
55        except:
56            cur.execute('''INSERT INTO Twitter (name, retrieved, friends) VALUES (?, 0, 1)''', (friend, ))
57            countnew += 1
58
59    print("New accounts = ", countnew, "retrieved=", countold)
60    conn.commit()
61
62cur.close()