esgAwtvQ

1# Must change database parameter in mysql section of database.ini to test before test.
2
3from config import config
4from multiprocessing.dummy import Pool
5from mysql.connector import MySQLConnection
6import sys
7
8urls = ([
9         'google.com',
10         'facebook.com',
11         'instragram.com',
12         'twitter.com',
13         'hotmail.com',
14         'gmail.com',
15         'joojle.com',
16         'line.com',
17         'afaps.ac.th',
18         'kmutnb.ac.th',
19         'crma.ac.th',
20         'rtna.ac.th',
21         'rtaf.ac.th',
22         'rpca.ac.th'
23     ])
24
25
26def insert(url):
27    sql = "INSERT INTO test_pool (url) VALUES (%s);"
28    conn = MySQLConnection(**params)
29    cursor = conn.cursor()
30    cursor.execute(sql, (url,))
31    cursor.close()
32    conn.commit()
33    sql = "SELECT MAX(url_id) FROM test_pool;"
34    cursor = conn.cursor()
35    cursor.execute(sql)
36    url_id = cursor.fetchone()[0]
37    cursor.close()
38    conn.close()
39    return (url_id, url)
40
41
42def create():
43    sql = "CREATE TABLE IF NOT EXISTS test_pool (url_id INT NOT NULL AUTO_INCREMENT, url VARCHAR(255) NOT NULL, PRIMARY KEY (url_id)) ;"
44    conn = MySQLConnection(**params)
45    cursor = conn.cursor()
46    cursor.execute(sql)
47    cursor.close()
48    conn.commit()
49    conn.close()
50
51
52def drop():
53    sql = "DROP TABLE test_pool;"
54    conn = MySQLConnection(**params)
55    cursor = conn.cursor()
56    cursor.execute(sql)
57    cursor.close()
58    conn.commit()
59    conn.close()
60
61
62def select():
63    sql = "SELECt * FROM test_pool;"
64    conn = MySQLConnection(**params)
65    cursor = conn.cursor()
66    cursor.execute(sql)
67    rows = cursor.fetchall()
68    cursor.close()
69    conn.close()
70    return set(rows)
71
72
73p = Pool(2)
74log = dict()
75result_list = list()
76params = config(section='mysql')
77chunk_size = sys.argv[1]
78
79if chunk_size <= sys.maxsize and chunk_size >=0:
80    create()
81    for res in p.imap_unordered(insert, urls, chunk_size):
82        result_list.append((res[0], res[1]))
83    r = select()
84#    print(r)
85    print(r == set(result_list))
86    drop()
87else:
88    print("chunk_size must >=0 and <= {}".format(sys.maxsize))
89# Notice if chunksize of imap_unordered greater than or equal to nmber of elements of url it will result as True otherwise false.