· 4 years ago · Apr 03, 2021, 09:04 AM
1import requests
2import concurrent.futures
3from Items import itemName
4from Realms import realmNames
5from Pets import petName
6import time
7import os
8from datetime import datetime
9import pickle
10import json
11
12# how to install:
13# If you don't have a file for any of the imports above this, you need to install it through cmd.
14# If you try to run the program and something is missing or it doesn't run, install what's missing and try again.
15# Press your windows key + r, type "cmd" without quotes, type in the following:
16# pip install python
17# pip install pickle
18# pip install time
19
20
21# To run the program, install any IDLE that supports Python, I recommend Pycharm: https://www.jetbrains.com/pycharm/
22# Make sure all the associated files are located in the same file as your main file or else it won't work.
23
24# change this if your PC can't handle 83 threads:
25MAX_THREADS = 83
26
27itemDict = {}
28petDict = {}
29# This is used for your dictionaries:
30for block in itemName:
31 for item in itemName[block]:
32 itemDict[item] = {'price': block, 'name': itemName[block][item]}
33
34for block in petName:
35 for item in petName[block]:
36 petDict[item] = {'price': block, 'name': petName[block][item]}
37
38
39# This is the multithreading section, it controls the download of urls and the search function:
40def download_url(url):
41 global realm_dict
42
43 if int(url.split("?")[0].split("/")[-2]) in realmNames:
44 realm_name = realmNames[int(url.split("?")[0].split("/")[-2])]
45 else:
46 realm_name = "unknown"
47 try:
48 realmid = int(url.split("?")[0].split("/")[-2])
49 if realmid in realm_dict:
50 resp = requests.get(url, headers={'If-Modified-Since': realm_dict[realmid][0]})
51 else:
52 resp = requests.get(url)
53 if resp.status_code == 304:
54 pass # print("not modified")
55 else:
56 data = {}
57 res = requests.get(url)
58 auctions = res.json()['auctions']
59
60 realm_dict[realmid] = [res.headers['Last-Modified'],
61 datetime.strptime(res.headers['Last-Modified'][4:-4].strip(), '%d %b %Y %H:%M:%S')]
62
63 with open("realm_timestamps.pickle", "wb") as f:
64 pickle.dump(realm_dict, f)
65
66 for i in auctions:
67 if 'unit_price' in i:
68 price = i['unit_price']
69 elif 'buyout' in i:
70 price = i['buyout']
71 else:
72 price = 0 # should never happen
73
74 item_id = i["item"]['id']
75
76 if price > 0:
77 if item_id in itemDict:
78 if price <= (itemDict[item_id]['price'] * 10000):
79 try:
80 print(f"- ITEM - {realm_name}: {itemDict[item_id]['name']} | {price / 10000}g")
81 except Exception as msg:
82 print(str(msg))
83 elif 'pet_species_id' in i['item']:
84 item_id = i['item']['pet_species_id']
85 if item_id in petDict:
86 if price <= (petDict[item_id]['price'] * 10000):
87 try:
88 if int(url.split("?")[0].split("/")[-2]) in realmNames:
89 print(f"- ITEM - {realm_name}: {petDict[item_id]['name']} | {price / 10000}g")
90 else:
91 print(f"- ITEM - Unknown Realm {petDict[item_id]['name']} | {price / 10000}g")
92 except Exception as msg:
93 print("pet id missing -----(" + str(msg) + ")" + str(item_id) + "<<")
94 except Exception as msg:
95 print(str(msg))
96 return data
97
98
99# Thread controller:
100def download_realm_data(urls):
101 threads = min(MAX_THREADS, len(urls))
102
103 with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
104 executor.map(download_url, urls)
105
106
107# End multithreading functions
108
109
110# This creates your access token:
111def create_access_token(client_id, client_secret, region='us'):
112 data = {'grant_type': 'client_credentials'}
113 response = requests.post('https://%s.battle.net/oauth/token' % region, data=data, auth=(client_id, client_secret))
114 return response.json()
115
116
117# ends here
118
119# This grabs urls for downloading:
120def get_realm_urls(token):
121 url = "https://us.api.blizzard.com/data/wow/connected-realm/index?namespace=dynamic-us&locale=en_US&access_token=" + token
122 res = requests.get(url)
123 realm_data = json.loads(res.content)
124 links = []
125 for i in (realm_data['connected_realms']):
126 links.append(i['href'].replace("?", "/auctions?") + "&locale=en_US&access_token=" + token)
127 return links
128
129
130# ends here
131
132# Add your client ID and secret here, obtain both from https://develop.battle.net/access/.
133# Create a client, generate a secret, plug them into create_access_token('client id goes here', 'secret goes here').
134
135response = create_access_token('client id here', 'secret here')
136token = response['access_token']
137
138# ends here
139
140
141# Open up the realm_times dict from file:
142if os.path.exists("realm_timestamps.pickle"):
143 try:
144 with open("realm_timestamps.pickle", "rb") as f:
145 realm_dict = pickle.load(f)
146 except:
147 realm_dict = {}
148else:
149 realm_dict = {}
150 # ...or build it on initial scrape.
151
152
153# ends here
154
155# This shows the api update time for each realm, you can comment this out if you want.
156# def sorter(e):
157# t = realm_dict[e][0]
158#
159# base_time = datetime(1980, 1, 1, 1, 1, 1)
160# key_time = datetime.strptime(t, "%a, %d %b %Y %H:%M:%S %Z")
161# difference = key_time - base_time
162# seconds_in_day = 24 * 60 * 60
163# x, y = divmod(difference.days * seconds_in_day + difference.seconds, 60)
164# x = x + y
165#
166# return x
167#
168#
169# keys = realm_dict.keys()
170# sort = sorted(realm_dict, key=sorter)
171# print(f"-- {len(sort)} Total Realms --")
172# for value in sort:
173# print(f">> {value}, {realm_dict[value][0]}")
174
175# ends here
176
177print("Start Scan:")
178
179# This cycles the downloads of the realms as they go through their hourly updates:
180while 1:
181 items_to_print = []
182 realms = get_realm_urls(token)
183 realms_to_download = []
184 for realm in realms:
185 realmid = int(realm.split("?")[0].split("/")[-2])
186 if realmid not in realm_dict or (datetime.utcnow() - realm_dict[realmid][1]).seconds >= 3600:
187 realms_to_download.append(realm)
188 # print(realms_to_download)
189 # print(len(realms_to_download))
190 stime = time.time()
191 if len(realms_to_download) > 0:
192 download_realm_data(realms_to_download)
193 # print(datetime.now().strftime('%H:%M:%S'))
194