JtDSvpwD

1import requests
2from requests_html import HTMLSession
3from loguru import logger
4from log import log as log
5import time
6from threading import Thread
7from datetime import datetime
8import random
9import json
10import sqlite3
11from bs4 import BeautifulSoup as soup
12from dhooks import Webhook, Embed
13from multiprocessing import Process
14from time import sleep
15import datetime
16from multiprocessing import Pool
17
18
19def read_from_txt(path):
20    '''
21    (None) -> list of str
22    Loads up all sites from the sitelist.txt file in the root directory.
23    Returns the sites as a list
24    '''
25    # Initialize variables
26    raw_lines = []
27    lines = []
28
29    # Load data from the txt file
30    try:
31        f = open(path, "r")
32        raw_lines = f.readlines()
33        f.close()
34
35    # Raise an error if the file couldn't be found
36    except:
37        log('e', "Couldn't locate <" + path + ">.")
38        raise FileNotFound()
39
40    if(len(raw_lines) == 0):
41        raise NoDataLoaded()
42
43    # Parse the data
44    for line in raw_lines:
45        lines.append(line.strip("\n"))
46    
47    # Parse Username and Password
48    for i in lines:
49        a = i.split(':')
50        if len(a) == 4:
51            username = a[2]
52            password = a[3]
53            ip = a[0]
54            port = a[1]
55            
56            final = '%s:%s@%s:%s'%(username,password,ip,port)
57
58            lines.remove(i)
59            lines.append(final)
60
61    # Return the data
62    return lines
63
64def send_embed(alert_type, link, sizes, site, image, product, price):
65    '''
66    (str, str, list, str, str, str) -> None
67    Sends a discord alert based on info provided.
68    '''
69    iconfooter = ['https://pbs.twimg.com/profile_images/1094367268216156162/xFSrCj3S_400x400.jpg',]
70    footertext = ['By Raven']
71    webhooks = ["https://discordapp.com/api/webhooks/553171697994563584/LGCG5x9bvYahM6N4R9lG7Htjygofxam5BHAzLZBgbUVV3_Fx4TPWFXg3xeS1t38fA_su"]
72
73    ts = time.time()
74    timenow = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
75
76
77    print(link)
78    print(sizes)
79    print(site)
80    print(image)
81    print(product)
82    print(price)
83
84    i = 0
85    for webhook in webhooks:
86
87        logger.debug('Sending Nontification')
88        url = Webhook(webhook)
89
90        em1 = Embed(description=link)
91
92        if(alert_type == "NEW_site"):
93            status = "New"
94        elif(alert_type == "RESTOCK_site"):
95            status = "Restock"
96        value = ''
97
98        for size in sizes:
99            if(alert_type == "NEW_site" or alert_type == "RESTOCK_site"):
100                # cart_link = site + "/cart/" + str(field[1]) + ":1"
101                value = value + '%s\n'%(str(size))
102
103        em1.color = 0x00FF00  # colors should be a hexadecimal value
104
105        em1.set_author(
106            name=link.split('/')[2],
107            icon_url='https://pbs.twimg.com/profile_images/1094367268216156162/xFSrCj3S_400x400.jpg',
108        )
109
110        em1.set_title(
111            title='**%s**'%(product),
112            url=link
113        )
114
115        em1.add_field(
116            name="Nontification Type",
117            value=status,
118            inline=True
119        )
120
121        em1.add_field(
122            name="Price",
123            value=price,
124            inline=True
125        )
126
127        em1.add_field(
128            name="Available Sizes",
129            value=str(value),
130            inline=False
131        )
132
133        em1.add_field(
134            name="API",
135            value="[Cyber](https://cybersole.io/dashboard/quicktask?url=%s) / [PD](https://api.destroyerbots.io/quicktask?url=%s) / [Dashe](https://api.dashe.io/v1/actions/quicktask?url=%s) / [Wop](http://client.wopbot.com/createTask?url=%s) / [TKS](https://thekickstationapi.com/quick-task.php?link=%s&autostart=true)"%(link,link,link,link,link),
136            inline=False
137        )
138
139        em1.set_footer(
140            text='%s | %s'%(footertext[i],timenow),
141            icon_url=iconfooter[i]
142        )
143        
144        if(image is not None):
145            em1.set_thumbnail(image)
146        
147        
148        url.send(embed=em1)
149        i += 1
150    
151def get_proxy(proxy_list):
152    '''
153    (list) -> dict
154    Given a proxy list <proxy_list>, a proxy is selected and returned.
155    '''
156    # Choose a random proxy
157    proxy = random.choice(proxy_list)
158
159    # Set up the proxy to be used
160    proxies = {
161        "http": 'http://'+str(proxy),
162        "https": 'https://'+str(proxy)
163    }
164
165    # Return the proxy
166    return proxies
167
168def update_site_db(keyword, proxy_list, session):
169    page_num = 1999
170    while(True):
171        logger.info("Monitoring keyword: <%s>."%(keyword))
172        
173        # Create link to monitor (Kith is a special case)
174        baselink = "https://caliroots.com"
175        link = "https://caliroots.com/search/searchbytext/%s/%s"%(keyword,page_num)
176
177        # Get a proxy
178        proxies = get_proxy(proxy_list)
179        
180        # Get the products on the site
181        try:
182            r = session.get(link, proxies=proxies, timeout=3, verify=False)
183        except:
184            try:
185                proxies = get_proxy(proxy_list)
186                logger.debug(proxies)
187                r = session.get(link, proxies=proxies, timeout=5, verify=False)
188            except Exception as e:
189                logger.warning("Connection to URL failed. Proxy Error! Finding Another Proxy")
190                continue
191        
192        logger.debug('Current Page: %s'%(page_num))
193        # Check validation of pages
194
195        # Check If Page Available
196        try:
197            if "Sorry" in str(r.html.find('div.simple-page')[0].text):
198                print('Product Not Found!')
199                if page_num > 1:
200                    page_num = 1
201                    continue
202        except:
203            pass
204
205        # GET PAGE
206        product_page = r.html.find('li.product')
207        if bool(product_page) == False:
208            continue
209        # Itterate Per Product in One Page
210        for product_page_preview in product_page: 
211
212            """
213            NEEDED INFORMATION!! FOR PRODUCT
214            """
215            BASELINK = baselink
216            SIZE = []
217            PRICE = product_page_preview.find('p.price')[0].find('span')[1].text
218            IMAGE = product_page_preview.find('img')[0].attrs['src']
219            TITLE = product_page_preview.find('p.brand')[0].text + ' ' +  product_page_preview.find('p.name')[0].text
220            LINK = BASELINK + product_page_preview.find('a')[0].attrs['href']
221            # CHECK IF PRODUCT IN STOCK
222            if bool(product_page_preview.find('div.sold-out')) is True:
223                logger.warning('Product Outstock')
224                continue
225
226            logger.success('Product Instock')
227
228            # Get Product Link
229
230            product_link = session.get(str(LINK), proxies=proxies, verify=False)
231            
232            # Get Size
233            sizes_field = product_link.html.find('div.select-product.row')
234            sizes = sizes_field[0].find('option')
235            for size in sizes:
236                if 'Choose size' in size.text:
237                    continue
238                else:
239                    SIZE.append(size.text)
240
241            # Store the product in the database
242            product_info = (TITLE, LINK, SIZE)
243            alert = add_to_site_db(product_info)
244            product_found = True
245
246            try:
247                # Send a Discord alert if the product is new
248                if alert == 'new':
249                    send_embed("NEW_site", LINK, SIZE, BASELINK, IMAGE, TITLE, PRICE)
250                elif alert == 'restock':
251                    send_embed("RESTOCK_site", LINK, SIZE, BASELINK, IMAGE, TITLE, PRICE)
252
253            except:
254                logger.warning('Failed when Sending the nontification')
255        # Wait the specified timeframe before checking the site again
256        logger.warning('Monitoring Next Page')
257        page_num += 1
258        time.sleep(delay)
259
260def add_to_site_db(product):
261    # Initialize variables
262    title = product[0]
263    link = product[1]
264    stock = str(product[2])
265    alert = False
266
267    # Create database
268    conn = sqlite3.connect('products.db')
269    c = conn.cursor()
270    c.execute("""CREATE TABLE IF NOT EXISTS products_site(link TEXT UNIQUE, product TEXT, stock TEXT)""")
271
272    # Add product to database if it's unique
273    try:
274        c.execute("""INSERT INTO products_site (link, product, stock) VALUES (?, ?, ?)""", (link, title, stock))
275        logger.success("Found new product <%s>."%(title))
276        alert = 'new'
277    except:
278        logger.warning("Product <%s> already exists in the database."%(title))
279
280        # RESTOCK function
281        c.execute('SELECT stock FROM products_site WHERE link = (?)',[link])
282        getstock = c.fetchall()
283        if getstock[0][0] != stock:
284            print('RESTOCK!!')
285            # c.execute("""INSERT OR REPLACE INTO products_site (link, product, stock) VALUES (?, ?, ?)""", (link, title, stock))
286            c.execute('UPDATE products_site SET stock = (?) WHERE link = (?)',(stock,link))
287            conn.commit()
288            alert = 'restock'
289
290    # Close database
291    conn.commit()
292    c.close()
293    conn.close()
294
295    # Return whether or not it's a new product
296    return alert
297
298
299if(__name__ == "__main__"):
300    # Ignore insecure messages
301    requests.packages.urllib3.disable_warnings()
302
303    # Initialize settings
304    session = HTMLSession()
305    session.browser
306    delay = 0 # Lots of sites + few proxies = longer delay to avoid bans
307
308    # Load proxies
309    proxies = read_from_txt("proxies.txt")
310    log('i', str(len(proxies)) + " proxies loaded.")
311
312    keywords = ['nike']
313
314    # Loop through each site site
315    for keyword in keywords:
316        # Monitor for new products
317        t = Thread(target=update_site_db, args=(keyword, proxies, session))
318        t.start()
319        time.sleep(5)