Cis6JzrD

1import re
2import urllib
3import smtplib
4import sqlite3
5from collections import deque
6from urllib.parse import urlsplit
7from sqlite3 import Error
8from email.mime.multipart import MIMEMultipart
9from email.mime.text import MIMEText
10
11import requests
12import requests.exceptions
13from bs4 import BeautifulSoup
14from selenium import webdriver
15from selenium.common.exceptions import NoSuchElementException
16
17## INIT ##
18browser = webdriver.Chrome('C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe')
19browser.get('https://www.yellowpages.com.au/search/listings?clue=restaurants&eventType=pagination&locationClue=Mackay+Region%2C+QLD&pageNumber=1&referredBy=UNKNOWN')
20
21
22results = {}
23search_category_list = ['restaurants'] # , 'hairdresser', 'engineering' # 'clothing', 'computer', 'electronic', 'gifts', 'animals', 'sports', 'toys', 'vehicle', 'religion', 'garden', 'health', 'non profit', 'books'
24search_location_list = ['Mackay QLD'] # , 'Brisbane, QLD 4000', 'Townsville, QLD' # 'Mackay Region, QLD', 'Greater Brisbane, QLD', 'Townsville Region, QLD'
25
26
27################# DETERMINE FUNCTIONS #####################
28
29def create_connection(db_file):
30    """ create a database connection to a SQLite database """
31    try:
32        conn = sqlite3.connect(db_file)
33        return conn
34    except IOError:
35        open(db_file, "w+")
36
37    return None
38
39conn = create_connection("database.db")
40cursor = conn.cursor()
41
42def get_contact_info(contact_class, div, attribute='href'):
43    try:
44        return div.find_element_by_class_name(contact_class).get_attribute(attribute)
45    except:
46        return None
47
48
49def get_business_name(contact_class, div):
50    try:
51        return div.find_element_by_class_name(contact_class).text
52    except:
53        return None
54
55def get_business_logo(contact_class, div, attribute='src'):
56    try:
57        return div.find_element_by_class_name(contact_class).get_attribute(attribute)
58    except:
59        return None
60
61def get_search(contact_class, div, attribute='value'):
62    try:
63        return div.find_element_by_class_name(contact_class).get_attribute(attribute)
64    except NoSuchElementException:
65        return None
66
67def next_page():
68    element = browser.find_element_by_css_selector('div.button-pagination-container a.navigation:last-of-type').get_attribute('href')
69    browser.get(element)
70
71def check_button_exists(contact_class, attribute='href'):
72    try:
73        browser.find_element_by_css_selector(contact_class).get_attribute(attribute)
74    except NoSuchElementException:
75        return False
76    return True
77
78def search(category, location):
79    cat = urllib.parse.quote(category)
80    loc = urllib.parse.quote(location)
81    browser.get(f'https://www.yellowpages.com.au/search/listings?clue={cat}&locationClue={loc}')
82
83
84def store_dict(cursor):
85    create_table(cursor)
86    for business_name in results:  # how to get the business name
87        website = results[business_name]['website']
88        email = results[business_name]['email']
89        location = results[business_name]['location']
90        type = results[business_name]['type']
91        logo = results[business_name]['logo']
92        if not duplicate_check(cursor, business_name, email):
93            cursor.execute("INSERT INTO Listings(business_name, website, email, location, type, logo) VALUES(:business_name, :website, :email, :location, :type, :logo)",
94                   {'business_name': business_name, 'website': website, 'email': email, 'location': location, 'type':type, 'logo':logo})
95            conn.commit()
96
97    conn.close()
98
99
100def create_table(cursor):
101    cursor.execute("""CREATE TABLE IF NOT EXISTS Listings (
102                            business_name text,
103                            website text,
104                            email text,
105                            location text,
106                            type text,
107                            logo text,
108                            sent int DEFAULT 0
109                        ); """)
110
111    blacklist(cursor, ["Roshni Indian Restaurant"], ["bookings@roshni.com.au"])      # BLACKLIST {name}, {email}
112
113def duplicate_check(cursor, name, email):
114    name = cursor.execute("SELECT business_name FROM Listings WHERE business_name=:name;", {'name': name}).fetchone()
115    email = cursor.execute("SELECT business_name FROM Listings WHERE email=:email;", {'email': email}).fetchone()
116
117    if name or email:
118        return True
119    return False
120
121def blacklist(cursor, names: list, emails: list):
122    for name, email in zip(names, emails):
123        name_exists = cursor.execute("SELECT business_name FROM Listings WHERE business_name=:name;", {'name': name}).fetchone()
124        email_exists = cursor.execute("SELECT business_name FROM Listings WHERE email=:email;", {'email': email}).fetchone()
125        if name_exists or email_exists:
126            if name_exists == email_exists:
127                cursor.execute("UPDATE Listings SET sent = 1 WHERE business_name = :name", {'business_name': name})
128            else:
129                cursor.execute("UPDATE Listings SET sent = 1 WHERE business_name = :name OR email = :email", {'business_name': name, 'email': email})
130        else:
131            cursor.execute("INSERT INTO Listings(business_name, website, email, location, type, logo, sent) VALUES(:business_name, :website, :email, :location, :type, :logo, :sent)", {'business_name': name, 'website': "", 'email': email, 'location': "", 'type':"", 'logo': "", 'sent': "1"})
132            conn.commit()
133
134################# SCRAPE #####################
135
136for location in search_location_list:
137    for category in search_category_list:
138        search(category, location)
139
140        # loop until there are no more pages
141        while True:
142            listing_divs = browser.find_elements_by_class_name('search-contact-card-table-div')
143
144            # sort through contact divs - fetch data
145            for div in listing_divs:
146                business_info = {}
147
148                logo = get_business_logo('listing-logo', div)
149                business_info['logo'] = logo
150
151                business_info['location'] = location
152                business_info['type'] = category
153
154                # Get website
155                website = get_contact_info('contact-url', div, attribute='href')
156                if website:
157                    business_info['website'] = website
158                else:
159                    business_info['website'] = ""
160
161                # Get email
162                email = get_contact_info('contact-email', div, attribute='data-email')
163                if email:
164                    business_info['email'] = email
165                else:
166                    continue
167
168
169
170
171                # Get business name
172                name = get_business_name('listing-name', div)
173                if name:
174                    results[name] = business_info
175
176
177
178
179
180            # if the next button exists, move to the next page.
181            if check_button_exists('div.button-pagination-container a.navigation:last-of-type', attribute='href'):
182                #next_button_exists = True
183                next_page()
184            else:
185                break
186
187
188print(results)
189store_dict(cursor)