X7j3TD0L

1# author: long nguyen (nguyenhailong253@gmail.com)
2
3import os
4import re
5import sys
6import time
7import smtplib
8import logging
9import requests
10import multiprocessing
11from settings import (BASE_URL,
12                      ATTRIBUTES_DICT,
13                      STATES,
14                      CATEGORY_LIST,
15                      FIXED_HEADERS)
16from bs4 import BeautifulSoup
17from RedisQueue import RedisQueue
18from base import IndeedScraperBase
19
20
21class IndeedJobContentScraper(IndeedScraperBase):
22    ''' A scraper to look for full description of jobs '''
23
24    def __init__(self):
25        super().__init__()
26        self.rqueue = RedisQueue('indeed_jobs_queuetest')
27        self.exception_queue = RedisQueue('indeed_exceptions_queuetest')
28        self.proxies = self.get_proxies()
29        self.headers = self.get_headers()
30
31    def _init_log(self):
32        logging.basicConfig(
33            filename="./logs/content_scrape_log_{}.log".format(
34                datetime.now().strftime("%Y-%m-%d %H.%M")),
35            format="%(levelname)s:%(message)s",
36            level=logging.DEBUG)
37
38        self.log = logging.getLogger(__name__)
39
40    def create_exception_table(self):
41        sql = """
42            CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
43            CREATE TABLE IF NOT EXISTS jobs.indeedjobsexception
44            (
45                id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
46                jobid VARCHAR NOT NULL,
47                scraped_at TIMESTAMP,
48                success BOOLEAN NOT NULL
49            );
50            """
51        with self.cursor() as cur:
52            cur.execute(sql)
53
54    def to_exception_tabel(self, jobid, time):
55        ''' Store jobid that jd not found'''
56        sql = """
57            INSERT INTO jobs.indeedjobsexception (jobid, scraped_at, success)
58            VALUES ('{}', '{}', FALSE)
59            """.format(jobid, time)
60        with self.cursor() as cur:
61            cur.execute(sql)
62
63    def to_table_db(self, jobid, content):
64        ''' Save jd to db'''
65        sql = """
66            UPDATE jobs.indeedjobstest
67            SET jd = '{}'
68            WHERE (info ->> 'jobid') = '{}';
69            """.format(content, jobid)
70        with self.cursor() as cur:
71            cur.execute(sql)
72
73    def check_scraped_jobid(self, jobid):
74        ''' Check if jd of jobid already scraped'''
75        sql = """
76            SELECT jd FROM jobs.seekjobs
77            WHERE (info ->> 'jobid') = '{}'
78            AND jd IS NOT NULL;
79            """.format(jobid)
80        result = self.query_one(sql)
81        if result:
82            print("scraped: {}".format(jobid))
83            return True
84        return False
85
86    def scrape_job_content(self, jobid):
87        ''' Scrape content of jobid'''
88
89        existed_id = 0
90        loop_count = 0
91        proxy_failed = 0
92
93        if self.check_scraped_jobid(jobid):
94            existed_id += 1
95            self.log.info("---jobid already scraped: {} - #{} \n"
96                          .format(jobid, existed_id))
97            return
98
99        finished = False
100        while not finished:
101            try:
102                url = "{}{}".format(BASE_URL, jobid)
103                html_page = requests.get(url,
104                                         headers=self.headers,
105                                         proxies=self.proxies,
106                                         timeout=5)
107                finished = False
108                status = html_page.status_code
109
110                if 300 <= status < 400:
111                    self.reset_proxy_pool()
112                    self.proxies = self.get_proxies()
113                    self.headers = self.get_headers()
114                elif status == 404:
115                    self.to_exception_tabel(jobid, self.NOW)
116                elif status == 200:
117                    soup = BeautifulSoup(
118                        re.sub("<!--|-->", "", html_page.text), "html5lib")
119
120                    jd = soup.find("div",
121                                   class_="jobsearch-JobComponent icl-u-xs-mt--md")
122                    if not jd:
123                        jd = soup.find("span",
124                                       class_="summary")
125                        if not jd:
126                            jd = soup.find("span",
127                                           id="job_summary")
128                            if not jd:
129                                loop_count += 1
130                                if loop_count == 5:
131                                    self.log.debug(
132                                        "-Unable to retrieve jd, saving to exception table for jobid: {} \n".format(jobid))
133                                    finished = True
134                                continue
135                    content = jd.get_text(separator="\n\n",
136                                          strip=True).replace("'", "''")
137                    self.to_table_db(jobid, content)
138                    finished = True
139                    print("-saved: {}".format(jobid))
140                else:
141                    self.to_exception_tabel(jobid, self.NOW)
142                    finished = True
143
144            except requests.exceptions.SSLError as s:
145                self.proxies = self.get_proxies()
146                self.log.info("-SSL Error: {}".format(s))
147
148            except requests.exceptions.ProxyError as p:
149                proxy_failed += 1
150                self.log.debug("-Proxy Error: {}".format(p))
151                if proxy_failed >= 3:
152                    self.reset_proxy_pool()
153                    self.log.debug("-Re-downloading proxies \n")
154                self.headers = self.get_headers()
155                self.proxies = self.get_proxies()
156
157            except requests.exceptions.RequestException as e:
158                time.sleep(180)
159                self.log.debug("-Request failed: {} \n".format(e))
160
161            except KeyboardInterrupt:
162                self.log.debug(
163                    "-Keyboard Interrupted: wait putting jobid back to queue")
164                break
165
166    def scraper(self):
167        ''' Run scraper'''
168
169        self.connect_db()
170        info_scraper_finished = 0
171        while True:
172            jobid = self.rqueue.pop(timeout=10)
173            if jobid:
174                if jobid == "Done":
175                    info_scraper_finished += 1
176                else:
177                    self.scrape_job_content(jobid.decode("utf-8"))
178            else:
179                time.sleep(3)
180
181            if info_scraper_finished == len(CATEGORY_LIST):
182                if self.rqueue.empty():
183                    self.exception_queue.put("Done")
184                    break
185
186    def run(self, slaves=15):
187        ''' Create a pool of workers to scrape '''
188
189        workers = []
190        for i in range(slaves):
191            w = multiprocessing.Process(target=self.scraper)
192            w.start()
193            workers.append(w)
194            print("Start #", i)
195
196        for w in workers:
197            w.join()
198
199
200if __name__ == '__main__':
201    s = IndeedJobContentScraper()
202    s.run()