· 9 years ago · Nov 17, 2016, 07:24 PM
1from sys import stdout
2
3from Stock import Stock
4import Pickler
5import Scraper
6import Rankings
7import Fixer
8import Writer
9
10# HTML error code handler - importing data is a chore, and getting a connection
11# error halfway through is horribly demotivating. Use a pickler to serialize
12# imported data into a hot-startable database.
13pklFileName = 'tmpstocks.pkl'
14pickler = Pickler.Pickler()
15
16# Check if a pickled file exists. Load it if the user requests. If no file
17# loaded, stocks is an empty list.
18stocks = pickler.loadPickledFile(pklFileName)
19
20# Scrape data from FINVIZ. Certain presets have been established (see direct
21# link for more details)
22url = 'http://finviz.com/screener.ashx?v=152&f=cap_smallover&' +
23 'ft=4&c=0,1,2,6,7,10,11,13,14,45,65'
24html = Scraper.importHtml(url)
25
26# Parse the HTML for the number of pages from which we'll pull data
27nPages = -1
28for line in html:
29 if line[0:40] == '<option selected="selected" value=1>Page':
30 # Find indices
31 b1 = line.index('/') + 1
32 b2 = b1 + line[b1:].index('<')
33 # Number of pages containing stock data
34 nPages = int(line[b1:b2])
35 break
36
37# Parse data from table on the first page of stocks and store in the database,
38# but only if no data was pickled
39if pickler.source == Pickler.PickleSource.NOPICKLE:
40 Scraper.importFinvizPage(html, stocks)
41
42# The first page of stocks (20 stocks) has been imported. Now import the
43# rest of them
44source = Pickler.PickleSource.FINVIZ
45iS = pickler.getIndex(source, 1, nPages + 1)
46
47for i in range(iS, nPages + 1):
48 try:
49 # Print dynamic progress message
50 print('Importing FINVIZ metrics from page ' + str(i) + ' of ' +
51 str(nPages) + '...', file=stdout, flush=True)
52
53 # Scrape data as before
54 url = 'http://finviz.com/screener.ashx?v=152&f=cap_smallover&ft=4&r=' +
55 str(i*20+1) + '&c=0,1,2,6,7,10,11,13,14,45,65'
56 html = Scraper.importHtml(url)
57
58 # Import stock metrics from page into a buffer
59 bufferList = []
60 Scraper.importFinvizPage(html, bufferList)
61
62 # If no errors encountered, extend buffer to stocks list
63 stocks.extend(bufferList)
64 except:
65 # Error encountered. Pickle stocks for later loading
66 pickler.setError(source, i, stocks)
67 break
68
69
70# FINVIZ stock metrics successfully imported
71print('n')
72
73# Store number of stocks in list
74nStocks = len(stocks)
75
76# Handle pickle file
77source = Pickler.PickleSource.YHOOEV
78iS = pickler.getIndex(source, 0, nStocks)
79
80# Grab EV/EBITDA metrics from Yahoo! Finance
81for i in range(iS, nStocks):
82 try:
83 # Print dynamic progress message
84 print('Importing Key Statistics for ' + stocks[i].tick +
85 ' (' + str(i) + '/' + str(nStocks - 1) + ') from Yahoo! Finance...',
86 file=stdout, flush=True)
87
88 # Scrape data from Yahoo! Finance
89 url = 'http://finance.yahoo.com/q/ks?s=' + stocks[i].tick + '+Key+Statistics'
90 html = Scraper.importHtml(url)
91
92 # Parse data
93 for line in html:
94 # Check no value
95 if 'There is no Key Statistics' in line or
96 'Get Quotes Results for' in line or
97 'Changed Ticker Symbol' in line or
98 '</html>' in line:
99 # Non-financial file (e.g. mutual fund) or
100 # Ticker not located or
101 # End of html page
102 stocks[i].evebitda = 1000
103 break
104 elif 'Enterprise Value/EBITDA' in line:
105 # Line contains EV/EBITDA data
106 evebitda = Scraper.readYahooEVEBITDA(line)
107 stocks[i].evebitda = evebitda
108 break
109 except:
110 # Error encountered. Pickle stocks for later loading
111 pickler.setError(source, i, stocks)
112 break
113
114
115# Yahoo! Finance EV/EBITDA successfully imported
116print('n')
117
118# Handle pickle file
119source = Pickler.PickleSource.YHOOBBY
120iS = pickler.getIndex(source, 0, nStocks)
121
122# Grab BBY metrics from Yahoo! Finance
123for i in range(iS, nStocks):
124 try:
125 # Print dynamic progress message
126 print('Importing Cash Flow for ' + stocks[i].tick +
127 ' (' + str(i) + '/' + str(nStocks - 1) + ') from Yahoo! Finance...',
128 file=stdout, flush=True)
129
130 # Scrape data from Yahoo! Finance
131 url = 'http://finance.yahoo.com/q/cf?s=' + stocks[i].tick + '&ql=1'
132 html = Scraper.importHtml(url)
133
134 # Parse data
135 totalBuysAndSells = 0
136 for line in html:
137 # Check no value
138 if 'There is no Cash Flow' in line or
139 'Get Quotes Results for' in line or
140 'Changed Ticker Symbol' in line or
141 '</html>' in line:
142 # Non-financial file (e.g. mutual fund) or
143 # Ticker not located or
144 # End of html page
145 break
146 elif 'Sale Purchase of Stock' in line:
147 # Line contains Sale/Purchase of Stock information
148 totalBuysAndSells = Scraper.readYahooBBY(line)
149 break
150
151 # Calculate BBY as a percentage of current market cap
152 bby = round(-totalBuysAndSells / stocks[i].mktcap * 100, 2)
153 stocks[i].bby = bby
154 except:
155 # Error encountered. Pickle stocks for later loading
156 pickler.setError(source, i, stocks)
157 break
158
159
160# Yahoo! Finance BBY successfully imported
161
162if not pickler.hasErrorOccurred:
163 # All data imported
164 print('n')
165 print('Fixing screener errors...')
166
167 # A number of stocks may have broken metrics. Fix these (i.e. assign out-of-
168 # bounds values) before sorting
169 stocks = Fixer.fixBrokenMetrics(stocks)
170
171 print('Ranking stocks...')
172
173 # Calculate shareholder Yield
174 for i in range(nStocks):
175 stocks[i].shy = stocks[i].div + stocks[i].bby
176
177 # Time to rank! Lowest value gets 100
178 rankPE = 100 * (1 - Rankings.rankByValue([o.pe for o in stocks]) / nStocks)
179 rankPS = 100 * (1 - Rankings.rankByValue([o.ps for o in stocks]) / nStocks)
180 rankPB = 100 * (1 - Rankings.rankByValue([o.pb for o in stocks]) / nStocks)
181 rankPFCF = 100 * (1 - Rankings.rankByValue([o.pfcf for o in stocks]) / nStocks)
182 rankEVEBITDA = 100 * (1 - Rankings.rankByValue([o.evebitda for o in stocks]) / nStocks)
183
184 # Shareholder yield ranked with highest getting 100
185 rankSHY = 100 * (Rankings.rankByValue([o.shy for o in stocks]) / nStocks)
186
187 # Rank total stock valuation
188 rankStock = rankPE + rankPS + rankPB + rankPFCF + rankEVEBITDA + rankSHY
189
190 # Rank 'em
191 rankOverall = Rankings.rankByValue(rankStock)
192 # Calculate Value Composite - higher the better
193 valueComposite = 100 * rankOverall / len(rankStock)
194 # Reverse indices - lower index -> better score
195 rankOverall = [len(rankStock) - 1 - x for x in rankOverall]
196
197 # Assign to stocks
198 for i in range(nStocks):
199 stocks[i].rank = rankOverall[i]
200 stocks[i].vc = round(valueComposite[i], 2)
201
202 print('Sorting stocks...')
203
204 # Sort all stocks by normalized rank
205 stocks = [x for (y, x) in sorted(zip(rankOverall, stocks))]
206
207 # Sort top decile by momentum factor. O'Shaughnessey historically uses 25
208 # stocks to hold. The top decile is printed, and the user may select the top 25
209 # (or any n) from the .csv file.
210 dec = int(nStocks / 10)
211 topDecile = []
212
213 # Store temporary momentums from top decile for sorting reasons
214 moms = [o.mom for o in stocks[:dec]]
215
216 # Sort top decile by momentum
217 for i in range(dec):
218 # Get index of top momentum performer in top decile
219 topMomInd = moms.index(max(moms))
220 # Sort
221 topDecile.append(stocks[topMomInd])
222 # Remove top momentum performer from further consideration
223 moms[topMomInd] = -100
224
225 print('Saving stocks...')
226
227 # Save momentum-weighted top decile
228 topCsvPath = 'top.csv'
229 Writer.writeCSV(topCsvPath, topDecile)
230
231 # Save results to .csv
232 allCsvPath = 'stocks.csv'
233 Writer.writeCSV(allCsvPath, stocks)
234
235 print('n')
236 print('Complete.')
237 print('Top decile (sorted by momentum) saved to: ' + topCsvPath)
238 print('All stocks (sorted by trending value) saved to: ' + allCsvPath)
239
240import re
241from urllib.request import urlopen
242
243from Stock import Stock
244
245def importHtml(url):
246 "Scrapes the HTML file from the given URL and returns line break delimited
247 strings"
248
249 response = urlopen(url, data = None)
250 html = response.read().decode('utf-8').split('n')
251
252 return html
253
254def importFinvizPage(html, stocks):
255 "Imports data from a FINVIZ HTML page and stores in the list of Stock
256 objects"
257
258 isFound = False
259
260 for line in html:
261 if line[0:15] == '<td height="10"':
262 isFound = True
263 # Import data line into stock database
264 _readFinvizLine(line, stocks)
265
266 if isFound and len(line) < 10:
267 break
268
269 return
270
271def _readFinvizLine(line, stocks):
272 "Imports stock metrics from the data line and stores it in the list of
273 Stock objects"
274
275 # Parse html
276 (stkraw, dl) = _parseHtml(line)
277
278 # Create new stock object
279 stock = Stock()
280
281 # Get ticker symbol
282 stock.tick = stkraw[dl[1] + 1: dl[2]]
283 # Get company name
284 stock.name = stkraw[dl[2] + 1 : dl[3]]
285
286 # Get market cap multiplier (either MM or BB)
287 if stkraw[dl[4] - 1] == 'B':
288 capmult = 1000000000
289 else:
290 capmult = 1000000
291
292 # Get market cap
293 stock.mktcap = capmult * _toFloat(stkraw[dl[3] + 1 : dl[4] - 1])
294 # Get P/E ratio
295 stock.pe = _toFloat(stkraw[dl[4] + 1 : dl[5]])
296 # Get P/S ratio
297 stock.ps = _toFloat(stkraw[dl[5] + 1 : dl[6]])
298 # Get P/B ratio
299 stock.pb = _toFloat(stkraw[dl[6] + 1 : dl[7]])
300 # Get P/FCF ratio
301 stock.pfcf = _toFloat(stkraw[dl[7] + 1 : dl[8]])
302 # Get Dividend Yield
303 stock.div = _toFloat(stkraw[dl[8] + 1 : dl[9] - 1])
304 # Get 6-mo Relative Price Strength
305 stock.mom = _toFloat(stkraw[dl[9] + 1 : dl[10] - 1])
306 # Get Current Stock Price
307 stock.price = _toFloat(stkraw[dl[11] + 1 : dl[12]])
308
309 # Append stock to list of stocks
310 stocks.append(stock)
311
312 return
313
314def _toFloat(line):
315 "Converts a string to a float. Returns NaN if the line can't be converted"
316
317 try:
318 num = float(line)
319 except:
320 num = float('NaN')
321
322 return num
323
324def readYahooEVEBITDA(line):
325 "Returns EV/EBITDA data from Yahoo! Finance HTML line"
326
327 # Parse html
328 (stkraw, dl) = _parseHtml(line)
329
330 for i in range(0, len(dl)):
331 if (stkraw[dl[i] + 1 : dl[i] + 24] == 'Enterprise Value/EBITDA'):
332 evebitda = stkraw[dl[i + 1] + 1 : dl[i + 2]]
333 break
334
335 return _toFloat(evebitda)
336
337def readYahooBBY(line):
338 "Returns total buys and sells from Yahoo! Finance HTML line. Result will
339 still need to be divided by market cap"
340
341 # Line also contains Borrowings details - Remove it all
342 if 'Net Borrowings' in line:
343 # Remove extra data
344 line = line[:line.find('Net Borrowings')]
345
346 # Trim prior data
347 line = line[line.find('Sale Purchase of Stock'):]
348
349 # Determine if buys or sells, replace open parantheses:
350 # (#,###) -> -#,###
351 line = re.sub(r'[(]', '-', line)
352
353 # Eliminate commas and close parantheses: -#,### -> -####
354 line = re.sub(r'[,|)]', '', line)
355
356 # Remove HTML data and markup, replacing with commas
357 line = re.sub(r'[<.*?>|]', ',', line)
358 line = re.sub(' ', ',', line)
359
360 # Locate the beginnings of each quarterly Sale Purchase points
361 starts = [m.start() for m in re.finditer(',d+,|,.d+', line)]
362
363 # Locate the ends of each quarterly Sale Purchase points
364 ends = [m.start() for m in re.finditer('d,', line)]
365
366 # Sum all buys and sells across year
367 tot = 0
368 for i in range(0, len(starts)):
369 # x1000 because all numbers are in thousands
370 tot = tot + float(line[starts[i] + 1 : ends[i] + 1]) * 1000
371
372 return tot
373
374def _parseHtml(line):
375 "Parses the HTML line by </td> breaks and returns the delimited string"
376
377 # Replace </td> breaks with placeholder, '`'
378 ph = '`'
379 rem = re.sub('</td>', ph, line)
380
381 # The ticker symbol initial delimiter is different
382 # Remove all other remaining HTML data
383 stkraw = re.sub('<.*?>', '', rem)
384
385 # Replace unbalanced HTML
386 stkraw = re.sub('">', '`', stkraw)
387
388 # Find the placeholders
389 dl = [m.start() for m in re.finditer(ph, stkraw)]
390
391 return (stkraw, dl)