· 9 years ago · Dec 07, 2016, 08:48 PM
1#!/usr/bin/python
2
3
4
5# CS288 Homework 9
6# Read the skeleton code carefully and try to follow the structure
7# You may modify the code but try to stay within the framework.
8
9import libxml2
10import sys
11import os #This is used to make command line calls/statements
12import commands
13import re
14import sys
15
16
17
18from xml.dom.minidom import parse, parseString
19
20# for converting dict to xml
21from cStringIO import StringIO
22from xml.parsers import expat
23#import MySQLdb "error code : no _mysqldb
24import mysql.connector
25from mysql.connector import errorcode
26
27# replace whitespace chars
28def replace_white_space(str):
29 p = re.compile(r'\s+')
30 new = p.sub(' ',str) # a lot of \n\t\t\t\t\t\t
31 return new.strip()
32
33# replace but these chars including ':'
34def replace_non_alpha_numeric(s):
35 p = re.compile(r'[^a-zA-Z0-9:-]+')
36 # p = re.compile(r'\W+') # replace whitespace chars
37 new = p.sub(' ',s)
38 return new.strip()
39
40# convert to xhtml
41# use: java -jar tagsoup-1.2.jar --files html_file
42#Context for using tagsoup : https://groups.google.com/forum/#!topic/tagsoup-friends/X6jOobR7jMs
43
44def html_to_xml(fn):
45 # ............
46 #The OS.system call returns an integer not the filename
47 xhtml_file=os.system("java -jar tagsoup-1.2.1.jar --files %s" % (fn)); #Similar to C's way of putting a string into a print statement; except you use % instead of ,
48 xhtml_file=fn.replace(".html",".xhtml");
49 return xhtml_file
50
51def get_elms_for_atr_val(tag,atr,val):
52
53 lst = []
54 elms = dom.getElementsByTagName(tag)
55 for i in elms:
56 if len(i.childNodes) == val: #if children == 6
57 lst.append(i) #Will only have tr with six children
58
59 return lst;
60
61
62# get all text recursively to the bottom
63def get_text(e):
64 lst=[]
65
66 if (e.nodeType == 3) or (e.nodeType == 4):
67 return [e.data] #need brackets for multiple returns
68 else:
69 for i in e.childNodes:
70 lst.append(get_text(i))
71
72 return lst;
73
74def extract_values(dm):
75 lst = []
76 l = get_elms_for_atr_val('tr','class',6)
77 #l has 101 entries now
78
79 for e in l:
80 lst.append(get_text(e));
81
82 #lst=l;
83 return lst
84
85 #mysql> describe most_active;
86def insert_to_db(l,tbl):
87
88
89 connect = mysql.connector.connect(user="root", password="Cu&,lXA<e3df",
90 host="localhost",
91 database=tbl)
92
93 db=tbl
94 #Using dictionaries table; kind of overkill for just one DB
95
96 cursor = connect.cursor()
97
98 cursor.execute(
99 "CREATE TABLE IF NOT EXISTS `StockExchangeInfoPython` ("
100 " `Symbol` varchar(20) NOT NULL,"
101 " `Title` varchar(25) NOT NULL,"
102 " `Volume` varchar(125) NOT NULL,"
103 " `Price` varchar(25) NOT NULL,"
104 " `Change` varchar(25) NOT NULL,"
105 " `PChnge` varchar(25) NOT NULL,"
106 " Primary Key (`Symbol`)"
107 ") ENGINE=InnoDB")
108
109 connect.commit()
110
111 cursor.close()
112
113 connect.close()
114
115
116# show databases;
117# show tables;
118
119def clean(lst):
120
121 i=0
122 while i < len(lst):
123 a= lst[i][1][1]
124 lst[i][1]=0
125 lst[i][1]=a
126 i+=1
127
128 return lst
129
130
131
132def main():
133 html_fn = sys.argv[1] #First argument on command line after program
134 fn = html_fn.replace('.html','')
135 xhtml_fn = html_to_xml(html_fn)
136
137 print "Hello World!"
138 # print "File name : ",xhtml_fn,".";
139 global dom
140 dom = parse(xhtml_fn)
141 lst=[];
142 lst = extract_values(dom)
143
144 #print len(lst); #first entry is the header
145 #https://www.tutorialspoint.com/python/list_pop.htm
146 #can simply pop the header off of the list
147 lst.pop(0); #Pop the first entry off of the lst
148
149 print len(lst[0][1])
150 print lst[0][1] #Use regular expressions to "clean" the list
151
152 print len(lst[0][1][1])
153 print lst[0][1][1]
154 lst= clean(lst);
155 print len(lst[0])
156 print len(lst[0][1])
157 print lst[0][1]
158 print lst[4]
159
160 stockdb = "StockExchangeInfo"
161
162
163 # make sure your mysql server is up and running
164 cursor = insert_to_db(lst,stockdb) # fn = table name for mysql
165
166 # comment l = select_from_db(cursor,fn) # display the table on the screen
167
168 # make sure the Apache web server is up and running
169 # write a PHP script to display the table(s) on your browser
170
171 #comment return xml
172# end of main()
173
174if __name__ == "__main__":
175 main()
176
177# end of hw7.py