· 8 years ago · Apr 06, 2018, 12:50 PM
1from bs4 import BeautifulSoup
2from datetime import datetime
3from pathlib import Path
4import urllib.request
5import pandas as pd
6import schedule
7import pickle
8import re
9import requests
10import time
11
12#ip-adres of webserver PLC
13ip ="192.168.0.1"
14
15#script loops every hour
16loopTime = 1
17
18if not ip:
19 print('First insert an ip adres')
20
21#echeck in the p.pickle file if there was an file downloaded before.
22def previousDownloadFile():
23 previousFile = Path('p.pickle')
24 if previousFile.exists():
25 print('previous logfile exists')
26 f = open('p.pickle', 'rb')
27 previousFileNumber = pickle.load(f)
28 #previousFileNumber = 166
29 print('previousFileNumber:',previousFileNumber)
30 getLastModifiedFile(previousFileNumber)
31 #if the p.pickle file dont exists, create a new one
32 else:
33 print('No previous downloadfile store found, create new one')
34 f = open('p.pickle', 'wb')
35 pickle.dump(0 ,f)
36 getLastModifiedFile(0)
37 print('---------------------------------------------')
38
39def getLastModifiedFile(previousFileNumber):
40 print('checking for new files')
41
42 #download de html pagina's om uit te zoeken welke files die dag zijn geupload
43 list_of_pages = [1,161, 201, 241, 281, 321] #getallen staan voor pagina nummer van webserver
44 data = []
45 for pages in list_of_pages:
46 global html
47 try:
48 html = urllib.request.urlopen("http://" + ip + "/Portal/Portal.mwsl?PriNav=FileBrowser&Path=%2FDataLogs%2F&FileStart="+ str(pages))
49
50 #als er iets fout gaat, print dan de fout en wacht 60 seconden voordat je het nog maals probeer
51 except urllib.error.URLError as e:
52 print(e.reason)
53 print('timeout 60 seconds')
54 time.sleep(60)
55 previousDownloadFile()
56
57 print("read pages", pages)
58 file1 = html.read()
59 html.close()
60
61 #soup is een library die de html analyseerd en er bepaalde componenten uit kan filteren
62 soup = BeautifulSoup(file1, 'html.parser')
63 #zoekt hier de systeem datum op en zet het in het goede format
64 table = soup.find_all("table", id="fileBrowserTable")[0]
65 plcDate = soup.find("span", class_="systemTime", id="systemTime")
66 plcDateFormat = pd.to_datetime(plcDate.text[14:]).strftime("%d-%m-%Y")
67 print("plcDate:", plcDateFormat)
68 filename = 0
69 info = 0
70
71 for soup in table.find_all("tr"):
72 #zoekt hier in de tabel naar filenaam
73 for names in soup.find_all("td", class_="fileBrowserName"):
74 linkfilename = names.a
75 filename = linkfilename.text
76 filenameFormat = filename[8:]
77 #print("filenameFormat:", filenameFormat)
78 #zoekt hier naar datum en tijd van dezelfde rij
79 for info in soup.find_all("span", class_="systemTime"):
80 timeDate = info.find("span", class_="fbTime")
81 try:
82 downloadTime = timeDate.text[:11]
83 timeFormat = pd.to_datetime(downloadTime).strftime("%H:%M:%S")
84 date = timeDate.text[13:]
85 dateFormat = pd.to_datetime(date).strftime("%d-%m-%Y")
86 #print("dateFormat:",dateFormat)
87
88 #als datum overeenkomt en het sensordata is
89 if dateFormat == plcDateFormat and filenameFormat[:6] == "Sensor":
90 print(filename, timeFormat)
91 data.append(filename)
92
93 #als datum overeenkomt en het een errorlog is
94 if dateFormat == plcDateFormat and filenameFormat[:5] == "Error":
95 print(filename, timeFormat)
96 downloadErrorlog(filename)
97 except:
98 pass
99
100 if data:
101 print('last fully modified file', data[-1])
102 lastModifiedFile = data[-1]
103 filenumber = int(re.search(r'\d+', lastModifiedFile).group())
104 lastModifiedFileNumber = filenumber -1
105
106 #check if there were any new files uploaded
107 if lastModifiedFileNumber == previousFileNumber:
108 print('no new file found on server')
109 pass
110 elif abs(lastModifiedFileNumber - previousFileNumber) > 1:
111 print('multiple new files found')
112 print('download files:', previousFileNumber,'/', lastModifiedFileNumber)
113 downloadFiles(previousFileNumber+1, lastModifiedFileNumber)
114 else:
115 print('download file:', lastModifiedFileNumber)
116 downloadFiles(previousFileNumber+1,lastModifiedFileNumber)
117
118 #schrijf laatste volledig geuploaden file in p.pickle
119 f = open('p.pickle', 'wb')
120 pickle.dump(lastModifiedFileNumber, f)
121
122def downloadFiles(previousFileNumber, lastModifiedFileNumber):
123 #download sensorlogs
124 for fileNumber in range(previousFileNumber, (lastModifiedFileNumber+1)):
125 fileNumber = '{:03d}'.format(fileNumber)
126 url = 'http://'+ip+ '/FileBrowser/Download?Path=/DataLogs/Datalog_Sensor'+ str(fileNumber) + '.csv'
127 local_filename = url.split('/')[-1]
128 print("local_filename:", local_filename)
129
130 s = requests.Session()
131 s.headers.update({'Referer':'http://'+ip+'/Portal/Portal.mwsl?PriNav=FileBrowser&Path=%2FDataLogs%2F&FileStart=1'})
132 try:
133 download=s.get(url)
134 #als er een fout optreed, 60 seconden timeout + retry
135 except requests.exceptions.RequestException as e:
136 print(e)
137 print('timeout 60 seconds')
138 time.sleep(60)
139 downloadFiles()
140
141 with open(local_filename, 'wb') as output_file:
142 output_file.write(download.content)
143 print('files downloaded')
144 print('timeout for 1 hour')
145
146
147def downloadErrorlog(filename):
148 #download errorlog
149 url = 'http://'+ip+'/FileBrowser/Download?Path=/DataLogs/'+filename
150 local_filename = url.split('/')[-1]
151
152 s = requests.Session()
153 s.headers.update({'Referer':'http://'+ip+'/Portal/Portal.mwsl?PriNav=FileBrowser&Path=%2FDataLogs%2F&FileStart=1'})
154
155 try:
156 download = s.get(url)
157 except requests.exceptions.RequestException as e:
158 print(e)
159 print('timeout 60 seconds')
160 time.sleep(60)
161 downloadFiles()
162
163
164 with open(local_filename, 'wb') as output_file:
165 output_file.write(download.content)
166
167
168previousDownloadFile()
169schedule.every(1).hours.do(previousDownloadFile)
170#schedule.every(5).seconds.do(getData)
171
172while 1:
173 schedule.run_pending()
174 time.sleep(1)