· 5 years ago · Jun 24, 2020, 05:46 PM
1#
2# Python 3 program to:
3# 1. Download a days worth of data from the WeatherFlow API in CSV format (or use a provided CSV file)
4# 2. Upadte the weewx.sdb file for the data in the CSV file:
5# 2a. lightning_strike_count = Total number of lightning strikes during the 5 minute time period
6# 2b. lightning_distance = AverageLightning Distance during the 5 minute time period
7# 2c. rain = Amount of rain in millimeters during the 5 minute time period
8# 2d. rainrate = Amount of rain if received over an hour period (rain * 12)
9# 2e. rainBatteryStatus = Precipitation Type (0=None; 1=Rain; 2=Hail)
10# 2f. signal7 = Closest (Min) Lightning Distance during the 5 minute time period
11# 2g. signal8 = Furthest (Max) Lightning Distance during the 5 minute time period
12#
13# By James Bellanca
14#
15
16#
17# Can take one of two command line arguments.
18# Both cannot be used at the same time. The last one entered wins.
19# If no argument is specified, it assumes -d 1 to retrieve and process yesterday's data.
20#
21# -d, -DaysAgo x Retrieves data from the WeatherFlow API for x days ago and processes it.
22# The file is saved in the folder that this program is in.
23# -f, -File x Processes the file x. It must be a properly formatted WeatherFlow API csv file.
24
25from datetime import datetime, timedelta
26import sys, getopt
27import csv
28import sqlite3
29import urllib.request
30
31# Set to the path and file of your WeeWx SDB database file
32DEFAULT_PATH = "/Users/FamilyMini/Downloads/weewx.sdb"
33
34# Set the Tempest Device ID to your value
35stationID = "#####" # IMPORTANT -- Put you Device ID here!!!
36
37# This is the demo API key which works for now for individual use
38APIkey = "20c70eae-e62f-4d3b-b3a4-8586e90f3ac8"
39
40# Function to update the values in the WeeWx database
41def updateRow(nextTimeInterval, loopIndex, timestamp, averagingStrikeIndex, strike_distance, precip_final, local_daily_precip_final, strike_count, precipTypeMax, strikeDistanceMin, strikeDistanceMax):
42 nextTimeInterval = str(int(timestamp)+300)
43 if strikeDistanceMin == 999:
44 strikeDistanceMin = 0
45 if averagingStrikeIndex > 0:
46 strike_distance = round(strike_distance / averagingStrikeIndex, 2)
47 precip_final = round(precip_final/25.4, 6) #convert from mm to inches and round
48 rainRate = round(precip_final*12, 6) #convert from mm to inches and round
49 local_daily_precip_final = round(local_daily_precip_final/25.4, 6) #convert mm to inches and round
50 sqltoexecute = "update archive set lightning_strike_count = " + str(strike_count) + ", lightning_distance = " + str(strike_distance) + ", rain = " + str(precip_final) + ", rainrate = " + str(rainRate) + ", rainBatteryStatus = " + str(precipTypeMax)+ ", signal7 = " + str(strikeDistanceMin)+ ", signal8 = " + str(strikeDistanceMax) + " where dateTime = " + timestamp
51 print (sqltoexecute)
52 cursorObj.execute(sqltoexecute)
53 con.commit()
54 return
55
56argumentList = sys.argv[1:]
57
58# Options
59options = "hd:f:"
60
61# Long options
62long_options = ["Help", "Days Ago", "File ="]
63
64try:
65 # Parsing argument
66 arguments, values = getopt.getopt(argumentList, options, long_options)
67
68 daysAgo = 1
69 processingMode = 0 # 0 = not set, assume 1 day ago; 1 = retrieve file from x days ago; 2 = Process filen from command line arguments
70
71 # checking each argument
72 for currentArgument, currentValue in arguments:
73
74 if currentArgument in ("-h", "--Help"):
75 print ("usage: -d | -f | -h")
76 print ("")
77 print ("Options:")
78 print (" -d x, --Days Ago x Retrieves data from the WeatherFlow API for x days ago")
79 print (" -f x, --File x Processes the file x which must be a properly formatted WeatherFlow API csv file")
80 print (" -h, --Help Displays help")
81 print (" No arguments Same as running -d 1")
82 sys.exit()
83
84 elif currentArgument in ("-d", "--DaysAgo"):
85 daysAgo = int(currentValue)
86 if daysAgo < 1 or daysAgo > 7:
87 print("Days Ago argument must be between 1 and 7.")
88 sys.exit()
89 else:
90 processingMode = 1
91
92 elif currentArgument in ("-f", "--File"):
93 processingMode = 2
94 print (("File to process (% s)") % (currentValue))
95 filename = str(currentValue)
96 print(filename)
97
98except getopt.error as err:
99 # output error, and return with an error code
100 print (str(err))
101
102if processingMode == 0 or processingMode == 1:
103 if daysAgo == 1:
104 print("Retrieving data from yesterday.")
105 else:
106 print ("Retrieving data from %s day(s) ago." % (daysAgo))
107 # URL uses the demo API key from WeatherFlow
108 url = 'https://swd.weatherflow.com/swd/rest/observations/device/' + stationID + '?api_key=' + APIkey + '&day_offset=' + str(daysAgo) + '&format=csv'
109 fileDate = datetime.now() - timedelta(hours=(daysAgo*24))
110 filename = "WF_"+str(fileDate.strftime("%Y-%m-%d"))+".csv"
111 print(url)
112 print(filename)
113 urllib.request.urlretrieve(url, filename)
114
115# Open WeeWx SQLite Database
116
117con = sqlite3.connect(DEFAULT_PATH)
118cursorObj = con.cursor()
119
120# Main processing loop for the CSV file
121
122with open(filename) as csvfile:
123
124 # Set default values
125 rowIndex = -1
126 loopIndex = 0
127 averagingStrikeIndex = 0
128 precip_type = 0
129 nextTimeInterval = 0
130 timestamp = ""
131 strike_distance = float(0)
132 strike_count = int(0)
133 precip_final = float(0)
134 strikeDistanceMin = float(999)
135 strikeDistanceMax = float(0)
136 precipTypeMax = float(0)
137
138 # Read and process each row in the CSV file
139 readCSV = csv.reader(csvfile, delimiter=',')
140 for row in readCSV:
141 #print(row)
142 rowIndex+=1
143 if rowIndex > 0:
144 timestamp = row[3]
145 # Check to see if the current row has a time later than the next row to update in WeeWx.
146 # This could happen if the csv file doesn't have an entry for the exact 5 minute entry that will be in WeeWx.
147 if (int(timestamp) > int(nextTimeInterval)) and (int(nextTimeInterval) > 0):
148 print("Timestamp missing! ", nextTimeInterval)
149 # Call function to do the database update
150 updateRow(nextTimeInterval, loopIndex, nextTimeInterval, averagingStrikeIndex, strike_distance, precip_final, local_daily_precip_final, strike_count, precipTypeMax, strikeDistanceMin, strikeDistanceMax)
151 # Determine what the next WeeWx row should be to make sure we compute values at that row even if it's not present in the file
152 nextTimeInterval = str(int(timestamp)+300)
153 # Reset all values
154 strike_distance = 0
155 strike_count = 0
156 precip_final = 0
157 loopIndex = 0
158 averagingStrikeIndex = 0
159 strikeDistanceMin = 999
160 strikeDistanceMax = 0
161 precip_type = 0
162 precipTypeMax = 0
163 # Sum or set values as appropriate
164 strike_distance += float(row[17])
165 strike_count += int(row[18])
166 precip_final += float(row[22])
167 local_daily_precip_final = float(row[23])
168 precip_type = int(row[16])
169 loopIndex+=1
170 # Compute number of rows with lightning to compute the average, and record min/max distance over the period, and precip type (0=none; 1=rain; 2=hail)
171 if int(row[18]) != 0:
172 averagingStrikeIndex +=1
173 if (int(row[17]) < strikeDistanceMin) and (int(row[18]) > 0):
174 strikeDistanceMin = float(row[17])
175 if int(row[17]) > strikeDistanceMax:
176 strikeDistanceMax = float(row[17])
177 if int(row[16]) > precipTypeMax:
178 precipTypeMax = int(row[16])
179 if int(row[3])%300 == 0:
180 # Call function to do the database update
181 updateRow(nextTimeInterval, loopIndex, timestamp, averagingStrikeIndex, strike_distance, precip_final, local_daily_precip_final, strike_count, precipTypeMax, strikeDistanceMin, strikeDistanceMax)
182 # Determine what the next WeeWx row should be to make sure we compute values at that row even if it's not present in the file
183 nextTimeInterval = str(int(timestamp)+300)
184 # Reset all values
185 strike_distance = 0
186 strike_count = 0
187 precip_final = 0
188 loopIndex = 0
189 averagingStrikeIndex = 0
190 strikeDistanceMin = 999
191 strikeDistanceMax = 0
192 precip_type = 0
193 precipTypeMax = 0