· 8 years ago · Oct 02, 2017, 03:28 PM
1#This script will be run via cron to check for messages from various applications
2#such as ntbackup, zmanda backup. these notifcation emmails will be parsed
3#and insterted into mysql for later usage for monitoring and rrd graphs
4
5require 'rmail'
6require 'active_record'
7require 'logger'
8require 'date'
9require 'time'
10require 'mysql'
11
12#mbox containing the messages to be parsed
13mailbox="/var/spool/mail/monitor"
14
15#our log file
16log_file="/var/log/monitor_import"
17log = Logger.new(log_file)
18log.level = Logger::INFO
19log.datetime_format = "%Y-%m-%d %H:%M "
20
21#database settings and ActiveRecord models
22ActiveRecord::Base.establish_connection(:adapter=>"mysql",:host=>"localhost",:database=>"monitoring",:username=>"monitor",:password=>"m0nit0r")
23
24class Customer < ActiveRecord::Base
25 has_many :customer_sites
26end
27
28class CustomerSite < ActiveRecord::Base
29 has_many :customer_hosts
30end
31
32class CustomerHost < ActiveRecord::Base
33 belongs_to :customer_site
34 has_many :customer_ntb_datasets
35 has_many :customer_zcb_datasets
36end
37
38class CustomerMessage < ActiveRecord::Base
39 has_many :customer_ntb_jobs
40 has_many :customer_zcb_jobs
41end
42
43class CustomerNtbDataset < ActiveRecord::Base
44 belongs_to :customer_host
45 has_many :customer_ntb_jobs
46end
47
48class CustomerNtbJob < ActiveRecord::Base
49 belongs_to :customer_ntb_dataset
50 belongs_to :customer_message
51end
52
53class CustomerZcbDataset < ActiveRecord::Base
54 belongs_to :customer_host
55 has_many :customer_zcb_jobs
56end
57
58class CustomerZcbJob < ActiveRecord::Base
59 belongs_to :customer_zcb_dataset
60 belongs_to :customer_message
61end
62
63
64#load all messages from mbox and insert them into an array
65log.info "loading messages from #{mailbox}"
66messages = Array.new
67File.open(mailbox) do |mbox|
68 RMail::Mailbox.parse_mbox(mbox) do |raw|
69 messages.push(raw)
70 end
71end
72
73#Main Loop - check to see if message has already been parsed and inserted into DB
74messages.each do |message|
75 if( (message_id = /^Message-ID:\s\<(.*)\>/i.match(message) ) && (customer_message = CustomerMessage.find_by_message_id(message_id[1])) )
76
77 elsif(message_id = /^Message-ID:\s\<(.*)\>/i.match(message))
78 customer_message = CustomerMessage.create(:message_id=>message_id[1],:body=>message)
79 #this looks like a new message, lets get the host, customer
80 if( (site = /^From\s*(?:[0-9a-z\-_]*\.)?([0-9a-z\-_]*)\@/i.match(message)) && (host = /^Subject:\s*(?:nt-backup log host:\s*|\[zcb:\s*)([0-9a-z_\-]*)/i.match(message)) )
81 #does this customer site exist?
82 if (!customer_site = CustomerSite.find_by_name(site[1]))
83 if(!customer=Customer.find_by_name(site[1]))
84 #customer does not exist, lets create this first
85 customer=Customer.create(:name=>site[1],:description=>"Created by monitor_import")
86 log.info "Created customer: #{site[1]}"
87 customer_site = customer.customer_sites.create(:name=>site[1],:description=>"Created by monitor_import")
88 log.info "Created customer-site: #{site[1]}"
89 else
90 #customer does exist, lets place the site under this customer
91 customer_site = customer.customer_sites.create(:name=>site[1],:description=>"Created by monitor_import")
92 log.info "Created customer: #{site[1]}"
93 end
94
95 end
96 #does host exist? if not lets create it
97 if(!customer_host = customer_site.customer_hosts.find_by_name(host[1]))
98 customer_host = customer_site.customer_hosts.create(:name=>host[1], :description=>"Created by monitor_import")
99 log.info "Created customer host #{host[1]}"
100 else
101 log.info "Customer host exists #{host[1]}"
102 end
103 end
104 #We should now have customer, customer site and customer host
105 #We now need to determine what to do with this message by looking at the subject line
106
107 #zmanda cloud backup
108 if(/^Subject:\s*\[zcb:/i.match(message))
109 #Pick dataset name from message
110 if (job_info = /(^Backup|^Upload) of backup set "(.*)" (?:is )?(failed|successful)/.match(message))
111 dataset=job_info[2]
112 task=job_info[1]
113 status=job_info[3]
114 #Check to see if this dataset is already in the customer_zcb_datasets table
115 if(customer_dataset = customer_host.customer_zcb_datasets.find_by_name(dataset))
116
117 else
118 #create this dataset
119 customer_dataset = customer_host.customer_zcb_datasets.create(:name=>dataset,:description=>"Created by zmanda_import")
120 log.info "Created dataset: #{customer_dataset[:name]} withd id: #{customer_dataset[:id]}, for host #{customer_host[:name]}"
121 end
122 else
123 log.warn "Something went wrong, unable to find dataset in message #{message_id}"
124 end
125
126 #determine if this is an Upload or a Backup job
127 if (task == "Backup")
128 #gather backup job details
129 start_time = /^Backup start time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
130 end_time = /^Backup end time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
131 files_on_disk = /^Files on disk : (\d*)/.match(message)
132 files_in_backup = /^Files in backup : (\d*)/.match(message)
133 details = /^Error Details : (.*)/.match(message)
134 backup_size = /^Total Bytes backed up : (\d.*) (KB|MB|GB)/.match(message)
135
136 #convert all sizes to KB
137 size = 0
138 if (backup_size && backup_size.length > 1)
139 size = backup_size[1]
140 size = size.to_f
141 size = size * 1024 if backup_size[2] == "MB"
142 size = size * 1024 * 1024 if backup_size[2] == "GB"
143 size = size.to_i
144 end
145
146 #format time stamps for db format is YYYY/MM/DD HH:MM:SS
147 start_datetime = start_time[1] << " " << start_time[2]
148 end_datetime = end_time[1] << " " << end_time[2]
149
150 #add code here to calcuate backup rate, may not be possible if we dont have compression sizes?
151
152 zcb_job = customer_dataset.customer_zcb_jobs.create(
153 :task=>"Backup",
154 :size=>size,
155 :start_time=>start_datetime,
156 :end_time=>end_datetime,
157 :files_on_disk=>files_on_disk[1],
158 :files_in_backup=>files_in_backup[1],
159 :details=>details[1],
160 :customer_message_id => customer_message[:id],
161 :status=>status
162 )
163 log.info "Imported Zmanda backup job with for #{customer_dataset[:name]}, Status: #{zcb_job[:status]}, for host: #{customer_host[:name]}"
164
165 elsif (task == "Upload")
166 #gather upload job details
167 start_time = /^Upload start time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
168 end_time = /^Upload end time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
169 details = /^Error Details : (.*)/.match(message)
170 upload_rate = /^Upload Rate : (\d*) Kbps/.match(message)
171 upload_size = /^Bytes uploaded : ([\d\.]*) (KB|MB|GB)/.match(message)
172
173 #convert all sizes to KB
174 if (upload_size && upload_size.length > 1)
175 size = upload_size[1]
176 size = size.to_f
177 size = size * 1024 if upload_size[2] == "MB"
178 size = size * 1024 * 1024 if upload_size[2] == "GB"
179 size = size.to_i
180 end
181
182 #format time stamps for db format is YYYY/MM/DD HH:MM:SS
183 start_datetime = start_time[1] << " " << start_time[2]
184 end_datetime = end_time[1] << " " << end_time[2]
185
186 zcb_job = customer_dataset.customer_zcb_jobs.create(
187 :task=>"Upload",
188 :size=>size,
189 :start_time=>start_datetime,
190 :end_time=>end_datetime,
191 :rate=>upload_rate[1],
192 :size=>size,
193 :details=>details[1],
194 :customer_message_id => customer_message[:id],
195 :status=>status
196 )
197 log.info "Imported Zmanda upload job with for #{customer_dataset[:name]}, Status: #{zcb_job[:status]}, for host: #{customer_host[:name]}"
198 #match upload with backup job, for now will assume it was the previous backup for this dataset
199 else
200 #not sure what job type this is break out here
201 log.warn "Unkown job type - #{task}"
202 end
203
204
205 #nt backup
206 elsif(/^Subject:\s*nt-backup/i.match(message))
207
208
209 #Pick dataset names from message
210 datasets=message.scan(/^Backup .* of \"(.*)\"/)
211 backups=message.scan(/^Backup .* of \"(.*)\"\n^Backup set (.*)\n^Backup description: \"(.*)\"\n^Media name: \"(.*)\"\n\n^Backup Type: (.*)\n\n^Backup started on (.*) at (.*) (AM|PM).\n^Backup completed on (.*) at (.*) (AM|PM).\n^Directories: (.*)\n^Files: (.*)\n^Bytes: (.*)/)
212
213 #did this job finish without error?
214 if (status = /^NTBackup finished the backup with no errors./.match(message))
215 job_status="successful"
216 else
217 job_status="failed"
218 end
219
220 #backups array layout, format location) description - example
221 # 0) Dataset name - E:
222 # 1) Backup set - #3 on media #1
223 # 2) Backup Descrition - SBS Backup created on 1/26/2011 at 11:00 PM
224 # 3) Media Name - Media created 1/26/2011 at 11:00 PM
225 # 4) Backup Type - Normal
226 # 5) Backup Start date - 1/27/2011
227 # 6) Backup Start time - 1:41
228 # 7) Backup Start time am or pm? - AM
229 # 8) Backup End date - 1/27/2011
230 # 9) Backup End time - 1:43
231 #10) Backup End time am or pm? - AM
232 #11) Directories - 71
233 #12) Files - 895
234 #13) Bytes - 281,723,426
235
236
237 #check to see if datasets already exist, if not create them
238 if(datasets)
239 0.upto((datasets.count)-1) {|i|
240 #does dataset already exist?
241 if(customer_dataset = customer_host.customer_ntb_datasets.find_by_name(datasets[i][0]))
242
243 elsif(customer_dataset = customer_host.customer_ntb_datasets.create(:name=>datasets[i][0]))
244 log.info "Created NT-Backup dataset #{customer_dataset[:name]} with id of : #{customer_dataset[:id]} "
245 else
246
247 end
248 }
249 end
250
251
252 #insert each ntbackup job
253 backups.each do |job|
254 #this size == 14 shit needs to go
255 if(job.size ==14 && customer_dataset=customer_host.customer_ntb_datasets.find_by_name(job[0]))
256 backupset=job[1]
257 description=job[2]
258 media_name=job[3]
259 type=job[4]
260 start_time=DateTime.parse(job[5] + job[6] + job[7])
261 end_time=DateTime.parse(job[8]+job[9]+job[10])
262 directories=job[11].gsub(",","").to_i
263 files=job[12].gsub(",","").to_i
264 size=job[13].gsub(",","").to_i/1024
265
266 customer_job = customer_dataset.customer_ntb_jobs.create(
267 :start_time=>start_time,
268 :end_time=>end_time,
269 :size=>size,
270 :directories=>directories,
271 :files=>files,
272 :status=>job_status,
273 :backupset=>backupset,
274 :description=>description,
275 :media_name=>media_name,
276 :customer_message_id=>customer_message[:id],
277 :type=>type
278 )
279 end
280 end
281
282
283 end
284
285 end
286end #end of Main Loop
287log.info "Application Clean Exit"