· 8 years ago · Oct 14, 2017, 04:30 AM
1#This script will be run via cron to check for messages from various applications
2#such as ntbackup, zmanda backup. these notifcation emmails will be parsed
3#and insterted into mysql for later usage for monitoring and rrd graphs
4
5require 'rmail'
6require 'active_record'
7require 'logger'
8require 'date'
9require 'time'
10
11#mbox containing the messages to be parsed
12mailbox="/var/spool/mail/monitor"
13
14#our log file
15log_file="/var/log/monitor_import"
16log = Logger.new(log_file)
17log.level = Logger::INFO
18log.datetime_format = "%Y-%m-%d %H:%M "
19
20#database settings and ActiveRecord models
21ActiveRecord::Base.establish_connection(:adapter=>"mysql",:host=>"localhost",:database=>"sciens_internal_dev",:username=>"monitor",:password=>"m0nit0r")
22
23class Customer < ActiveRecord::Base
24 has_many :customer_sites
25end
26
27class CustomerSite < ActiveRecord::Base
28 has_many :customer_hosts
29end
30
31class CustomerHost < ActiveRecord::Base
32 belongs_to :customer_site
33 has_many :customer_ntb_datasets
34 has_many :customer_zcb_datasets
35end
36
37class CustomerMessage < ActiveRecord::Base
38 has_many :customer_ntb_jobs
39 has_many :customer_zcb_jobs
40end
41
42class CustomerNtbDataset < ActiveRecord::Base
43 belongs_to :customer_host
44 has_many :customer_ntb_jobs
45end
46
47class CustomerNtbJob < ActiveRecord::Base
48 belongs_to :customer_ntb_dataset
49 belongs_to :customer_message
50end
51
52class CustomerZcbDataset < ActiveRecord::Base
53 belongs_to :customer_host
54 has_many :customer_zcb_jobs
55end
56
57class CustomerZcbJob < ActiveRecord::Base
58 belongs_to :customer_zcb_dataset
59 belongs_to :customer_message
60end
61
62
63#load all messages from mbox and insert them into an array
64log.info "loading messages from #{mailbox}"
65messages = Array.new
66File.open(mailbox) do |mbox|
67 RMail::Mailbox.parse_mbox(mbox) do |raw|
68 messages.push(raw)
69 end
70end
71
72#Main Loop - check to see if message has already been parsed and inserted into DB
73messages.each do |message|
74 if( (message_id = /^Message-ID:\s\<(.*)\>/i.match(message) ) && (customer_message = CustomerMessage.find_by_message_id(message_id[1])) )
75
76 elsif(message_id = /^Message-ID:\s\<(.*)\>/i.match(message))
77 customer_message = CustomerMessage.create(:message_id=>message_id[1],:body=>message)
78 #this looks like a new message, lets get the host, customer
79 if( (site = /^From\s*(?:[0-9a-z\-_]*\.)?([0-9a-z\-_]*)\@/i.match(message)) && (host = /^Subject:\s*(?:nt-backup log host:\s*|\[zcb:\s*)([0-9a-z_\-]*)/i.match(message)) )
80 #does this customer site exist?
81 if (!customer_site = CustomerSite.find_by_name(site[1]))
82 if(!customer=Customer.find_by_name(site[1]))
83 #customer does not exist, lets create this first
84 customer=Customer.create(:name=>site[1],:description=>"Created by monitor_import")
85 log.info "Created customer: #{site[1]}"
86 customer_site = customer.customer_sites.create(:name=>site[1],:description=>"Created by monitor_import")
87 log.info "Created customer-site: #{site[1]}"
88 else
89 #customer does exist, lets place the site under this customer
90 customer_site = customer.customer_sites.create(:name=>site[1],:description=>"Created by monitor_import")
91 log.info "Created customer: #{site[1]}"
92 end
93
94 end
95 #does host exist? if not lets create it
96 if(!customer_host = customer_site.customer_hosts.find_by_name(host[1]))
97 customer_host = customer_site.customer_hosts.create(:name=>host[1], :description=>"Created by monitor_import")
98 log.info "Created customer host #{host[1]}"
99 else
100 log.info "Customer host exists #{host[1]}"
101 end
102 end
103 #We should now have customer, customer site and customer host
104 #We now need to determine what to do with this message by looking at the subject line
105
106 #zmanda cloud backup
107 if(/^Subject:\s*\[zcb:/i.match(message))
108 #Pick dataset name from message
109 if (job_info = /(^Backup|^Upload) of backup set "(.*)" (?:is )?(failed|successful)/.match(message))
110 dataset=job_info[2]
111 task=job_info[1]
112 status=job_info[3]
113 #Check to see if this dataset is already in the customer_zcb_datasets table
114 if(customer_dataset = customer_host.customer_zcb_datasets.find_by_name(dataset))
115
116 else
117 #create this dataset
118 customer_dataset = customer_host.customer_zcb_datasets.create(:name=>dataset,:description=>"Created by zmanda_import")
119 log.info "Created dataset: #{customer_dataset[:name]} withd id: #{customer_dataset[:id]}, for host #{customer_host[:name]}"
120 end
121 else
122 log.warn "Something went wrong, unable to find dataset in message from #{from_address}"
123 end
124
125 #determine if this is an Upload or a Backup job
126 if (task == "Backup")
127 #gather backup job details
128 start_time = /^Backup start time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
129 end_time = /^Backup end time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
130 files_on_disk = /^Files on disk : (\d*)/.match(message)
131 files_in_backup = /^Files in backup : (\d*)/.match(message)
132 details = /^Error Details : (.*)/.match(message)
133 backup_size = /^Total Bytes backed up : (\d.*) (KB|MB|GB)/.match(message)
134
135 #convert all sizes to KB
136 size = 0
137 if (backup_size && backup_size.length > 1)
138 size = backup_size[1]
139 size = size.to_f
140 size = size * 1024 if backup_size[2] == "MB"
141 size = size * 1024 * 1024 if backup_size[2] == "GB"
142 size = size.to_i
143 end
144
145 #format time stamps for db format is YYYY/MM/DD HH:MM:SS
146 start_datetime = start_time[1] << " " << start_time[2]
147 end_datetime = end_time[1] << " " << end_time[2]
148
149 #add code here to calcuate backup rate, may not be possible if we dont have compression sizes?
150
151 zcb_job = customer_dataset.customer_zcb_jobs.create(
152 :task=>"Backup",
153 :size=>size,
154 :start_time=>start_datetime,
155 :end_time=>end_datetime,
156 :files_on_disk=>files_on_disk[1],
157 :files_in_backup=>files_in_backup[1],
158 :details=>details[1],
159 :customer_message_id => customer_message[:id],
160 :status=>status
161 )
162 log.info "Imported Zmanda backup job with for #{customer_dataset[:name]}, Status: #{zcb_job[:status]}, for host: #{customer_host[:name]}"
163
164 elsif (task == "Upload")
165 #gather upload job details
166 start_time = /^Upload start time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
167 end_time = /^Upload end time : (\d{4}\/\d\d\/\d\d) (\d\d:\d\d:\d\d)/.match(message)
168 details = /^Error Details : (.*)/.match(message)
169 upload_rate = /^Upload Rate : (\d*) Kbps/.match(message)
170 upload_size = /^Bytes uploaded : ([\d\.]*) (KB|MB|GB)/.match(message)
171
172 #convert all sizes to KB
173 if (upload_size && upload_size.length > 1)
174 size = upload_size[1]
175 size = size.to_f
176 size = size * 1024 if upload_size[2] == "MB"
177 size = size * 1024 * 1024 if upload_size[2] == "GB"
178 size = size.to_i
179 end
180
181 #format time stamps for db format is YYYY/MM/DD HH:MM:SS
182 start_datetime = start_time[1] << " " << start_time[2]
183 end_datetime = end_time[1] << " " << end_time[2]
184
185 zcb_job = customer_dataset.customer_zcb_jobs.create(
186 :task=>"Upload",
187 :size=>size,
188 :start_time=>start_datetime,
189 :end_time=>end_datetime,
190 :rate=>upload_rate[1],
191 :size=>size,
192 :details=>details[1],
193 :customer_message_id => customer_message[:id],
194 :status=>status
195 )
196 log.info "Imported Zmanda upload job with for #{customer_dataset[:name]}, Status: #{zcb_job[:status]}, for host: #{customer_host[:name]}"
197 #match upload with backup job, for now will assume it was the previous backup for this dataset
198 else
199 #not sure what job type this is break out here
200 log.warn "Unkown job type - #{task}"
201 end
202
203
204 #nt backup
205 elsif(/^Subject:\s*nt-backup/i.match(message))
206
207
208 #Pick dataset names from message
209 datasets=message.scan(/^Backup .* of \"(.*)\"/)
210 backups=message.scan(/^Backup .* of \"(.*)\"\n^Backup set (.*)\n^Backup description: \"(.*)\"\n^Media name: \"(.*)\"\n\n^Backup Type: (.*)\n\n^Backup started on (.*) at (.*) (AM|PM).\n^Backup completed on (.*) at (.*) (AM|PM).\n^Directories: (.*)\n^Files: (.*)\n^Bytes: (.*)/)
211
212 #did this job finish without error?
213 if (status = /^NTBackup finished the backup with no errors./.match(message))
214 job_status="successful"
215 else
216 job_status="failed"
217 end
218
219 #backups array layout, format location) description - example
220 # 0) Dataset name - E:
221 # 1) Backup set - #3 on media #1
222 # 2) Backup Descrition - SBS Backup created on 1/26/2011 at 11:00 PM
223 # 3) Media Name - Media created 1/26/2011 at 11:00 PM
224 # 4) Backup Type - Normal
225 # 5) Backup Start date - 1/27/2011
226 # 6) Backup Start time - 1:41
227 # 7) Backup Start time am or pm? - AM
228 # 8) Backup End date - 1/27/2011
229 # 9) Backup End time - 1:43
230 #10) Backup End time am or pm? - AM
231 #11) Directories - 71
232 #12) Files - 895
233 #13) Bytes - 281,723,426
234
235
236 #check to see if datasets already exist, if not create them
237 if(datasets)
238 0.upto((datasets.count)-1) {|i|
239 #does dataset already exist?
240 if(customer_dataset = customer_host.customer_ntb_datasets.find_by_name(datasets[i][0]))
241
242 elsif(customer_dataset = customer_host.customer_ntb_datasets.create(:name=>datasets[i][0]))
243 log.info "Created NT-Backup dataset #{customer_dataset[:name]} with id of : #{customer_dataset[:id]} "
244 else
245
246 end
247 }
248 end
249
250
251 #insert each ntbackup job
252 backups.each do |job|
253 #this size == 14 shit needs to go
254 if(job.size ==14 && customer_dataset=customer_host.customer_ntb_datasets.find_by_name(job[0]))
255 backupset=job[1]
256 description=job[2]
257 media_name=job[3]
258 type=job[4]
259 start_time=DateTime.parse(job[5] + job[6] + job[7])
260 end_time=DateTime.parse(job[8]+job[9]+job[10])
261 directories=job[11].gsub(",","").to_i
262 files=job[12].gsub(",","").to_i
263 size=job[13].gsub(",","").to_i/1024
264
265 customer_job = customer_dataset.customer_ntb_jobs.create(
266 :start_time=>start_time,
267 :end_time=>end_time,
268 :size=>size,
269 :directories=>directories,
270 :files=>files,
271 :status=>job_status,
272 :backupset=>backupset,
273 :description=>description,
274 :media_name=>media_name,
275 :customer_message_id=>customer_message[:id],
276 :type=>type
277 )
278 end
279 end
280
281
282 end
283
284 end
285end #end of Main Loop
286log.info "Application Clean Exit"