WL8Wh6kf

1var PerspectiumReplicator = Class.create();
2
3// Function that returns whether a table is replicatable, called by condition of
4// Perspectium Replicate business rules
5PerspectiumReplicator.isReplicatedTable = function(strTable, strDirection, strOperation) {
6    if (strOperation == "insert") {
7        strOperation = "create";
8    }
9
10    var grPSPReplConf = new GlideRecord("psp_replicate_conf");
11    grPSPReplConf.addQuery("action_" + strOperation, true);
12    grPSPReplConf.addQuery("sync_direction", strDirection);
13    grPSPReplConf.addQuery("active", true);
14    grPSPReplConf.addQuery("table_name", strTable);
15    grPSPReplConf.query();
16    if(grPSPReplConf.next()) {
17        return true;
18    }
19
20    return false;
21};
22
23PerspectiumReplicator.prototype = {  
24    initialize: function () {
25		this.tableCounter = {};
26		this.bulk_count = 0;
27		this.bulk_byte_count = 0;
28		this.bulk_query_count = 0;
29    	this.psp = new Perspectium();
30		this.logger = new PerspectiumLogger();
31		this.encryption = new PerspectiumEncryption();
32		this.key = gs.getProperty("instance_name", "unregistered");
33						
34		this.pspMS = new PerspectiumMessageSet();
35		this.messageSetCounter = {};
36		this.sentFirstMessage = false;
37		this.flagLastMessage = false;
38		this.bulk_share_query_limit = 1000;
39		this.bulk_share_all_records = false;
40		this.encryptionType = {"NONE":"0","TRIPLE_DES":"1","AES_128":"2", "BASE64_ONLY":"3"};
41
42		this.pspUtil = new PerspectiumUtil();
43		this.dynSysJournalLimit = this.pspUtil.getPspPropertyValue('com.perspectium.dynamic.sys_journal_field.limit', '100');
44		this.dynSysAuditLimit = this.pspUtil.getPspPropertyValue('com.perspectium.dynamic.sys_audit.limit', '200');		
45		this.domainSeparated = gs.getProperty('glide.sys.restrict_global_domain_processes', 'false');		
46		this.messageStatsMod = "Subscribing";
47		this.fieldsToReplicate = [];
48
49		this.maxBytes = parseInt(this.pspUtil.getPspPropertyValue("com.perspectium.output_bytes_limit", "5000000")); // max bytes to cap the send string limit of 16MB for Java String
50    },
51	
52	getKey:function(grShareConfig){
53		if(!grShareConfig.u_target_queue_group.isNil())
54			return this.key + "." + grShareConfig.u_target_queue_group;
55		else
56			return this.key;
57	},
58	
59	// function called by Replicator functions so we can keep track of first message sent
60	createPSPOut: function(topic, type, key, name, value, target_queue, extra, attributes, shareGR, record_sys_id) {
61		// if value blank we don't send out
62		if(value == ""){
63			this.logger.logError("Not sending message with topic " + topic + ", type " + type + ", key " + key + ", name " + name + " as value is empty", "PerspectiumReplicator.createPSPOut", shareGR);
64			return;	
65		}
66		
67		// if first message hasn't been sent out yet we'll set flag in attributes.
68		if(!this.sentFirstMessage){					
69			attributes = this.appendAttribute("psp_flag", "first", attributes);			
70			this.sentFirstMessage = true;
71		}
72
73		// siam add attribute for tracking in audit
74		if (topic == 'siam') {
75			var siamMessageId;
76			if (record_sys_id && record_sys_id != null && record_sys_id != "") {
77				siamMessageId = record_sys_id;				
78			}
79			else {
80				siamMessageId = this.generateUUID();
81			}
82			
83			var currentMS = (new Date).getTime();
84			siamMessageId += '-' + String(currentMS);
85			
86			// create a SIAM summary record so we know this message went outbound from servicenow
87			/*var sasgr = new GlideRecord("u_psp_siam_audit");
88			sasgr.u_message_id = siamMessageId;
89			sasgr.u_message_timestamp = String(currentMS);
90			sasgr.u_psp_timestamp = gs.nowDateTime();
91			sasgr.u_direction = 'Outbound';
92			sasgr.u_component_type = 'ServiceNow';
93			var sasSysId = sasgr.insert();*/
94			
95			attributes = this.appendAttribute("SIAM_message_id", siamMessageId, attributes);	
96		}
97		
98		// check if we have deferred messages to put in deferred state
99		var state = "ready";
100		if (name.indexOf(".deferred") > 0) {
101			// set name back to proper name of update
102			name = name.replace("deferred", "update");
103			if (topic != 'counter') {
104				state = "deferred";			
105			}
106		}
107		
108		if (name.contains("sys_attachment_doc.") || name.contains("sys_attachment.")) {
109			var pspM = new PerspectiumMessage("replicator", "servicenow", key, name, value, target_queue, "", attributes, shareGR, "",state, false, "u_psp_attachment_out_message", record_sys_id);					
110			//places the message in the attachments outbound table
111			return pspM.enqueue();
112		}
113		else if (name.contains("sys_audit.")) {
114			var pspM = new PerspectiumMessage("replicator", "servicenow", key, name, value, target_queue, extra, attributes, shareGR, "", state, false, "u_psp_audit_out_message");
115				
116			//places the message in the sys audit outbound table
117			return pspM.enqueue();
118		 }		
119		else {	
120			// call actual Perspectium function to create outbound message
121			this.psp.createPSPOut(topic, type, key, name, value, target_queue, extra, attributes, shareGR, state, record_sys_id);
122		}
123	},
124
125	scheduleBulkShare : function(schd_sys_id) {
126		this.logger.logDebug("Starting scheduled bulk share: " + schd_sys_id, "PerspectiumReplicator.scheduleBulkShare");
127		var bgr = new GlideRecord("psp_bulk_share");
128		bgr.addQuery("u_run_schedule", schd_sys_id);
129		bgr.query();
130		while(bgr.next()) {
131			this.logger.logDebug("scheduled bulk sharing: " + bgr.sys_id, "PerspectiumReplicator.scheduleBulkShare");
132			var startTime = "";
133			if (bgr.u_share_updates_since_then) {
134				startTime = bgr.getValue("started");
135			}
136
137			this.scheduleOneBulkShare(bgr, startTime);
138		}
139		this.logger.logDebug("Finished scheduled bulk share: " + schd_sys_id, "PerspectiumReplicator.scheduleBulkShare");
140	},
141	
142	scheduleOneBulkShare : function(bgr, startTime) {
143		// first check if we can find this bulk share so we don't create duplicates
144		var bgr2 = new GlideRecord('psp_bulk_share');
145		bgr2.addQuery('sys_id', bgr.sys_id);
146		bgr2.queryNoDomain();
147		if (!bgr2.next()) {	
148	    	return;
149		}
150		
151		// copy fields to save to a history of the bulk share and display as a related list
152		var nr = new GlideRecord("u_psp_previous_bulk_shares");
153		nr.name = bgr.u_name;
154		nr.u_table_name = bgr.table_name;
155		nr.u_started = bgr.started;
156		nr.u_completed = bgr.completed;
157		nr.u_duration = bgr.u_duration;
158		nr.u_condition_script = "sys_updated_on>javascript:gs.dateGenerate('" + nr.u_started + "')";
159		nr.u_records_processed = bgr.u_records_processed;
160		nr.u_status = bgr.status;
161		nr.u_bulk_share = bgr.sys_id;
162		if (!nr.u_started.isNil())
163			nr.insert();
164		
165		this.scheduleOnceBulkShareJob(bgr, startTime);	
166	},
167	
168	scheduleOnceBulkShareJob: function(bgr, startTime) {		
169		bgr.status = "Scheduled";
170		bgr.u_cancel = false;
171		bgr.started = gs.nowDateTime();		
172		bgr.completed = "";		
173		bgr.u_duration = "";
174		bgr.u_records_processed = "";
175		bgr.u_records_per_second = "";		
176		bgr.update();
177		
178		// clear out previous history of records processed if any so we don't have old data in this table
179		var ogr = new GlideRecord("u_psp_records_processed");
180		ogr.addQuery("u_source_table", bgr.getTableName());
181		ogr.addQuery("u_source", bgr.sys_id);
182		ogr.deleteMultiple();
183		
184		var pspSO;
185		// if advanced is selected or user selected to distribute bulk share, we use the custom script include to schedule job
186		if ((bgr.u_advanced == true || bgr.u_advanced == "true") || 
187			   (bgr.u_distribute_bulk_share_workload == true || bgr.u_distribute_bulk_share_workload == "true")) {
188			var jobPriority;
189			if (!bgr.u_scheduled_job_priority.nil() && bgr.u_scheduled_job_priority > 0) {
190				jobPriority = parseInt(bgr.u_scheduled_job_priority);
191			}
192			else {
193				jobPriority = parseInt(this.pspUtil.getPspPropertyValue("com.perspectium.replicator.scheduled_job.priority", "100"));
194			}
195			
196			var systemId = "";
197			if (bgr.u_distribute_bulk_share_workload == true || bgr.u_distribute_bulk_share_workload == "true") {
198				// get the last node we ran with so we query for one that isn't that one
199				var lastSystemId = this.pspUtil.getPspPropertyValue('com.perspectium.replicator.last_system_id', '');
200				var scgr = new GlideRecord("sys_cluster_state");
201				scgr.addQuery("system_id", "!=", lastSystemId);
202				scgr.query();
203				if (scgr.next()) {
204					systemId = scgr.system_id;
205					this.pspUtil.setPspPropertyValue('com.perspectium.replicator.last_system_id', systemId);
206				}							
207			}
208
209			pspSO = new PerspectiumScheduleOnce(jobPriority, systemId);
210		}
211		else {
212			pspSO = new ScheduleOnce();
213		}		
214				
215		pspSO.setLabel("Perspectium Replicator Bulk Share " + bgr.table_name);
216		var gdt = new GlideDateTime();
217		gdt.addSeconds(10); // schedule in 10 seconds
218		pspSO.setTime(gdt.getValue()); 
219		pspSO.script =  "var pspR = new PerspectiumReplicator();";
220		pspSO.script += "pspR.bulkShareRecords('" + bgr.sys_id + "', '" + startTime + "');";		
221		pspSO.schedule();	
222	},
223	
224	executeBeforeBulkShareScript: function(script, bgr){
225		var gc = (typeof GlideController != 'undefined') ? GlideController : Packages.com.glide.script.GlideController;
226		var cancel = false;
227
228		// put bulk share configuration global for reference in script
229        gc.putGlobal('bulkshare_gr', bgr);   
230		// put a cancel var global so user can cancel running bulk share
231        gc.putGlobal('cancel', cancel);
232		
233		var rc = gc.evaluateString(script);        		
234		
235		cancel = gc.getGlobal('cancel');
236        gc.removeGlobal('cancel');
237        gc.removeGlobal('bulkshare_gr');		
238				
239		return cancel;			
240	},
241	
242	executeAfterBulkShareScript: function(script, bgr){
243		var gc = (typeof GlideController != 'undefined') ? GlideController : Packages.com.glide.script.GlideController;
244		
245		// put bulk share configuration global for reference in script
246        gc.putGlobal('bulkshare_gr', bgr); 
247		
248		var rc = gc.evaluateString(script);        		
249		gc.removeGlobal('bulkshare_gr');		
250	},
251
252	generateUUID: function () { // Public Domain/MIT
253		var d = new Date().getTime();
254		if (typeof performance !== 'undefined' && typeof performance.now === 'function'){
255			d += performance.now(); //use high-precision timer if available
256		}
257		return 'xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
258			var r = (d + Math.random() * 16) % 16 | 0;
259			d = Math.floor(d / 16);
260			return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
261		});
262	},
263	
264	etsBulkShareRecords: function(bgr) {
265		if (bgr.status == 'Cancelling')
266			this.cancelledBulkShare(bgr);
267		
268		var conditions = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"];
269		var jobs = parseInt(this.pspUtil.getPspPropertyValue("com.perspectium.bulk_share_scheduled_jobs", "4"));
270		
271		bgr.u_set_id = this.generateUUID();
272		bgr.update();
273		
274		for (var j = 0; j < jobs; j++){
275			var pspSO = new ScheduleOnce();
276			var jobName = "Perspectium Replicator Bulk Share " + (j + 1) + " " + bgr.table_name;
277			pspSO.setLabel(jobName);
278
279			var condition = "";
280			// create conditions based on number of jobs so we can evenly spread out records between the jobs
281			// for ex. job #1 'sys_idSTARTSWITH0^ORsys_idSTARTSWITH4^ORsys_idSTARTSWITH8^ORsys_idSTARTSWITHc^EQ'
282			if (jobs > 1) {
283				condition = "sys_idSTARTSWITH" + conditions[j];
284				var conditionCount = j + jobs;
285				while (conditionCount < 16) {
286					condition += "^ORsys_idSTARTSWITH" + conditions[conditionCount];
287					conditionCount += jobs;
288				}				
289			}
290			
291			var gdt = new GlideDateTime();
292			gdt.addSeconds(5); // schedule in 5 seconds
293			
294			pspSO.setTime(gdt.getValue()); 			
295			pspSO.script = "var pspB = new PerspectiumBulkShare('" + bgr.sys_id + "', '" + condition + "^EQ', '" + jobName + "');";
296			pspSO.script += "pspB.execute();";			
297			pspSO.schedule();			
298		}
299	},
300	
301    bulkShareRecords : function(bsysid, startTime) { 
302		var bgr = new GlideRecord('psp_bulk_share');
303		bgr.addQuery('sys_id', bsysid);
304		bgr.queryNoDomain();
305		if(!bgr.next()) {	
306	    	return;
307		}
308		
309		this.logger.logDebug("bulkShareRecords("+ bsysid +")", "PerspectiumReplicator.bulkShareRecords", bgr);
310		
311		if (bgr.status == "Cancelling") {
312			this.cancelledBulkShare(bgr);
313			return;
314		}
315		
316		if (bgr.status == "Cancelled") {
317			return;
318		}
319		
320		this.logger.logDebug("bulkShareRecords: " + bgr.table_name, "PerspectiumReplicator.bulkShareRecords", bgr);
321
322		// check if a "run as" user chosen to run bulk share as
323		var curUser = "";
324		if (!bgr.u_run_as.isNil()) {
325			curUser = gs.getSession().impersonate(bgr.u_run_as);	
326		}
327
328		bgr.status = 'Running';
329		bgr.update();
330		this.psp.createPSPOut("monitor", "label", this.getKey(bgr), "Perspectium^PSP", "Bulk share started : '" + bgr.table_name + "'", null, null, null, bgr);
331		// before bulk sharing any records we'll run before script if specified
332		if (!bgr.u_before_bulk_share_script.nil()) {
333			var cancel = this.executeBeforeBulkShareScript(bgr.u_before_bulk_share_script, bgr);
334			if (cancel == true || cancel == "true") {
335				// user cancels bulk share so we'll update to cancelled status
336				this.cancelledBulkShare(bgr);
337				this.logger.logDebug(bgr.u_name + " bulk share (" + bgr.sys_id + ") cancelled by Before Bulk Share Script", "PerspectiumReplicator.bulkShareRecords");
338				
339				// before returning set back to original user
340				if (!bgr.u_run_as.isNil() && curUser != "") {
341					gs.getSession().impersonate(curUser);	
342				}
343
344				return;
345			}			
346		}
347				
348		this.logger.logDebug("bulkShareRecords: after post label", "PerspectiumReplicator.bulkShareRecords", bgr);
349
350		var action = "bulk";
351        if (bgr.u_insert_only == "true" || bgr.u_insert_only == true) {
352        	action = "insert";
353        }
354
355		if (bgr.u_share_only_selected_fields == "true" || bgr.u_share_only_selected_fields == true){
356			var pspSF = new PerspectiumShareFields();
357			this.fieldsToReplicate = pspSF.getFieldsToReplicate(bgr);
358		}
359		
360		// check for ETS mode
361		if (bgr.u_ets_active == "true" || bgr.u_ets_active == true) {
362			this.etsBulkShareRecords(bgr);
363			
364			// finished bulk share so we'll execute any after script if specified
365			if (!bgr.u_after_bulk_share_script.nil()) {
366				this.executeAfterBulkShareScript(bgr.u_after_bulk_share_script, bgr);
367			}
368			
369			return;
370		}
371		else if (bgr.u_share_only_sys_ids_listed == "true" || bgr.u_share_only_sys_ids_listed == true) {
372			this.logger.logDebug("bulkShareRecords: " + bgr.table_name + " sharing only sys ids listed", "PerspectiumReplicator.bulkShareRecords", bgr);
373			
374			// go through related list of sys ids and share each sys id
375			var tgr = new GlideRecord('u_psp_bulk_share_sys_id');
376			tgr.addQuery('u_bulk_share', bgr.sys_id);
377			tgr.query();
378						
379			// save total row count for attributes
380			this.bulk_query_count = tgr.getRowCount();
381
382			while(tgr.next()){
383	    		var gr=new GlideRecord(bgr.table_name);
384			
385				gr.addQuery('sys_id', tgr.u_record_sys_id);
386				
387				// if a run as user selected we'll want to do a regular query()
388				// to honor domain separation since people will only choose a run as user to get the results that user only sees
389				if (!bgr.u_run_as.isNil()) {
390					gr.query();
391				}
392				else {
393					gr.queryNoDomain();
394				}
395				
396				if(!gr.next()) {	
397				//if(!gr.get(tgr.u_record_sys_id)) {
398					this.logger.logDebug("No record with sys id " + tgr.u_record_sys_id + " found in table " + bgr.table_name, "PerspectiumReplicator.bulkShareRecords", bgr);
399					continue;
400				}
401				
402				this.logger.logDebug("Bulk sharing record: " + tgr.u_record_sys_id + " from table " + bgr.table_name, "PerspectiumReplicator.bulkShareRecords", bgr);
403				
404				this.bulkShareRecord(bgr, gr, action);
405				
406				if (this.bulk_count % 1000 == 0) {					
407					bgr.addQuery('sys_id', bsysid);
408					bgr.queryNoDomain();
409					if(bgr.next()) {
410						// only update count if we found bulk share
411						// so we don't insert a new record
412						bgr.u_records_processed = this.bulk_count;
413		    			bgr.update();
414						
415						// check if cancelled
416						if (bgr.status == "Cancelling") {
417							this.cancelledBulkShare(bgr);
418							
419							// before returning set back to original user
420							if (!bgr.u_run_as.isNil() && curUser != "") {
421								gs.getSession().impersonate(curUser);	
422							}
423
424							return; // bail
425						}
426						
427						if (bgr.status == "Cancelled") {							
428							// before returning set back to original user
429							if (!bgr.u_run_as.isNil() && curUser != "") {
430								gs.getSession().impersonate(curUser);	
431							}
432
433							return;
434						}
435					}
436				}				
437			}	    	        	
438        }		
439		else if (bgr.condition.nil()) {
440	    	var more = this.bulkShareAllRecords(bgr, null, startTime);
441	    	while(more != null) {
442				more = this.bulkShareAllRecords(bgr, more, startTime);
443	    	}
444		} else {	
445			this.logger.logDebug("bulkShareRecords: " + bgr.table_name + " with condition: " + bgr.condition, "PerspectiumReplicator.bulkShareRecords", bgr);
446			
447	    	var gr=new GlideRecord(bgr.table_name);
448	    	gr.addEncodedQuery(bgr.condition);
449	    	if(bgr.u_share_updates_since_then && startTime && startTime != 'undefined' && startTime != "") {
450				gr.addQuery('sys_updated_on', '>=', new GlideDateTime(startTime));
451			}
452			
453			if (bgr.isValidField("u_limit_number_of_records_shared") && !bgr.u_limit_number_of_records_shared.nil()) {
454				gr.setLimit(bgr.getValue("u_limit_number_of_records_shared"));
455			}
456			else if(bgr.isValidField("u_limit_number_of_records_shar") && !bgr.u_limit_number_of_records_shar.nil()) {
457				gr.setLimit(bgr.getValue("u_limit_number_of_records_shar"));
458			}
459						
460			// if a run as user selected we'll want to do a regular query()
461			// to honor domain separation since people will only choose a run as user to get the results that user only sees
462			if (!bgr.u_run_as.isNil()) {
463				gr.query();
464			}
465			else {
466                gr.queryNoDomain(); 
467			}	
468				
469			// save total row count for attributes
470			this.bulk_query_count = gr.getRowCount();
471
472			// use _next to account for tables such as sys_template that have "next" column
473	    	while(gr._next()) {
474				this.bulkShareRecord(bgr, gr, action);
475				
476				if (this.bulk_count % this.bulk_share_query_limit == 0) {					
477					bgr.addQuery('sys_id', bsysid);
478					bgr.queryNoDomain();
479					if(bgr.next()) {
480						// only update count if we found bulk share
481						// so we don't insert a new record
482			    		bgr.u_records_processed = this.bulk_count;
483			    		bgr.update();
484
485						// check if cancelled
486						if (bgr.status == "Cancelling") {
487							this.cancelledBulkShare(bgr);				
488							// before returning set back to original user
489							if (!bgr.u_run_as.isNil() && curUser != "") {
490								gs.getSession().impersonate(curUser);	
491							}
492							
493							return; // bail
494						}
495						
496						if (bgr.status == "Cancelled") {
497							// before returning set back to original user
498							if (!bgr.u_run_as.isNil() && curUser != "") {
499								gs.getSession().impersonate(curUser);	
500							}
501							
502							return;
503						}
504					}
505				}
506	    	}
507		}
508		
509		// finished bulk share so we'll execute any after script if specified
510		if (!bgr.u_after_bulk_share_script.nil()) {
511			this.executeAfterBulkShareScript(bgr.u_after_bulk_share_script, bgr);
512		}
513		
514		// check if we can find this bulk share so we don't create duplicates
515		var bgr2 = new GlideRecord('psp_bulk_share');
516		bgr2.addQuery('sys_id', bgr.sys_id);
517		bgr2.queryNoDomain();
518		if (bgr2.next()) {	
519			bgr2.completed = gs.nowDateTime();
520			if (bgr2.u_cancel == true) {
521				bgr2.status = 'Cancelled';
522			}
523			else {
524				bgr2.status = 'Completed';
525			}
526			bgr2.u_records_processed = this.bulk_count;
527			bgr2.update();
528		}
529
530		// send out # processed message set message
531		this.pspMS.createMessageSetProcessed(this.messageSetCounter, bgr, this.getKey(bgr));
532									
533		// get total breakdown of records processed and update the related list
534		this.updateTotalRecordsProcessed(bgr, this.messageSetCounter);
535		
536		// send share counters
537		var counterName = bgr.table_name + "." + action;
538	    this.psp.sendCounter(counterName, this.bulk_count, "counter");
539	    this.psp.sendCounter(counterName + ".bytes", this.bulk_byte_count, "counter");
540		
541		this.psp.createPSPOut("monitor", "label", this.getKey(bgr), "Perspectium^PSP", "Bulk share completed : '" + bgr.table_name + "' ('" + this.bulk_count + "')", null, null, null, bgr);
542		
543		// set back to original user after finishing
544		if (!bgr.u_run_as.isNil() && curUser != "") {
545			gs.getSession().impersonate(curUser);	
546		}
547    },
548	
549	updateTotalRecordsProcessed: function (bgr, messageSetCounter) {
550		// clear out previous history of records processed if any so we don't have old data in this table
551		var ogr = new GlideRecord("u_psp_records_processed");
552		ogr.addQuery("u_source_table", bgr.getTableName());
553		ogr.addQuery("u_source", bgr.sys_id);
554		ogr.deleteMultiple();
555		
556		var totalCount = 0;
557		for (var m in messageSetCounter) {
558			var value = messageSetCounter[m];
559			if (value == 0) {
560				continue;
561			}
562			
563			var rpgr = new GlideRecord("u_psp_records_processed");
564			rpgr.u_source_table = bgr.getTableName();
565			rpgr.u_source = bgr.sys_id;
566			
567			var name = m;
568			// remove action such as .bulk as we don't need it
569			if (name.indexOf(".") > 0) {
570				name = name.substr(0, name.indexOf("."));
571			}
572			
573			rpgr.u_name = name;
574			rpgr.u_value = value;
575			rpgr.insert();	
576			
577			totalCount += value;
578		}
579		
580		if (totalCount == 0) {
581			return;
582		}
583		
584		// check if we can find this bulk share so we don't create duplicates
585		var bgr2 = new GlideRecord('psp_bulk_share');
586		bgr2.addQuery('sys_id', bgr.sys_id);
587		bgr2.queryNoDomain();
588		if (bgr2.next()) {	
589			bgr2.u_records_processed = totalCount;
590			bgr2.update();
591		}
592		
593	},
594	
595	cancelledBulkShare : function(bgr) {
596		// first check if we can find this bulk share so we don't create duplicates
597		var bgr2 = new GlideRecord('psp_bulk_share');
598		bgr2.addQuery('sys_id', bgr.sys_id);
599		bgr2.queryNoDomain();
600		if (!bgr2.next()) {	
601	    	return;
602		}
603		
604		bgr.status = "Cancelled";
605		bgr.u_records_processed = this.bulk_count;
606		bgr.update();
607		
608		// only send counter if we actually shared records
609		if(this.bulk_count > 0){		
610			var counterName = bgr.table_name + ".bulk";
611			this.psp.sendCounter(counterName, this.bulk_count, "counter");
612	    	this.psp.sendCounter(counterName + ".bytes", this.bulk_byte_count, "counter");
613			
614			// send out # processed message set message
615			this.pspMS.createMessageSetProcessed(this.messageSetCounter, bgr, this.getKey(bgr));
616		}
617	
618		this.psp.createPSPOut("monitor", "label", this.getKey(bgr), "Perspectium^PSP", "Bulk share cancelled : '" + bgr.table_name + "' ('" + this.bulk_count + "')", null, null, null, bgr);
619	},
620	
621    bulkShareAllRecords : function(bgr, starting_sysid, startTime) {	
622
623		var gr=new GlideRecord(bgr.table_name);
624		gr.orderBy('sys_id');
625		
626		// check if this is a database view
627		var sdv = new GlideRecord("sys_db_view");
628		sdv.addQuery("name", bgr.table_name);
629		sdv.queryNoDomain();		
630		// if not then we limit the query to 1000 records to optimize performance
631		if (!sdv.next()) {
632			this.logger.logDebug("Limiting query to  " + this.bulk_share_query_limit, "PerspectiumReplicator.bulkShareAllRecords", bgr);
633			if (bgr.isValidField("u_limit_number_of_records_shared") && !bgr.u_limit_number_of_records_shared.nil()) {
634				gr.setLimit(bgr.getValue("u_limit_number_of_records_shared"));
635			}
636			else if(bgr.isValidField("u_limit_number_of_records_shar") && !bgr.u_limit_number_of_records_shar.nil()) {
637				gr.setLimit(bgr.getValue("u_limit_number_of_records_shar"));
638			}
639			else{
640				gr.setLimit(this.bulk_share_query_limit);		
641			}		
642		}
643			 					
644		if (starting_sysid != null) {
645			gr.addQuery('sys_id', '>', starting_sysid);
646		}
647		
648		if(bgr.u_share_updates_since_then && startTime && startTime != 'undefined' && startTime != "") {
649			gr.addQuery('sys_updated_on', '>=', new GlideDateTime(startTime));
650		}
651		
652		// if a run as user selected we'll want to do a regular query()
653		// to honor domain separation since people will only choose a run as user to get the results that user only sees
654		if (!bgr.u_run_as.isNil()) {
655			gr.query();
656		}
657		else {
658            gr.queryNoDomain();
659		}				
660
661		this.bulk_share_all_records = true;
662		// save total row count for attributes when we have complete total
663		if(gr.getRowCount() != this.bulk_share_query_limit)
664			this.bulk_query_count = this.bulk_count + gr.getRowCount();
665
666		var last_sys_id = null;
667		var kount = 0;
668	
669		var action = "bulk";
670		if (bgr.u_insert_only == "true" || bgr.u_insert_only == true) {
671			action = "insert";
672		}
673		
674		this.logger.logDebug("bulkShareAllRecords: " + bgr.table_name + " action=" + action, "PerspectiumReplicator.bulkShareAllRecords", bgr);
675	
676		// use _next to account for tables such as sys_template that have "next" column
677		while(gr._next()) {
678			last_sys_id = gr.sys_id;
679			this.bulkShareRecord(bgr, gr, action);
680			kount ++;
681			
682			// check every 1000 records if we want to cancel
683			if (kount % this.bulk_share_query_limit == 0) {								
684				bgr.addQuery('sys_id', bgr.sys_id);
685				bgr.queryNoDomain();
686				if(bgr.next()) {
687					// only update count if we found bulk share
688					// so we don't insert a new record
689					bgr.u_records_processed = this.bulk_count;
690					bgr.update();
691
692					// check if cancelled
693					if (bgr.status == "Cancelling") {
694						this.cancelledBulkShare(bgr);
695						return null; // bail
696					}
697					
698					var limitRecordsShared;
699					if(bgr.isValidField("u_limit_number_of_records_shared")){
700						limitRecordsShared = bgr.u_limit_number_of_records_shared;
701					}
702					else{
703						limitRecordsShared = bgr.u_limit_number_of_records_shar;
704					}
705					
706					if(limitRecordsShared != null && limitRecordsShared != "" && limitRecordsShared <= kount) {
707						return null;
708					}
709					
710					
711					if (bgr.status == "Cancelled") {
712						return null;
713					}
714				}				
715			}
716		}
717		
718		this.logger.logDebug("bulkShareAllRecords: " + bgr.table_name + " shared: " + kount + " records", "PerspectiumReplicator.bulkShareAllRecords", bgr);
719			
720		// reach query limit so we return last sys_id for next set of records
721		// if we queried all at once count will be greater so we won't return last sys_id
722		if (kount == this.bulk_share_query_limit) {
723			return last_sys_id;
724		}
725		
726		var bgr2 = new GlideRecord('psp_bulk_share');
727		bgr2.addQuery('sys_id', bgr.sys_id);
728		bgr2.queryNoDomain();
729		if (bgr2.next()) {			
730			bgr.u_records_processed = this.bulk_count;
731			bgr.update();
732		}
733		
734		return null;	
735    },
736
737	bulkShareRecord : function(bgr, gr, op) {				
738		if (bgr.u_include_child_only == "true" ||  bgr.u_include_child_only == true) {
739			var tname = bgr.table_name;
740			if (gr.sys_class_name && !gr.sys_class_name.isNil()) {
741				tname = gr.sys_class_name;
742				
743				// get the record from the new table name to get the right record and ensure we get any fields new table name only has
744				var tngr = new GlideRecord(tname);
745				
746				tngr.addQuery('sys_id', gr.sys_id);
747				
748				// if a run as user selected we'll want to do a regular query()
749				// to honor domain separation since people will only choose a run as user to get the results that user only sees
750				if (!bgr.u_run_as.isNil()) {
751					tngr.query();
752				}
753				else {
754					tngr.queryNoDomain();
755				}
756				
757				if(!tngr.next()){
758				//if(!tngr.get(gr.sys_id)){
759					this.logger.logDebug("Cannot find record " + gr.sys_id + " in " + tname, "PerspectiumReplicator.bulkShareRecord", bgr);
760					return;
761				}				
762				
763				this.bulkShareOneRecord(bgr, tname, tngr, op);				
764			}
765			else
766				this.bulkShareOneRecord(bgr, tname, gr, op);
767			
768			return;
769		} else if (bgr.u_include_child_tables == "true" ||  bgr.u_include_child_tables == true) {
770			var found = false;
771			var currentTable = "";
772			var parentTable = gr.sys_class_name;
773			
774			while(parentTable != -1 && currentTable != bgr.table_name){
775				currentTable = parentTable;
776				
777				// check we can get child table record and then share it instead of base table record
778				var ctgr = new GlideRecord(currentTable);	
779				
780				ctgr.addQuery('sys_id', gr.sys_id);
781				
782				// if a run as user selected we'll want to do a regular query()
783				// to honor domain separation since people will only choose a run as user to get the results that user only sees
784				if (!bgr.u_run_as.isNil()) {
785					ctgr.query();
786				}
787				else {
788					ctgr.queryNoDomain();
789				}
790				
791				if(!ctgr.next()){
792					continue;	
793				}
794				
795				this.logger.logDebug("bulkShareRecord " + gr.sys_id + " for child table " + currentTable, "PerspectiumReplicator.bulkShareRecord", bgr);
796				
797				
798				if(this.bulkShareOneRecord(bgr, currentTable, ctgr, op)) {				
799					found = true;
800				}
801				
802				parentTable = this.getParentTable(currentTable);
803			}
804			
805			if (found) {
806				this.logger.logDebug("bulkShareRecord there were child tables, no need to sync base","PerspectiumReplicator.bulkShareRecord", bgr);
807				return;
808			}
809		}
810		
811		this.bulkShareOneRecord(bgr, bgr.table_name, gr, op);
812	},
813	
814	getParentTable: function(table_name) {
815		var cgr = new GlideRecord('sys_db_object');
816		cgr.addQuery('name', table_name);
817		cgr.query();
818		if(cgr.next()){
819			if(cgr.super_class.nil()){
820				return -1;
821			}
822			else{
823				return cgr.super_class.name;
824			}
825		}
826		return -1;
827	},
828	
829	// override encryption key if target queue contains a custom key
830	setQueueKey : function (tqgr) {
831		if (tqgr.u_target_queue.isNil())
832			return;
833		
834		if (tqgr.u_target_queue.u_record_encryption_key.isNil())
835			return;
836		
837		var encrypter = (typeof GlideEncrypter != 'undefined') ? new GlideEncrypter() : new Packages.com.glide.util.Encrypter();
838		var qKey = encrypter.decrypt(tqgr.u_target_queue.u_record_encryption_key);
839				
840		this.logger.logDebug("setQueueKey using target queue key: " + qKey);
841		this.encryption = new PerspectiumEncryption(qKey);
842	},
843	
844    bulkShareOneRecord : function(bgr, table_name, gr, op) {
845        try {
846			var sys_id = gr.sys_id;
847            if(gr.sys_id.isNil() || sys_id == null || sys_id == "") {
848				// sys_id not found for this table
849				this.logger.logDebug("bulkShareOneRecord not found for " + table_name +  "," + sys_id, "PerspectiumReplicator.bulkShareOneRecord", bgr);
850				return false;
851			}
852			
853			if (!bgr.u_before_share_script.nil()) {
854				var jsonParams = {};
855				jsonParams.psp_action = op;
856				var ignore = this.psp.executeBeforeShareScript(bgr.u_before_share_script, gr,jsonParams);
857				if (ignore == true || ignore == "true") {
858					this.logger.logDebug("bulkShareOneRecord ignored by beforeShareScript", "PerspectiumReplicator.bulkShareOneRecord");
859					return false;
860				}
861				op = jsonParams.psp_action;
862			}
863						
864			// share history set first
865			if (bgr.u_include_history_set == "true" || bgr.u_include_history_set == true) {
866				this.shareHistorySet(table_name, sys_id, null, bgr);	
867			}
868			
869			// override encryption key if target queue contains a custom key
870			if (!bgr.u_target_queue.isNil() && !bgr.u_target_queue.u_record_encryption_key.isNil()) {
871				var encrypter = (typeof GlideEncrypter != 'undefined') ? new GlideEncrypter() : new Packages.com.glide.util.Encrypter();
872				var qKey = encrypter.decrypt(bgr.u_target_queue.u_record_encryption_key);
873				
874				this.logger.logDebug("bulkShareOneRecord using target queue key: " + qKey);
875				this.encryption = new PerspectiumEncryption(qKey);
876			}
877
878			this.setQueueKey(bgr);
879
880			var enc = "";
881			//var cipher = this.encryptionType.TRIPLE_DES;
882			var cipher = bgr.u_cipher;
883			var grTableMap;
884			if(bgr.isValidField("u_table_map") && !bgr.u_table_map.nil()){
885				grTableMap = this.encryption.getOutboundTableMap(bgr.u_table_map);
886				if(grTableMap.u_topic == "siam"){
887					cipher = this.encryptionType.BASE64_ONLY;
888					enc = this.encryption.encryptTableMap(gr, grTableMap, cipher); 
889				}
890				else{
891					cipher = bgr.u_cipher;
892			    	enc = this.encryption.encryptTableMap(gr, grTableMap, bgr.u_cipher);
893				}
894			} else if (!bgr.u_view_name.nil()) {
895				enc = this.encryption.encrypt(gr, bgr.u_view_name, op, bgr.u_target_queue, cipher);
896			} else if (bgr.u_share_only_selected_fields == "true" || bgr.u_share_only_selected_fields == true){
897				var pspSF = new PerspectiumShareFields();
898				var recordXmlStr = pspSF.createShareRecordXML(bgr, gr, table_name, this.fieldsToReplicate);
899				enc = this.encryption.encryptString(recordXmlStr);
900			} else {
901            	enc = this.encryption.encrypt(gr, null, op, bgr.u_target_queue, cipher);
902			}
903			
904			// if the value is too large then we'll error
905			// account for both if the value is bigger than our max sending size
906			// as well as the largest size allowed for a String
907			if (enc.length() > this.maxBytes || (enc.length() * 2) > 16000000) {					
908				this.logger.logError(op + " message for " + table_name + " <" + gr.sys_id + "> cannot be created in outbound queue as it is larger than message limit", "PerspectiumReplicator.bulkShareOneRecord");
909				return;
910			}
911
912            //topic, type, key, name, value
913			var topic = "replicator";
914			var type = "servicenow";
915			var bAttributes = "";
916
917			bAttributes = this.appendAttribute("cipher", cipher, bAttributes);
918			if(typeof(grTableMap) != "undefined" && !grTableMap.isNil()){
919				if(!grTableMap.u_topic.isNil())
920					topic = grTableMap.u_topic;
921				if(!grTableMap.u_type.isNil())
922					type = grTableMap.u_type;
923				if(!grTableMap.u_target_tablename.isNil())
924					table_name = grTableMap.u_target_tablename;
925				bAttributes = this.appendTableMapAttributes(gr, grTableMap, bAttributes);
926			}
927            //var name = table_name + "." + op;
928            //this.psp.createPSPOut("replicator", "servicenow", this.getKey(bgr), name, enc, bgr.u_target_queue);
929			//this.bulk_count++;
930			
931			// add count for bytes but getting length of value
932			// to ensure it's a string add ""
933			this.bulk_byte_count += enc.length();
934			
935			if (bgr.u_include_attachment == "true" || bgr.u_include_attachment == true) {
936				this.shareAttachments(sys_id, bgr, op);	
937			}
938			
939			if (bgr.u_share_journal == "true" || bgr.u_share_journal == true) {
940				this.shareJournal(sys_id, null, bgr.sys_id, true);				
941			}
942		
943			if (bgr.u_include_sys_audit == "true" || bgr.u_include_sys_audit == true) {
944				this.shareSysAudit(table_name, sys_id, null, bgr);
945			}
946						
947			if (bgr.u_include_embedded_images_vide == "true" || bgr.u_include_embedded_images_vide == true) {
948				this.shareEmbedded(gr, sys_id, bgr, op);	
949			}
950
951			// send table record last so we can set psp_flag last for last message in set
952			//topic, type, key, name, value, queue, extra, attributes
953            var name = table_name + "." + op;
954
955			// add attributes with bulk share conf id to use for message set id
956			bAttributes = this.appendAttribute("set_id", bgr.sys_id, bAttributes);
957			
958			// for case of bulk sharing all records and we're at our per query limit
959			// we'll check if this is the last record so we can set psp_flag last
960			// otherwise flag won't be set since we can' tell if we're at the 1000 record in the set or 1000 is the last record 
961			if(this.bulk_share_all_records && (this.bulk_count + 1) % this.bulk_share_query_limit == 0){
962				var trgr = new GlideRecord(bgr.table_name);
963				trgr.orderBy('sys_id');
964				trgr.addQuery('sys_id', '>', gr.sys_id);
965				trgr.query();
966				
967				if (!trgr.next()){
968					bAttributes = this.appendAttribute("psp_flag", "last", bAttributes);
969				}				
970			}			   
971			// set flag in attributes for last message in bulk share
972			else if((this.bulk_count + 1) == this.bulk_query_count){				
973				bAttributes = this.appendAttribute("psp_flag", "last", bAttributes);				
974			}
975			
976			this.createPSPOut(topic, type, this.getKey(bgr), name, enc, bgr.u_target_queue, "", bAttributes, bgr, gr.sys_id);
977			this.bulk_count++;
978			
979			// increment counter for each message sent
980			this.incrementMessageSetCounter(name);			
981			
982        } catch(e) {
983            if (String(e) == "java.lang.NullPointerException") {
984				this.logger.logWarn("bulkShareOneRecord failed due to invalid null characters, sys_id: " + gr.sys_id + ", error = " + e, "PerspectiumReplicator.bulkShareOneRecord", bgr);
985			} 
986			else {
987				this.logger.logWarn("bulkShareOneRecord " + table_name + ", record = " + gr.sys_id + ", error = " + e.message, "PerspectiumReplicator.bulkShareOneRecord", bgr);
988			}
989        }
990		
991		return true;
992    },
993	
994	// operation is passed if cgr was reconstituted from XML e.g. from sys_audit_delete business rule
995	shareRecord : function(cgr, table_name, operation, grShareConfigSysId, pspAttributes, sumCounts){
996		var sys_class_name = cgr.getTableName();
997		var sys_id = cgr.sys_id;
998		var op = "insert";
999		
1000		if (!table_name) {
1001			table_name = sys_class_name;
1002		}
1003			
1004		if (cgr.operation() != null) {
1005	    	op = cgr.operation().toString();
1006		}
1007		
1008		// override operation if passed one
1009		if (operation && operation != "") {
1010			op = operation;
1011		}
1012		
1013		// each time dynamic share called we'll re-set flag that we sent first message since each dynamic share
1014		// is the first message
1015		this.sentFirstMessage = false;
1016		
1017		var qc = new GlideRecord('psp_replicate_conf');
1018    	try {
1019            qc.addQuery('table_name', table_name);
1020	    	qc.addQuery("sync_direction", "share");
1021	    	qc.addQuery("active", "true");
1022			
1023			// if we were passed in a dynamic share config we'll query only for it
1024			if(grShareConfigSysId && grShareConfigSysId != null && grShareConfigSysId != ""){
1025	    		qc.addQuery("sys_id", grShareConfigSysId);				
1026			}
1027
1028            qc.query();
1029			
1030			if (!qc.hasNext()) {
1031				this.logger.logDebug("shareRecord no share definition found for " + sys_class_name + " " + op, "PerspectiumReplicator.shareRecord", qc);
1032				return; // nothing can be done
1033			}
1034
1035            while (qc.next()) {
1036		        this.logger.logDebug("sharing " + sys_class_name + " " + sys_id + " " + op + " using share config " + qc.sys_id, "PerspectiumReplicator.shareRecord", qc);
1037				
1038				if (!operation && op == "delete" && qc.u_use_listener == true) {
1039					this.logger.logDebug("this came in from business rule, but we are using audit listener so dont process share config " + qc.sys_id, "PerspectiumReplicator.shareRecord", qc);
1040					break;
1041				}
1042				
1043				if (qc.u_select_column_updates_to_ignore == "true" || qc.u_select_column_updates_to_ignore == true) {
1044					var isUpdated = this.ignoreUpdatedField(qc, cgr);
1045					if (isUpdated == true || isUpdated == "true"){
1046						this.logger.logDebug("shareRecord skipped for " + table_name + " - [" + cgr.sys_id + "] due to Ignore Updated Fields", "PerspectiumReplicator.shareRecord", qc);
1047						return;
1048					}
1049				}
1050				
1051				if (qc.u_share_only_selected_fields == "true" || qc.u_share_only_selected_fields == true && this.fieldsToReplicate.length == 0){
1052					var pspSF = new PerspectiumShareFields();
1053					this.fieldsToReplicate = pspSF.getFieldsToReplicate(qc);
1054				}
1055				
1056				// if not delete and we weren't pass share config sys id, then we'll ignore share configs that do have a reference to a business rule
1057				// since those were created in the new way and we only want those to run when share config sys id passed				
1058				if(!grShareConfigSysId && 
1059				   (qc.u_business_rule != null && qc.u_business_rule != "" && !qc.u_business_rule.isNil())){
1060					continue;
1061				}
1062				
1063    		    if (qc.u_share_base_table_only == true) {
1064			        sys_class_name = table_name;
1065		        }
1066				else if ((qc.u_include_child_tables == "true" ||  qc.u_include_child_tables == true) && sys_class_name != table_name) {	
1067					var parentTable = this.getParentTable(sys_class_name);
1068					while(parentTable != -1){
1069				
1070						// check if we can get child table record and then share it instead of base table record
1071						var ctgr = new GlideRecord(parentTable);	
1072						ctgr.addQuery('sys_id', sys_id);
1073						ctgr.queryNoDomain();
1074						
1075						this.logger.logDebug("shareRecord " + sys_id + " for child table " + parentTable, "PerspectiumReplicator.shareRecord", qc);
1076										
1077						if (ctgr._next()) {
1078							this.shareOneRecord(ctgr, qc, parentTable, op, null, pspAttributes, sumCounts);
1079						}
1080						
1081						if (parentTable == table_name) {
1082							break;
1083						}
1084						parentTable = this.getParentTable(parentTable);
1085					}
1086				}				
1087				
1088				// get start and finished datetime as we share record to send for messageset
1089				var startedDateTime = gs.nowNoTZ();
1090				
1091				this.shareOneRecord(cgr, qc, sys_class_name, op, null, pspAttributes, sumCounts);
1092
1093				var finishedDateTime = gs.nowNoTZ();
1094
1095				// for each share configuration, send message set 
1096				this.pspMS.createMessageSetProcessed(this.messageSetCounter, qc, this.getKey(qc), startedDateTime, finishedDateTime);
1097				
1098			}
1099    	} catch(e) {
1100            this.logger.logWarn("shareRecord " + sys_class_name + ", error = " + e, "PerspectiumReplicator.shareRecord", qc);
1101    	}
1102    },
1103	
1104	ignoreUpdatedField: function (qc, cgr) {
1105		var gru = GlideScriptRecordUtil.get(cgr);
1106		var changedFields = gru.getChangedFieldNames();
1107		gs.include('j2js');
1108		changedFields = j2js(changedFields);
1109		
1110		// Get Array of columns to ignore
1111		var exclusiveColumns = [];
1112		var excludeGR = new GlideRecord("u_updated_columns_to_ignore");
1113		excludeGR.addQuery("u_dyname_share", qc.sys_id);
1114		excludeGR.query();
1115		while (excludeGR.next()) {
1116			exclusiveColumns.push(excludeGR.u_column_name.toString());
1117		}
1118		
1119		for (var j = 0; j < changedFields.length; j++) {
1120			var logIndex = "Updated Column (" + (j+1) + " / " + changedFields.length + "): " + changedFields[j];
1121			if (exclusiveColumns.indexOf(changedFields[j]) == -1){
1122				// Updated column not in the list - replicate
1123				this.logger.logDebug(logIndex + " is NOT in the exclusion list: " + exclusiveColumns.toString(), "PerspectiumReplicator.ignoreUpdatedField", qc);
1124				return false;
1125			}
1126			// Updated column in the list - keep checking
1127			this.logger.logDebug(logIndex + " is in the exclusion list: " + exclusiveColumns.toString(), "PerspectiumReplicator.ignoreUpdatedField", qc);
1128		}
1129		return true;
1130	},
1131	
1132	shareOneRecord : function(cgr, qc, sys_class_name, op, pspTag, pspAttributes, sumCounts) {
1133		try {
1134			this.logger.logDebug("shareOneRecord " + sys_class_name + " : " + qc.table_name + " " + cgr.sys_id + " " + op, "PerspectiumReplicator.shareOneRecord", qc);
1135
1136			var filter = (typeof GlideFilter != 'undefined') ? GlideFilter : Packages.com.glide.script.Filter;
1137			if (!qc.condition.isNil() && !filter.checkRecord(cgr, qc.condition.toString())) {
1138				this.logger.logDebug(qc.condition.toString() + " not matched for sharing " + sys_class_name, "PerspectiumReplicator.shareOneRecord", qc);
1139				return;
1140			}
1141
1142			if (!qc.u_before_share_script.nil()) {
1143				var jsonParams = {};
1144				jsonParams.psp_action = op;
1145				var ignore = this.psp.executeBeforeShareScript(qc.u_before_share_script, cgr, jsonParams);
1146				if (ignore == true || ignore == "true") {
1147					this.logger.logDebug("shareOneRecord ignored by beforeShareScript", "PerspectiumReplicator.shareOneRecord");
1148					return;
1149				}
1150				op = jsonParams.psp_action;
1151			}
1152
1153			qc.u_shared++;
1154			qc.setWorkflow(false);
1155			qc.autoSysFields(false);
1156			qc.update();
1157
1158			var hasAudit = false;
1159
1160			// if insert we'll want to do share attachments since we we won't share when called by script action
1161			// since we can't get record to check conditions when we do attachment before first inserting the record
1162			if (op == 'insert' && (qc.u_share_attachment == "true" || qc.u_share_attachment == true)) {
1163				this.shareAttachments(cgr.sys_id, qc, "bulk");
1164			}
1165
1166			if (qc.u_share_journal == true && op != "delete") {
1167				this.shareJournalDelayed(cgr.sys_id, cgr.sys_updated_on, qc);
1168			}
1169
1170			if ((qc.u_include_sys_audit == "true" || qc.u_include_sys_audit == true) && op != "delete") {
1171				// check if it has any audit messages
1172				// if it does we'll share them and set flag so message set psp_flag=last 
1173				// will be sent on last sys_audit message when audit messages sent in background job 1 second later
1174				var agr = new GlideRecord("sys_audit");
1175				agr.addQuery("tablename", sys_class_name);
1176				agr.addQuery("documentkey", cgr.sys_id);
1177
1178				if (cgr.sys_updated_on != null && cgr.sys_updated_on != "" && !cgr.sys_updated_on.isNil()) {
1179					agr.addQuery("sys_created_on", ">=", cgr.sys_updated_on);
1180				}
1181
1182				agr.orderByDesc("sys_created_on");
1183				agr.query();
1184
1185				if(agr.getRowCount() > 0){	
1186					hasAudit = true;
1187					this.shareSysAuditDelayed(sys_class_name, cgr.sys_id, cgr.sys_updated_on, qc);
1188				}			
1189			}
1190
1191			if (qc.u_share_referenced_field_recor == true && op != "delete") {
1192				this.shareReferencedFieldRecords(cgr, qc);
1193			}
1194
1195			if ((qc.u_include_embedded_images_vide == "true" || qc.u_include_embedded_images_vide == true) && op != "delete") {
1196				this.shareEmbedded(cgr, cgr.sys_id, qc, "bulk");	
1197				// for embedded to work correctly 
1198				// (i.e. such as images referenced <img src="/sys_attachment.do?sys_id&#61;8e400b140f0b0a00a7f0c3ace1050eb8"/> 
1199				// we'll also want to share attachments for this record
1200				this.shareAttachments(cgr.sys_id, qc, "bulk");	
1201
1202			}
1203
1204			this.setQueueKey(qc);
1205
1206			var enc = "";
1207			//var cipher=this.encryptionType.TRIPLE_DES; 
1208			var cipher = qc.u_cipher;
1209			var grTableMap;
1210			if (qc.u_share_base_table_only == true && op != "delete") {
1211				var ngr = new GlideRecord(qc.table_name);
1212				ngr.setWorkflow(false);
1213
1214				ngr.addQuery('sys_id', cgr.sys_id);
1215				ngr.queryNoDomain();
1216				if (ngr.next()) {
1217					// Match any changes occured within the script
1218					if(!qc.u_before_share_script.nil()){
1219						for(var k in cgr){ 
1220							ngr[k] = cgr[k]; 
1221						}
1222					}
1223					if (qc.isValidField("u_table_map") && !qc.u_table_map.nil() && qc.u_conditional_share != true) {
1224						grTableMap = this.encryption.getOutboundTableMap(qc.u_table_map);	
1225						if(grTableMap.u_topic == "siam"){
1226							cipher = this.encryptionType.BASE64_ONLY;
1227							enc = this.encryption.encryptTableMap(ngr, grTableMap, cipher, pspTag);
1228						}
1229						else{
1230							cipher = qc.u_cipher;					
1231							enc = this.encryption.encryptTableMap(ngr, grTableMap, qc.u_cipher, pspTag);						
1232						}
1233					} else if (!qc.u_view_name.nil()) {
1234						enc = this.encryption.encrypt(ngr, qc.u_view_name, op, qc.u_target_queue, cipher);
1235					} else if (qc.u_share_only_selected_fields == "true" || qc.u_share_only_selected_fields == true){
1236						var pspSF = new PerspectiumShareFields();
1237						var recordXmlStr = pspSF.createShareRecordXML(qc, ngr, qc.table_name, this.fieldsToReplicate);
1238						enc = this.encryption.encryptString(recordXmlStr);
1239					} else {
1240						enc = this.encryption.encrypt(ngr, null, op, qc.u_target_queue, cipher);
1241					}
1242					sys_class_name = qc.table_name;
1243				}
1244			} else {
1245				if (qc.u_share_base_table_only == true && op == "delete") {
1246					sys_class_name = qc.table_name;
1247					var ngr = new GlideRecord(qc.table_name);
1248					for(var k in cgr) {
1249						ngr[k] = cgr[k];
1250					}
1251
1252					cgr = ngr;
1253				}
1254
1255				if (qc.isValidField("u_table_map") && !qc.u_table_map.nil() && qc.u_conditional_share != true){
1256					grTableMap = this.encryption.getOutboundTableMap(qc.u_table_map);	
1257					if(grTableMap.u_topic == "siam"){
1258						cipher = this.encryptionType.BASE64_ONLY;
1259						enc = this.encryption.encryptTableMap(cgr, grTableMap, "3", pspTag);
1260					}
1261					else{
1262						cipher = qc.u_cipher;
1263						enc = this.encryption.encryptTableMap(cgr, grTableMap, qc.u_cipher, pspTag);
1264					}
1265				} else if (!qc.u_view_name.nil()) {
1266					enc = this.encryption.encrypt(cgr, qc.u_view_name, op, qc.u_target_queue, cipher);
1267				} else if (qc.u_share_only_selected_fields == "true" || qc.u_share_only_selected_fields == true){
1268						var pspSF = new PerspectiumShareFields();
1269						var recordXmlStr = pspSF.createShareRecordXML(qc, cgr, sys_class_name, this.fieldsToReplicate);
1270						enc = this.encryption.encryptString(recordXmlStr);
1271				} else {
1272					enc = this.encryption.encrypt(cgr, null, op, qc.u_target_queue, cipher);
1273				}
1274			}
1275
1276			// if the value is too large then we'll error
1277			// account for both if the value is bigger than our max sending size
1278			// as well as the largest size allowed for a String
1279			if (enc.length() > this.maxBytes || (enc.length() * 2) > 16000000) {					
1280				this.logger.logError(op + " message for " + sys_class_name + " <" + cgr.sys_id + "> cannot be created in outbound queue as it is larger than message limit", "PerspectiumReplicator.shareOneRecord");
1281				return;
1282			}
1283
1284			//topic, type, key, name, value
1285			var topic = "replicator";
1286			var type = "servicenow";
1287			var name = sys_class_name;
1288			var attributes = "";
1289
1290			if(pspAttributes != null && pspAttributes.length > 0){
1291				attributes = pspAttributes;
1292			}
1293			attributes = this.appendAttribute("cipher", cipher, attributes);
1294			if(typeof(grTableMap) != "undefined" && !grTableMap.isNil()){
1295				if(!grTableMap.u_topic.nil()){
1296					topic = grTableMap.u_topic;
1297				}
1298				if(!grTableMap.u_type.nil()){
1299					type = grTableMap.u_type;
1300				}
1301				if(!grTableMap.u_target_tablename.nil()){
1302					name = grTableMap.u_target_tablename;
1303				}
1304				attributes = this.appendTableMapAttributes(cgr, grTableMap, attributes);
1305				this.logger.logDebug("Attributes: " + attributes);
1306			}
1307
1308			if (qc.u_update_or_insert == true && (op == "insert" || op == "update")) {
1309				name = name + ".bulk";
1310			}
1311			else
1312				name = name + "." + op;
1313
1314			// add dynamic share configuration id in attributes for message set
1315			attributes = this.appendAttribute("set_id", qc.sys_id, attributes);	
1316
1317			// also add psp_flag that this is last message if not audit messages sent
1318			if(!hasAudit){
1319				attributes = this.appendAttribute("psp_flag", "last", attributes);
1320			}
1321
1322			if (qc.u_conditional_share == 'true' || qc.u_conditional_share ==  true) {
1323				var ctmgr = new GlideRecord('u_psp_conditional_share');
1324				ctmgr.addQuery('u_dynamic_share', qc.sys_id);
1325				ctmgr.query();
1326				while (ctmgr.next()) {				
1327					if (GlideFilter.checkRecord(cgr, ctmgr.u_condition.toString()) == false || GlideFilter.checkRecord(cgr, ctmgr.u_condition.toString()) == 'false') 
1328						continue;
1329
1330					if (ctmgr.u_use_table_map == true || ctmgr.u_use_table_map == 'true') {
1331						grTableMap = this.encryption.getOutboundTableMap(ctmgr.u_table_map);
1332						if(grTableMap.u_topic == "siam"){
1333							topic = "siam";
1334							type = grTableMap.u_type;
1335							if (!grTableMap.u_target_tablename.nil()) {
1336								var recAction = name.substr(name.indexOf('.'));
1337								name = grTableMap.u_target_tablename + recAction;							
1338							}
1339
1340							cipher = this.encryptionType.BASE64_ONLY;
1341							enc = this.encryption.encryptTableMap(cgr, grTableMap, cipher, pspTag);
1342						}
1343						else{
1344							cipher = qc.u_cipher;					
1345							enc = this.encryption.encryptTableMap(cgr, grTableMap, qc.u_cipher, pspTag);						
1346						}
1347
1348						attributes = this.appendTableMapAttributes(cgr, grTableMap, attributes);
1349					}
1350					else if (!qc.u_view_name.nil()) 
1351						enc = this.encryption.encrypt(cgr, qc.u_view_name, op, ctmgr.u_target_queue, cipher);
1352					else 
1353						enc = this.encryption.encrypt(cgr, null, op, ctmgr.u_target_queue, cipher);
1354
1355					this.createAndSendOutbounds(topic, type, this.getKey(qc), name, enc, ctmgr.u_target_queue, null, attributes, qc, sumCounts);
1356				}
1357
1358				// bail out after this, we don't want to send duplicates
1359				return;
1360			}
1361
1362			var extra = "";
1363			if (op == "deferred" && cgr.isValidField("number")) {
1364				extra = "number=" + cgr.number;
1365			}
1366
1367			this.createAndSendOutbounds(topic, type, this.getKey(qc), name, enc, qc.u_target_queue, extra, attributes, qc, cgr.sys_id, sumCounts);
1368gs.log("before after share");
1369			if (!qc.u_after_share_script.nil()) {
1370				gs.log("inside after share");
1371				this.executeAfterShareScript(qc.u_after_share_script, cgr);
1372			}
1373		} catch (e) {
1374			if (String(e) == "java.lang.NullPointerException") {
1375				this.logger.logWarn("shareOneRecord failed due to invalid null characters, sys_id: " + cgr.sys_id + ", error = " + e, "PerspectiumReplicator.shareOneRecord", qc);
1376			}
1377			else {
1378				this.logger.logWarn("shareOneRecord " + sys_class_name + ", error = " + e, "PerspectiumReplicator.shareOneRecord", qc);
1379			}
1380		}
1381		
1382	},
1383	
1384	executeAfterShareScript: function(script, gr) {
1385		var gc = (typeof GlideController != 'undefined') ? GlideController : Packages.com.glide.script.GlideController;
1386		var oldCurrent = current;
1387        gc.putGlobal('current', gr);
1388		
1389        var rc = gc.evaluateString(script);
1390		
1391        gc.removeGlobal('current');
1392		current = oldCurrent;		
1393	},
1394	
1395	createAndSendOutbounds : function (topic, type, key, name, enc, targetQueue, extra, attributes, qc, record_sys_id, sumCounts) {
1396		this.createPSPOut(topic, type, key, name, enc, targetQueue, extra, attributes, qc, record_sys_id);
1397		// send share counter data for this record
1398		var aSumCounts = false;
1399		if (sumCounts && sumCounts != null && sumCounts != "") {
1400			aSumCounts = sumCounts;
1401		}
1402		// increment counter for each message sent
1403		this.incrementMessageSetCounter(name);
1404		if (aSumCounts) {
1405			this.incrementTableCounter(name, enc.length());
1406		} 
1407		else {
1408			this.psp.sendCounter(name, 1, "counter");
1409			this.psp.sendCounter(name + ".bytes", enc.length(), "counter");
1410		}
1411	},
1412	
1413	shareReferencedFieldRecords : function(cgr, qc) {
1414		this.logger.logDebug("sharing referenced records for " + cgr.getTableName() + "." + cgr.sys_id, "PerspectiumReplicator.shareReferencedFieldRecords", qc);
1415		var kounter = {};
1416		var bytesKounter = {};
1417        
1418		gs.log("cgr table : " + cgr.getTableName());
1419		var dgr = new GlideRecord("u_psp_dynamic_share_referenced");
1420		dgr.addQuery("u_dynamic_share", qc.sys_id);
1421		dgr.query();
1422		while(dgr.next()) {
1423			gs.log("dgr.referenced_table_name : " +dgr.u_referenced_table_name);
1424			gs.log("dgr.u_referenced_field: " + dgr.u_reference_field);
1425			gs.log("INFO : " + cgr.getElement(dgr.u_reference_field));
1426			
1427			
1428			//gs.log("dgr.tablename : " + dgr.getTableName());
1429		  // if referenced field does not contain a sys_id, do not send the record
1430		  if (cgr.getElement(dgr.u_reference_field).isNil()) {
1431			  gs.log("dgr.u_ref FAIL :  " + dgr.u_referenced_table_name);
1432				continue;
1433		  }
1434			
1435		  var refRecord = cgr.getElement(dgr.u_reference_field).getRefRecord();
1436		  var refTableName = refRecord.getTableName();
1437	
1438		  this.logger.logDebug("sharing referenced record " + refTableName + "." + refRecord.sys_id, "PerspectiumReplicator.shareReferencedFieldRecords", qc);
1439			
1440		  if (dgr.u_share_only_selected_fields == "true" || dgr.u_share_only_selected_fields == true){
1441              var pspSF = new PerspectiumShareFields();
1442			  gs.log("replicator, inside if selected fields");
1443             //alvin
1444              var ar = new GlideRecord("u_psp_share_field");
1445
1446			  ar.addQuery("u_source", dgr.sys_id);
1447              ar.query();
1448              var referencedFields = [];
1449              while(ar.next()){
1450                 referencedFields.push(ar.u_field.toString());
1451              }
1452			  gs.log("referencedFields: " + referencedFields);
1453       
1454              var recordXmlStr = pspSF.createShareRecordXML(dgr, refRecord, refTableName, referencedFields);
1455              var enc = this.encryption.encryptString(recordXmlStr);
1456          }
1457          else 
1458			  var enc = this.encryption.encrypt(refRecord, "", "", "", qc.u_cipher);
1459			
1460		  // add dynamic share configuration id to attributes for message set 		
1461		  var bAttributes = this.appendAttribute("set_id", qc.sys_id, "");		
1462		  bAttributes = this.appendAttribute("cipher", qc.u_cipher, bAttributes);
1463	
1464		  // always bulk share ref records
1465		  if ( cgr.u_target_queue != null && cgr.u_target_queue != "" && !cgr.u_target_queue.isNil()) {			  
1466		    	//this.psp.createPSPOut("replicator", "servicenow", this.getKey(qc), refTableName + ".bulk", enc, null, null, null, qc);
1467				this.createPSPOut("replicator", "servicenow", this.getKey(qc), refTableName + ".bulk", enc, null, null, bAttributes, qc, refRecord.sys_id);
1468		  } else {
1469				//this.psp.createPSPOut("replicator", "servicenow", this.getKey(qc), refTableName + ".bulk", enc, qc.u_target_queue, null, null, qc);
1470				this.createPSPOut("replicator", "servicenow", this.getKey(qc), refTableName + ".bulk", enc, qc.u_target_queue, null, bAttributes, qc, refRecord.sys_id);
1471		  }
1472			
1473		  // track share by table
1474		  var valueBytes = enc.length();
1475		  if (kounter[refTableName]) {
1476				kounter[refTableName] ++;
1477				bytesKounter[refTableName] = bytesKounter[refTableName] + valueBytes;
1478		  } else {
1479				kounter[refTableName] = 1;
1480				bytesKounter[refTableName] = valueBytes;
1481		  }
1482		}
1483		
1484		// send share counter data for each table
1485		for(n in kounter) {			
1486			this.psp.sendCounter(n + ".bulk", kounter[n], "counter");
1487			this.psp.sendCounter(n + ".bulk.bytes", bytesKounter[n], "counter");
1488
1489			// increment counter for each message sent
1490			this.incrementMessageSetCounter(n + ".bulk");			
1491		}
1492	},	
1493	
1494	// called from script action and shareRecords
1495	shareAttachments : function(sys_id,grShareConfig,op) {
1496		this.logger.logDebug("sharing attachments for " + sys_id, "PerspectiumReplicator.shareAttachments", grShareConfig);
1497		
1498		// to account for when shareAttachments() called from script action/custom ui actions directly
1499		// check if record passes filter conditions to continue sharing
1500		var cgr = new GlideRecord(grShareConfig.table_name);
1501		
1502		cgr.addQuery('sys_id', sys_id);
1503		
1504		// if a run as user selected we'll want to do a regular query()
1505		// to honor domain separation since people will only choose a run as user to get the results that user only sees
1506		if (grShareConfig.isValidField("u_run_as") && !grShareConfig.u_run_as.isNil()) {	   
1507			cgr.query();
1508		}
1509		else {
1510			cgr.queryNoDomain();
1511		}
1512		
1513		if (!cgr.next()) {	
1514			this.logger.logDebug("not sharing attachments immediately as " + sys_id + " not found in table " + grShareConfig.table_name, "PerspectiumReplicator.shareAttachments", grShareConfig);
1515			return;	
1516		}
1517						
1518		var filter = (typeof GlideFilter != 'undefined') ? GlideFilter : Packages.com.glide.script.Filter;
1519		// if this has a condition we'll need to get the actual record in order to check if it passes the condition
1520		if (!grShareConfig.condition.isNil() 
1521			// && cgr.next() 
1522			&& !filter.checkRecord(cgr, grShareConfig.condition.toString())) {
1523			this.logger.logDebug("skipping sharing attachments for share config " + grShareConfig.sys_id + " as " + grShareConfig.condition.toString() + " condition not met", "PerspectiumReplicator.shareAttachments", grShareConfig);
1524			return;
1525		}
1526												
1527		// check if conditional share is enabled to share out to its queue
1528		if (grShareConfig.isValidField("u_conditional_share") && grShareConfig.u_conditional_share == 'true' || grShareConfig.u_conditional_share ==  true) {
1529			var ctmgr = new GlideRecord('u_psp_conditional_share');
1530			ctmgr.addQuery('u_dynamic_share', grShareConfig.sys_id);
1531			ctmgr.query();
1532			while (ctmgr.next()) {	
1533				// doesn't match this conditinal share so don't share out record
1534				if (GlideFilter.checkRecord(cgr, ctmgr.u_condition.toString()) == false || GlideFilter.checkRecord(cgr, ctmgr.u_condition.toString()) == 'false') {
1535					this.logger.logDebug("skipping sharing attachments for conditional share " + ctmgr.sys_id + ", share config " + grShareConfig.sys_id + " as " + ctmgr.u_condition.toString() + " condition not met", "PerspectiumReplicator.shareAttachments", grShareConfig);
1536					continue;
1537				}
1538				
1539				// share out attachments to this conditional share's queue
1540				this.shareRecordAttachments(sys_id, grShareConfig, op, ctmgr.u_target_queue);
1541			}
1542			
1543			return;
1544		}
1545				
1546		this.shareRecordAttachments(sys_id, grShareConfig, op, grShareConfig.u_target_queue);		
1547	},
1548	
1549	shareRecordAttachments: function(sys_id, grShareConfig, op, targetQueue) {	
1550		var saRecCount = 0;
1551		var saByteCount = 0;
1552		var sadRecCount = 0;
1553		var sadByteCount = 0;
1554		
1555		// first get the sys_attachment
1556		var jgr = new GlideRecord("sys_attachment");
1557		jgr.addQuery("table_sys_id", sys_id);
1558		jgr.query();
1559		while (jgr.next()) {
1560			
1561			var enc = this.encryption.encrypt(jgr, "", "", "", grShareConfig.u_cipher);
1562			// always send as upsert to get all entries, potentially problematic if a lot !!
1563			var name = "sys_attachment." + op;
1564			var target_queue = targetQueue;
1565									
1566			// add attributes with bulk share conf id to use for message set id
1567			var bAttributes = "";
1568			bAttributes = this.appendAttribute("set_id", grShareConfig.sys_id, bAttributes);
1569			bAttributes = this.appendAttribute("cipher", grShareConfig.u_cipher, bAttributes);
1570
1571			this.logger.logDebug("sharing sys_attachment " + jgr.sys_id + " to queue " + target_queue, "PerspectiumReplicator.shareRecordAttachments", grShareConfig);			
1572			this.createPSPOut("replicator", "servicenow", this.getKey(grShareConfig), name, enc, target_queue, "", bAttributes, grShareConfig, jgr.sys_id);
1573			
1574			// increment counter for each message sent
1575			this.incrementMessageSetCounter(name);
1576					
1577			// increment counters
1578			saRecCount++;
1579			saByteCount += enc.length();
1580			
1581			// now for each sys_attachment, get the related sys_attachment_doc entries as well
1582			var dgr = new GlideRecord("sys_attachment_doc");
1583			dgr.addQuery("sys_attachment", jgr.sys_id);
1584			dgr.query();
1585			while(dgr.next()) {
1586				var enc = this.encryption.encrypt(dgr, "", "", "", grShareConfig.u_cipher);
1587				// always send as upsert to get all entries, potentially problematic if a lot !!
1588				var name = "sys_attachment_doc." + op;	
1589				
1590				this.createPSPOut("replicator", "servicenow", this.getKey(grShareConfig), name, enc, target_queue, "", bAttributes, grShareConfig, dgr.sys_id);
1591				
1592				// increment counter for each message sent
1593				this.incrementMessageSetCounter("sys_attachment_doc");
1594				
1595				// increment counters
1596				sadRecCount++;
1597				sadByteCount += enc.length();
1598			}
1599		}
1600		
1601		// send share counter data if we have one sys_attachment
1602		// since then we'll have byte data as well as at least one sys_attachment_doc record for it
1603		if (saRecCount > 0) {
1604			var sysAttachmentName = "sys_attachment." + op;
1605			var sysAttachmentDocName = "sys_attachment_doc." + op;
1606			
1607			this.psp.sendCounter(sysAttachmentName, saRecCount, "counter");
1608			this.psp.sendCounter(sysAttachmentName + ".bytes", saByteCount, "counter");
1609			this.psp.sendCounter(sysAttachmentDocName, sadRecCount, "counter");
1610			this.psp.sendCounter(sysAttachmentDocName + ".bytes", sadByteCount, "counter");
1611		}		
1612	},
1613		
1614	shareJournal : function(sys_id, sys_updated_on, grShareConfigSysId, isBulkShare) {
1615		var grShareConfig;
1616		if (isBulkShare) {
1617			grShareConfig = new GlideRecord('psp_bulk_share');
1618		}
1619		else {
1620			grShareConfig = new GlideRecord('psp_replicate_conf');
1621		}
1622		
1623		grShareConfig.addQuery('sys_id', grShareConfigSysId);
1624		grShareConfig.queryNoDomain();
1625		if(!grShareConfig.next()){
1626			this.logger.logDebug("no share definition found for " + grShareConfigSysId, "PerspectiumReplicator.shareJournal");
1627			return // nothing can be done
1628		}
1629		
1630		this.logger.logDebug("sharing journal for " + sys_id, "PerspectiumReplicator.shareJournal", grShareConfig);
1631		
1632		// add attributes with bulk share conf id to use for message set id
1633		var bAttributes = "";
1634		bAttributes = this.appendAttribute("set_id", grShareConfig.sys_id, bAttributes);
1635		bAttributes = this.appendAttribute("cipher", grShareConfig.u_cipher, bAttributes);
1636		
1637		var target_queue = grShareConfig.u_target_queue;
1638		
1639		var recCount = 0;
1640		var byteCount = 0;
1641
1642		var name = "sys_journal_field.bulk";
1643		
1644		// only send journal entries for this session, not all of them
1645		var jgr = new GlideRecord("sys_journal_field");
1646		jgr.addQuery("element_id", sys_id);
1647		
1648		if (sys_updated_on != null && sys_updated_on != "" && !sys_updated_on.isNil()) {
1649			jgr.addQuery("sys_created_on", ">=", sys_updated_on);
1650			jgr.setLimit(this.dynSysJournalLimit);
1651		}
1652		
1653		jgr.orderByDesc("sys_created_on");
1654		jgr.query();
1655		while (jgr.next()) {
1656			var enc = this.encryption.encrypt(jgr, "", "", "", grShareConfig.u_cipher);	
1657			this.createPSPOut("replicator", "servicenow", this.getKey(grShareConfig), name, enc, target_queue, "", bAttributes, grShareConfig, jgr.sys_id);
1658			
1659			// increment counter for each message sent
1660			this.incrementMessageSetCounter(name);
1661			
1662			// increment counters
1663			recCount++;
1664			byteCount += enc.length();
1665			
1666		}
1667		
1668		// send share counter data
1669		if(recCount > 0){
1670			this.psp.sendCounter(name, recCount, "counter");
1671			this.psp.sendCounter(name + ".bytes", byteCount, "counter");
1672		}
1673	},
1674	
1675	shareJournalDelayed : function(sys_id, sys_updated_on, grShareConfig) {
1676		this.logger.logDebug("sharing sys_audit delayed for " + sys_id, "PerspectiumReplicator.shareJournalDelayed", grShareConfig);
1677		//var target_queue = grShareConfig.u_target_queue;
1678		var pspSO = new ScheduleOnce();		
1679		var gdt = new GlideDateTime();
1680		gdt.addSeconds(1); // schedule in 1 second to make sure all audits are in
1681		
1682		pspSO.setTime(gdt.getValue()); 
1683		
1684		pspSO.script += "var pspR = new PerspectiumReplicator();";
1685		// set flag false so we don't send first message and to flag last message which will be a sys_audit message
1686		pspSO.script += "pspR.sentFirstMessage = true;";
1687		pspSO.script += "pspR.flagLastMessage = true;";
1688		
1689		pspSO.script += "pspR.shareJournal('" + sys_id + "','" + sys_updated_on + "','" + grShareConfig.sys_id + "');";
1690				
1691		pspSO.schedule();
1692	},
1693	
1694	shareSysAuditDelayed : function(table_name, sys_id, sys_updated_on, grShareConfig) {
1695		this.logger.logDebug("sharing sys_audit delayed for " + sys_id, "PerspectiumReplicator.shareSysAuditDelayed", grShareConfig);
1696		//var target_queue = grShareConfig.u_target_queue;
1697		var pspSO = new ScheduleOnce();		
1698		var gdt = new GlideDateTime();
1699		gdt.addSeconds(1); // schedule in 1 second to make sure all audits are in
1700		
1701		pspSO.setTime(gdt.getValue()); 
1702		
1703		pspSO.script += "var pspR = new PerspectiumReplicator();";
1704		// set flag false so we don't send first message and to flag last message which will be a sys_audit message
1705		pspSO.script += "pspR.sentFirstMessage = true;";
1706		pspSO.script += "pspR.flagLastMessage = true;";
1707		
1708		//pspSO.script += "pspR.shareSysAudit('" + table_name + "','" + sys_id + "','" + sys_updated_on + "','" + target_queue + "','" + this.getKey(grShareConfig) + "');";
1709		pspSO.script += "pspR.shareSysAuditDynamic('" + table_name + "','" + sys_id + "','" + sys_updated_on + "','" + grShareConfig.sys_id + "');";
1710				
1711		pspSO.schedule();
1712	},
1713	
1714	// background job created to share sys_audit records for a dynamic share
1715	shareSysAuditDynamic : function(table_name, sys_id, sys_updated_on, grShareConfigSysId){
1716		// get dynamic share configuration by sys_id to call shareSysAudit() to actually share sys_audit records
1717		var grShareConfig = new GlideRecord('psp_replicate_conf');
1718        	
1719		grShareConfig.addQuery('sys_id', grShareConfigSysId);
1720		grShareConfig.queryNoDomain();
1721		if(!grShareConfig.next()){
1722		//if (!grShareConfig.get(grShareConfigSysId)) {
1723			this.logger.logDebug("no share definition found for " + table_name + " " + grShareConfigSysId, "PerspectiumReplicator.shareSysAuditDynamic");
1724			return // nothing can be done
1725		}
1726		
1727		// since dynamically shared sys_audit records are done separate from dynamic share and its records
1728		// we'll send message set message for it separately
1729		var startedDateTime = gs.nowDateTime();
1730						
1731		this.shareSysAudit(table_name, sys_id, sys_updated_on, grShareConfig);
1732
1733		var finishedDateTime = gs.nowDateTime();
1734
1735		// for each share configuration, send message set 
1736		this.pspMS.createMessageSetProcessed(this.messageSetCounter, grShareConfig, this.getKey(grShareConfig), startedDateTime, finishedDateTime);		
1737	},
1738	
1739	shareSysAudit : function(table_name, sys_id, sys_updated_on, grShareConfig) {		
1740		this.logger.logDebug("sharing sys_audit for " + sys_id + ", " + table_name, "PerspectiumReplicator.shareSysAudit", grShareConfig);
1741		
1742		var target_queue = grShareConfig.u_target_queue;
1743		var target_key = this.getKey(grShareConfig);
1744		
1745		// add attributes with bulk share conf id to use for message set id
1746		var bAttributes = "";
1747		bAttributes = this.appendAttribute("set_id", grShareConfig.sys_id, bAttributes);
1748		bAttributes = this.appendAttribute("cipher", grShareConfig.u_cipher, bAttributes);
1749		
1750		var name = "sys_audit.bulk";
1751		var recCount = 0;
1752		var byteCount = 0;
1753		
1754		// only send journal entries for this session, not all of them
1755		var jgr = new GlideRecord("sys_audit");
1756		jgr.addQuery("tablename", table_name);
1757		jgr.addQuery("documentkey", sys_id);		
1758		
1759		if (sys_updated_on != null && sys_updated_on != "" && !sys_updated_on.isNil()) {
1760			jgr.addQuery("sys_created_on", ">=", sys_updated_on);
1761			jgr.setLimit(this.dynSysAuditLimit);
1762		}
1763		
1764		jgr.orderByDesc("sys_created_on");
1765		jgr.query();
1766		
1767		// save # of rows returned to flag last message in message set for dynamic share
1768		var auditCount = jgr.getRowCount();
1769		
1770		while (jgr.next()) {
1771			var enc = this.encryption.encrypt(jgr, "", "", "", grShareConfig.u_cipher);
1772			
1773			// last audit record we'll want set psp_flag=last if flag set
1774			if(recCount == (auditCount - 1) && this.flagLastMessage){
1775				bAttributes = this.appendAttribute("psp_flag", "last", bAttributes);
1776			}
1777			
1778			this.createPSPOut("replicator", "servicenow", target_key, name, enc, target_queue, "", bAttributes, grShareConfig, jgr.sys_id);
1779			
1780			// increment counter for each message sent
1781			this.incrementMessageSetCounter(name);
1782						
1783			// increment counters
1784			recCount++;
1785			byteCount += enc.length();
1786			
1787		}
1788		
1789		// send share counter data
1790		if(recCount > 0){
1791			this.psp.sendCounter(name, recCount, "counter");
1792			this.psp.sendCounter(name + ".bytes", byteCount, "counter");
1793		}
1794	},
1795	
1796	//helpder function for pausing the loop  
1797	pauseLoop : function(ms) {  
1798		ms += new Date().getTime();  
1799		while (new Date() < ms) {}  
1800	},  
1801	
1802	// the entire set is always shared
1803	shareHistorySet : function(table_name, sys_id, sys_updated_on, grShareConfig) {
1804		this.logger.logDebug("sharing history set for " + sys_id + "," + table_name, "PerspectiumReplicator.shareHistorySet", grShareConfig);
1805
1806		var target_queue = grShareConfig.u_target_queue;
1807		
1808		// add attributes with bulk share conf id to use for message set id
1809		var bAttributes = "";
1810		bAttributes = this.appendAttribute("set_id", grShareConfig.sys_id, bAttributes);
1811		bAttributes = this.appendAttribute("cipher", grShareConfig.u_cipher, bAttributes);
1812
1813		var jgr = new GlideRecord("sys_history_set");
1814		jgr.addQuery("table", table_name);
1815		jgr.addQuery("id", sys_id);
1816		jgr.query();
1817		// there is only one
1818		if (!jgr.next()) {
1819			this.logger.logDebug("ERROR - no history set for " + sys_id, "PerspectiumReplicator.shareHistorySet", grShareConfig);
1820			return;
1821		}
1822	
1823		var enc = this.encryption.encrypt(jgr, "", "", "", grShareConfig.u_cipher);
1824		var name = "sys_history_set.bulk";
1825		this.createPSPOut("replicator", "servicenow", this.key, name, enc, target_queue, "", bAttributes, grShareConfig, jgr.sys_id);
1826		
1827		// increment counter for each message sent
1828		this.incrementMessageSetCounter(name);
1829		
1830		// send share counter data
1831		this.psp.sendCounter(name, 1, "counter");
1832		this.psp.sendCounter(name + ".bytes", enc.length(), "counter");
1833
1834		// now share history line(s)
1835		var lgr = new GlideRecord("sys_history_line");
1836		lgr.addQuery("set", jgr.sys_id);
1837		if (sys_updated_on != null && sys_updated_on != "" && !sys_updated_on.isNil()) {
1838			lgr.addQuery("sys_created_on", ">=", sys_updated_on);
1839		}
1840		lgr.query();
1841		
1842		var recCount = 0;
1843		var byteCount = 0;
1844		while(lgr.next()) {
1845			enc = this.encryption.encrypt(lgr, "", "", "", grShareConfig.u_cipher);
1846			name = "sys_history_line.bulk";
1847			this.createPSPOut("replicator", "servicenow", this.key, name, enc, target_queue, "", bAttributes, grShareConfig, lgr.sys_id);
1848			
1849			
1850			// increment counter for each message sent
1851			this.incrementMessageSetCounter(name);			
1852			
1853			// increment counters
1854			recCount++;
1855			byteCount += enc.length();
1856		}
1857		
1858		// send share counter data
1859		if(recCount > 0){
1860			this.psp.sendCounter(name, recCount, "counter");
1861			this.psp.sendCounter(name + ".bytes", byteCount, "counter");
1862		}
1863	},
1864	
1865	// to share embedded images and videos stored in db_image and db_video
1866	shareEmbedded : function(grParam, sys_id, grShareConfig, op) {
1867		this.logger.logDebug("sharing embedded for " + sys_id, "PerspectiumReplicator.shareEmbedded", grShareConfig);
1868		
1869		// go through fields and see if any are html type
1870		var fields = grParam.getFields();
1871		for (i=0; i<fields.size(); i++) { 
1872			var field = fields.get(i);
1873			var descriptor = field.getED(); 			
1874			var internalType = descriptor.getInternalType();
1875			
1876			// check either field is a html type
1877			if(internalType && internalType != '' && internalType.toLowerCase().indexOf('html') >= 0){			
1878		    	this.logger.logDebug("checking embedded images/videos for field: type " + internalType + ", name " + field.getName() + " for table " + grParam.getTableName() + " sys_id " +  sys_id, "PerspectiumReplicator.shareEmbedded", grShareConfig);
1879				this.findEmbeddedElements(field, grShareConfig, op);								
1880			}
1881		}
1882	},
1883	
1884	findEmbeddedElements: function(geParam, grShareConfig, op){
1885		if(!geParam.hasValue())
1886			return;
1887
1888		// get field's value and use XML document to parse through to find <img>				
1889		var fieldValue = geParam.getHTMLValue();
1890		// get escaped value if it's encoded
1891		if(fieldValue.indexOf('&lt;') >= 0){
1892			fieldValue = geParam.getEscapedValue();
1893		}
1894				
1895		var xml_util = (typeof GlideXMLUtil != 'undefined') ? new GlideXMLUtil() : new Packages.com.glide.util.XMLUtil();
1896		//var xmlDoc = xml_util.parseHTML(geParam.getHTMLValue());
1897		var xmlDoc = xml_util.parseHTML(fieldValue);
1898		
1899        if (!xmlDoc) {
1900            this.logger.logWarn("deserializing: XML unparseable " + fieldValue, "PerspectiumReplicator.findEmbeddedElements", grShareConfig);
1901            return;
1902        }
1903		
1904		var root = xmlDoc.getDocumentElement();
1905		
1906		// start with root and iterate through to find img elements
1907		this.findEmbeddedElement(root, xml_util, grShareConfig, op);			
1908	},
1909	
1910	findEmbeddedElement: function(element, xml_util, grShareConfig, op){
1911		var it = xml_util.childElementIterator(element);
1912        while (it.hasNext()) {
1913            var el = it.next();
1914            
1915            var n = el.getNodeName();
1916            var v = xml_util.getText(el);
1917			if (v != null)
1918				v += "";
1919            
1920			//this.logger.logDebug(n + ", " + v, "Perspectium.findEmbeddedElement", grShareConfig);
1921			
1922			// found img now try to share it if we can get its name from its src attribute
1923			if(n == "img"){
1924				var imageName = xml_util.getAttribute(el, "src");
1925				if(imageName != null){
1926					// to make sure it's a string returned add + ""
1927					this.shareDBMedia(imageName + "", grShareConfig, op, "db_image");
1928				}
1929			}
1930			// video can be either in object or embed object depending on version of TinyMCE used to embed
1931			// object is more common with TinyMCE in Calgary
1932			else if(n == "object"){
1933				var videoName = xml_util.getAttribute(el, "data");
1934				if(videoName != null){
1935					// to make sure it's a string returned add + ""
1936					this.shareDBMedia(videoName + "", grShareConfig, op, "db_video");
1937				}
1938			}
1939			// embed and source more common with TinyMCE in Eureka
1940			else if(n == "embed" || n == "source"){
1941				var videoName = xml_util.getAttribute(el, "src");
1942				if(videoName != null){
1943					// to make sure it's a string returned add + ""
1944					this.shareDBMedia(videoName + "", grShareConfig, op, "db_video");
1945				}
1946			}			
1947			
1948			// go through this element's children
1949			this.findEmbeddedElement(el, xml_util, grShareConfig, op);        
1950		}		
1951	},
1952
1953	shareDBMedia: function(mediaFilename, grShareConfig, op, dbTableName){
1954		mediaFilename = this.cleanEmbeddedFilename(mediaFilename);
1955		
1956		// for ones with sys_attachment the filename is something like sys_attachment.do?sys_id=adb7ee106f93c200dbf856b21c3ee401 
1957		// so we pass the sys_id listed in the filename to share
1958		// for when user is sharing an embedded image that is not attached to the record (such as one that already exists in the system)
1959		if(mediaFilename.indexOf("sys_attachment") >= 0){			
1960			// convert to = if &#61; i.e. sys_attachment.do?sys_id&#61;a255dd140fc70a00a7f0c3ace1050e6e
1961			mediaFilename = mediaFilename.replace(/&#61;/g, "=");
1962			
1963			// get sys_id part only
1964			mediaFilename = mediaFilename.substring(mediaFilename.indexOf("sys_id=") + "sys_id=".length);
1965			
1966			// disregard any & after for any other parameters i.e. sys_attachment.do?sys_id=59e6a59c49cb420082f2053890c13db2&amp;view=true
1967			if(mediaFilename.indexOf("&") > 0){
1968				mediaFilename = mediaFilename.substring(0, mediaFilename.indexOf("&"));	
1969			}
1970				
1971			this.shareOneAttachment(mediaFilename, grShareConfig, op);
1972			return;
1973		}
1974		
1975		this.logger.logDebug("finding media with filename " + mediaFilename, "PerspectiumReplicator.shareDBMedia", grShareConfig);
1976		var recCount = 0;
1977		var byteCount = 0;
1978		var name = dbTableName + "." + op;
1979
1980		// query to find record with this name and share
1981		var dr = new GlideRecord(dbTableName);
1982		dr.addQuery('name', mediaFilename);
1983		dr.query();
1984		while (dr.next()) {
1985			var drSysId = dr.sys_id;
1986			this.logger.logDebug("sharing " + dbTableName + " " + drSysId, "PerspectiumReplicator.shareDBMedia", grShareConfig);
1987			
1988			var enc = this.encryption.encrypt(dr, "", "", "", grShareConfig.u_cipher);
1989			
1990			// add attributes with bulk share conf id to use for message set id
1991			var bAttributes = "";
1992			bAttributes = this.appendAttribute("set_id", grShareConfig.sys_id, bAttributes);
1993			bAttributes = this.appendAttribute("cipher", grShareConfig.u_cipher, bAttributes);
1994			
1995			var target_queue = grShareConfig.u_target_queue;
1996			/*if (target_queue == null || target_queue == "" || target_queue.isNil()) {
1997				this.psp.createPSPOut("replicator", "servicenow", this.getKey(grShareConfig), name, enc);
1998			} else {*/
1999				this.createPSPOut("replicator", "servicenow", this.getKey(grShareConfig), name, enc, target_queue, "", bAttributes, grShareConfig, dr.sys_id);
2000			//}
2001			
2002			// increment counter for each message sent
2003			this.incrementMessageSetCounter(name);
2004			
2005			// increment counters
2006			recCount++;
2007			byteCount += enc.length();
2008			
2009			// next share sys_attachment and sys_attachment_doc for this db_image
2010			this.shareAttachments(drSysId, grShareConfig, op);
2011			
2012		}
2013		
2014		// send share counter data
2015		if(recCount > 0){
2016			this.psp.sendCounter(name, recCount, "counter");
2017			this.psp.sendCounter(name + ".bytes", byteCount, "counter");
2018		}
2019	},
2020	
2021	cleanEmbeddedFilename: function(embeddedName){
2022		// if name begins with / or ends with "x" i.e. image.pngx		
2023		if(embeddedName.indexOf("/") == 0){
2024			embeddedName = embeddedName.substring(1);
2025		}
2026		
2027		if(this.psp.endsWith(embeddedName, "x")){
2028			embeddedName = embeddedName.substring(0, embeddedName.length - 1);
2029		}
2030					
2031		// replace with space to match stored in db
2032		embeddedName = embeddedName.replace(/%20/g, " ");
2033		
2034		return embeddedName;
2035	},
2036	
2037	// called from shareDBMedia for sharing embedded elements
2038	// passing in sys_attachment sys_id
2039	// for when user is sharing an embedded image that is not attached to the record (such as one that already exists in the system)
2040	shareOneAttachment : function(sys_id, grShareConfig, op) {
2041		this.logger.logDebug("sharing sys_attachment " + sys_id, "PerspectiumReplicator.shareOneAttachment", grShareConfig);
2042		
2043		var saRecCount = 0;
2044		var saByteCount = 0;
2045		var sadRecCount = 0;
2046		var sadByteCount = 0;
2047		// first get the sys_attachment
2048		var jgr = new GlideRecord("sys_attachment");
2049		
2050		jgr.addQuery('sys_id', sys_id);
2051		jgr.query();
2052		if(!jgr.next()){	
2053		//if(!jgr.get(sys_id)){
2054			this.logger.logDebug("no record found in sys_attachment for sys_id " + sys_id, "PerspectiumReplicator.shareOneAttachment", grShareConfig);
2055			return;			
2056		}
2057			
2058		var enc = this.encryption.encrypt(jgr, "", "", "", grShareConfig.u_cipher);
2059		// always send as upsert to get all entries, potentially problematic if a lot !!
2060		var name = "sys_attachment." + op;
2061						
2062		// add attributes with bulk share conf id to use for message set id
2063		var bAttributes = "";
2064		bAttributes = this.appendAttribute("set_id", grShareConfig.sys_id, bAttributes);
2065		bAttributes = this.appendAttribute("cipher", grShareConfig.u_cipher, bAttributes);
2066		
2067		var target_queue = grShareConfig.u_target_queue;
2068		this.createPSPOut("replicator", "servicenow", this.getKey(grShareConfig), name, enc, target_queue, "", bAttributes, grShareConfig, jgr.sys_id);
2069					
2070		// increment counter for each message sent
2071		this.incrementMessageSetCounter(name);
2072		
2073		// increment counters
2074		saRecCount++;
2075		saByteCount += enc.length();
2076			
2077		// now get the related sys_attachment_doc entries as well
2078		var dgr = new GlideRecord("sys_attachment_doc");
2079		dgr.addQuery("sys_attachment", jgr.sys_id);
2080		dgr.query();
2081		while(dgr.next()) {
2082			var enc = this.encryption.encrypt(dgr, "", "", "", grShareConfig.u_cipher);
2083			// always send as upsert to get all entries, potentially problematic if a lot !!
2084			var name = "sys_attachment_doc." + op;
2085			this.createPSPOut("replicator", "servicenow", this.getKey(grShareConfig), name, enc, target_queue, "", bAttributes, grShareConfig, dgr.sys_id);
2086						
2087			// increment counter for each message sent
2088			this.incrementMessageSetCounter(name);
2089			
2090			// increment counters
2091			sadRecCount++;
2092			sadByteCount += enc.length();
2093		}
2094				
2095		// send share counter data if we have one sys_attachment
2096		// since then we'll have byte data as well as at least one sys_attachment_doc record for it
2097		if(saRecCount > 0){
2098			var sysAttachmentName = "sys_attachment." + op;
2099			var sysAttachmentDocName = "sys_attachment_doc." + op;
2100			
2101			this.psp.sendCounter(sysAttachmentName, saRecCount, "counter");
2102			this.psp.sendCounter(sysAttachmentName + ".bytes", saByteCount, "counter");
2103			this.psp.sendCounter(sysAttachmentDocName, sadRecCount, "counter");
2104			this.psp.sendCounter(sysAttachmentDocName + ".bytes", sadByteCount, "counter");
2105		}
2106	},
2107	
2108    fetchFromDefaultQueue: function() {
2109		var encrypter = (typeof GlideEncrypter != 'undefined') ? new GlideEncrypter() : new Packages.com.glide.util.Encrypter();
2110		var queueName = "psp.out.servicenow." + this.key;
2111		var endpoint = this.psp.getQInputURL();
2112		var quser = this.psp.getQUser();
2113		var qpassword = this.psp.getQPassword();
2114		var instance = this.key;
2115		
2116		this.fetchFromOneQueue(endpoint, queueName, quser, qpassword, instance);
2117	},
2118	
2119    fetchFromQueue: function() {
2120		var encrypter = (typeof GlideEncrypter != 'undefined') ? new GlideEncrypter() : new Packages.com.glide.util.Encrypter();
2121		var qGR = new GlideRecord("u_psp_queues");
2122		qGR.addQuery("u_active", "true");
2123		qGR.addQuery("u_direction", "Subscribe");
2124		qGR.addQuery("u_ets", "false");
2125        qGR.orderBy("u_order");
2126		qGR.query();
2127		while(qGR.next()) {
2128	    	try{
2129	    		var p = encrypter.decrypt(qGR.u_queue_password);
2130				// pass subscribed queue GlideRecord object itself to use for messageset
2131	    		this.fetchFromOneQueue(qGR.u_endpoint_url, qGR.u_name, qGR.u_queue_user, p, qGR.u_instance, qGR);
2132			}
2133			catch(e) {
2134				this.logger.logError("Error fetching from queue " + qGR.u_name + " on " + qGR.u_endpoint_url + ": " + e, "PerspectiumReplicator.fetchFromQueue");
2135			}
2136		}
2137    },
2138    
2139    fetchFromOneQueue: function(endpoint, queueName, quser, qpassword, instance, subscribedQueue) {
2140		if (!subscribedQueue.u_instance_created_on.isNil() && subscribedQueue.u_instance_created_on != this.key) {
2141    		subscribedQueue.u_active = 'false'; 
2142			subscribedQueue.update();
2143    		this.logger.logWarn("The queue " + subscribedQueue.u_name + " was deactivated to protect against cloning", "PerspectiumReplicator.fetchFromOneQueue");
2144			return;
2145    	}
2146		
2147		if (quser == "" || qpassword == "" || endpoint == "") {
2148			this.logger.logError("Unable to run as the target server or credentials are not configured properly. Please check your Perspectium properties", "PerspectiumReplicator.fetchFromOneQueue");
2149			return;	
2150		}
2151		
2152		if (!this.endsWith(endpoint, "/")) {
2153			endpoint += "/";
2154		}
2155		
2156		this.logger.logDebug(endpoint + " " + queueName + " " + quser + " " + instance, "PerspectiumReplicator.fetchFromOneQueue");
2157		
2158		var sdt = new Date();
2159		var pspS = new Perspectium();
2160        
2161		var url = endpoint + "output/" + queueName;
2162        var getMethod = new Packages.org.apache.commons.httpclient.methods.GetMethod(url);
2163        getMethod.setRequestHeader("psp_quser", quser);
2164        getMethod.setRequestHeader("psp_qpassword", qpassword);
2165        getMethod.setRequestHeader("psp_instance", instance);
2166        var httpClient = this.psp.getHttpClient();
2167        var result = httpClient.executeMethod(getMethod);
2168        
2169        if (result != 200) {
2170			getMethod.releaseConnection();
2171            throw new Error("HTTP GET failed: " + result + " for URL: " + url);
2172        }
2173        
2174        var answer = getMethod.getResponseBodyAsString();
2175		// release connection after fetch
2176		getMethod.releaseConnection();
2177        
2178		var kounter = {};
2179		var bytesKounter = {};
2180        var json = new JSON();
2181        var obj = json.decode(answer);
2182		
2183		this.logger.logDebug(obj.length + ' records being processed for ' + queueName, "PerspectiumReplicator.fetchFromOneQueue");
2184		var csecs = pspS._getTimeDiffSecs(sdt);
2185		var rsecs = obj.length/csecs;		
2186        this.logger.logDebug(obj.length + " records (" + queueName + ") COLLECTED in " + csecs + " secs (" + rsecs.toFixed(2) + " recs/s)", "PerspectiumReplicator.fetchFromOneQueue");
2187		this.psp.addMessageStats(obj.length + " inbound records COLLECTED from " + queueName, csecs, this.messageStatsMod, '', "COLLECTED", obj.length);
2188		
2189		sdt = new Date();
2190		var numRecords = 0;
2191		
2192		// create new messageset object to reset and hold all the messageset messages for this group of mesages
2193		var pspMS2 = new PerspectiumMessageSet();
2194		var recTypes = [];
2195        for(var i = 0; i < obj.length; i++) {			
2196            var topic = obj[i].topic;
2197            var type = obj[i].type;
2198            var key = obj[i].key;
2199            var name = obj[i].name;
2200            var value = obj[i].value;
2201			var description = obj[i].description;
2202			var timestamp = obj[i].psp_timestamp;
2203			var extra = obj[i].extra;
2204			var priority = obj[i].priority;
2205			var attributes = obj[i].attributes;            
2206            recTypes.push(obj[i].name.toString());
2207            var igr = new GlideRecord("psp_in_message");
2208            igr.newRecord();
2209            igr.topic = topic;
2210			
2211			if (topic === "replicator") {
2212				numRecords += 1;
2213			}
2214			
2215            igr.type = type;
2216            igr.key = key;
2217            igr.name = name;
2218            igr.value = value;
2219			igr.u_description = description;
2220			var gmt = new GlideDateTime(timestamp);
2221			igr.u_timestamp = gmt.getDisplayValue();
2222			igr.u_extra = extra;
2223			igr.u_priority = priority;
2224			igr.u_attributes = attributes;
2225			if(subscribedQueue && subscribedQueue != null && subscribedQueue != "") {
2226				igr.u_subscribed_queue = subscribedQueue.sys_id;
2227			}
2228			
2229			var messageState = "ready";
2230			try {		
2231				if (pspS.processPSPInMessage(igr)) {
2232    				messageState = "received";						
2233	    		} else if(igr.state != "error") {
2234					messageState = "skipped";
2235	    		} else if (igr.state == "error") {
2236					messageState = "error";
2237				}
2238			} catch (e) {
2239				messageState = "error";
2240				igr.u_state_info = e;
2241				this.logger.logWarn("error: " + e, "PerspectiumReplicator.fetchFromOneQueue");
2242			} finally {
2243				igr.state = messageState;
2244				igr.insert();
2245				
2246				// handle counters and messageset for replicator messages being subscribed (not errors coming back to show in outbound)
2247				if (topic == "replicator" && type != "error") {
2248					
2249					// only update counters if message was received and processed
2250					if(messageState == "received"){
2251						// get # of bytes for value field
2252						var valueBytes;
2253						// try to get from decoded value field first
2254						if(pspS.decodeData != null && pspS.decodeData != '')
2255							valueBytes = pspS.decodeData.length(); // dloo
2256						// otherwise use encrypted value field for # of bytes
2257						else
2258							valueBytes = value.length(); // dloo
2259												
2260						if (kounter[name]) {
2261							kounter[name] ++;
2262							bytesKounter[name] = bytesKounter[name] + valueBytes;
2263						} else {
2264							kounter[name] = 1;
2265							bytesKounter[name] = valueBytes;
2266						}
2267					}
2268					
2269					// handle messageset for this message if we have a set_id
2270					var pspMsg = new PerspectiumMessage(topic, type, key, name, value, "", extra, attributes, "", "");
2271					var messageSetId = pspMsg.getAttribute("set_id");
2272					this.logger.logDebug("set_id: " + messageSetId, "PerspectiumReplicator.fetchFromOneQueue");
2273					if(messageSetId == null){
2274						continue;	
2275					}
2276					
2277					// increment count based on state of message
2278					pspMS2.incrementMessageSetCount(messageSetId, messageState);
2279					
2280					var messagePspFlag = pspMsg.getAttribute("psp_flag");
2281					if(messagePspFlag == null){
2282						continue;	
2283					}
2284					
2285					// update if we received first or last message
2286					if(messagePspFlag.indexOf("first") >= 0){
2287						pspMS2.setMessageSetFirst(messageSetId);
2288					}
2289					
2290					if(messagePspFlag.indexOf("last") >= 0){
2291						pspMS2.setMessageSetLast(messageSetId);
2292					}
2293					
2294				}
2295			}
2296        }
2297		
2298		csecs = pspS._getTimeDiffSecs(sdt);
2299		rsecs = obj.length/csecs;	
2300        this.logger.logDebug(obj.length + " records (" + queueName + ") PROCESSED in " + csecs + " secs (" + rsecs.toFixed(2) + " recs/s)", "PerspectiumReplicator.fetchFromOneQueue");
2301		this.psp.addMessageStats(obj.length + " inbound records PROCESSED from " + queueName, csecs, this.messageStatsMod, recTypes.toString(), "PROCESSED", numRecords);
2302		
2303		// insert all history set into table for scheduled job to process
2304		pspS.pspRHS.insertIntoTable();
2305		
2306		for(n in kounter) {
2307			pspS.sendOutboundCounter(n, kounter[n]);
2308			// send bytes as well
2309			pspS.sendOutboundCounter(n + ".bytes", bytesKounter[n]);			
2310		}
2311		
2312		// send out messageset messages for all replicator messages processed in this job
2313		if(subscribedQueue && subscribedQueue != null && subscribedQueue != ""){
2314			pspMS2.createMessageSetMessages(this.key, subscribedQueue, "snc-subscribed");
2315		}
2316		else{
2317			pspMS2.createMessageSetMessages(this.key, "", "snc-subscribed");			
2318		}
2319    },
2320	
2321	createListener : function(table_name, rgr) {
2322		this.createSysAuditDeleteListener(table_name, rgr);
2323	},
2324	
2325	createSysAuditDeleteListener : function(table_name, rgr) {
2326		var grBR = new GlideRecord("sys_script");
2327		grBR.addQuery("collection", "sys_audit_delete");
2328		grBR.addQuery("name", "Perspectium DL - " + table_name);
2329		// to make sure we find right one look for the one with the dynamic share's sys_id in its script
2330		grBR.addQuery("script", "CONTAINS", rgr.sys_id);			
2331		grBR.query();
2332		if(!grBR.next()) {
2333			this.logger.logDebug("Creating Perspectium Delete Listener for " + table_name, "PerspectiumReplicator.createSysAuditDeleteListener");
2334			grBR.name = "Perspectium DL - " + table_name;
2335			grBR.collection = "sys_audit_delete";
2336			grBR.script = "var psp = new Perspectium();var pspR = new PerspectiumReplicator();var egr = psp.deserialize(current.tablename, current.payload.toString());pspR.shareRecord(egr, '" + table_name + "', 'delete', '" + rgr.sys_id + "');";
2337			grBR.when = "after";
2338			grBR.order = "50";
2339			grBR.advanced = "true";
2340			grBR.execute_function = "false";
2341			grBR.active = "true";
2342			grBR.action_insert = "true";
2343			grBR.sys_domain = "global";
2344			grBR.condition = "current.tablename == '" + table_name + "' || (new PerspectiumUtil().isChildTable(current.tablename, '" + table_name + "'))";	
2345			
2346			grBR.setWorkflow(false);
2347			grBR.insert();
2348		} else {
2349			this.logger.logDebug("Perspectium Delete Listener already exists for " + table_name, "PerspectiumReplicator.createSysAuditDeleteListener");
2350		}
2351	},
2352	
2353	deleteListener : function(table_name, rgr) {
2354		this.deleteSysAuditDeleteListener(table_name, rgr);
2355	},
2356	
2357	deleteSysAuditDeleteListener : function(table_name, rgr) {
2358		this.logger.logDebug("deleting SysAuditDeleteListener for " + table_name, "PerspectiumReplicator.deleteSysAuditDeleteListener");
2359		var grBR = new GlideRecord("sys_script");
2360		grBR.addQuery("collection", "sys_audit_delete");
2361		grBR.addQuery("name", "Perspectium DL - " + table_name);
2362		// to make sure we find right one look for the one with the dynamic share's sys_id in its script
2363		grBR.addQuery("script", "CONTAINS", rgr.sys_id);					
2364		grBR.deleteMultiple();
2365	},
2366	
2367	updateReplicatorBusinessRule : function(rgr, createNew) {
2368		this.logger.logDebug(rgr.table_name, "PerspectiumReplicator.updateReplicatorBusinessRule", rgr);
2369		
2370		if (rgr.u_use_listener == true && rgr.action_delete == true && rgr.active == true) {
2371			this.logger.logDebug("creating delete listener for " + rgr.table_name, "PerspectiumReplicator.updateReplicatorBusinessRule", rgr);
2372		    this.createListener(rgr.table_name, rgr);
2373		} else {
2374		    this.deleteListener(rgr.table_name, rgr);
2375		}
2376		
2377		// flag if we should not update and always create a new business rule
2378		var createNewBR = false;
2379		if(typeof createNew != 'undefined' && createNew != null)
2380			createNewBR = createNew;
2381		
2382		//Flip the active flag on the appropriate Business Rule
2383		var grBR = new GlideRecord("sys_script");
2384		grBR.addQuery("collection", rgr.table_name);
2385		grBR.addQuery("name", "Perspectium Replicate");
2386		
2387		// use the business rule referenced in share config
2388		if (rgr.u_business_rule != null && rgr.u_business_rule != "" && !rgr.u_business_rule.isNil()) {
2389			grBR.addQuery("sys_id", rgr.u_business_rule);
2390		}
2391		grBR.query();
2392		
2393		// for async we'll always send as bulk since we can't tell if record is update or insert
2394		var grOperation = "";
2395		if (rgr.u_business_rule_when == "async") {
2396			grOperation = "bulk";
2397		}
2398		
2399		// Build the condition string
2400		var replicateCondition = "current.operation() != null && PerspectiumReplicator.isReplicatedTable('" + rgr.table_name + "', 'share', current.operation().toString())";
2401		if (rgr.u_interactive_only){
2402			replicateCondition += " && gs.isInteractive()";
2403		}
2404		if (!rgr.u_run_on_subscribe){
2405			replicateCondition += " && current.psp_subscribed_record != true";
2406		}
2407
2408		if(grBR.next() && !createNewBR) {
2409			this.logger.logDebug("updateReplicatorBusinessRule updating", "PerspectiumReplicator.updateReplicatorBusinessRule", rgr);
2410			grBR.active = rgr.active;
2411			grBR.action_delete = rgr.action_delete;
2412			grBR.action_insert = rgr.action_create;
2413			grBR.action_update = rgr.action_update;
2414		
2415			if ( rgr.u_business_rule_order > 0 ) {
2416				grBR.order = rgr.u_business_rule_order;
2417			}
2418
2419            grBR.condition = replicateCondition;
2420			grBR.script =  "var pspR = new PerspectiumReplicator();";
2421			// pass share sys_id so we can reference it when sharing record
2422			// to support multiple share configurations with different business order rule and when for the same table
2423            grBR.script += "pspR.shareRecord(current, '" + rgr.table_name + "', '" + grOperation + "', '" + rgr.sys_id + "');";
2424				
2425			// always set when to run business rule based on share config as default value on share config form is "after"
2426			grBR.when = rgr.u_business_rule_when;
2427			
2428			grBR.setWorkflow(false);
2429			grBR.update();			
2430		} else {
2431			this.logger.logDebug("updateReplicatorBusinessRule inserting", "PerspectiumReplicator.updateReplicatorBusinessRule", rgr);
2432			grBR = new GlideRecord("sys_script");
2433			grBR.name = "Perspectium Replicate";
2434			grBR.collection = rgr.table_name;
2435			
2436			grBR.order = "50";
2437			if( rgr.u_business_rule_order > 0 )
2438				grBR.order = rgr.u_business_rule_order;
2439			
2440			grBR.condition = replicateCondition;
2441			grBR.script =  "var pspR = new PerspectiumReplicator();";
2442			// pass share sys_id so we can reference it when sharing record
2443			// to support multiple share configurations with different business order rule and when for the same table
2444            grBR.script += "pspR.shareRecord(current, '" + rgr.table_name + "', '" + grOperation + "', '" + rgr.sys_id + "');";
2445				
2446			// always set when to run business rule based on share config as default value on share config form is "after"
2447			grBR.when = rgr.u_business_rule_when;			
2448			
2449			grBR.active = rgr.active;
2450			grBR.advanced = "true";
2451			grBR.execute_function = "false";
2452			grBR.action_delete = rgr.action_delete;
2453			grBR.action_insert = rgr.action_create;
2454			grBR.action_update = rgr.action_update;
2455			grBR.sys_domain = "global";
2456			grBR.setWorkflow(false);
2457			var ior = grBR.insert();
2458			
2459			// save business rule's sys_id into the share config so we can reference for next time we update
2460			// setworkflow false so it doesn't fire business rule again and do a cyclical update
2461			rgr.u_business_rule = ior;
2462			rgr.setWorkflow(false);
2463			rgr.update();
2464			
2465			this.logger.logDebug("updateReplicatorBusinessRule after insert =" + ior, "PerspectiumReplicator.updateReplicatorBusinessRule", rgr);
2466		}
2467	},
2468
2469	resetReplicatorBusinessRule: function(rgr){
2470		// delete business rule and then recreate it from scratch so it has all the proper settings
2471		this.deleteReplicatorBusinessRule(rgr);		
2472
2473		this.deleteListener(rgr.table_name, rgr);
2474		
2475		if (rgr.u_business_rule != null && rgr.u_business_rule != "" && !rgr.u_business_rule.isNil()) {
2476			this.updateReplicatorBusinessRule(rgr);		
2477		}
2478		// if this share config doesn't have a business rule reference then we'll want to delete 
2479		// all share configs for this table and re-create them
2480		// since there will only be one business rule for the table previously and resetting one will
2481		// cause the other ones to not have a business rule to use anymore
2482		else{
2483			var qc = new GlideRecord('psp_replicate_conf');
2484            qc.addQuery('table_name', rgr.table_name);
2485	    	qc.addQuery("sync_direction", "share");
2486	    	qc.addQuery("active", "true");
2487            qc.query();
2488			
2489			while(qc.next()){
2490				this.updateReplicatorBusinessRule(qc, true);
2491			}			
2492		}
2493	},
2494	
2495	deleteReplicatorBusinessRule : function(rgr) {
2496		//Flip the active flag on the appropriate Business Rule
2497		var grBR = new GlideRecord("sys_script");
2498		grBR.addQuery("collection", rgr.table_name);
2499		grBR.addQuery("name", "Perspectium Replicate");
2500		
2501		// use the business rule referenced in share config
2502		if (rgr.u_business_rule != null && rgr.u_business_rule != "" && !rgr.u_business_rule.isNil()) {
2503			grBR.addQuery("sys_id", rgr.u_business_rule);
2504		}
2505		
2506		grBR.query();
2507		if(grBR.next()) {
2508			grBR.deleteRecord();
2509		}
2510	},
2511	
2512	synchDynShareRules : function() {		
2513		// delete all Perspectium Replicate business rules
2514		var grBR = new GlideRecord("sys_script");
2515		grBR.addQuery("name", "Perspectium Replicate");
2516		grBR.deleteMultiple();
2517		
2518		// delete all Perspectium DL - x business rules
2519		var grDL = new GlideRecord("sys_script");
2520		grDL.addQuery("name", "STARTSWITH", "Perspectium DL");
2521		grDL.deleteMultiple();
2522		
2523		//gs.print("deleted existing rules and recreating new ones");
2524		
2525		// recreate them
2526		var rgr2 = new GlideRecord("psp_replicate_conf");
2527		rgr2.addQuery("sync_direction", "share");
2528		rgr2.addActiveQuery();
2529		rgr2.query();
2530		while(rgr2.next()) {
2531			gs.print("creating rule for " + rgr2.table_name);
2532			this.updateReplicatorBusinessRule(rgr2, true);
2533		}
2534	},
2535	
2536	appendAttribute : function(attrKey, attrVal, attrString){
2537		if(attrString.length > 0)
2538			attrString += ",";
2539		attrString += attrKey + "=" + attrVal;
2540		return attrString;
2541	},
2542	
2543	appendTableMapAttributes : function(grParam, grTableMap, attrString){
2544		var fmap = new GlideRecord('u_psp_table_field_map');
2545		fmap.addQuery('u_parentid' , grTableMap.sys_id);
2546		fmap.query();
2547		while(fmap.next()) {
2548			if(fmap.u_target_field.startsWith("@")){
2549				if(fmap.u_use_script){
2550					var gc = (typeof GlideController != 'undefined') ? GlideController : Packages.com.glide.script.GlideController;
2551					var oldCurrent = current;
2552					gc.putGlobal('current', grParam);
2553					var answer = gc.evaluateString(fmap.u_source_script);
2554					gc.removeGlobal('current');		
2555					current = oldCurrent;				
2556					this.logger.logDebug("fmap.u_target_field = " + fmap.u_target_field.substring(1));
2557					attrString = this.appendAttribute(fmap.u_target_field.substring(1), answer, attrString);					
2558				}
2559				else{
2560					this.logger.logDebug("fmap.u_target_field = " + fmap.u_target_field.substring(1));
2561					attrString = this.appendAttribute(fmap.u_target_field.substring(1), grParam.getValue(fmap.u_source_field), attrString);
2562				}
2563			}
2564		}
2565		return attrString;
2566	},
2567		
2568	incrementMessageSetCounter : function(name) {
2569		if (this.messageSetCounter[name]) {
2570			this.messageSetCounter[name] ++;
2571		} 
2572		else {
2573			this.messageSetCounter[name] = 1;
2574		}		
2575	},
2576		
2577	endsWith : function(str, suffix) {
2578    	return str.indexOf(suffix, str.length - suffix.length) !== -1;
2579	},
2580	
2581	incrementTableCounter : function(tableName, recordByte) {
2582		if(this.tableCounter[tableName]) {
2583			var byteTotal = 0;
2584			this.tableCounter[tableName].bytes = recordByte + this.tableCounter[tableName].bytes;
2585			this.tableCounter[tableName].count++;
2586		}
2587		else {
2588			this.tableCounter[tableName] = {
2589				count: 1,
2590				bytes: recordByte
2591			};
2592		}
2593	},
2594    
2595    type: 'PerspectiumReplicator'
2596};