· 6 years ago · Apr 08, 2020, 04:40 PM
1// Copyright (c) Nucleuz Inc. All rights reserved.
2//
3// This program is intended to illustrate a basic method of using the
4// Nucleuz DLP Engine SDK in C#. Only a small portion of the SDK is covered
5// with the primary goal to get a basic application up and running.
6//
7// The code in this file and all accompanying files is offered as-is for use and
8// modification. Nucleuz Inc makes no warranties, express or implied, and
9// hereby disclaims all implied warranties, including any warranty of
10// merchantability and warranty of fitness for a particular purpose.
11
12using System;
13using System.Runtime.InteropServices;
14using NucleuzDotNET;
15
16namespace SampleCSharpApp
17{
18 class Program
19 {
20 // Document similarity rule pack is stored in a separate variable so it can be
21 // updated to demonstrate Document Similarity.
22 // Production would likely store the generated rule pack to persistent storage.
23 public static System.String strExampleDocSimilarityPolicyId = "63b0f83b-0724-11a8-6eeb-5641cef9dbf5";
24 public static System.String strDocSimilarityCustomRuleId = "e17af494-2fdb-4b26-b1b5-cdd8c6228c85";
25 private static System.Xml.XmlDocument xmlDocSimilarityCustomRulePack = null;
26
27 static void Main(string[] args)
28 {
29 System.Console.WriteLine("Nucleuz DLP Engine .NET Sample Application");
30 System.Console.WriteLine("Nucleuz .NET SDK/API version: " + NucleuzDotNET.NucleuzDlpEngineDotNetApiVersion.GetNucleuzDotNetSDKVersion() );
31
32 NucleuzDotNET.Nucleuz_DlpEngine dlpEngine = null;
33
34 try
35 {
36 // Get license key from user
37 System.Console.Write("Please enter your Nucleuz DLP Engine license key: ");
38 String strLicenseKey = System.Console.ReadLine();
39
40 System.Console.Write("Constructing Nucleuz_DlpEngine...");
41
42 // Get logging options
43 NucleuzDotNET.Nucleuz_DLP_LoggingOptions loggingOptions = GetLoggingOptions();
44
45 dlpEngine = new NucleuzDotNET.Nucleuz_DlpEngine(
46 strLicenseKey,
47 loggingOptions,
48 new My_Custom_Policy_Loader(), // Custom policy loader
49 new My_Custom_RulePack_Loader() // Custom rule pack loader
50 );
51 System.Console.WriteLine("done.");
52
53 System.Console.Write("Getting Nucleuz DLP Engine version...");
54 String strNucleuzEngineVersion = dlpEngine.getEngineVersion();
55 System.Console.WriteLine("done. Version: " + strNucleuzEngineVersion);
56
57 // Configure evaluation options
58 NucleuzDotNET.Nucleuz_DLP_EvaluationOptions evaluationOptions = new NucleuzDotNET.Nucleuz_DLP_EvaluationOptions()
59 {
60 evaluateFlags = NucleuzDotNET.Nucleuz_DLP_EvaluateFlags.NUCLEUZ_EVAL_DEFAULT_FLAGS,
61 maxRegexMatches = 500,
62 };
63
64 // Evaluate File
65 //------------------------------------------------------------------
66 {
67 System.Console.Write("\nEvaluating file...");
68
69 System.Xml.XmlDocument xmlEvalContextDoc = GetEvaluationContext();
70
71 System.Xml.XmlDocument xmlResultDoc = dlpEngine.evaluateFile(
72 "NO_TENANT", // Tenant ID
73 "7060a43a-63a8-b14a-31b5-37e511ab2ce2", // Policy ID
74 "InputFiles\\TestInput.txt", // File To Evaluate
75 xmlEvalContextDoc, // evaluation context
76 evaluationOptions // evaluation options
77 );
78
79 System.Console.WriteLine("done. Result:\n[[[\n" + GetStringifiedXml(xmlResultDoc) + "\n]]]");
80 }
81
82 // Evaluate Buffer
83 //------------------------------------------------------------------
84 {
85 System.Console.Write("\nEvaluating buffer...");
86
87 String strContent = @"
88IPv4 address: 192.168.1.1
89
90IPV6 address: 1234:5678::1234:ABCD
91
92Payment Information:
93Visa: 4111-1111-1111-1111
94";
95 System.Byte[] bufferToEvaluate = System.Text.UTF8Encoding.UTF8.GetBytes(strContent);
96
97 System.Xml.XmlDocument xmlEvalContextDoc = GetEvaluationContext();
98
99 System.Xml.XmlDocument xmlResultDoc = dlpEngine.evaluateBuffer(
100 "NO_TENANT", // Tenant ID
101 "7060a43a-63a8-b14a-31b5-37e511ab2ce2", // Policy ID
102 bufferToEvaluate, // buffer To Evaluate
103 xmlEvalContextDoc, // evaluation context
104 evaluationOptions // evaluation options
105 );
106
107 System.Console.WriteLine("done. Result:\n[[[\n" + GetStringifiedXml(xmlResultDoc) + "\n]]]");
108 }
109 // Use "Push" model
110 //------------------------------------------------------------------
111 {
112 System.Console.Write("\nEvaluating buffer using \"Push\" API...");
113
114 // Use 'using' directive so session's internal references are released promptly.
115 using(NucleuzDotNET.PushDataSession pushDataSession = dlpEngine.createPushSession(
116 "NO_TENANT", // Tenant ID
117 "7060a43a-63a8-b14a-31b5-37e511ab2ce2", // Policy ID
118 evaluationOptions // evaluation options
119 ) )
120 {
121 // This is a super-basic example just to illustrate API.
122 // Normally each buffer will be larger.
123 String[] strContent = new string[]
124 {
125 "IPv4 address: 192.168.1.1",
126 "",
127 "IPV6 address: 1234:5678::1234:ABCD",
128 "",
129 "Payment Information:",
130 "Visa: 4111-1111-1111-1111",
131 };
132
133 // Push each line of buffer
134 for(int i = 0; i < strContent.Length; i++)
135 {
136 System.Byte[] bufferToEvaluate = System.Text.UTF8Encoding.UTF8.GetBytes(strContent[i]);
137
138 pushDataSession.pushData(bufferToEvaluate);
139 }
140
141 System.Xml.XmlDocument xmlEvalContextDoc = GetEvaluationContext();
142
143 System.Xml.XmlDocument xmlResultDoc = pushDataSession.evaluate(
144 xmlEvalContextDoc // evaluation context
145 );
146
147 System.Console.WriteLine("done. Result:\n[[[\n" + GetStringifiedXml(xmlResultDoc) + "\n]]]");
148 }
149 }
150
151 // Demonstrate basic Document Similarity
152 //------------------------------------------------------------------
153 // Part 1: Generate Similarity classification rule (ie. fingerprint)
154 {
155 System.Console.Write("\nGenerating fingerprint for document similarity...");
156
157 System.String strOriginalDocument = @"
158World Cup Russia 2018: How and where to buy tickets to next summer's tournament
159
160The world's biggest tournament is less than a year away
161
162World Cup 2018 tickets are in such high demand that according to FIFA, more
163than 500,000 tickets have been requested by fans through the governing soccer
164body's official website.
165
166Russians have topped the list for most requests, followed by Mexico, Argentina,
167Brazil, China, USA, Colombia, Germany and English. 50,000 tickets have been
168requested for the final, while 40,000 have been requested for the opener, which
169will feature the Russian national team.
170
171It remains to be seen if this World Cup will break attendance records, but the
172highest average attendance for a cup per match is 67,991, from the 1994 World
173Cup in the United States, which also holds highest overall attendance at 3.65
174million.
175";
176
177 NucleuzDotNET.Nucleuz_DLP_FingerprintingOptions fingerprintingOptions = new NucleuzDotNET.Nucleuz_DLP_FingerprintingOptions(
178 NucleuzDotNET.Nucleuz_DLP_DocumentSimilarityComparisonModes.NUCLEUZ_DOCSIM_COMPARE_FULL,
179 2, // shingleSize
180 4, // windowSize
181 false // do not include original content
182 );
183
184 System.Byte[] bufferToFingerprint = System.Text.UTF8Encoding.UTF8.GetBytes(strOriginalDocument);
185
186 xmlDocSimilarityCustomRulePack = dlpEngine.generateDocumentSimilarityClassificationRule(
187 strDocSimilarityCustomRuleId,
188 40, // default Similarity Threshold
189 "World Cup Russia 2018 Tickets", // name
190 "Detect content that's similar to a snippet of an article about tickets for the 2018 World Cup", // description
191 "en", // language code
192 bufferToFingerprint, // content To Fingerprint
193 NucleuzDotNET.Nucleuz_DLP_DocumentSimilarityAlgorithms.NUCLEUZ_DOCSIM_SHINDOW_FINGERPRINTING_ALGO,
194 fingerprintingOptions,
195 null // no initial rule pack
196 );
197
198 System.Console.WriteLine("done.");
199
200 // NOTE: In a production setting the resulting rule pack will likely be stored
201 // in persistent storage.
202
203 // Part 2: Compare content using Document Similarity
204 // Evaluate policy which references similarity
205 System.Console.Write("\nEvaluating document for similarity...");
206
207 System.String strDerivedDocument = @"
208World Cup Russia 2018: Buy tickets to next summer's tournament
209
210World's biggest tournament one year away
211
212World Cup 2018 tickets are in such high demand that more than 500,000 tickets
213have been requested by fans through the official website.
214
215Russians have topped the list for most requests, followed by Mexico, Argentina,
216Brazil, China, USA, Colombia, Germany and England. 50,000 tickets have been
217requested for the final game, while 40,000 have been requested for the opening
218game, which will feature the Russian national team.
219
220It's questionable if this World Cup will break attendance records. The
221highest average attendance for a cup is 68,991 per match, from the 1994 World
222Cup in the United States. The same cup also holds the highest overall
223attendance at 3.75 million.
224";
225 System.Byte[] derivedDocumentBuffer = System.Text.UTF8Encoding.UTF8.GetBytes(strDerivedDocument);
226
227 System.Xml.XmlDocument xmlEvalContextDoc = GetEvaluationContext();
228
229 System.Xml.XmlDocument xmlResultDoc = dlpEngine.evaluateBuffer(
230 "NO_TENANT", // Tenant ID
231 strExampleDocSimilarityPolicyId, // policy ID
232 derivedDocumentBuffer, // buffer to evaluate
233 xmlEvalContextDoc, // evaluation context
234 evaluationOptions // evaluation options
235 );
236
237 System.String strXmlResults = GetStringifiedXml(xmlResultDoc);
238 System.Console.WriteLine(@"done. Result:
239------------------------------------------------------------
240Document Similarity evaluation results (length = " + strXmlResults.Length + @"):
241------------------------------------------------------------
242" + strXmlResults);
243 }
244
245 // Get information about matching classification rule(s) (display to users)
246 //------------------------------------------------------------------
247 {
248 System.Console.Write("\nGetting classification info...");
249
250 System.Xml.XmlDocument xmlResultDoc = dlpEngine.getClassificationInfo(
251 "NO_TENANT", // Tenant ID
252 "\"dfbe8315-b32d-2f0c-99e6-3fad6c4faf82\",\"ac439e77-7b7b-228b-cfb6-e1c9c5413131\"", // Classification rule IDs (obtain from evaluation result's XML)
253 "en" // lang code
254 );
255
256 System.IO.StringWriter strWriter = new System.IO.StringWriter();
257 xmlResultDoc.Save(strWriter);
258 System.String strEvaluationResult = strWriter.ToString();
259 System.Console.WriteLine("done. Information:\n[[[\n" + strEvaluationResult + "\n]]]");
260 }
261
262 // Unloading a policy or classification rule is not required since they will
263 // automatically be unloaded at program end.
264 // When an update is available on the system, the new policy and/or
265 // classification rules can be utilized without restarting the application, by
266 // simply unloading them. The next evaluation to need them will request the
267 // new one.
268
269 System.Console.Write("Calling unloadPolicy...");
270 dlpEngine.unloadPolicy(
271 "NO_TENANT", // tenant ID
272 "7060a43a-63a8-b14a-31b5-37e511ab2ce2" // policy ID
273 );
274 System.Console.WriteLine("done.");
275
276 System.Console.Write("Calling unloadAllPolicies...");
277 dlpEngine.unloadAllPolicies(
278 );
279 System.Console.WriteLine("done.");
280
281 System.Console.Write("Calling unloadClassificationRulesForTenant...");
282 dlpEngine.unloadClassificationRulesForTenant(
283 "NO_TENANT" // tenant ID
284 );
285 System.Console.WriteLine("done.");
286
287 System.Console.Write("Calling unloadClassificationRulesForPolicy...");
288 dlpEngine.unloadClassificationRulesForPolicy(
289 "NO_TENANT", // tenant ID
290 "7060a43a-63a8-b14a-31b5-37e511ab2ce2" // policy ID
291 );
292 System.Console.WriteLine("done.");
293 }
294 catch(System.DllNotFoundException dnfe)
295 {
296 System.Console.Error.WriteLine("FAILED.");
297 System.Console.Error.WriteLine("Failed to locate Nucleuz DLP Engine DLL.");
298 System.Console.Error.WriteLine("Error details:");
299 System.Console.Error.WriteLine(dnfe);
300 }
301 catch(System.Exception e)
302 {
303 System.Console.Error.WriteLine("FAILED.");
304 System.Console.Error.WriteLine("Exception caught. Details:");
305 System.Console.Error.WriteLine(e);
306 }
307 finally
308 {
309 // Uninitialize engine
310 if(dlpEngine != null)
311 {
312 // The uninitialization configuration.
313 NucleuzDotNET.Nucleuz_DLP_UninitializeOptions uninitOptions = new NucleuzDotNET.Nucleuz_DLP_UninitializeOptions(
314 // At this point there should be no outstanding evaluations.
315 NucleuzDotNET.Nucleuz_DLP_Uninit_Mode.NUCLEUZ_UNINIT_ERROR_OUTSTANDING_EVALUATIONS,
316 // Shouldn't be a need for grace period since nothing should be
317 // running in the engine at shutdown.
318 2
319 );
320
321 try
322 {
323 System.Console.Write("Uninitializing Nucleuz DLP Engine...");
324 dlpEngine.uninitialize(uninitOptions);
325 System.Console.WriteLine("done.");
326 }
327 catch(System.Exception e)
328 {
329 System.Console.Error.WriteLine("FAILED.");
330 System.Console.Error.WriteLine("Exception caught. Details:");
331 System.Console.Error.WriteLine(e);
332 }
333 }
334 }
335 Console.Read();
336 }
337 //------------------------------------------------------------------------
338
339 // Custom logging example.
340 // This implementation simply outputs to stdout.
341 public class My_Custom_Logger : NucleuzDotNET.INucleuzLoggingHandler
342 {
343 public void log(NucleuzDotNET.Nucleuz_DLP_LoggingEntry logEntry)
344 {
345 System.Console.WriteLine("LOG: [[" + logEntry.year + "-" + logEntry.month + "-" + logEntry.day + " " + logEntry.hour + ":" + logEntry.minute + ":" + logEntry.second + ":" + logEntry.microsecond + "|" + logEntry.psTraceLevel + "|" + logEntry.psTraceData + "]]");
346 }
347 }
348 //------------------------------------------------------------------------
349
350 // Setup logging options
351 public static NucleuzDotNET.Nucleuz_DLP_LoggingOptions GetLoggingOptions()
352 {
353 NucleuzDotNET.Nucleuz_DLP_LoggingOptions loggingOptions;
354
355 // Use this to disable logging
356 loggingOptions = new NucleuzDotNET.LoggingOptions_NoLogging();
357
358 // Use this to log all levels to a file on disk
359 loggingOptions = new NucleuzDotNET.LoggingOptions_FileLogging(
360 "NucleuzTraceLogFromDotNet.log",
361 ulong.MaxValue, // no limit on log file size
362 "FATAL|ERROR|WARNING|PROGRESS|INFO|DEBUG"
363 );
364
365 // Use this to log all levels with a custom-defined logging callback
366 /*
367 loggingOptions = new NucleuzDotNET.LoggingOptions_Custom(
368 new My_Custom_Logger(),
369 "FATAL|ERROR|WARNING|PROGRESS|INFO|DEBUG"
370 );
371 */
372
373 return loggingOptions;
374 }
375 //------------------------------------------------------------------------
376
377 public class My_Custom_Policy_Loader : NucleuzDotNET.INucleuzPolicyLoader
378 {
379 // Basic sample implementation of a custom policy loader.
380 // NOTE: Typically production will get the policies from some interface
381 // which is backed by persistent storage like a database or file system.
382
383 private System.String strExampleDocSimilarityPolicy = @"<?xml version=""1.0"" encoding=""UTF-8""?>
384<DlpPolicyTemplate xmlns=""http://schemas.nucleuz.com/2013/dlp_policy"">
385
386 <Metadata id=""" + strExampleDocSimilarityPolicyId + @""" formatVersion=""100"" >
387 <Name default=""true"" langcode=""en"">Basic Sample Document Similarity Policy</Name>
388 <Description default=""true"" langcode=""en-us"">This sample policy looks for document similarity in a very basic form.</Description>
389 <Version major=""2017"" minor=""10"" build=""0"" revision=""1""/>
390 </Metadata>
391
392 <Rules>
393 <Rule id=""World_Cup_2018_Tickets_Rule"">
394 <FormatVersion version=""1"">
395 <Condition>
396 <!-- World Cup 2018 Document Similarity -->
397 <ClassificationRule idRef=""e17af494-2fdb-4b26-b1b5-cdd8c6228c85"" minConfidence=""40"" />
398 </Condition>
399
400 <Actions>
401 <Notify recipients=""soccerfans@example.com"" />
402 </Actions>
403
404 </FormatVersion>
405 </Rule>
406 </Rules>
407</DlpPolicyTemplate>
408";
409
410 // Default/built-in policy loader to fall back on
411 private NucleuzDotNET.Nucleuz_DLP_Utils_Load_Policy_From_File_PolicyLoader m_BuiltInPolicyLoader = new NucleuzDotNET.Nucleuz_DLP_Utils_Load_Policy_From_File_PolicyLoader();
412
413 public System.Byte[] getPolicy(
414 System.String strTenantId,
415 System.String strPolicyId)
416 {
417// System.Console.WriteLine("Custom policy loader called for tenant ID '" + strTenantId + "' and policy ID '" + strPolicyId + "'");
418
419 if(strPolicyId == strExampleDocSimilarityPolicyId)
420 {
421 // Get raw bytes for policy XML
422 System.Byte[] pbPolicyXml = System.Text.UTF8Encoding.UTF8.GetBytes(strExampleDocSimilarityPolicy);
423
424 return(pbPolicyXml);
425 }
426 else
427 {
428 // Requested some other policy.
429 // This sample implementation will defer to the built-in default
430 // mechanism to search for the policy on the file system.
431 return(
432 m_BuiltInPolicyLoader.getPolicy(
433 strTenantId,
434 strPolicyId
435 )
436 );
437 }
438 }
439 }
440 //------------------------------------------------------------------------
441
442 public class My_Custom_RulePack_Loader : NucleuzDotNET.INucleuzClassificationRuleLoader
443 {
444 // Default/built-in classification rule loader to fall back on
445 private NucleuzDotNET.Nucleuz_DLP_Utils_Load_Classification_Rule_From_File_ClassificationRuleLoader m_BuiltInClassificationRuleLoader = new NucleuzDotNET.Nucleuz_DLP_Utils_Load_Classification_Rule_From_File_ClassificationRuleLoader();
446
447 byte[] INucleuzClassificationRuleLoader.getClassificationRule(
448 System.String strTenantId,
449 System.String strClassificationRuleId)
450 {
451// System.Console.WriteLine("Custom classification rule loader called for tenant ID '" + strTenantId + "' and classification rule ID '" + strClassificationRuleId + "'");
452
453 if(strClassificationRuleId == strDocSimilarityCustomRuleId)
454 {
455 // NOTE: Typically a similarity rule will have been pre-created via
456 // some user action and then stored in persistent storage
457 // (ex. database, file system, etc). The custom rule pack loader will
458 // then fetch the stored rule pack.
459
460 // Verify rule pack was previously created (via call to )
461 if(xmlDocSimilarityCustomRulePack == null)
462 {
463 System.Console.Error.WriteLine("ERROR: Similarity classification rule not generated before requesting rule pack.");
464 throw new NucleuzDotNET.NucleuzException(
465 Nucleuz_DLP_ErrorCodes.NUCLEUZ_E_CLASSIFICATION_RULE_NOT_FOUND,
466 "Similarity classification rule not generated before requesting rule pack."
467 );
468 }
469
470 // Get raw bytes for rule pack
471 System.IO.MemoryStream memStream = new System.IO.MemoryStream(1024 * 1024); // start with 1MB capacity
472 xmlDocSimilarityCustomRulePack.Save(memStream);
473 System.Byte[] pbRulePackXml = memStream.ToArray();
474
475 return(pbRulePackXml);
476 }
477 else
478 {
479 // Request for some other classification rule.
480 // This sample implementation will defer to the built-in default
481 // mechanism to search for the classification rule on the file system.
482 return(
483 m_BuiltInClassificationRuleLoader.getClassificationRule(
484 strTenantId,
485 strClassificationRuleId
486 )
487 );
488 }
489 }
490 }
491 //----------------------------------------------------------------------------
492
493 protected static System.Xml.XmlDocument GetEvaluationContext()
494 {
495 String strEvaluationContextXml = @"<?xml version=""1.0"" encoding=""UTF-8""?>
496<PolicyInput>
497 <Sender address=""sender@example.com"" />
498
499 <Recipients>
500 <Recipient address=""recipient1@example.com"" />
501 <Recipient address=""recipient1@domain.com"" />
502 </Recipients>
503
504 <Pages value=""10"" />
505
506 <Author>John Doe, John Smith</Author>
507
508 <LastEdited value=""14:05:23"" />
509
510 <DateStarted value=""2016-02-02"" />
511
512 <FileCreated value=""2016-02-02T14:05:23"" />
513
514</PolicyInput>
515";
516
517 System.Xml.XmlDocument xmlEvalContextDoc = new System.Xml.XmlDocument();
518 xmlEvalContextDoc.Load(new System.IO.StringReader(strEvaluationContextXml) );
519
520 return xmlEvalContextDoc;
521 }
522 //----------------------------------------------------------------------------
523
524 protected static System.String GetStringifiedXml(System.Xml.XmlDocument xmlDoc)
525 {
526 System.IO.StringWriter strWriter = new System.IO.StringWriter();
527 xmlDoc.Save(strWriter);
528 return strWriter.ToString();
529 }
530 //----------------------------------------------------------------------------
531
532 }
533}