· 7 years ago · Sep 09, 2018, 09:50 PM
1using System;
2using System.Collections.Generic;
3using System.Linq;
4using System.Threading.Tasks;
5
6//needed refference files
7using HtmlAgilityPack;
8using System.Net.Http;
9using Xceed.Words.NET;
10
11// jokūbas varnagiris
12// iff-7/10
13// KTU informatikos fakultetas
14
15namespace CrawlerDemo
16{
17 class Program
18 {
19 static void Main(string[] args)
20 {
21 //making a seperate "main" since the real main cant be async
22 RunAsync();
23 //to stop program from exiting
24 Console.ReadLine();
25 }
26 static async void RunAsync()
27 {
28 //words to look up, should probably be imported from a file
29 List<string> words = new List<string>();
30 Console.WriteLine("the operation might take a few seconds.");
31 Console.WriteLine("enter words seperated by spaces or leave empty to use default ones :");
32 string input = Console.ReadLine();
33 if (input.Equals("")) //default words to test program
34 {
35 words.Add("programming");
36 words.Add("penis");
37 words.Add("car");
38 words.Add("mitochondria");
39 }
40 else
41 {
42 string[] values = input.Split(' ');
43 foreach (var item in values)
44 {
45 words.Add(item);
46 }
47 }
48
49 //returned list of objects containing a word and that words definitions
50 List<GlossItem> Glossary = await startCrawlerasync(words); //await means the async method will pause the program until its done (basically syncs it)
51
52 //create docx file
53 string filename = "test.docx";
54 var doc = DocX.Create(filename);
55 //add title and create table
56 doc.InsertParagraph("Glossary");
57 Table t = doc.AddTable(Glossary.Count, 2);
58 t.Alignment = Alignment.left;
59 t.Design = TableDesign.TableGrid;
60 t.AutoFit = AutoFit.Contents;
61
62 //populate table and output to console
63 for ( int k = 0; k < Glossary.Count; k++)
64 {
65 Console.WriteLine();
66 Console.WriteLine(" {0} - \n", Glossary[k].Word);
67 t.Rows[k].Cells[0].Paragraphs.First().Append(Glossary[k].Word);
68 t.Rows[k].Cells[0].Paragraphs.First().Bold();
69 string defs = "";
70 for (int i = 0; i < Glossary[k].definitions.Count && i < 5; i++) // max 5 definitions per word
71 {
72 defs = defs + " * " + Glossary[k].definitions[i] + "\n";
73 }
74 Console.WriteLine(" * {0}", defs);
75 t.Rows[k].Cells[1].Paragraphs.First().Append(defs);
76 }
77 //add table and save document
78 doc.InsertTable(t);
79 doc.Save();
80 }
81
82 public class GlossItem
83 {
84 public string Word = "default"; //will change
85 public List<string> definitions; //list of definitions
86
87 public GlossItem(string word) //init
88 {
89 Word = word;
90 definitions = new List<string>();
91 }
92
93 public void AddDeff(string words) //add definitions to word
94 {
95 definitions.Add(words);
96 }
97 }
98 // the following method name looks weird but imagine its just
99 // private static List<GlossItem> startCrawlerasync(List<string> WordsToDefine)
100 // it has to look like it does because crawling isn't a synched process
101 private static async Task<List<GlossItem>> startCrawlerasync(List<string> WordsToDefine)
102 {
103 //init
104 List<GlossItem> Glossary = new List<GlossItem>();
105 //for each word:
106 // 1. form a url, get the page source file
107 foreach (string word in WordsToDefine)
108 {
109 GlossItem WordToDefine = new GlossItem(word);
110 var httpClient = new HttpClient();
111 var htmlDocument = new HtmlDocument();
112 var url = "https://www.dictionary.com/browse/" + word + "?s=t"; // will not add any definitions to non-english words
113 var html = await httpClient.GetStringAsync(url);
114 htmlDocument.LoadHtml(html);
115 // 2. extract all tags inside "ordered list" tags
116 var results = htmlDocument.DocumentNode.Descendants("ol").ToList();
117
118 bool done = false; //this exists to not add too many definitions to a word
119 foreach (var item in results)
120 {
121 if (done) //this exists to not add too many definitions to a word
122 {
123 break;
124 }
125 foreach (var ListItem in item.ChildNodes)
126 {
127 if (done) //this exists to not add too many definitions to a word
128 {
129 break;
130 }
131 if (ListItem.Name == "li") //we only care about "list item" tags
132 {
133 for (int i = 0; (ListItem.ChildNodes.Count > i); i++)
134 {
135 if (ListItem.ChildNodes[i].Name == "span") //we only need the text inside the "span" tags
136 {
137 WordToDefine.AddDeff(ListItem.ChildNodes[i].InnerText); //add text as definition
138
139 if (WordToDefine.definitions.Count > 9) //this exists to not add too many definitions to a word
140 {
141 done = true;
142 break;
143 }
144 }
145 }
146 }
147 }
148 }
149 //add object to glossary
150 Glossary.Add(WordToDefine);
151 }
152 return Glossary; //return it
153 }
154 }
155}