LXNMmyyj

1#!/bin/bash
2#Use this script to clean up your merged wordlists before adding them to your main wordlist.
3#Sometimes the wordlists you get are too big to be able to be cleaned by my other script, you need to split them.
4#Read the comments and customize this script according to your needs.
5
6#Author: IssouLinux
7#Date of creation: 2019/11/01
8#If you have any suggestion to improve this script, please let me know on Twitter:
9#https://twitter.com/IssouLinux
10
11if [ -f "partaa" ]
12then
13partaa="partaa"
14main_wordlist="/home/seifer/HackTools/BIG_wordlist"
15
16#Delete the lines with 25 characters or more
17echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
18LC_ALL=C sed -i -r "/.{25,}/d" $partaa
19
20echo -e "\e[32;1mLooking for email addresses...\e[0m"
21#Delete the email addresses
22if LC_ALL=C fgrep --quiet @aol.com $partaa; then
23echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
24LC_ALL=C sed -i '/@aol.com/d' $partaa
25fi
26
27if LC_ALL=C fgrep --quiet @yahoo.fr $partaa; then
28echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
29LC_ALL=C sed -i '/@yahoo.fr/d' $partaa
30fi
31
32if LC_ALL=C fgrep --quiet @yahoo.com $partaa; then
33echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
34LC_ALL=C sed -i '/@yahoo.com/d' $partaa
35fi
36
37if LC_ALL=C fgrep --quiet @msn.fr $partaa; then
38echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
39LC_ALL=C sed -i '/@msn.fr/d' $partaa
40fi
41
42if LC_ALL=C fgrep --quiet @msn.com $partaa; then
43echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
44LC_ALL=C sed -i '/@msn.com/d' $partaa
45fi
46
47if LC_ALL=C fgrep --quiet @hotmail.com $partaa; then
48echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
49LC_ALL=C sed -i '/hotmail.com/d' $partaa
50fi
51
52if LC_ALL=C fgrep --quiet @hotmail.fr $partaa; then
53echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
54LC_ALL=C sed -i '/@hotmail.fr/d' $partaa
55fi
56
57if LC_ALL=C fgrep --quiet @gmail.com $partaa; then
58echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
59LC_ALL=C sed -i '/@gmail.com/d' $partaa
60fi
61
62#Delete the non-ASCII readable characters
63echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
64tr -cd '\11\12\15\40-\176' < $partaa > newfile001
65
66#Remove both leading and trailing whitespaces
67LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partaa
68
69#Delete the lines with 3 characters or less
70echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
71LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
72
73#Delete the blank lines
74echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
75LC_ALL=C sed -i '/^$/d' newfile001
76
77#Delete the duplicate lines
78echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
79LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
80
81#Compare if words already exist in the main wordlist
82LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
83
84cat output001 >> newfile002
85
86#Erase words already existing in main wordlist
87sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
88
89#Count new words to be add to the main wordlist
90nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
91cat NewWords_$(date +%Y%m%d) >> $main_wordlist
92echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
93
94rm -rf NewWords_$(date +%Y%m%d)
95rm -rf newfile001
96rm -rf $partaa
97rm -rf newfile002
98rm -rf output001
99else
100echo -e "\e[31;1mNo partaa file here\e[0m"
101fi
102
103
104if [ -f "partab" ]
105then
106partab="partab"
107main_wordlist="/home/seifer/HackTools/BIG_wordlist"
108
109#Delete the lines with 25 characters or more
110echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
111LC_ALL=C sed -i -r "/.{25,}/d" $partab
112
113echo -e "\e[32;1mLooking for email addresses...\e[0m"
114#Delete the email addresses
115if LC_ALL=C fgrep --quiet @aol.com $partab; then
116echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
117LC_ALL=C sed -i '/@aol.com/d' $partab
118fi
119
120if LC_ALL=C fgrep --quiet @yahoo.fr $partab; then
121echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
122LC_ALL=C sed -i '/@yahoo.fr/d' $partab
123fi
124
125if LC_ALL=C fgrep --quiet @yahoo.com $partab; then
126echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
127LC_ALL=C sed -i '/@yahoo.com/d' $partab
128fi
129
130if LC_ALL=C fgrep --quiet @msn.fr $partab; then
131echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
132LC_ALL=C sed -i '/@msn.fr/d' $partab
133fi
134
135if LC_ALL=C fgrep --quiet @msn.com $partab; then
136echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
137LC_ALL=C sed -i '/@msn.com/d' $partab
138fi
139
140if LC_ALL=C fgrep --quiet @hotmail.com $partab; then
141echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
142LC_ALL=C sed -i '/hotmail.com/d' $partab
143fi
144
145if LC_ALL=C fgrep --quiet @hotmail.fr $partab; then
146echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
147LC_ALL=C sed -i '/@hotmail.fr/d' $partab
148fi
149
150if LC_ALL=C fgrep --quiet @gmail.com $partab; then
151echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
152LC_ALL=C sed -i '/@gmail.com/d' $partab
153fi
154
155#Delete the non-ASCII readable characters
156echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
157tr -cd '\11\12\15\40-\176' < $partab > newfile001
158
159#Remove both leading and trailing whitespaces
160LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partab
161
162#Delete the lines with 3 characters or less
163echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
164LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
165
166#Delete the blank lines
167echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
168LC_ALL=C sed -i '/^$/d' newfile001
169
170#Delete the duplicate lines
171echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
172LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
173
174#Compare if words already exist in the main wordlist
175LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
176
177cat output001 >> newfile002
178
179#Erase words already existing in main wordlist
180sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
181
182#Count new words to be add to the main wordlist
183nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
184cat NewWords_$(date +%Y%m%d) >> $main_wordlist
185echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
186
187rm -rf NewWords_$(date +%Y%m%d)
188rm -rf newfile001
189rm -rf $partab
190rm -rf newfile002
191rm -rf output001
192
193else
194echo -e "\e[31;1mNo partab file here\e[0m"
195fi
196
197
198if [ -f "partac" ]
199then
200partac="partac"
201main_wordlist="/home/seifer/HackTools/BIG_wordlist"
202
203#Delete the lines with 25 characters or more
204echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
205LC_ALL=C sed -i -r "/.{25,}/d" $partac
206
207echo -e "\e[32;1mLooking for email addresses...\e[0m"
208#Delete the email addresses
209if LC_ALL=C fgrep --quiet @aol.com $partac; then
210echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
211LC_ALL=C sed -i '/@aol.com/d' $partac
212fi
213
214if LC_ALL=C fgrep --quiet @yahoo.fr $partac; then
215echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
216LC_ALL=C sed -i '/@yahoo.fr/d' $partac
217fi
218
219if LC_ALL=C fgrep --quiet @yahoo.com $partac; then
220echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
221LC_ALL=C sed -i '/@yahoo.com/d' $partac
222fi
223
224if LC_ALL=C fgrep --quiet @msn.fr $partac; then
225echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
226LC_ALL=C sed -i '/@msn.fr/d' $partac
227fi
228
229if LC_ALL=C fgrep --quiet @msn.com $partac; then
230echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
231LC_ALL=C sed -i '/@msn.com/d' $partac
232fi
233
234if LC_ALL=C fgrep --quiet @hotmail.com $partac; then
235echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
236LC_ALL=C sed -i '/hotmail.com/d' $partac
237fi
238
239if LC_ALL=C fgrep --quiet @hotmail.fr $partac; then
240echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
241LC_ALL=C sed -i '/@hotmail.fr/d' $partac
242fi
243
244if LC_ALL=C fgrep --quiet @gmail.com $partac; then
245echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
246LC_ALL=C sed -i '/@gmail.com/d' $partac
247fi
248
249#Delete the non-ASCII readable characters
250echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
251tr -cd '\11\12\15\40-\176' < $partac > newfile001
252
253#Remove both leading and trailing whitespaces
254LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partac
255
256#Delete the lines with 3 characters or less
257echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
258LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
259
260#Delete the blank lines
261echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
262LC_ALL=C sed -i '/^$/d' newfile001
263
264#Delete the duplicate lines
265echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
266LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
267
268#Compare if words already exist in the main wordlist
269LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
270
271cat output001 >> newfile002
272
273#Erase words already existing in main wordlist
274sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
275
276#Count new words to be add to the main wordlist
277nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
278cat NewWords_$(date +%Y%m%d) >> $main_wordlist
279echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
280
281rm -rf NewWords_$(date +%Y%m%d)
282rm -rf newfile001
283rm -rf $partac
284rm -rf newfile002
285rm -rf output001
286else
287echo -e "\e[31;1mNo partac file here\e[0m"
288fi
289
290
291
292if [ -f "partad" ]
293then
294partad="partad"
295main_wordlist="/home/seifer/HackTools/BIG_wordlist"
296
297#Delete the lines with 25 characters or more
298echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
299LC_ALL=C sed -i -r "/.{25,}/d" $partad
300
301echo -e "\e[32;1mLooking for email addresses...\e[0m"
302#Delete the email addresses
303if LC_ALL=C fgrep --quiet @aol.com $partad; then
304echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
305LC_ALL=C sed -i '/@aol.com/d' $partad
306fi
307
308if LC_ALL=C fgrep --quiet @yahoo.fr $partad; then
309echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
310LC_ALL=C sed -i '/@yahoo.fr/d' $partad
311fi
312
313if LC_ALL=C fgrep --quiet @yahoo.com $partad; then
314echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
315LC_ALL=C sed -i '/@yahoo.com/d' $partad
316fi
317
318if LC_ALL=C fgrep --quiet @msn.fr $partad; then
319echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
320LC_ALL=C sed -i '/@msn.fr/d' $partad
321fi
322
323if LC_ALL=C fgrep --quiet @msn.com $partad; then
324echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
325LC_ALL=C sed -i '/@msn.com/d' $partad
326fi
327
328if LC_ALL=C fgrep --quiet @hotmail.com $partad; then
329echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
330LC_ALL=C sed -i '/hotmail.com/d' $partad
331fi
332
333if LC_ALL=C fgrep --quiet @hotmail.fr $partad; then
334echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
335LC_ALL=C sed -i '/@hotmail.fr/d' $partad
336fi
337
338if LC_ALL=C fgrep --quiet @gmail.com $partad; then
339echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
340LC_ALL=C sed -i '/@gmail.com/d' $partad
341fi
342
343#Delete the non-ASCII readable characters
344echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
345tr -cd '\11\12\15\40-\176' < $partad > newfile001
346
347#Remove both leading and trailing whitespaces
348LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partad
349
350#Delete the lines with 3 characters or less
351echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
352LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
353
354#Delete the blank lines
355echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
356LC_ALL=C sed -i '/^$/d' newfile001
357
358#Delete the duplicate lines
359echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
360LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
361
362#Compare if words already exist in the main wordlist
363LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
364
365cat output001 >> newfile002
366
367#Erase words already existing in main wordlist
368sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
369
370#Count new words to be add to the main wordlist
371nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
372cat NewWords_$(date +%Y%m%d) >> $main_wordlist
373echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
374
375rm -rf NewWords_$(date +%Y%m%d)
376rm -rf newfile001
377rm -rf $partad
378rm -rf newfile002
379rm -rf output001
380else
381echo -e "\e[31;1mNo partad file here\e[0m"
382fi
383
384
385if [ -f "partae" ]
386then
387partae="partae"
388main_wordlist="/home/seifer/HackTools/BIG_wordlist"
389
390#Delete the lines with 25 characters or more
391echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
392LC_ALL=C sed -i -r "/.{25,}/d" $partae
393
394echo -e "\e[32;1mLooking for email addresses...\e[0m"
395#Delete the email addresses
396if LC_ALL=C fgrep --quiet @aol.com $partae; then
397echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
398LC_ALL=C sed -i '/@aol.com/d' $partae
399fi
400
401if LC_ALL=C fgrep --quiet @yahoo.fr $partae; then
402echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
403LC_ALL=C sed -i '/@yahoo.fr/d' $partae
404fi
405
406if LC_ALL=C fgrep --quiet @yahoo.com $partae; then
407echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
408LC_ALL=C sed -i '/@yahoo.com/d' $partae
409fi
410
411if LC_ALL=C fgrep --quiet @msn.fr $partae; then
412echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
413LC_ALL=C sed -i '/@msn.fr/d' $partae
414fi
415
416if LC_ALL=C fgrep --quiet @msn.com $partae; then
417echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
418LC_ALL=C sed -i '/@msn.com/d' $partae
419fi
420
421if LC_ALL=C fgrep --quiet @hotmail.com $partae; then
422echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
423LC_ALL=C sed -i '/hotmail.com/d' $partae
424fi
425
426if LC_ALL=C fgrep --quiet @hotmail.fr $partae; then
427echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
428LC_ALL=C sed -i '/@hotmail.fr/d' $partae
429fi
430
431if LC_ALL=C fgrep --quiet @gmail.com $partae; then
432echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
433LC_ALL=C sed -i '/@gmail.com/d' $partae
434fi
435
436#Delete the non-ASCII readable characters
437echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
438tr -cd '\11\12\15\40-\176' < $partae > newfile001
439
440#Remove both leading and trailing whitespaces
441LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partae
442
443#Delete the lines with 3 characters or less
444echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
445LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
446
447#Delete the blank lines
448echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
449LC_ALL=C sed -i '/^$/d' newfile001
450
451#Delete the duplicate lines
452echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
453LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
454
455#Compare if words already exist in the main wordlist
456LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
457
458cat output001 >> newfile002
459
460#Erase words already existing in main wordlist
461sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
462
463#Count new words to be add to the main wordlist
464nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
465cat NewWords_$(date +%Y%m%d) >> $main_wordlist
466echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
467
468rm -rf NewWords_$(date +%Y%m%d)
469rm -rf newfile001
470rm -rf $partae
471rm -rf newfile002
472rm -rf output001
473else
474echo -e "\e[31;1mNo partae file here\e[0m"
475fi
476
477
478
479if [ -f "partaf" ]
480then
481partaf="partaf"
482main_wordlist="/home/seifer/HackTools/BIG_wordlist"
483
484#Delete the lines with 25 characters or more
485echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
486LC_ALL=C sed -i -r "/.{25,}/d" $partaf
487
488echo -e "\e[32;1mLooking for email addresses...\e[0m"
489#Delete the email addresses
490if LC_ALL=C fgrep --quiet @aol.com $partaf; then
491echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
492LC_ALL=C sed -i '/@aol.com/d' $partaf
493fi
494
495if LC_ALL=C fgrep --quiet @yahoo.fr $partaf; then
496echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
497LC_ALL=C sed -i '/@yahoo.fr/d' $partaf
498fi
499
500if LC_ALL=C fgrep --quiet @yahoo.com $partaf; then
501echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
502LC_ALL=C sed -i '/@yahoo.com/d' $partaf
503fi
504
505if LC_ALL=C fgrep --quiet @msn.fr $partaf; then
506echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
507LC_ALL=C sed -i '/@msn.fr/d' $partaf
508fi
509
510if LC_ALL=C fgrep --quiet @msn.com $partaf; then
511echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
512LC_ALL=C sed -i '/@msn.com/d' $partaf
513fi
514
515if LC_ALL=C fgrep --quiet @hotmail.com $partaf; then
516echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
517LC_ALL=C sed -i '/hotmail.com/d' $partaf
518fi
519
520if LC_ALL=C fgrep --quiet @hotmail.fr $partaf; then
521echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
522LC_ALL=C sed -i '/@hotmail.fr/d' $partaf
523fi
524
525if LC_ALL=C fgrep --quiet @gmail.com $partaf; then
526echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
527LC_ALL=C sed -i '/@gmail.com/d' $partaf
528fi
529
530#Delete the non-ASCII readable characters
531echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
532tr -cd '\11\12\15\40-\176' < $partaf > newfile001
533
534#Remove both leading and trailing whitespaces
535LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partaf
536
537#Delete the lines with 3 characters or less
538echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
539LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
540
541#Delete the blank lines
542echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
543LC_ALL=C sed -i '/^$/d' newfile001
544
545#Delete the duplicate lines
546echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
547LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
548
549#Compare if words already exist in the main wordlist
550LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
551
552cat output001 >> newfile002
553
554#Erase words already existing in main wordlist
555sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
556
557#Count new words to be add to the main wordlist
558nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
559cat NewWords_$(date +%Y%m%d) >> $main_wordlist
560echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
561
562rm -rf NewWords_$(date +%Y%m%d)
563rm -rf newfile001
564rm -rf $partaf
565rm -rf newfile002
566rm -rf output001
567
568else
569echo -e "\e[31;1mNo partaf file here\e[0m"
570fi
571
572
573if [ -f "partag" ]
574then
575partag="partag"
576main_wordlist="/home/seifer/HackTools/BIG_wordlist"
577
578#Delete the lines with 25 characters or more
579echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
580LC_ALL=C sed -i -r "/.{25,}/d" $partag
581
582echo -e "\e[32;1mLooking for email addresses...\e[0m"
583#Delete the email addresses
584if LC_ALL=C fgrep --quiet @aol.com $partag; then
585echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
586LC_ALL=C sed -i '/@aol.com/d' $partag
587fi
588
589if LC_ALL=C fgrep --quiet @yahoo.fr $partag; then
590echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
591LC_ALL=C sed -i '/@yahoo.fr/d' $partag
592fi
593
594if LC_ALL=C fgrep --quiet @yahoo.com $partag; then
595echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
596LC_ALL=C sed -i '/@yahoo.com/d' $partag
597fi
598
599if LC_ALL=C fgrep --quiet @msn.fr $partag; then
600echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
601LC_ALL=C sed -i '/@msn.fr/d' $partag
602fi
603
604if LC_ALL=C fgrep --quiet @msn.com $partag; then
605echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
606LC_ALL=C sed -i '/@msn.com/d' $partag
607fi
608
609if LC_ALL=C fgrep --quiet @hotmail.com $partag; then
610echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
611LC_ALL=C sed -i '/hotmail.com/d' $partag
612fi
613
614if LC_ALL=C fgrep --quiet @hotmail.fr $partag; then
615echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
616LC_ALL=C sed -i '/@hotmail.fr/d' $partag
617fi
618
619if LC_ALL=C fgrep --quiet @gmail.com $partag; then
620echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
621LC_ALL=C sed -i '/@gmail.com/d' $partag
622fi
623
624#Delete the non-ASCII readable characters
625echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
626tr -cd '\11\12\15\40-\176' < $partag > newfile001
627
628#Remove both leading and trailing whitespaces
629LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partag
630
631#Delete the lines with 3 characters or less
632echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
633LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
634
635#Delete the blank lines
636echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
637LC_ALL=C sed -i '/^$/d' newfile001
638
639#Delete the duplicate lines
640echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
641LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
642
643#Compare if words already exist in the main wordlist
644LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
645
646cat output001 >> newfile002
647
648#Erase words already existing in main wordlist
649sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
650
651#Count new words to be add to the main wordlist
652nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
653cat NewWords_$(date +%Y%m%d) >> $main_wordlist
654echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
655
656rm -rf NewWords_$(date +%Y%m%d)
657rm -rf newfile001
658rm -rf $partag
659rm -rf newfile002
660rm -rf output001
661else
662echo -e "\e[31;1mNo partag file here\e[0m"
663fi
664
665
666if [ -f "partah" ]
667then
668partah="partah"
669main_wordlist="/home/seifer/HackTools/BIG_wordlist"
670
671#Delete the lines with 25 characters or more
672echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
673LC_ALL=C sed -i -r "/.{25,}/d" $partah
674
675echo -e "\e[32;1mLooking for email addresses...\e[0m"
676#Delete the email addresses
677if LC_ALL=C fgrep --quiet @aol.com $partah; then
678echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
679LC_ALL=C sed -i '/@aol.com/d' $partah
680fi
681
682if LC_ALL=C fgrep --quiet @yahoo.fr $partah; then
683echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
684LC_ALL=C sed -i '/@yahoo.fr/d' $partah
685fi
686
687if LC_ALL=C fgrep --quiet @yahoo.com $partah; then
688echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
689LC_ALL=C sed -i '/@yahoo.com/d' $partah
690fi
691
692if LC_ALL=C fgrep --quiet @msn.fr $partah; then
693echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
694LC_ALL=C sed -i '/@msn.fr/d' $partah
695fi
696
697if LC_ALL=C fgrep --quiet @msn.com $partah; then
698echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
699LC_ALL=C sed -i '/@msn.com/d' $partah
700fi
701
702if LC_ALL=C fgrep --quiet @hotmail.com $partah; then
703echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
704LC_ALL=C sed -i '/hotmail.com/d' $partah
705fi
706
707if LC_ALL=C fgrep --quiet @hotmail.fr $partah; then
708echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
709LC_ALL=C sed -i '/@hotmail.fr/d' $partah
710fi
711
712if LC_ALL=C fgrep --quiet @gmail.com $partah; then
713echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
714LC_ALL=C sed -i '/@gmail.com/d' $partah
715fi
716
717#Delete the non-ASCII readable characters
718echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
719tr -cd '\11\12\15\40-\176' < $partah > newfile001
720
721#Remove both leading and trailing whitespaces
722LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partah
723
724#Delete the lines with 3 characters or less
725echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
726LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
727
728#Delete the blank lines
729echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
730LC_ALL=C sed -i '/^$/d' newfile001
731
732#Delete the duplicate lines
733echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
734LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
735
736#Compare if words already exist in the main wordlist
737LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
738
739cat output001 >> newfile002
740
741#Erase words already existing in main wordlist
742sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
743
744#Count new words to be add to the main wordlist
745nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
746cat NewWords_$(date +%Y%m%d) >> $main_wordlist
747echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
748
749rm -rf NewWords_$(date +%Y%m%d)
750rm -rf newfile001
751rm -rf $partah
752rm -rf newfile002
753rm -rf output001
754
755else
756echo -e "\e[31;1mNo partah file here\e[0m"
757fi
758
759
760if [ -f "partai" ]
761then
762partai="partai"
763main_wordlist="/home/seifer/HackTools/BIG_wordlist"
764
765#Delete the lines with 25 characters or more
766echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
767LC_ALL=C sed -i -r "/.{25,}/d" $partai
768
769echo -e "\e[32;1mLooking for email addresses...\e[0m"
770#Delete the email addresses
771if LC_ALL=C fgrep --quiet @aol.com $partai; then
772echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
773LC_ALL=C sed -i '/@aol.com/d' $partai
774fi
775
776if LC_ALL=C fgrep --quiet @yahoo.fr $partai; then
777echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
778LC_ALL=C sed -i '/@yahoo.fr/d' $partai
779fi
780
781if LC_ALL=C fgrep --quiet @yahoo.com $partai; then
782echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
783LC_ALL=C sed -i '/@yahoo.com/d' $partai
784fi
785
786if LC_ALL=C fgrep --quiet @msn.fr $partai; then
787echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
788LC_ALL=C sed -i '/@msn.fr/d' $partai
789fi
790
791if LC_ALL=C fgrep --quiet @msn.com $partai; then
792echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
793LC_ALL=C sed -i '/@msn.com/d' $partai
794fi
795
796if LC_ALL=C fgrep --quiet @hotmail.com $partai; then
797echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
798LC_ALL=C sed -i '/hotmail.com/d' $partai
799fi
800
801if LC_ALL=C fgrep --quiet @hotmail.fr $partai; then
802echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
803LC_ALL=C sed -i '/@hotmail.fr/d' $partai
804fi
805
806if LC_ALL=C fgrep --quiet @gmail.com $partai; then
807echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
808LC_ALL=C sed -i '/@gmail.com/d' $partai
809fi
810
811#Delete the non-ASCII readable characters
812echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
813tr -cd '\11\12\15\40-\176' < $partai > newfile001
814
815#Remove both leading and trailing whitespaces
816LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partai
817
818#Delete the lines with 3 characters or less
819echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
820LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
821
822#Delete the blank lines
823echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
824LC_ALL=C sed -i '/^$/d' newfile001
825
826#Delete the duplicate lines
827echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
828LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
829
830#Compare if words already exist in the main wordlist
831LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
832
833cat output001 >> newfile002
834
835#Erase words already existing in main wordlist
836sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
837
838#Count new words to be add to the main wordlist
839nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
840cat NewWords_$(date +%Y%m%d) >> $main_wordlist
841echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
842
843rm -rf NewWords_$(date +%Y%m%d)
844rm -rf newfile001
845rm -rf $partai
846rm -rf newfile002
847rm -rf output001
848
849else
850echo -e "\e[31;1mNo partai file here\e[0m"
851fi
852
853
854if [ -f "partaj" ]
855then
856partaj="partaj"
857main_wordlist="/home/seifer/HackTools/BIG_wordlist"
858
859#Delete the lines with 25 characters or more
860echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
861LC_ALL=C sed -i -r "/.{25,}/d" $partaj
862
863echo -e "\e[32;1mLooking for email addresses...\e[0m"
864#Delete the email addresses
865if LC_ALL=C fgrep --quiet @aol.com $partaj; then
866echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
867LC_ALL=C sed -i '/@aol.com/d' $partaj
868fi
869
870if LC_ALL=C fgrep --quiet @yahoo.fr $partaj; then
871echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
872LC_ALL=C sed -i '/@yahoo.fr/d' $partaj
873fi
874
875if LC_ALL=C fgrep --quiet @yahoo.com $partaj; then
876echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
877LC_ALL=C sed -i '/@yahoo.com/d' $partaj
878fi
879
880if LC_ALL=C fgrep --quiet @msn.fr $partaj; then
881echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
882LC_ALL=C sed -i '/@msn.fr/d' $partaj
883fi
884
885if LC_ALL=C fgrep --quiet @msn.com $partaj; then
886echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
887LC_ALL=C sed -i '/@msn.com/d' $partaj
888fi
889
890if LC_ALL=C fgrep --quiet @hotmail.com $partaj; then
891echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
892LC_ALL=C sed -i '/hotmail.com/d' $partaj
893fi
894
895if LC_ALL=C fgrep --quiet @hotmail.fr $partaj; then
896echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
897LC_ALL=C sed -i '/@hotmail.fr/d' $partaj
898fi
899
900if LC_ALL=C fgrep --quiet @gmail.com $partaj; then
901echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
902LC_ALL=C sed -i '/@gmail.com/d' $partaj
903fi
904
905#Delete the non-ASCII readable characters
906echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
907tr -cd '\11\12\15\40-\176' < $partaj > newfile001
908
909#Remove both leading and trailing whitespaces
910LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partaj
911
912#Delete the lines with 3 characters or less
913echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
914LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
915
916#Delete the blank lines
917echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
918LC_ALL=C sed -i '/^$/d' newfile001
919
920#Delete the duplicate lines
921echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
922LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
923
924#Compare if words already exist in the main wordlist
925LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
926
927cat output001 >> newfile002
928
929#Erase words already existing in main wordlist
930sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
931
932#Count new words to be add to the main wordlist
933nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
934cat NewWords_$(date +%Y%m%d) >> $main_wordlist
935echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
936
937rm -rf NewWords_$(date +%Y%m%d)
938rm -rf newfile001
939rm -rf $partaj
940rm -rf newfile002
941rm -rf output001
942else
943echo -e "\e[31;1mNo partaj file here\e[0m"
944fi
945
946
947if [ -f "partak" ]
948then
949partak="partak"
950main_wordlist="/home/seifer/HackTools/BIG_wordlist"
951
952#Delete the lines with 25 characters or more
953echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
954LC_ALL=C sed -i -r "/.{25,}/d" $partak
955
956echo -e "\e[32;1mLooking for email addresses...\e[0m"
957#Delete the email addresses
958if LC_ALL=C fgrep --quiet @aol.com $partak; then
959echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
960LC_ALL=C sed -i '/@aol.com/d' $partak
961fi
962
963if LC_ALL=C fgrep --quiet @yahoo.fr $partak; then
964echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
965LC_ALL=C sed -i '/@yahoo.fr/d' $partak
966fi
967
968if LC_ALL=C fgrep --quiet @yahoo.com $partak; then
969echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
970LC_ALL=C sed -i '/@yahoo.com/d' $partak
971fi
972
973if LC_ALL=C fgrep --quiet @msn.fr $partak; then
974echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
975LC_ALL=C sed -i '/@msn.fr/d' $partak
976fi
977
978if LC_ALL=C fgrep --quiet @msn.com $partak; then
979echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
980LC_ALL=C sed -i '/@msn.com/d' $partak
981fi
982
983if LC_ALL=C fgrep --quiet @hotmail.com $partak; then
984echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
985LC_ALL=C sed -i '/hotmail.com/d' $partak
986fi
987
988if LC_ALL=C fgrep --quiet @hotmail.fr $partak; then
989echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
990LC_ALL=C sed -i '/@hotmail.fr/d' $partak
991fi
992
993if LC_ALL=C fgrep --quiet @gmail.com $partak; then
994echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
995LC_ALL=C sed -i '/@gmail.com/d' $partak
996fi
997
998#Delete the non-ASCII readable characters
999echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
1000tr -cd '\11\12\15\40-\176' < $partak > newfile001
1001
1002#Remove both leading and trailing whitespaces
1003LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partak
1004
1005#Delete the lines with 3 characters or less
1006echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
1007LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
1008
1009#Delete the blank lines
1010echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
1011LC_ALL=C sed -i '/^$/d' newfile001
1012
1013#Delete the duplicate lines
1014echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
1015LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
1016
1017#Compare if words already exist in the main wordlist
1018LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
1019
1020cat output001 >> newfile002
1021
1022#Erase words already existing in main wordlist
1023sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
1024
1025#Count new words to be add to the main wordlist
1026nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
1027cat NewWords_$(date +%Y%m%d) >> $main_wordlist
1028echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
1029
1030rm -rf NewWords_$(date +%Y%m%d)
1031rm -rf newfile001
1032rm -rf $partak
1033rm -rf newfile002
1034rm -rf output001
1035
1036else
1037echo -e "\e[31;1mNo partak file here\e[0m"
1038fi
1039
1040
1041if [ -f "partal" ]
1042then
1043partal="partal"
1044main_wordlist="/home/seifer/HackTools/BIG_wordlist"
1045
1046#Delete the lines with 25 characters or more
1047echo -e "\e[32;1mSuppression of 25 characters long lines or more...\e[0m"
1048LC_ALL=C sed -i -r "/.{25,}/d" $partal
1049
1050echo -e "\e[32;1mLooking for email addresses...\e[0m"
1051#Delete the email addresses
1052if LC_ALL=C fgrep --quiet @aol.com $partal; then
1053echo -e '\e[31;1mSuppresion of lines containing @aol.com...\e[0m'
1054LC_ALL=C sed -i '/@aol.com/d' $partal
1055fi
1056
1057if LC_ALL=C fgrep --quiet @yahoo.fr $partal; then
1058echo -e '\e[31;1mSuppresion of lines containing @yahoo.fr...\e[0m'
1059LC_ALL=C sed -i '/@yahoo.fr/d' $partal
1060fi
1061
1062if LC_ALL=C fgrep --quiet @yahoo.com $partal; then
1063echo -e '\e[31;1mSuppresion of lines containing @yahoo.com...\e[0m'
1064LC_ALL=C sed -i '/@yahoo.com/d' $partal
1065fi
1066
1067if LC_ALL=C fgrep --quiet @msn.fr $partal; then
1068echo -e '\e[31;1mSuppresion of lines containing @msn.fr...\e[0m'
1069LC_ALL=C sed -i '/@msn.fr/d' $partal
1070fi
1071
1072if LC_ALL=C fgrep --quiet @msn.com $partal; then
1073echo -e '\e[31;1mSuppresion of lines containing @msn.com...\e[0m'
1074LC_ALL=C sed -i '/@msn.com/d' $partal
1075fi
1076
1077if LC_ALL=C fgrep --quiet @hotmail.com $partal; then
1078echo -e '\e[31;1mSuppresion of lines containing @hotmail.com...\e[0m'
1079LC_ALL=C sed -i '/hotmail.com/d' $partal
1080fi
1081
1082if LC_ALL=C fgrep --quiet @hotmail.fr $partal; then
1083echo -e '\e[31;1mSuppresion of lines containing @hotmail.fr...\e[0m'
1084LC_ALL=C sed -i '/@hotmail.fr/d' $partal
1085fi
1086
1087if LC_ALL=C fgrep --quiet @gmail.com $partal; then
1088echo -e '\e[31;1mSuppresion of lines containing @gmail.com...\e[0m'
1089LC_ALL=C sed -i '/@gmail.com/d' $partal
1090fi
1091
1092#Delete the non-ASCII readable characters
1093echo -e "\e[32;1mSuppression of non-ASCII readable characters...\e[0m"
1094tr -cd '\11\12\15\40-\176' < $partal > newfile001
1095
1096#Remove both leading and trailing whitespaces
1097LC_ALL=C sed -i 's/^[\t ]*//g;s/[\t ]*$//g' $partal
1098
1099#Delete the lines with 3 characters or less
1100echo -e "\e[32;1mSuppression of 3 characters long lines or less...\e[0m"
1101LC_ALL=C sed -i -r '/^.{,3}$/d' newfile001
1102
1103#Delete the blank lines
1104echo -e "\e[32;1mSuppression of all blank lines...\e[0m"
1105LC_ALL=C sed -i '/^$/d' newfile001
1106
1107#Delete the duplicate lines
1108echo -e "\e[32;1mSuppression of all duplicate lines...\e[0m"
1109LC_ALL=C awk '!(count[$0]++)' newfile001 > newfile002
1110
1111#Compare if words already exist in the main wordlist
1112LC_ALL=C fgrep -x -f newfile002 $main_wordlist > output001
1113
1114cat output001 >> newfile002
1115
1116#Erase words already existing in main wordlist
1117sort newfile002 | uniq -u > NewWords_$(date +%Y%m%d)
1118
1119#Count new words to be add to the main wordlist
1120nb_newwords=$(< "NewWords_$(date +%Y%m%d)" wc -l)
1121cat NewWords_$(date +%Y%m%d) >> $main_wordlist
1122echo -e "\e[31;1m $nb_newwords new words have been added to your main wordlist\e[0m"
1123
1124rm -rf NewWords_$(date +%Y%m%d)
1125rm -rf newfile001
1126rm -rf $partal
1127rm -rf newfile002
1128rm -rf output001
1129else
1130echo -e "\e[31;1mNo partal file here\e[0m"
1131fi