kpqAFfXw

1package SquidAnalyzer;
2#------------------------------------------------------------------------------
3# Project  : Squid Log Analyzer
4# Name     : SquidAnalyzer.pm
5# Language : Perl 5
6# OS       : All
7# Copyright: Copyright (c) 2001-2016 Gilles Darold - All rights reserved.
8# Licence  : This program is free software; you can redistribute it
9#            and/or modify it under the same terms as Perl itself.
10# Author   : Gilles Darold, gilles _AT_ darold _DOT_ net
11# Function : Main perl module for Squid Log Analyzer
12# Usage    : See documentation.
13#------------------------------------------------------------------------------
14use strict qw/vars/;
15
16BEGIN {
17	use Exporter();
18	use vars qw($VERSION $COPYRIGHT $AUTHOR @ISA @EXPORT $ZCAT_PROG $BZCAT_PROG $XZCAT_PROG $RM_PROG);
19	use POSIX qw/ strftime sys_wait_h /;
20	use IO::File;
21	use Socket;
22	use Time::HiRes qw/ualarm/;
23	use Time::Local qw/timelocal_nocheck timegm_nocheck/;
24	use Fcntl qw(:flock);
25	use IO::Handle;
26	use FileHandle;
27	use POSIX qw(locale_h);
28	setlocale(LC_NUMERIC, '');
29	setlocale(LC_ALL,     'C');
30
31	# Set all internal variable
32	$VERSION = '6.5';
33	$COPYRIGHT = 'Copyright (c) 2001-2016 Gilles Darold - All rights reserved.';
34	$AUTHOR = "Gilles Darold - gilles _AT_ darold _DOT_ net";
35
36	@ISA = qw(Exporter);
37	@EXPORT = qw//;
38
39	$| = 1;
40
41}
42
43$ZCAT_PROG  = "/bin/zcat";
44$BZCAT_PROG = "/bin/bzcat";
45$RM_PROG    = "/bin/rm";
46$XZCAT_PROG = "/bin/xzcat";
47
48# DNS Cache
49my %CACHE = ();
50
51# Color used to draw grpahs
52my @GRAPH_COLORS = ('#6e9dc9', '#f4ab3a', '#ac7fa8', '#8dbd0f');
53
54# Default translation srings
55my %Translate = (
56	'CharSet' => 'utf-8',
57	'01' => 'Jan',
58	'02' => 'Feb',
59	'03' => 'Mar',
60	'04' => 'Apr',
61	'05' => 'May',
62	'06' => 'Jun',
63	'07' => 'Jul',
64	'08' => 'Aug',
65	'09' => 'Sep',
66	'10' => 'Oct',
67	'11' => 'Nov',
68	'12' => 'Dec',
69	'KB' => 'Kilo bytes',
70	'MB' => 'Mega bytes',
71	'GB' => 'Giga bytes',
72	'Bytes' => 'Bytes',
73	'Total' => 'Total',
74	'Years' => 'Years',
75	'Users' => 'Users',
76	'Sites' => 'Sites',
77	'Cost' => 'Cost',
78	'Requests' => 'Requests',
79	'Megabytes' => 'Mega bytes',
80	'Months' => 'Months',
81	'Days' => 'Days',
82	'Hit' => 'Hit',
83	'Miss' => 'Miss',
84	'Denied' => 'Denied',
85	'Domains' => 'Domains',
86	'Requests_graph' => 'Requests',
87	'Megabytes_graph' => 'Mega bytes',
88	'Months_graph' => 'Months',
89	'Days_graph' => 'Days',
90	'Hit_graph' => 'Hit',
91	'Miss_graph' => 'Miss',
92	'Denied_graph' => 'Denied',
93	'Total_graph' => 'Total',
94	'Domains_graph' => 'Domains',
95	'Users_help' => 'Total number of different users for this period',
96	'Sites_help' => 'Total number of different visited sites for this period',
97	'Domains_help' => 'Total number of different second level visited domain for this period',
98	'Hit_help' => 'Objects found in cache',
99	'Miss_help' => 'Objects not found in cache',
100	'Denied_help' => 'Objects with denied access',
101	'Cost_help' => '1 Mega byte =',
102	'Generation' => 'Report generated on',
103	'Main_cache_title' => 'Cache Statistics',
104	'Cache_title' => 'Cache Statistics on',
105	'Stat_label' => 'Stat',
106	'Mime_link' => 'Mime Types',
107	'Network_link' => 'Networks',
108	'User_link' => 'Users',
109	'Top_url_link' => 'Top Urls',
110	'Top_domain_link' => 'Top Domains',
111	'Back_link' => 'Back',
112	'Graph_cache_hit_title' => '%s Requests statistics on',
113	'Graph_cache_byte_title' => '%s Mega Bytes statistics on',
114	'Hourly' => 'Hourly',
115	'Hours' => 'Hours',
116	'Daily' => 'Daily',
117	'Days' => 'Days',
118	'Monthly' => 'Monthly',
119	'Months' => 'Months',
120	'Mime_title' => 'Mime Type Statistics on',
121	'Mime_number' => 'Number of mime type',
122	'Network_title' => 'Network Statistics on',
123	'Network_number' => 'Number of network',
124	'Duration' => 'Duration',
125	'Time' => 'Time',
126	'Largest' => 'Largest',
127	'Url' => 'Url',
128	'User_title' => 'User Statistics on',
129	'User_number' => 'Number of user',
130	'Url_Hits_title' => 'Top %d Url hits on',
131	'Url_Bytes_title' => 'Top %d Url bytes on',
132	'Url_Duration_title' => 'Top %d Url duration on',
133	'Url_number' => 'Number of Url',
134	'Domain_Hits_title' => 'Top %d Domain hits on',
135	'Domain_Bytes_title' => 'Top %d Domain bytes on',
136	'Domain_Duration_title' => 'Top %d Domain duration on',
137	'Domain_number' => 'Number of domain',
138	'Domain_graph_hits_title' => 'Domain Hits Statistics on',
139	'Domain_graph_bytes_title' => 'Domain Bytes Statistiques on',
140	'Second_domain_graph_hits_title' => 'Second level Hits Statistics on',
141	'Second_domain_graph_bytes_title' => 'Second level Bytes Statistiques on',
142	'First_visit' => 'First visit',
143	'Last_visit' => 'Last visit',
144	'Globals_Statistics' => 'Globals Statistics',
145	'Legend' => 'Legend',
146	'File_Generated' => 'File generated by',
147	'Up_link' => 'Up',
148	'Click_year_stat' => 'Click on year\'s statistics link for details',
149	'Mime_graph_hits_title' => 'Mime Type Hits Statistics on',
150	'Mime_graph_bytes_title' => 'Mime Type MBytes Statistiques on',
151	'User' => 'User',
152	'Count' => 'Count',
153	'WeekDay' => 'Su Mo Tu We Th Fr Sa',
154	'Week' => 'Week',
155	'Top_denied_link' => 'Top Denied',
156	'Blocklist_acl_title' => 'Blocklist ACL use',
157	'Throughput' => 'Throughput',
158	'Graph_throughput_title' => '%s throughput on',
159	'Throughput_graph' => 'Bytes/sec',
160);
161
162my @TLD1 = (
163	'\.com\.ac','\.net\.ac','\.gov\.ac','\.org\.ac','\.mil\.ac','\.co\.ae',
164	'\.net\.ae','\.gov\.ae','\.ac\.ae','\.sch\.ae','\.org\.ae','\.mil\.ae','\.pro\.ae',
165	'\.name\.ae','\.com\.af','\.edu\.af','\.gov\.af','\.net\.af','\.org\.af','\.com\.al',
166	'\.edu\.al','\.gov\.al','\.mil\.al','\.net\.al','\.org\.al','\.ed\.ao','\.gv\.ao',
167	'\.og\.ao','\.co\.ao','\.pb\.ao','\.it\.ao','\.com\.ar','\.edu\.ar','\.gob\.ar',
168	'\.gov\.ar','\.gov\.ar','\.int\.ar','\.mil\.ar','\.net\.ar','\.org\.ar','\.tur\.ar',
169	'\.gv\.at','\.ac\.at','\.co\.at','\.or\.at','\.com\.au','\.net\.au','\.org\.au',
170	'\.edu\.au','\.gov\.au','\.csiro\.au','\.asn\.au','\.id\.au','\.org\.ba','\.net\.ba',
171	'\.edu\.ba','\.gov\.ba','\.mil\.ba','\.unsa\.ba','\.untz\.ba','\.unmo\.ba','\.unbi\.ba',
172	'\.unze\.ba','\.co\.ba','\.com\.ba','\.rs\.ba','\.co\.bb','\.com\.bb','\.net\.bb',
173	'\.org\.bb','\.gov\.bb','\.edu\.bb','\.info\.bb','\.store\.bb','\.tv\.bb','\.biz\.bb',
174	'\.com\.bh','\.info\.bh','\.cc\.bh','\.edu\.bh','\.biz\.bh','\.net\.bh','\.org\.bh',
175	'\.gov\.bh','\.com\.bn','\.edu\.bn','\.gov\.bn','\.net\.bn','\.org\.bn','\.com\.bo',
176	'\.net\.bo','\.org\.bo','\.tv\.bo','\.mil\.bo','\.int\.bo','\.gob\.bo','\.gov\.bo',
177	'\.edu\.bo','\.adm\.br','\.adv\.br','\.agr\.br','\.am\.br','\.arq\.br','\.art\.br',
178	'\.ato\.br','\.b\.br','\.bio\.br','\.blog\.br','\.bmd\.br','\.cim\.br','\.cng\.br',
179	'\.cnt\.br','\.com\.br','\.coop\.br','\.ecn\.br','\.edu\.br','\.eng\.br','\.esp\.br',
180	'\.etc\.br','\.eti\.br','\.far\.br','\.flog\.br','\.fm\.br','\.fnd\.br','\.fot\.br',
181	'\.fst\.br','\.g12\.br','\.ggf\.br','\.gov\.br','\.imb\.br','\.ind\.br','\.inf\.br',
182	'\.jor\.br','\.jus\.br','\.lel\.br','\.mat\.br','\.med\.br','\.mil\.br','\.mus\.br',
183	'\.net\.br','\.nom\.br','\.not\.br','\.ntr\.br','\.odo\.br','\.org\.br','\.ppg\.br',
184	'\.pro\.br','\.psc\.br','\.psi\.br','\.qsl\.br','\.rec\.br','\.slg\.br','\.srv\.br',
185	'\.tmp\.br','\.trd\.br','\.tur\.br','\.tv\.br','\.vet\.br','\.vlog\.br','\.wiki\.br',
186	'\.zlg\.br','\.com\.bs','\.net\.bs','\.org\.bs','\.edu\.bs','\.gov\.bs','com\.bz',
187	'edu\.bz','gov\.bz','net\.bz','org\.bz','\.ab\.ca','\.bc\.ca','\.mb\.ca','\.nb\.ca',
188	'\.nf\.ca','\.nl\.ca','\.ns\.ca','\.nt\.ca','\.nu\.ca','\.on\.ca','\.pe\.ca','\.qc\.ca',
189	'\.sk\.ca','\.yk\.ca','\.co\.ck','\.org\.ck','\.edu\.ck','\.gov\.ck','\.net\.ck',
190	'\.gen\.ck','\.biz\.ck','\.info\.ck','\.ac\.cn','\.com\.cn','\.edu\.cn','\.gov\.cn',
191	'\.mil\.cn','\.net\.cn','\.org\.cn','\.ah\.cn','\.bj\.cn','\.cq\.cn','\.fj\.cn','\.gd\.cn',
192	'\.gs\.cn','\.gz\.cn','\.gx\.cn','\.ha\.cn','\.hb\.cn','\.he\.cn','\.hi\.cn','\.hl\.cn',
193	'\.hn\.cn','\.jl\.cn','\.js\.cn','\.jx\.cn','\.ln\.cn','\.nm\.cn','\.nx\.cn','\.qh\.cn',
194	'\.sc\.cn','\.sd\.cn','\.sh\.cn','\.sn\.cn','\.sx\.cn','\.tj\.cn','\.tw\.cn','\.xj\.cn',
195	'\.xz\.cn','\.yn\.cn','\.zj\.cn','\.com\.co','\.org\.co','\.edu\.co','\.gov\.co',
196	'\.net\.co','\.mil\.co','\.nom\.co','\.ac\.cr','\.co\.cr','\.ed\.cr','\.fi\.cr','\.go\.cr',
197	'\.com\.cu','\.edu\.cu','\.gov\.cu','\.net\.cu','\.org\.cu',
198	'\.or\.cr','\.sa\.cr','\.cr','\.ac\.cy','\.net\.cy','\.gov\.cy','\.org\.cy',
199	'\.pro\.cy','\.name\.cy','\.ekloges\.cy','\.tm\.cy','\.ltd\.cy','\.biz\.cy','\.press\.cy',
200	'\.parliament\.cy','\.com\.cy','\.edu\.do','\.gob\.do','\.gov\.do','\.com\.do','\.sld\.do',
201	'\.org\.do','\.net\.do','\.web\.do','\.mil\.do','\.art\.do','\.com\.dz','\.org\.dz',
202	'\.net\.dz','\.gov\.dz','\.edu\.dz','\.asso\.dz','\.pol\.dz','\.art\.dz','\.com\.ec',
203	'\.info\.ec','\.net\.ec','\.fin\.ec','\.med\.ec','\.pro\.ec','\.org\.ec','\.edu\.ec',
204	'\.gov\.ec','\.mil\.ec','\.com\.eg','\.edu\.eg','\.eun\.eg','\.gov\.eg','\.mil\.eg',
205	'\.name\.eg','\.net\.eg','\.org\.eg','\.sci\.eg','\.com\.er','\.edu\.er','\.gov\.er',
206	'\.mil\.er','\.net\.er','\.org\.er','\.ind\.er','\.rochest\.er','\.w\.er','\.com\.es',
207	'\.nom\.es','\.org\.es','\.gob\.es','\.edu\.es','\.com\.et','\.gov\.et','\.org\.et',
208	'\.edu\.et','\.net\.et','\.biz\.et','\.name\.et','\.info\.et','\.ac\.fj','\.biz\.fj',
209	'\.com\.fj','\.info\.fj','\.mil\.fj','\.name\.fj','\.net\.fj','\.org\.fj','\.pro\.fj',
210	'\.co\.fk','\.org\.fk','\.gov\.fk','\.ac\.fk','\.nom\.fk','\.net\.fk','\.fr','\.tm\.fr',
211	'\.asso\.fr','\.nom\.fr','\.prd\.fr','\.presse\.fr','\.com\.fr','\.gouv\.fr','\.co\.gg',
212	'\.net\.gg','\.org\.gg','\.com\.gh','\.edu\.gh','\.gov\.gh','\.org\.gh','\.mil\.gh',
213	'\.com\.gn','\.ac\.gn','\.gov\.gn','\.org\.gn','\.net\.gn','\.com\.gr','\.edu\.gr','\.net\.gr',
214	'\.org\.gr','\.gov\.gr','\.mil\.gr','\.com\.gt','\.edu\.gt','\.net\.gt','\.gob\.gt',
215	'\.org\.gt','\.mil\.gt','\.ind\.gt','\.com\.gu','\.net\.gu','\.gov\.gu','\.org\.gu','\.edu\.gu',
216	'\.com\.hk','\.edu\.hk','\.gov\.hk','\.idv\.hk','\.net\.hk','\.org\.hk','\.ac\.id','\.co\.id',
217	'\.net\.id','\.or\.id','\.web\.id','\.sch\.id','\.mil\.id','\.go\.id','\.war\.net\.id','\.ac\.il',
218	'\.co\.il','\.org\.il','\.net\.il','\.k12\.il','\.gov\.il','\.muni\.il','\.idf\.il','\.in',
219	'\.co\.in','\.firm\.in','\.net\.in','\.org\.in','\.gen\.in','\.ind\.in','\.ac\.in','\.edu\.in',
220	'\.res\.in','\.ernet\.in','\.gov\.in','\.mil\.in','\.nic\.in','\.nic\.in','\.iq','\.gov\.iq',
221	'\.edu\.iq','\.com\.iq','\.mil\.iq','\.org\.iq','\.net\.iq','\.ir','\.ac\.ir','\.co\.ir',
222	'\.gov\.ir','\.id\.ir','\.net\.ir','\.org\.ir','\.sch\.ir','\.dnssec\.ir','\.gov\.it',
223	'\.edu\.it','\.co\.je','\.net\.je','\.org\.je','\.com\.jo','\.net\.jo','\.gov\.jo','\.edu\.jo',
224	'\.org\.jo','\.mil\.jo','\.name\.jo','\.sch\.jo','\.ac\.jp','\.ad\.jp','\.co\.jp','\.ed\.jp',
225	'\.go\.jp','\.gr\.jp','\.lg\.jp','\.ne\.jp','\.or\.jp','\.co\.ke','\.or\.ke','\.ne\.ke','\.go\.ke',
226	'\.ac\.ke','\.sc\.ke','\.me\.ke','\.mobi\.ke','\.info\.ke','\.per\.kh','\.com\.kh','\.edu\.kh',
227	'\.gov\.kh','\.mil\.kh','\.net\.kh','\.org\.kh','\.com\.ki','\.biz\.ki','\.de\.ki','\.net\.ki',
228	'\.info\.ki','\.org\.ki','\.gov\.ki','\.edu\.ki','\.mob\.ki','\.tel\.ki','\.km','\.com\.km',
229	'\.coop\.km','\.asso\.km','\.nom\.km','\.presse\.km','\.tm\.km','\.medecin\.km','\.notaires\.km',
230	'\.pharmaciens\.km','\.veterinaire\.km','\.edu\.km','\.gouv\.km','\.mil\.km','\.net\.kn',
231	'\.org\.kn','\.edu\.kn','\.gov\.kn','\.kr','\.co\.kr','\.ne\.kr','\.or\.kr','\.re\.kr','\.pe\.kr',
232	'\.go\.kr','\.mil\.kr','\.ac\.kr','\.hs\.kr','\.ms\.kr','\.es\.kr','\.sc\.kr','\.kg\.kr',
233	'\.seoul\.kr','\.busan\.kr','\.daegu\.kr','\.incheon\.kr','\.gwangju\.kr','\.daejeon\.kr',
234	'\.ulsan\.kr','\.gyeonggi\.kr','\.gangwon\.kr','\.chungbuk\.kr','\.chungnam\.kr','\.jeonbuk\.kr',
235	'\.jeonnam\.kr','\.gyeongbuk\.kr','\.gyeongnam\.kr','\.jeju\.kr','\.edu\.kw','\.com\.kw',
236	'\.net\.kw','\.org\.kw','\.gov\.kw','\.com\.ky','\.org\.ky','\.net\.ky','\.edu\.ky','\.gov\.ky',
237	'\.com\.kz','\.edu\.kz','\.gov\.kz','\.mil\.kz','\.net\.kz','\.org\.kz','\.com\.lb','\.edu\.lb',
238	'\.gov\.lb','\.net\.lb','\.org\.lb','\.gov\.lk','\.sch\.lk','\.net\.lk','\.int\.lk','\.com\.lk',
239	'\.org\.lk','\.edu\.lk','\.ngo\.lk','\.soc\.lk','\.web\.lk','\.ltd\.lk','\.assn\.lk','\.grp\.lk',
240	'\.hotel\.lk','\.com\.lr','\.edu\.lr','\.gov\.lr','\.org\.lr','\.net\.lr','\.com\.lv','\.edu\.lv',
241	'\.gov\.lv','\.org\.lv','\.mil\.lv','\.id\.lv','\.net\.lv','\.asn\.lv','\.conf\.lv','\.com\.ly',
242	'\.net\.ly','\.gov\.ly','\.plc\.ly','\.edu\.ly','\.sch\.ly','\.med\.ly','\.org\.ly','\.id\.ly',
243	'\.ma','\.net\.ma','\.ac\.ma','\.org\.ma','\.gov\.ma','\.press\.ma','\.co\.ma','\.tm\.mc',
244	'\.asso\.mc','\.co\.me','\.net\.me','\.org\.me','\.edu\.me','\.ac\.me','\.gov\.me','\.its\.me',
245	'\.priv\.me','\.org\.mg','\.nom\.mg','\.gov\.mg','\.prd\.mg','\.tm\.mg','\.edu\.mg','\.mil\.mg',
246	'\.com\.mg','\.com\.mk','\.org\.mk','\.net\.mk','\.edu\.mk','\.gov\.mk','\.inf\.mk','\.name\.mk',
247	'\.pro\.mk','\.com\.ml','\.net\.ml','\.org\.ml','\.edu\.ml','\.gov\.ml','\.presse\.ml','\.gov\.mn',
248	'\.edu\.mn','\.org\.mn','\.com\.mo','\.edu\.mo','\.gov\.mo','\.net\.mo','\.org\.mo','\.com\.mt',
249	'\.org\.mt','\.net\.mt','\.edu\.mt','\.gov\.mt','\.aero\.mv','\.biz\.mv','\.com\.mv','\.coop\.mv',
250	'\.edu\.mv','\.gov\.mv','\.info\.mv','\.int\.mv','\.mil\.mv','\.museum\.mv','\.name\.mv','\.net\.mv',
251	'\.org\.mv','\.pro\.mv','\.ac\.mw','\.co\.mw','\.com\.mw','\.coop\.mw','\.edu\.mw','\.gov\.mw',
252	'\.int\.mw','\.museum\.mw','\.net\.mw','\.org\.mw','\.com\.mx','\.net\.mx','\.org\.mx','\.edu\.mx',
253	'\.gob\.mx','\.com\.my','\.net\.my','\.org\.my','\.gov\.my','\.edu\.my','\.sch\.my','\.mil\.my',
254	'\.name\.my','\.com\.nf','\.net\.nf','\.arts\.nf','\.store\.nf','\.web\.nf','\.firm\.nf',
255	'\.info\.nf','\.other\.nf','\.per\.nf','\.rec\.nf','\.com\.ng','\.org\.ng','\.gov\.ng','\.edu\.ng',
256	'\.net\.ng','\.sch\.ng','\.name\.ng','\.mobi\.ng','\.biz\.ng','\.mil\.ng','\.gob\.ni','\.co\.ni',
257	'\.com\.ni','\.ac\.ni','\.edu\.ni','\.org\.ni','\.nom\.ni','\.net\.ni','\.mil\.ni','\.com\.np',
258	'\.edu\.np','\.gov\.np','\.org\.np','\.mil\.np','\.net\.np','\.edu\.nr','\.gov\.nr','\.biz\.nr',
259	'\.info\.nr','\.net\.nr','\.org\.nr','\.com\.nr','\.com\.om','\.co\.om','\.edu\.om','\.ac\.om',
260	'\.sch\.om','\.gov\.om','\.net\.om','\.org\.om','\.mil\.om','\.museum\.om','\.biz\.om','\.pro\.om',
261	'\.med\.om','\.edu\.pe','\.gob\.pe','\.nom\.pe','\.mil\.pe','\.sld\.pe','\.org\.pe','\.com\.pe',
262	'\.net\.pe','\.com\.ph','\.net\.ph','\.org\.ph','\.mil\.ph','\.ngo\.ph','\.i\.ph','\.gov\.ph',
263	'\.edu\.ph','\.com\.pk','\.net\.pk','\.edu\.pk','\.org\.pk','\.fam\.pk','\.biz\.pk','\.web\.pk',
264	'\.gov\.pk','\.gob\.pk','\.gok\.pk','\.gon\.pk','\.gop\.pk','\.gos\.pk','\.pwr\.pl','\.com\.pl',
265	'\.biz\.pl','\.net\.pl','\.art\.pl','\.edu\.pl','\.org\.pl','\.ngo\.pl','\.gov\.pl','\.info\.pl',
266	'\.mil\.pl','\.waw\.pl','\.warszawa\.pl','\.wroc\.pl','\.wroclaw\.pl','\.krakow\.pl','\.katowice\.pl',
267	'\.poznan\.pl','\.lodz\.pl','\.gda\.pl','\.gdansk\.pl','\.slupsk\.pl','\.radom\.pl','\.szczecin\.pl',
268	'\.lublin\.pl','\.bialystok\.pl','\.olsztyn\.pl','\.torun\.pl','\.gorzow\.pl','\.zgora\.pl',
269	'\.biz\.pr','\.com\.pr','\.edu\.pr','\.gov\.pr','\.info\.pr','\.isla\.pr','\.name\.pr','\.net\.pr',
270	'\.org\.pr','\.pro\.pr','\.est\.pr','\.prof\.pr','\.ac\.pr','\.com\.ps','\.net\.ps','\.org\.ps',
271	'\.edu\.ps','\.gov\.ps','\.plo\.ps','\.sec\.ps','\.co\.pw','\.ne\.pw','\.or\.pw','\.ed\.pw','\.go\.pw',
272	'\.belau\.pw','\.arts\.ro','\.com\.ro','\.firm\.ro','\.info\.ro','\.nom\.ro','\.nt\.ro','\.org\.ro',
273	'\.rec\.ro','\.store\.ro','\.tm\.ro','\.www\.ro','\.co\.rs','\.org\.rs','\.edu\.rs','\.ac\.rs',
274	'\.gov\.rs','\.in\.rs','\.com\.sb','\.net\.sb','\.edu\.sb','\.org\.sb','\.gov\.sb','\.com\.sc',
275	'\.net\.sc','\.edu\.sc','\.gov\.sc','\.org\.sc','\.co\.sh','\.com\.sh','\.org\.sh','\.gov\.sh',
276	'\.edu\.sh','\.net\.sh','\.nom\.sh','\.com\.sl','\.net\.sl','\.org\.sl','\.edu\.sl','\.gov\.sl',
277	'\.gov\.st','\.saotome\.st','\.principe\.st','\.consulado\.st','\.embaixada\.st','\.org\.st',
278	'\.edu\.st','\.net\.st','\.com\.st','\.store\.st','\.mil\.st','\.co\.st','\.edu\.sv','\.gob\.sv',
279	'\.com\.sv','\.org\.sv','\.red\.sv','\.co\.sz','\.ac\.sz','\.org\.sz','\.com\.tr','\.gen\.tr',
280	'\.org\.tr','\.biz\.tr','\.info\.tr','\.av\.tr','\.dr\.tr','\.pol\.tr','\.bel\.tr','\.tsk\.tr',
281	'\.bbs\.tr','\.k12\.tr','\.edu\.tr','\.name\.tr','\.net\.tr','\.gov\.tr','\.web\.tr','\.tel\.tr',
282	'\.tv\.tr','\.co\.tt','\.com\.tt','\.org\.tt','\.net\.tt','\.biz\.tt','\.info\.tt','\.pro\.tt',
283	'\.int\.tt','\.coop\.tt','\.jobs\.tt','\.mobi\.tt','\.travel\.tt','\.museum\.tt','\.aero\.tt',
284	'\.cat\.tt','\.tel\.tt','\.name\.tt','\.mil\.tt','\.edu\.tt','\.gov\.tt','\.edu\.tw','\.gov\.tw',
285	'\.mil\.tw','\.com\.tw','\.net\.tw','\.org\.tw','\.idv\.tw','\.game\.tw','\.ebiz\.tw','\.club\.tw',
286	'\.com\.mu','\.gov\.mu','\.net\.mu','\.org\.mu','\.ac\.mu','\.co\.mu','\.or\.mu','\.ac\.mz',
287	'\.co\.mz','\.edu\.mz','\.org\.mz','\.gov\.mz','\.com\.na','\.co\.na','\.ac\.nz','\.co\.nz',
288	'\.cri\.nz','\.geek\.nz','\.gen\.nz','\.govt\.nz','\.health\.nz','\.iwi\.nz','\.maori\.nz',
289	'\.mil\.nz','\.net\.nz','\.org\.nz','\.parliament\.nz','\.school\.nz','\.abo\.pa','\.ac\.pa',
290	'\.com\.pa','\.edu\.pa','\.gob\.pa','\.ing\.pa','\.med\.pa','\.net\.pa','\.nom\.pa','\.org\.pa',
291	'\.sld\.pa','\.com\.pt','\.edu\.pt','\.gov\.pt','\.int\.pt','\.net\.pt','\.nome\.pt','\.org\.pt',
292	'\.publ\.pt','\.com\.py','\.edu\.py','\.gov\.py','\.mil\.py','\.net\.py','\.org\.py','\.com\.qa',
293	'\.edu\.qa','\.gov\.qa','\.mil\.qa','\.net\.qa','\.org\.qa','\.asso\.re','\.com\.re','\.nom\.re',
294	'\.ac\.ru','\.adygeya\.ru','\.altai\.ru','\.amur\.ru','\.arkhangelsk\.ru','\.astrakhan\.ru',
295	'\.bashkiria\.ru','\.belgorod\.ru','\.bir\.ru','\.bryansk\.ru','\.buryatia\.ru','\.cbg\.ru',
296	'\.chel\.ru','\.chelyabinsk\.ru','\.chita\.ru','\.chita\.ru','\.chukotka\.ru','\.chuvashia\.ru',
297	'\.com\.ru','\.dagestan\.ru','\.e-burg\.ru','\.edu\.ru','\.gov\.ru','\.grozny\.ru','\.int\.ru',
298	'\.irkutsk\.ru','\.ivanovo\.ru','\.izhevsk\.ru','\.jar\.ru','\.joshkar-ola\.ru','\.kalmykia\.ru',
299	'\.kaluga\.ru','\.kamchatka\.ru','\.karelia\.ru','\.kazan\.ru','\.kchr\.ru','\.kemerovo\.ru',
300	'\.khabarovsk\.ru','\.khakassia\.ru','\.khv\.ru','\.kirov\.ru','\.koenig\.ru','\.komi\.ru',
301	'\.kostroma\.ru','\.kranoyarsk\.ru','\.kuban\.ru','\.kurgan\.ru','\.kursk\.ru','\.lipetsk\.ru',
302	'\.magadan\.ru','\.mari\.ru','\.mari-el\.ru','\.marine\.ru','\.mil\.ru','\.mordovia\.ru',
303	'\.mosreg\.ru','\.msk\.ru','\.murmansk\.ru','\.nalchik\.ru','\.net\.ru','\.nnov\.ru','\.nov\.ru',
304	'\.novosibirsk\.ru','\.nsk\.ru','\.omsk\.ru','\.orenburg\.ru','\.org\.ru','\.oryol\.ru','\.penza\.ru',
305	'\.perm\.ru','\.pp\.ru','\.pskov\.ru','\.ptz\.ru','\.rnd\.ru','\.ryazan\.ru','\.sakhalin\.ru','\.samara\.ru',
306	'\.saratov\.ru','\.simbirsk\.ru','\.smolensk\.ru','\.spb\.ru','\.stavropol\.ru','\.stv\.ru',
307	'\.surgut\.ru','\.tambov\.ru','\.tatarstan\.ru','\.tom\.ru','\.tomsk\.ru','\.tsaritsyn\.ru',
308	'\.tsk\.ru','\.tula\.ru','\.tuva\.ru','\.tver\.ru','\.tyumen\.ru','\.udm\.ru','\.udmurtia\.ru','\.ulan-ude\.ru',
309	'\.vladikavkaz\.ru','\.vladimir\.ru','\.vladivostok\.ru','\.volgograd\.ru','\.vologda\.ru',
310	'\.voronezh\.ru','\.vrn\.ru','\.vyatka\.ru','\.yakutia\.ru','\.yamal\.ru','\.yekaterinburg\.ru',
311	'\.yuzhno-sakhalinsk\.ru','\.ac\.rw','\.co\.rw','\.com\.rw','\.edu\.rw','\.gouv\.rw','\.gov\.rw',
312	'\.int\.rw','\.mil\.rw','\.net\.rw','\.com\.sa','\.edu\.sa','\.gov\.sa','\.med\.sa','\.net\.sa',
313	'\.org\.sa','\.pub\.sa','\.sch\.sa','\.com\.sd','\.edu\.sd','\.gov\.sd','\.info\.sd','\.med\.sd',
314	'\.net\.sd','\.org\.sd','\.tv\.sd','\.a\.se','\.ac\.se','\.b\.se','\.bd\.se','\.c\.se','\.d\.se',
315	'\.e\.se','\.f\.se','\.g\.se','\.h\.se','\.i\.se','\.k\.se','\.l\.se','\.m\.se','\.n\.se','\.o\.se',
316	'\.org\.se','\.p\.se','\.parti\.se','\.pp\.se','\.press\.se','\.r\.se','\.s\.se','\.t\.se','\.tm\.se',
317	'\.u\.se','\.w\.se','\.x\.se','\.y\.se','\.z\.se','\.com\.sg','\.edu\.sg','\.gov\.sg','\.idn\.sg',
318	'\.net\.sg','\.org\.sg','\.per\.sg','\.art\.sn','\.com\.sn','\.edu\.sn','\.gouv\.sn','\.org\.sn',
319	'\.perso\.sn','\.univ\.sn','\.com\.sy','\.edu\.sy','\.gov\.sy','\.mil\.sy','\.net\.sy','\.news\.sy',
320	'\.org\.sy','\.ac\.th','\.co\.th','\.go\.th','\.in\.th','\.mi\.th','\.net\.th','\.or\.th','\.ac\.tj',
321	'\.biz\.tj','\.co\.tj','\.com\.tj','\.edu\.tj','\.go\.tj','\.gov\.tj','\.info\.tj','\.int\.tj',
322	'\.mil\.tj','\.name\.tj','\.net\.tj','\.nic\.tj','\.org\.tj','\.test\.tj','\.web\.tj','\.agrinet\.tn',
323	'\.com\.tn','\.defense\.tn','\.edunet\.tn','\.ens\.tn','\.fin\.tn','\.gov\.tn','\.ind\.tn','\.info\.tn',
324	'\.intl\.tn','\.mincom\.tn','\.nat\.tn','\.net\.tn','\.org\.tn','\.perso\.tn','\.rnrt\.tn','\.rns\.tn',
325	'\.rnu\.tn','\.tourism\.tn','\.ac\.tz','\.co\.tz','\.go\.tz','\.ne\.tz','\.or\.tz','\.biz\.ua',
326	'\.cherkassy\.ua','\.chernigov\.ua','\.chernovtsy\.ua','\.ck\.ua','\.cn\.ua','\.co\.ua','\.com\.ua',
327	'\.crimea\.ua','\.cv\.ua','\.dn\.ua','\.dnepropetrovsk\.ua','\.donetsk\.ua','\.dp\.ua','\.edu\.ua',
328	'\.gov\.ua','\.if\.ua','\.in\.ua','\.ivano-frankivsk\.ua','\.kh\.ua','\.kharkov\.ua','\.kherson\.ua',
329	'\.khmelnitskiy\.ua','\.kiev\.ua','\.kirovograd\.ua','\.km\.ua','\.kr\.ua','\.ks\.ua','\.kv\.ua',
330	'\.lg\.ua','\.lugansk\.ua','\.lutsk\.ua','\.lviv\.ua','\.me\.ua','\.mk\.ua','\.net\.ua','\.nikolaev\.ua',
331	'\.od\.ua','\.odessa\.ua','\.org\.ua','\.pl\.ua','\.poltava\.ua','\.pp\.ua','\.rovno\.ua','\.rv\.ua',
332	'\.sebastopol\.ua','\.sumy\.ua','\.te\.ua','\.ternopil\.ua','\.uzhgorod\.ua','\.vinnica\.ua','\.vn\.ua',
333	'\.zaporizhzhe\.ua','\.zhitomir\.ua','\.zp\.ua','\.zt\.ua','\.ac\.ug','\.co\.ug','\.go\.ug','\.ne\.ug',
334	'\.or\.ug','\.org\.ug','\.sc\.ug','\.ac\.uk','\.bl\.uk','\.british-library\.uk','\.co\.uk','\.cym\.uk',
335	'\.gov\.uk','\.govt\.uk','\.icnet\.uk','\.jet\.uk','\.lea\.uk','\.ltd\.uk','\.me\.uk','\.mil\.uk',
336	'\.mod\.uk','\.mod\.uk','\.national-library-scotland\.uk','\.nel\.uk','\.net\.uk','\.nhs\.uk',
337	'\.nhs\.uk','\.nic\.uk','\.nls\.uk','\.org\.uk','\.orgn\.uk','\.parliament\.uk','\.parliament\.uk',
338	'\.plc\.uk','\.police\.uk','\.sch\.uk','\.scot\.uk','\.soc\.uk','\.dni\.us','\.fed\.us','\.isa\.us',
339	'\.kids\.us','\.nsn\.us','\.com\.uy','\.edu\.uy','\.gub\.uy','\.mil\.uy','\.net\.uy','\.org\.uy',
340	'\.co\.ve','\.com\.ve','\.edu\.ve','\.gob\.ve','\.info\.ve','\.mil\.ve','\.net\.ve','\.org\.ve',
341	'\.web\.ve','\.co\.vi','\.com\.vi','\.k12\.vi','\.net\.vi','\.org\.vi','\.ac\.vn','\.biz\.vn',
342	'\.com\.vn','\.edu\.vn','\.gov\.vn','\.health\.vn','\.info\.vn','\.int\.vn','\.name\.vn','\.net\.vn',
343	'\.org\.vn','\.pro\.vn','\.co\.ye','\.com\.ye','\.gov\.ye','\.ltd\.ye','\.me\.ye','\.net\.ye',
344	'\.org\.ye','\.plc\.ye','\.ac\.yu','\.co\.yu','\.edu\.yu','\.gov\.yu','\.org\.yu','\.ac\.za',
345	'\.agric\.za','\.alt\.za','\.bourse\.za','\.city\.za','\.co\.za','\.cybernet\.za','\.db\.za',
346	'\.ecape\.school\.za','\.edu\.za','\.fs\.school\.za','\.gov\.za','\.gp\.school\.za','\.grondar\.za',
347	'\.iaccess\.za','\.imt\.za','\.inca\.za','\.kzn\.school\.za','\.landesign\.za','\.law\.za',
348	'\.lp\.school\.za','\.mil\.za','\.mpm\.school\.za','\.ncape\.school\.za','\.net\.za','\.ngo\.za',
349	'\.nis\.za','\.nom\.za','\.nw\.school\.za','\.olivetti\.za','\.org\.za','\.pix\.za','\.school\.za',
350	'\.tm\.za','\.wcape\.school\.za','\.web\.za','\.ac\.zm','\.co\.zm','\.com\.zm','\.edu\.zm','\.gov\.zm',
351	'\.net\.zm','\.org\.zm','\.sch\.zm'
352);
353
354my @TLD2 = (
355	'\.ac','\.ad','\.ae','\.af','\.ag','\.ai','\.al','\.am','\.ao','\.aq',
356	'\.ar','\.as','\.at','\.au','\.aw','\.ax','\.az','\.ba','\.bb','\.bd',
357	'\.be','\.bf','\.bg','\.bh','\.bi','\.bj','\.bm','\.bn','\.bo','\.br',
358	'\.bs','\.bt','\.bw','\.by','\.bz','\.ca','\.cc','\.cd','\.cf','\.cg',
359	'\.ch','\.ci','\.ck','\.cl','\.cm','\.cn','\.co','\.cr','\.cu','\.cv',
360	'\.cw','\.cx','\.cy','\.cz','\.de','\.dj','\.dk','\.dm','\.do','\.dz',
361	'\.ec','\.ee','\.eg','\.er','\.es','\.et','\.eu','\.fi','\.fj','\.fk',
362	'\.fm','\.fo','\.fr','\.ga','\.gd','\.ge','\.gf','\.gg','\.gh','\.gi',
363	'\.gl','\.gm','\.gn','\.gp','\.gq','\.gr','\.gs','\.gt','\.gu','\.gw',
364	'\.gy','\.hk','\.hm','\.hn','\.hr','\.ht','\.hu','\.id','\.ie','\.il',
365	'\.im','\.in','\.io','\.iq','\.ir','\.is','\.it','\.je','\.jm','\.jo',
366	'\.jp','\.ke','\.kg','\.kh','\.ki','\.km','\.kn','\.kp','\.kr','\.kw',
367	'\.ky','\.kz','\.la','\.lb','\.lc','\.li','\.lk','\.lr','\.ls','\.lt',
368	'\.lu','\.lv','\.ly','\.ma','\.mc','\.md','\.me','\.mg','\.mh','\.mk',
369	'\.ml','\.mm','\.mn','\.mo','\.mp','\.mq','\.mr','\.ms','\.mt','\.mu',
370	'\.mv','\.mw','\.mx','\.my','\.mz','\.na','\.nc','\.ne','\.nf','\.ng',
371	'\.ni','\.nl','\.no','\.np','\.nr','\.nu','\.nz','\.om','\.pa','\.pe',
372	'\.pf','\.pg','\.ph','\.pk','\.pl','\.pm','\.pn','\.pr','\.ps','\.pt',
373	'\.pw','\.py','\.qa','\.re','\.ro','\.rs','\.ru','\.rw','\.sa','\.sb',
374	'\.sc','\.sd','\.se','\.sg','\.sh','\.si','\.sk','\.sl','\.sm','\.sn',
375	'\.so','\.sr','\.ss','\.st','\.su','\.sv','\.sx','\.sy','\.sz','\.tc',
376	'\.td','\.tf','\.tg','\.th','\.tj','\.tk','\.tl','\.tm','\.tn','\.to',
377	'\.tr','\.tt','\.tv','\.tw','\.tz','\.ua','\.ug','\.uk','\.us','\.uy',
378	'\.uz','\.va','\.vc','\.ve','\.vg','\.vi','\.vn','\.vu','\.wf','\.ws',
379	'\.ye','\.za','\.zm','\.zw','\.com','\.info','\.net','\.org','\.biz',
380	'\.name','\.pro','\.xxx','\.aero','\.asia','\.bzh','\.cat','\.coop',
381	'\.edu','\.gov','\.int','\.jobs','\.mil','\.mobi','\.museum','\.paris',
382	'\.sport','\.tel','\.travel','\.kids','\.mail','\.post','\.arpa','\.example',
383	'\.invalid','\.localhost','\.test','\.bitnet','\.csnet','\.lan','\.local',
384	'\.onion','\.root','\.uucp','\.tld','\.nato'
385);
386
387my %month_number = (
388	'Jan' => '01',
389	'Feb' => '02',
390	'Mar' => '03',
391	'Apr' => '04',
392	'May' => '05',
393	'Jun' => '06',
394	'Jul' => '07',
395	'Aug' => '08',
396	'Sep' => '09',
397	'Oct' => '10',
398	'Nov' => '11',
399	'Dec' => '12',
400);
401
402# Regex to match ipv4 and ipv6 address
403my $ip_regexp = qr/^([a-fA-F0-9\.\:]+)$/;
404my $cidr_regex = qr/^[a-fA-F0-9\.\:]+\/\d+$/;
405
406# Native log format squid %ts.%03tu %6tr %>a %Ss/%03>Hs %<st %rm %ru %un %Sh/%<A %mt
407my $native_format_regex1 = qr/^(\d+\.\d{3})\s+(\d+)\s+([^\s]+)\s+([^\s]+)\s+(\d+)\s+([^\s]+)\s+(.*)/;
408my $native_format_regex2 = qr/^([^\s]+?)\s+([^\s]+)\s+([^\s]+\/[^\s]+)\s+([^\s]+)\s*/;
409#logformat common     %>a %[ui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %<st %Ss:%Sh
410#logformat combined   %>a %[ui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
411my $common_format_regex1 = qr/([^\s]+)\s([^\s]+)\s([^\s]+)\s\[(\d+\/...\/\d+:\d+:\d+:\d+\s[\d\+\-]+)\]\s"([^\s]+)\s([^\s]+)\s([^\s]+)"\s(\d+)\s+(\d+)(.*)\s([^\s:]+:[^\s]+)\s*([^\/]+\/[^\s]+|-)?$/;
412# Log format for SquidGuard logs
413my $sg_format_regex1 = qr/^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}) .* Request\(([^\/]+\/[^\/]+)\/[^\)]*\) ([^\s]+) ([^\s\\]+)\/[^\s]+ ([^\s]+) ([^\s]+) ([^\s]+)/;
414# Log format for ufdbGuard logs: BLOCK user clienthost aclname category url method
415my $ug_format_regex1 = qr/^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}) .* (BLOCK) ([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)$/;
416
417sub new
418{
419	my ($class, $conf_file, $log_file, $debug, $rebuild, $pid_dir, $pidfile, $timezone) = @_;
420
421	# Construct the class
422	my $self = {};
423	bless $self, $class;
424
425	# Initialize all variables
426	$self->_init($conf_file, $log_file, $debug, $rebuild, $pid_dir, $pidfile, $timezone);
427
428	# Return the instance
429	return($self);
430
431}
432
433sub localdie
434{
435	my ($self, $msg) = @_;
436
437	print STDERR "$msg";
438	unlink("$self->{pidfile}");
439
440	# Cleanup old temporary files
441	foreach my $tmp_file ('last_parsed.tmp', 'sg_last_parsed.tmp') {
442		unlink("$self->{pid_dir}/$tmp_file");
443	}
444
445	exit 1;
446}
447
448####
449# method used to fork as many child as wanted
450##
451sub spawn
452{
453	my $self = shift;
454	my $coderef = shift;
455
456	unless (@_ == 0 && $coderef && ref($coderef) eq 'CODE') {
457		print "usage: spawn CODEREF";
458		exit 0;
459	}
460
461	my $pid;
462	if (!defined($pid = fork)) {
463		print STDERR "Error: cannot fork: $!\n";
464		return;
465	} elsif ($pid) {
466		$self->{running_pids}{$pid} = 1;
467		return; # the parent
468	}
469	# the child -- go spawn
470	$< = $>;
471	$( = $); # suid progs only
472
473	exit &$coderef();
474}
475
476sub wait_all_childs
477{
478	my $self = shift;
479
480	while (scalar keys %{$self->{running_pids}} > 0) {
481		my $kid = waitpid(-1, WNOHANG);
482		if ($kid > 0) {
483			delete $self->{running_pids}{$kid};
484		}
485		sleep(1);
486	}
487}
488
489sub manage_queue_size
490{
491	my ($self, $child_count) = @_;
492
493	while ($child_count >= $self->{queue_size}) {
494		my $kid = waitpid(-1, WNOHANG);
495		if ($kid > 0) {
496			$child_count--;
497			delete $self->{running_pids}{$kid};
498		}
499		sleep(1);
500	}
501
502	return $child_count;
503}
504
505sub save_current_line
506{
507	my $self = shift;
508
509	if ($self->{end_time}) {
510		my $current = new IO::File;
511		$current->open(">$self->{Output}/SquidAnalyzer.current") or $self->localdie("FATAL: Can't write to file $self->{Output}/SquidAnalyzer.current, $!\n");
512		print $current "$self->{end_time}\t$self->{end_offset}";
513		$current->close;
514	}
515	if ($self->{sg_end_time}) {
516		my $current = new IO::File;
517		$current->open(">$self->{Output}/SquidGuard.current") or $self->localdie("FATAL: Can't write to file $self->{Output}/SquidGuard.current, $!\n");
518		print $current "$self->{sg_end_time}\t$self->{sg_end_offset}";
519		$current->close;
520	}
521	if ($self->{ug_end_time}) {
522		my $current = new IO::File;
523		$current->open(">$self->{Output}/ufdbGuard.current") or $self->localdie("FATAL: Can't write to file $self->{Output}/ufdbGuard.current, $!\n");
524		print $current "$self->{ug_end_time}\t$self->{ug_end_offset}";
525		$current->close;
526	}
527}
528
529# Extract number of seconds since epoch from timestamp in log line
530sub look_for_timestamp
531{
532	my ($self, $line) = @_;
533
534	my $time = 0;
535	my $tz = ((0-$self->{TimeZone})*3600);
536	# Squid native format
537	if ( $line =~ $native_format_regex1 ) {
538		$time = $1;
539		$self->{is_squidguard_log} = 0;
540		$self->{is_ufdbguard_log} = 0;
541	# Squid common HTTP format
542	} elsif ( $line =~ $common_format_regex1 ) {
543		$time = $4;
544		$time =~ /(\d+)\/(...)\/(\d+):(\d+):(\d+):(\d+)\s/;
545		if (!$self->{TimeZone}) {
546			$time = timelocal_nocheck($6, $5, $4, $1, $month_number{$2} - 1, $3 - 1900);
547		} else {
548			$time = timegm_nocheck($6, $5, $4, $1, $month_number{$2} - 1, $3 - 1900) + $tz;
549		}
550		$self->{is_squidguard_log} = 0;
551		$self->{is_ufdbguard_log} = 0;
552	# SquidGuard log format
553	} elsif ( $line =~ $sg_format_regex1 ) {
554		$self->{is_squidguard_log} = 1;
555		$self->{is_ufdbguard_log} = 0;
556		if (!$self->{TimeZone}) {
557			$time = timelocal_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900);
558		} else {
559			$time = timegm_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900) + $tz;
560		}
561	# ufdbGuard log format
562	} elsif ( $line =~ $ug_format_regex1 ) {
563		$self->{is_ufdbguard_log} = 1;
564		$self->{is_squidguard_log} = 0;
565		if (!$self->{TimeZone}) {
566			$time = timelocal_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900);
567		} else {
568			$time = timegm_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900) + $tz;
569		}
570	}
571
572	return $time;
573}
574
575# Detect if log file is a squidGuard log or not
576sub get_log_format
577{
578	my ($self, $file) = @_;
579
580	my $logfile = new IO::File;
581	$logfile->open($file) || $self->localdie("ERROR: Unable to open log file $file. $!\n");
582	my $max_line = 10000;
583	my $i = 0;
584	while (my $line = <$logfile>) {
585		chomp($line);
586
587		# SquidGuard log format
588		if ( $line =~ $sg_format_regex1 ) {
589			$self->{is_squidguard_log} = 1;
590			$self->{is_ufdbguard_log} = 0;
591			last;
592		# ufdbGuard log format
593		} elsif ( $line =~ $ug_format_regex1 ) {
594			$self->{is_ufdbguard_log} = 1;
595			$self->{is_squidguard_log} = 0;
596			last;
597		# Squid native format
598		} elsif ( $line =~ $native_format_regex1 ) {
599			$self->{is_squidguard_log} = 0;
600			$self->{is_ufdbguard_log} = 0;
601			last;
602		# Squid common HTTP format
603		} elsif ( $line =~ $common_format_regex1 ) {
604			$self->{is_squidguard_log} = 0;
605			$self->{is_ufdbguard_log} = 0;
606			last;
607		} else {
608			last if ($i > $max_line);
609		}
610		$i++;
611	}
612	$logfile->close();
613}
614
615
616sub parseFile
617{
618	my ($self) = @_;
619
620	my $line_count = 0;
621	my $line_processed_count = 0;
622	my $line_stored_count = 0;
623	my $saved_queue_size = $self->{queue_size};
624	my $history_offset = $self->{end_offset};
625
626	foreach my $lfile (@{$self->{LogFile}}) {
627
628		# Detect if log file is from squid or squidguard
629		$self->get_log_format($lfile);
630		if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
631			$history_offset = $self->{end_offset};
632		} elsif (!$self->{is_squidguard_log}) {
633			$history_offset = $self->{ug_end_offset};
634		} else {
635			$history_offset = $self->{sg_end_offset};
636		}
637
638		print STDERR "Starting to parse logfile $lfile.\n" if (!$self->{QuietMode});
639		if ((!-f $lfile) || (-z $lfile)) {
640			print STDERR "DEBUG: bad or empty log file $lfile.\n" if (!$self->{QuietMode});
641			next;
642		}
643		# Restore the right multiprocess queue
644		$self->{queue_size} = $saved_queue_size;
645
646		# Compressed file do not allow multiprocess
647		if ($lfile =~ /\.(gz|bz2)$/) {
648			$self->{queue_size} = 1;
649		}
650
651		# Search the last position in logfile
652		if ($history_offset) {
653
654			# Initialize start offset for each file
655			if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
656				$self->{end_offset} = $history_offset;
657			} elsif (!$self->{is_squidguard_log}) {
658				$self->{ug_end_offset} = $history_offset;
659			} else {
660				$self->{sg_end_offset} = $history_offset;
661			}
662
663			# Compressed file are always read from the begining
664			if ($lfile =~ /\.(gz|bz2)$/i) {
665				if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
666					$self->{end_offset} = 0;
667				} elsif (!$self->{is_squidguard_log}) {
668					$self->{ug_end_offset} = 0;
669				} else {
670					$self->{sg_end_offset} = 0;
671				}
672			} else {
673				# Look at first line to see if the file should be parse from the begining.
674				my $logfile = new IO::File;
675				$logfile->open($lfile) || $self->localdie("ERROR: Unable to open log file $lfile. $!\n");
676				my $line = <$logfile>;
677				chomp($line);
678
679				my $curtime = $self->look_for_timestamp($line);
680
681				my $hist_time = $self->{history_time};
682				if ($self->{is_squidguard_log}) {
683					$hist_time = $self->{sg_history_time};
684				} elsif ($self->{is_ufdbguard_log}) {
685					$hist_time = $self->{ug_history_time};
686				}
687				# if the first timestamp is higher that the history time, start from the beginning
688				if ($curtime > $hist_time) {
689					print STDERR "DEBUG: new file: $lfile, start from the beginning.\n" if (!$self->{QuietMode});
690					if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
691						$self->{end_offset} = 0;
692					} elsif (!$self->{is_squidguard_log}) {
693						$self->{ug_end_offset} = 0;
694					} else {
695						$self->{sg_end_offset} = 0;
696					}
697				# If the size of the file is lower than the history offset, parse this file from the beginning
698				} elsif ((lstat($lfile))[7] <= $history_offset) {
699					# move at begining of the file to see if this is a new one
700					$logfile->seek(0, 0);
701					for (my $i = 1; $i <= 10000; $i++) {
702						$line = <$logfile>;
703						chomp($line);
704						$curtime = $self->look_for_timestamp($line);
705						if ($curtime) {
706							if ($hist_time > $curtime) {
707								print STDERR "DEBUG: this file will not been parsed: $lfile, size lower than expected and $curtime is lower than history time $self->{history_time}.\n" if (!$self->{QuietMode});
708								$line = 'NOK';
709								last;
710							}
711						}
712					}
713					$logfile->close;
714					# This file should be ommitted jump to the next file
715					next if ($line eq 'NOK');
716					
717					print STDERR "DEBUG: new file: $lfile, start from the beginning.\n" if (!$self->{QuietMode});
718					if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
719						$self->{end_offset} = 0;
720					} elsif (!$self->{is_squidguard_log}) {
721						$self->{ug_end_offset} = 0;
722					} else {
723						$self->{sg_end_offset} = 0;
724					}
725				} else {
726					# move at offset and see if next line is older than history time
727					$logfile->seek($history_offset, 0);
728					for (my $i = 1; $i <= 10; $i++) {
729						$line = <$logfile>;
730						chomp($line);
731						$curtime = $self->look_for_timestamp($line);
732						if ($curtime) {
733							if ($curtime < $hist_time) {
734								my $tmp_time = CORE::localtime($curtime);
735								print STDERR "DEBUG: this file will not been parsed: $lfile, line after offset is older than expected: $curtime < $hist_time.\n" if (!$self->{QuietMode});
736								$line = 'NOK';
737								last;
738							}
739						}
740					}
741					$logfile->close;
742					# This file should be ommitted jump to the next file
743					next if ($line eq 'NOK');
744				}
745				$logfile->close;
746			}
747
748		} else {
749			print STDERR "DEBUG: this file will be parsed, no history found.\n" if (!$self->{QuietMode});
750			# Initialise start offset for each file
751			if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
752				$self->{end_offset} = 0;
753			} elsif (!$self->{is_squidguard_log}) {
754				$self->{ug_end_offset} = 0;
755			} else {
756				$self->{sg_end_offset} = 0;
757			}
758		}
759
760		if ($self->{queue_size} <= 1) {
761			if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
762				$self->_parse_file_part($lfile, $self->{end_offset});
763			} elsif (!$self->{is_squidguard_log}) {
764				$self->_parse_file_part($lfile, $self->{ug_end_offset});
765			} else {
766				$self->_parse_file_part($lfile, $self->{sg_end_offset});
767			}
768		} else {
769			# Create multiple processes to parse one log file by chunks of data
770			my @chunks = $self->split_logfile($lfile);
771			my $child_count = 0;
772			for (my $i = 0; $i < $#chunks; $i++) {
773				if ($self->{interrupt}) {
774					print STDERR "FATAL: Abort signal received when processing to next chunk\n";
775					return;
776				}
777				$self->spawn(sub {
778					$self->_parse_file_part($lfile, $chunks[$i], $chunks[$i+1], $i);
779				});
780				$child_count = $self->manage_queue_size(++$child_count);
781			}
782		}
783	}
784
785	# Wait for last child stop
786	$self->wait_all_childs() if ($self->{queue_size} > 1);
787
788	# Get the last information parsed in this file part
789	foreach my $tmp_file ('last_parsed.tmp', 'sg_last_parsed.tmp') {
790
791		if (-e "$self->{pid_dir}/$tmp_file") {
792
793			if (open(IN, "$self->{pid_dir}/$tmp_file")) {
794				my %history_tmp = ();
795				while (my $l = <IN>) {
796					chomp($l);
797					my @data = split(/\s/, $l);
798					$history_tmp{"$data[0]$data[1]$data[2]"}{$data[4]} = join(' ', @data);
799					$line_stored_count += $data[5];
800					$line_processed_count += $data[6];
801					$line_count += $data[7];
802					if (!$self->{first_year} || ("$data[8]$data[9]" lt "$self->{first_year}$self->{first_month}{$data[8]}}") ) {
803						$self->{first_year} = $data[8];
804						$self->{first_month}{$data[8]} = $data[9];
805					}
806					my @tmp = split(/,/, $data[10]);
807					foreach my $w (@tmp) {
808						if (!grep(/^$w$/, @{$self->{week_parsed}})) {
809							push(@{$self->{week_parsed}}, $w);
810						}
811					}
812				}
813				close(IN);
814				foreach my $date (sort {$b <=> $a} keys %history_tmp) {
815					foreach my $offset (sort {$b <=> $a} keys %{$history_tmp{$date}}) {
816						my @data = split(/\s/, $history_tmp{$date}{$offset});
817						$self->{last_year} = $data[0];
818						$self->{last_month}{$data[0]} = $data[1];
819						$self->{last_day}{$data[0]} = $data[2];
820						if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
821							$self->{end_time} = $data[3];
822							$self->{end_offset} = $data[4];
823						} elsif (!$self->{is_squidguard_log}) {
824							$self->{ug_end_time} = $data[3];
825							$self->{ug_end_offset} = $data[4];
826						} else {
827							$self->{sg_end_time} = $data[3];
828							$self->{sg_end_offset} = $data[4];
829						}
830						last;
831					}
832					last;
833				}
834			} else {
835				print STDERR "ERROR: can't read last parsed line from $self->{pid_dir}/$tmp_file, $!\n";
836			}
837		}
838	}
839
840	if (!$self->{last_year}) {
841
842		print STDERR "No new log registered...\n" if (!$self->{QuietMode});
843
844	} else {
845
846		if (!$self->{QuietMode}) {
847			print STDERR "SQUID LOG END TIME  : ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($self->{end_time})), "\n" if ($self->{end_time});
848			print STDERR "SQUIGUARD LOG END TIME  : ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($self->{sg_end_time})), "\n" if ($self->{sg_end_time});
849			print STDERR "UFDBGUARD LOG END TIME  : ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($self->{ug_end_time})), "\n" if ($self->{ug_end_time});
850			print STDERR "Read $line_count lines, matched $line_processed_count and found $line_stored_count new lines\n";
851		}
852
853		# Set the current start time into history file
854		$self->save_current_line();
855
856		# Force reordering and unique sorting of data files
857		my $child_count = 0;
858		if (!$self->{rebuild}) {
859			if (!$self->{QuietMode}) {
860				print STDERR "Reordering daily data files now...\n";
861			}
862			for my $date ("$self->{first_year}$self->{first_month}{$self->{first_year}}" .. "$self->{last_year}$self->{last_month}{$self->{last_year}}") {
863				$date =~ /^(\d{4})(\d{2})$/;
864				my $y = $1;
865				my $m = $2;
866				next if (($m < 1) || ($m > 12));
867				if ($self->{interrupt}) {
868					print STDERR "FATAL: Abort signal received\n";
869					return;
870				}
871				if (-d "$self->{Output}/$y/$m") {
872					foreach my $d ("01" .. "31") {
873						if (-d "$self->{Output}/$y/$m/$d") {
874							if ($self->{queue_size} > 1) {
875								$self->spawn(sub {
876									$self->_save_stat($y, $m, $d);
877								});
878								$child_count = $self->manage_queue_size(++$child_count);
879							} else {
880								$self->_save_stat($y, $m, $d);
881							}
882							$self->_clear_stats();
883						}
884					}
885				}
886			}
887			# Wait for last child stop
888			$self->wait_all_childs() if ($self->{queue_size} > 1);
889			$child_count = 0;
890		}
891
892		# Compute week statistics
893		if (!$self->{no_week_stat}) {
894			if (!$self->{QuietMode}) {
895				print STDERR "Generating weekly data files now...\n";
896			}
897
898			foreach my $week (@{$self->{week_parsed}}) {
899				my ($y, $m, $wn) = split(/\//, $week);
900				my @wd = &get_wdays_per_month($wn, "$y-$m");
901				$wn++;
902
903				print STDERR "Compute and dump weekly statistics for week $wn on $y\n" if (!$self->{QuietMode});
904				if ($self->{queue_size} > 1) {
905					$self->spawn(sub {
906						$self->_save_data($y, $m, undef, sprintf("%02d", $wn), @wd);
907					});
908					$child_count = $self->manage_queue_size(++$child_count);
909				} else {
910					$self->_save_data($y, $m, undef, sprintf("%02d", $wn), @wd);
911				}
912				$self->_clear_stats();
913			}
914		}
915		# Wait for last child stop
916		$self->wait_all_childs() if ($self->{queue_size} > 1);
917		$child_count = 0;
918
919		# Compute month statistics
920		if (!$self->{QuietMode}) {
921			print STDERR "Generating monthly data files now...\n";
922		}
923
924		for my $date ("$self->{first_year}$self->{first_month}{$self->{first_year}}" .. "$self->{last_year}$self->{last_month}{$self->{last_year}}") {
925			$date =~ /^(\d{4})(\d{2})$/;
926			my $y = $1;
927			my $m = $2;
928			next if (($m < 1) || ($m > 12));
929			if ($self->{interrupt}) {
930				print STDERR "FATAL: Abort signal received\n";
931				return;
932			}
933			if (-d "$self->{Output}/$y/$m") {
934				print STDERR "Compute and dump month statistics for $y/$m\n" if (!$self->{QuietMode});
935				if ($self->{queue_size} > 1) {
936					$self->spawn(sub {
937						$self->_save_data("$y", "$m");
938					});
939					$child_count = $self->manage_queue_size(++$child_count);
940				} else {
941					$self->_save_data("$y", "$m");
942				}
943				$self->_clear_stats();
944			}
945		}
946
947		# Wait for last child stop
948		$self->wait_all_childs() if ($self->{queue_size} > 1);
949
950		# Compute year statistics
951		$child_count = 0;
952		if (!$self->{QuietMode}) {
953			print STDERR "Generating yearly data files now...\n";
954		}
955		for my $year ($self->{first_year} .. $self->{last_year}) {
956			if ($self->{interrupt}) {
957				print STDERR "FATAL: Abort signal received\n";
958				return;
959			}
960			if (-d "$self->{Output}/$year") {
961				print STDERR "Compute and dump year statistics for $year\n" if (!$self->{QuietMode});
962				if ($self->{queue_size} > 1) {
963					$self->spawn(sub {
964						$self->_save_data("$year");
965					});
966					$child_count = $self->manage_queue_size(++$child_count);
967				} else {
968					$self->_save_data("$year");
969				}
970				$self->_clear_stats();
971			}
972		}
973
974		# Wait for last child stop
975		$self->wait_all_childs() if ($self->{queue_size} > 1);
976
977	}
978
979}
980
981sub split_logfile
982{
983	my ($self, $logf) = @_;
984
985	my @chunks = (0);
986
987	# get file size
988	my $totalsize = (stat("$logf"))[7] || 0;
989
990	my $offsplit = $self->{end_offset};
991	if ($self->{is_squidguard_log}) {
992		$offsplit = $self->{sg_end_offset};
993	} elsif ($self->{is_ufdbguard_log}) {
994		$offsplit = $self->{ug_end_offset};
995	}
996
997	# If the file is very small, many jobs actually make the parsing take longer
998	if ( ($totalsize <= 16777216) || ($totalsize <= $offsplit)) { #16MB
999		push(@chunks, $totalsize);
1000		return @chunks;
1001	}
1002
1003	# Split and search the right position in file corresponding to the number of jobs
1004	my $i = 1;
1005	if ($offsplit && ($offsplit < $totalsize)) {
1006		$chunks[0] = $offsplit;
1007	}
1008	my $lfile = undef;
1009	open($lfile, $logf) || die "FATAL: cannot read log file $logf. $!\n";
1010	while ($i < $self->{queue_size}) {
1011		my $pos = int(($totalsize/$self->{queue_size}) * $i);
1012		if ($pos > $chunks[0]) {
1013			$lfile->seek($pos, 0);
1014			#Move the offset to the BEGINNING of each line, because the logic in process_file requires so
1015			$pos = $pos + length(<$lfile>) - 1;
1016			push(@chunks, $pos) if ($pos < $totalsize);
1017		}
1018		last if ($pos >= $totalsize);
1019		$i++;
1020	}
1021	$lfile->close();
1022
1023	push(@chunks, $totalsize);
1024
1025	return @chunks;
1026}
1027
1028sub check_exclusions
1029{
1030	my ($self, $login, $client_ip, $url) = @_;
1031
1032	return 0 if (!exists $self->{Exclude}{users} && !exists $self->{Exclude}{clients} && !exists $self->{Exclude}{networks} && !exists $self->{Exclude}{uris});
1033
1034	# check for user exclusion
1035	if (exists $self->{Exclude}{users} && $login) {
1036		foreach my $e (@{$self->{Exclude}{users}}) {
1037			# look for users using the following format: user@domain.tld, domain\user and user 
1038			if ( ($login =~ m#^$e$#i) || ($login =~ m#^$e\@#i) || ($login =~ m#\\$e$#i) ) {
1039				return 1;
1040			}
1041		}
1042	}
1043
1044	# check for client exclusion
1045	if (exists $self->{Exclude}{clients} && $client_ip) {
1046		foreach my $e (@{$self->{Exclude}{clients}}) {
1047			if ($client_ip =~ m#^$e$#i) {
1048				return 1;
1049			}
1050		}
1051	}
1052
1053	# check for Network exclusion
1054	if (exists $self->{Exclude}{networks} && $client_ip) {
1055		foreach my $e (@{$self->{Include}{networks}}) {
1056			if (&check_ip($client_ip, $e)) {
1057				return 1;
1058			}
1059		}
1060	}
1061
1062	# check for URL exclusion
1063	if (exists $self->{Exclude}{uris} && $url) {
1064		foreach my $e (@{$self->{Exclude}{uris}}) {
1065			if ($url =~ m#^$e$#i) {
1066				return 1;
1067			}
1068		}
1069	}
1070
1071	return 0;
1072}
1073
1074sub check_inclusions
1075{
1076	my ($self, $login, $client_ip) = @_;
1077
1078	return 1 if (!exists $self->{Include}{users} && !exists $self->{Include}{clients} && !exists $self->{Include}{networks});
1079
1080	# check for user inclusion
1081	if (exists $self->{Include}{users} && $login) {
1082		foreach my $e (@{$self->{Include}{users}}) {
1083			# look for users using the following format: user@domain.tld, domain\user and user 
1084			if ( ($login =~ m#^$e$#i) || ($login =~ m#^$e\@#i) || ($login =~ m#\\$e$#i) ) {
1085				return 1;
1086			}
1087		}
1088	}
1089
1090	# If login is a client ip, checked login against clients and networks filters
1091	if (!$client_ip && ($login =~ /^\d+\.\d+\.\d+\.\d+$/)) {
1092		$client_ip = $login;
1093	}
1094
1095	# check for client inclusion
1096	if (exists $self->{Include}{clients} && $client_ip) {
1097		foreach my $e (@{$self->{Include}{clients}}) {
1098			if ($client_ip =~ m#^$e$#i) {
1099				return 1;
1100			}
1101		}
1102	}
1103
1104	# check for Network inclusion
1105	if (exists $self->{Include}{networks} && $client_ip) {
1106		foreach my $e (@{$self->{Include}{networks}}) {
1107			if (&check_ip($client_ip, $e)) {
1108				return 1;
1109			}
1110		}
1111	}
1112
1113	return 0;
1114}
1115
1116sub _parse_file_part
1117{
1118	my ($self, $file, $start_offset, $stop_offset) = @_;
1119
1120	print STDERR "Reading file $file from offset $start_offset to ", ($stop_offset||'end'), ".\n" if (!$self->{QuietMode});
1121
1122	# Open logfile
1123	my $logfile = new IO::File;
1124	if ($file =~ /\.gz/) {
1125		# Open a pipe to zcat program for compressed log
1126		$logfile->open("$ZCAT_PROG $file |") || $self->localdie("ERROR: cannot read from pipe to $ZCAT_PROG $file. $!\n");
1127	} elsif ($file =~ /\.bz2/) {
1128		# Open a pipe to bzcat program for compressed log
1129		$logfile->open("$BZCAT_PROG $file |") || $self->localdie("ERROR: cannot read from pipe to $BZCAT_PROG $file. $!\n");
1130	} elsif ($file =~ /\.xz/) {
1131		# Open a pipe to xzcat program for compressed log
1132		$logfile->open("$XZCAT_PROG $file |") || $self->localdie("ERROR: cannot read from pipe to $XZCAT_PROG $file. $!\n");
1133	} else {
1134		$logfile->open($file) || $self->localdie("ERROR: Unable to open Squid access.log file $file. $!\n");
1135	}
1136
1137	my $line = '';
1138	my $time = 0;
1139	my $elapsed = 0;
1140	my $client_ip = '';
1141	my $client_name = '';
1142	my $code = '';
1143	my $bytes = 0;
1144	my $method = '';
1145	my $url = '';
1146	my $login = '';
1147	my $status = '';
1148	my $mime_type = '';
1149
1150	my $acl = '';
1151
1152	my $line_count = 0;
1153	my $line_processed_count = 0;
1154	my $line_stored_count = 0;
1155
1156	# Move directly to the start position
1157	if ($start_offset) {
1158		$logfile->seek($start_offset, 0);
1159		if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
1160			$self->{end_offset} = $start_offset;
1161		} elsif (!$self->{is_squidguard_log}) {
1162			$self->{ug_end_offset} = $start_offset;
1163		} else {
1164			$self->{sg_end_offset} = $start_offset;
1165		}
1166	}
1167
1168	# Set timezone in seconds
1169	my $tz = ((0-$self->{TimeZone})*3600);
1170
1171	# The log file format must be :
1172	# 	time elapsed client code/status bytes method URL rfc931 peerstatus/peerhost type
1173	# This is the default format of squid access log file.
1174
1175	# Read and parse each line of the access log file
1176	while ($line = <$logfile>) {
1177
1178		# quit this log if we reach the ending offset
1179		if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
1180			last if ($stop_offset && ($self->{end_offset}>= $stop_offset));
1181			# Store the current position in logfile
1182			$self->{end_offset} += length($line);
1183		} elsif (!$self->{is_squidguard_log}) {
1184			last if ($stop_offset && ($self->{ug_end_offset}>= $stop_offset));
1185			# Store the current position in logfile
1186			$self->{ug_end_offset} += length($line);
1187		} else {
1188			last if ($stop_offset && ($self->{sg_end_offset}>= $stop_offset));
1189			# Store the current position in logfile
1190			$self->{sg_end_offset} += length($line);
1191		}
1192
1193		chomp($line);
1194		next if (!$line);
1195
1196		# skip immediately lines that squid is not able to tag.
1197		next if ($line =~ / TAG_NONE(_ABORTED)?\//);
1198
1199		# Number of log lines parsed
1200		$line_count++;
1201
1202		# SquidAnalyzer supports the following squid log format:
1203		#logformat squid      %ts.%03tu %6tr %>a %Ss/%03>Hs %<st %rm %ru %un %Sh/%<A %mt
1204		#logformat squidmime  %ts.%03tu %6tr %>a %Ss/%03>Hs %<st %rm %ru %un %Sh/%<A %mt [%>h] [%<h]
1205		#logformat common     %>a %[ui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %<st %Ss:%Sh
1206		#logformat combined   %>a %[ui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
1207		# Parse log with format: time elapsed client code/status bytes method URL rfc931 peerstatus/peerhost mime_type
1208		my $format = 'native';
1209		if ( $line =~ $native_format_regex1 ) {
1210			$time = $1;
1211			$time += $tz;
1212			$elapsed = abs($2);
1213			$client_ip = $3;
1214			$code = $4;
1215			$bytes = $5;
1216			$method = $6;
1217			$line = $7;
1218		} elsif ( $line =~ $common_format_regex1 ) {
1219			$format = 'http';
1220			$client_ip = $1;
1221			$elapsed = abs($2);
1222			$login = lc($3);
1223			$time = $4;
1224			$method = $5;
1225			$url = lc($6);
1226			$status = $8;
1227			$bytes = $9;
1228			$line = $10;
1229			$code = $11;
1230			$mime_type = $12;
1231			$time =~ /(\d+)\/(...)\/(\d+):(\d+):(\d+):(\d+)\s/;
1232			if (!$self->{TimeZone}) {
1233				$time = timelocal_nocheck($6, $5, $4, $1, $month_number{$2} - 1, $3 - 1900);
1234			} else {
1235				$time = timegm_nocheck($6, $5, $4, $1, $month_number{$2} - 1, $3 - 1900) + $tz;
1236			}
1237			# Some site has corrupted mime_type, try to remove nasty characters
1238			$mime_type =~ s/[^\-\/\.\(\)\+\_,\=a-z0-9]+//igs;
1239                } elsif ($line =~ $sg_format_regex1) {
1240                        $format = 'squidguard';
1241			$self->{is_squidguard_log} = 1;
1242			$acl = $7;
1243                        $client_ip = $9;
1244                        $elapsed = 0;
1245                        $login = lc($10);
1246                        $method = $11;
1247                        $url = lc($8);
1248                        $status = 301;
1249                        $bytes = 0;
1250                        $code = $12 . ':';
1251                        $mime_type = '';
1252			if (!$self->{TimeZone}) {
1253				$time = timelocal_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900);
1254			} else {
1255				$time = timegm_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900) + $tz;
1256			}
1257		# Log format for ufdbGuard logs: timestamp [pid] BLOCK user clienthost aclname category url method
1258		} elsif ($line =~ $ug_format_regex1) {
1259                        $format = 'ufdbguard';
1260			$self->{is_ufdbguard_log} = 1;
1261			$acl = "$10/$11";
1262                        $client_ip = $9;
1263                        $elapsed = 0;
1264                        $login = lc($8);
1265                        $method = $13;
1266                        $url = lc($12);
1267                        $status = 301;
1268                        $bytes = 0;
1269                        $code = 'REDIRECT:';
1270                        $mime_type = '';
1271			if (!$self->{TimeZone}) {
1272				$time = timelocal_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900);
1273			} else {
1274				$time = timegm_nocheck($6, $5, $4, $3, $2 - 1, $1 - 1900) + $tz;
1275			}
1276		} else {
1277			next;
1278		}
1279		
1280		if ($time) {
1281			# Do not parse some unwanted method
1282			my $qm_method = quotemeta($method) || '';
1283			next if (($#{$self->{ExcludedMethods}} >= 0) && grep(/^$qm_method$/, @{$self->{ExcludedMethods}}));
1284
1285			# Do not parse some unwanted code; e.g. TCP_DENIED/403
1286			my $qm_code = quotemeta($code) || '';
1287			next if (($#{$self->{ExcludedCodes}} >= 0) && grep(m#^$code$#, @{$self->{ExcludedCodes}}));
1288
1289			# Go to last parsed date (incremental mode)
1290			if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
1291				next if ($self->{history_time} && ($time <= $self->{history_time}));
1292			} elsif (!$self->{is_squidguard_log}) {
1293				next if ($self->{ug_history_time} && ($time <= $self->{ug_history_time}));
1294			} else {
1295				next if ($self->{sg_history_time} && ($time <= $self->{sg_history_time}));
1296			}
1297
1298			# Register the last parsing time and last offset position in logfile
1299			if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
1300				$self->{end_time} = $time if (!$time || ($self->{end_time} < $time));
1301				# Register the first parsing time
1302				if (!$self->{begin_time} || ($self->{begin_time} > $time)) {
1303					$self->{begin_time} = $time;
1304					print STDERR "SQUID LOG SET START TIME: ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($time)), "\n" if (!$self->{QuietMode});
1305				}
1306			} elsif (!$self->{is_squidguard_log}) {
1307				$self->{ug_end_time} = $time if (!$time || ($self->{ug_end_time} < $time));
1308				# Register the first parsing time
1309				if (!$self->{ug_begin_time} || ($self->{ug_begin_time} > $time)) {
1310					$self->{ug_begin_time} = $time;
1311					print STDERR "UFDBGUARD LOG SET START TIME: ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($time)), "\n" if (!$self->{QuietMode});
1312				}
1313			} else {
1314				$self->{sg_end_time} = $time if (!$time || ($self->{sg_end_time} < $time));
1315				# Register the first parsing time
1316				if (!$self->{sg_begin_time} || ($self->{sg_begin_time} > $time)) {
1317					$self->{sg_begin_time} = $time;
1318					print STDERR "SQUIDGUARD LOG SET START TIME: ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($time)), "\n" if (!$self->{QuietMode});
1319				}
1320			}
1321
1322			# Only store (HIT|UNMODIFIED)/(MISS|MODIFIED|TUNNEL)/(DENIED|REDIRECT) status
1323			# and peer CD_SIBLING_HIT/ aswell as peer SIBLING_HIT/...
1324			if ( ($code =~ m#(HIT|UNMODIFIED)[:/]#) || ($self->{SiblingHit} && ($line =~ / (CD_)?SIBLING_HIT/)) ) {
1325				$code = 'HIT';
1326			} elsif ($code =~ m#(MISS|MODIFIED|TUNNEL)[:/]#) {
1327				$code = 'MISS';
1328			} elsif ($code =~ m#(DENIED|REDIRECT)[:/]#) {
1329				$code = 'DENIED';
1330			} else {
1331				next;
1332			}
1333
1334			# With common and combined log format those fields have already been parsed
1335			if (($format eq 'native') && ($line =~ $native_format_regex2) ) {
1336				$url = lc($1);
1337				$login = lc($2);
1338				$status = lc($3);
1339				$mime_type = lc($4);
1340				# Some site has corrupted mime_type, try to remove nasty characters
1341				$mime_type =~ s/[^\-\/\.\(\)\+\_,\=a-z0-9]+//igs;
1342			}
1343
1344			if ($url) {
1345				if (!$mime_type || ($mime_type eq '-')) {
1346					$mime_type = 'none';
1347				}
1348
1349				# Do not parse some unwanted method
1350				next if (($#{$self->{ExcludedMimes}} >= 0) && map {$mime_type =~ m#^$_$#} @{$self->{ExcludedMimes}});
1351
1352				# Remove extra space character in username
1353				$login =~ s/\%20//g;
1354
1355				my $id = $client_ip || '';
1356				if ($login ne '-') {
1357					$id = $login;
1358				}
1359				next if (!$id || (!$bytes && ($code ne 'DENIED')));
1360
1361				#####
1362				# If there's some mandatory inclusion, check the entry against the definitions
1363				# The entry is skipped directly if it is not in an inclusion list
1364				#####
1365				next if (!$self->check_inclusions($login, $client_ip));
1366
1367				#####
1368				# Check the entry against the exclusion definitions. The entry
1369				# is skipped directly when it match an exclusion definition.
1370				#####
1371				next if ($self->check_exclusions($login, $client_ip, $url));
1372
1373				# Set default user login to client ip address
1374				# Anonymize all users
1375				if ($self->{AnonymizeLogin} && ($client_ip ne $id)) {
1376					if (!exists $self->{AnonymizedId}{$id}) {
1377						$self->{AnonymizedId}{$id} = &anonymize_id();
1378					}
1379					$id = $self->{AnonymizedId}{$id};
1380				}
1381
1382				# Now parse data and generate statistics
1383				$self->_parseData($time, $elapsed, $client_ip, $code, $bytes, $url, $id, $mime_type, $acl);
1384				$line_stored_count++;
1385
1386			}
1387			$line_processed_count++;
1388		}
1389	}
1390	$logfile->close();
1391
1392	if ($self->{cur_year}) {
1393		# Save last parsed data
1394		$self->_append_data($self->{cur_year}, $self->{cur_month}, $self->{cur_day});
1395		# Stats can be cleared
1396		$self->_clear_stats();
1397
1398		# Stores last week to process
1399		my $wn = &get_week_number($self->{cur_year}, $self->{cur_month}, $self->{cur_day});
1400		if (!grep(/^$self->{cur_year}\/$self->{cur_month}\/$wn$/, @{$self->{week_parsed}})) {
1401			push(@{$self->{week_parsed}}, "$self->{cur_year}/$self->{cur_month}/$wn");
1402		}
1403
1404		# Save the last information parsed in this file part
1405		my $tmp_file = 'last_parsed.tmp';
1406		if ($self->{is_squidguard_log}) {
1407			$tmp_file = 'sg_last_parsed.tmp';
1408		} elsif ($self->{is_ufdbguard_log}) {
1409			$tmp_file = 'ug_last_parsed.tmp';
1410		}
1411		if (open(OUT, ">>$self->{pid_dir}/$tmp_file")) {
1412			flock(OUT, 2) || die "FATAL: can't acquire lock on file $tmp_file, $!\n";
1413			if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
1414				print OUT "$self->{last_year} $self->{last_month}{$self->{last_year}} $self->{last_day}{$self->{last_year}} $self->{end_time} $self->{end_offset} $line_stored_count $line_processed_count $line_count $self->{first_year} $self->{first_month}{$self->{first_year}} ", join(',', @{$self->{week_parsed}}), "\n";
1415			} elsif (!$self->{is_squidguard_log}) {
1416				print OUT "$self->{last_year} $self->{last_month}{$self->{last_year}} $self->{last_day}{$self->{last_year}} $self->{ug_end_time} $self->{ug_end_offset} $line_stored_count $line_processed_count $line_count $self->{first_year} $self->{first_month}{$self->{first_year}} ", join(',', @{$self->{week_parsed}}), "\n";
1417			} else {
1418				print OUT "$self->{last_year} $self->{last_month}{$self->{last_year}} $self->{last_day}{$self->{last_year}} $self->{sg_end_time} $self->{sg_end_offset} $line_stored_count $line_processed_count $line_count $self->{first_year} $self->{first_month}{$self->{first_year}} ", join(',', @{$self->{week_parsed}}), "\n";
1419			}
1420			close(OUT);
1421		} else {
1422			print STDERR "ERROR: can't save last parsed line into $self->{pid_dir}/$tmp_file, $!\n";
1423		}
1424	}
1425
1426}
1427
1428sub _clear_stats
1429{
1430	my $self = shift;
1431
1432	# Hashes to store user statistics
1433	$self->{stat_user_hour} = ();
1434	$self->{stat_user_day} = ();
1435	$self->{stat_user_month} = ();
1436	$self->{stat_usermax_hour} = ();
1437	$self->{stat_usermax_day} = ();
1438	$self->{stat_usermax_month} = ();
1439	$self->{stat_user_url_hour} = ();
1440	$self->{stat_user_url_day} = ();
1441	$self->{stat_user_url_month} = ();
1442
1443	# Hashes to store network statistics
1444	$self->{stat_network_hour} = ();
1445	$self->{stat_network_day} = ();
1446	$self->{stat_network_month} = ();
1447	$self->{stat_netmax_hour} = ();
1448	$self->{stat_netmax_day} = ();
1449	$self->{stat_netmax_month} = ();
1450
1451	# Hashes to store user / network statistics
1452	$self->{stat_netuser_hour} = ();
1453	$self->{stat_netuser_day} = ();
1454	$self->{stat_netuser_month} = ();
1455
1456	# Hashes to store cache status (hit/miss)
1457	$self->{stat_code_hour} = ();
1458	$self->{stat_code_day} = ();
1459	$self->{stat_code_month} = ();
1460
1461	# Hashes to store throughput statsœ
1462	$self->{stat_throughput_hour} = ();
1463	$self->{stat_throughput_day} = ();
1464	$self->{stat_throughput_month} = ();
1465
1466	# Hashes to store mime type
1467	$self->{stat_mime_type_hour} = ();
1468	$self->{stat_mime_type_day} = ();
1469	$self->{stat_mime_type_month} = ();
1470
1471}
1472
1473sub _init
1474{
1475	my ($self, $conf_file, $log_file, $debug, $rebuild, $pid_dir, $pidfile, $timezone) = @_;
1476
1477	# Set path to pid file
1478	$pidfile = $pid_dir . '/' . $pidfile;
1479
1480	# Prevent for a call without instance
1481	if (!ref($self)) {
1482		print STDERR "ERROR - init : Unable to call init without an object instance.\n";
1483		unlink("$pidfile");
1484		exit(0);
1485	}
1486	$self->{pidfile} = $pidfile || '/tmp/squid-analyzer.pid';
1487
1488	# Load configuration information
1489	if (!$conf_file) {
1490		if (-f '/etc/squidanalyzer/squidanalyzer.conf') {
1491			$conf_file = '/etc/squidanalyzer/squidanalyzer.conf';
1492		} elsif (-f '/etc/squidanalyzer.conf') {
1493			$conf_file = '/etc/squidanalyzer.conf';
1494		} elsif (-f 'squidanalyzer.conf') {
1495			$conf_file = 'squidanalyzer.conf';
1496		}
1497	}
1498	my %options = $self->parse_config($conf_file, $log_file, $rebuild);
1499
1500	# Configuration options
1501	$self->{MinPie} = $options{MinPie} || 2;
1502	$self->{QuietMode} = $options{QuietMode} || 0;
1503	$self->{UrlReport} = $options{UrlReport} || 0;
1504	$self->{UrlHitsOnly} = $options{UrlHitsOnly} || 0;
1505	$self->{MaxFormatError} = $options{MaxFormatError} || 0;
1506	if (defined $options{UserReport}) {
1507		$self->{UserReport} = $options{UserReport};
1508	} else {
1509		# Assure backward compatibility after update otherwize
1510		# data files will lost users information if directive
1511		# is not found in the configuration file
1512		$self->{UserReport} = 1;
1513	}
1514	$self->{Output} = $options{Output} || '';
1515	$self->{WebUrl} = $options{WebUrl} || '';
1516	$self->{WebUrl} .= '/' if ($self->{WebUrl} && ($self->{WebUrl} !~ /\/$/));
1517	$self->{DateFormat} = $options{DateFormat} || '%y-%m-%d';
1518	$self->{Lang} = $options{Lang} || '';
1519	$self->{AnonymizeLogin} = $options{AnonymizeLogin} || 0;
1520	$self->{SiblingHit} = $options{SiblingHit} || 1;
1521	$self->{ImgFormat} = $options{ImgFormat} || 'png';
1522	$self->{Locale} = $options{Locale} || '';
1523	$self->{TopUrlUser} = $options{TopUrlUser} || 0;
1524	$self->{no_year_stat} = 0;
1525	$self->{no_week_stat} = 0;
1526	$self->{UseClientDNSName} = $options{UseClientDNSName} || 0;
1527	$self->{DNSLookupTimeout} = $options{DNSLookupTimeout} || 0.0001;
1528	$self->{DNSLookupTimeout} = int($self->{DNSLookupTimeout} * 1000000);
1529	$self->{LogFile} = ();
1530	$self->{queue_size} = 1;
1531	$self->{running_pids} = ();
1532	$self->{pid_dir} = $pid_dir || '/tmp';
1533	$self->{child_count} = 0;
1534	$self->{rebuild} = $rebuild || 0;
1535	$self->{is_squidguard_log} = 0;
1536	$self->{TimeZone} = $options{TimeZone} || $timezone || 0;
1537
1538	# Cleanup old temporary files
1539	foreach my $tmp_file ('last_parsed.tmp', 'sg_last_parsed.tmp') {
1540		unlink("$self->{pid_dir}/$tmp_file");
1541	}
1542
1543	$self->{CustomHeader} = $options{CustomHeader} || qq{<a href="$self->{WebUrl}"><img src="$self->{WebUrl}images/logo-squidanalyzer.png" title="SquidAnalyzer $VERSION" border="0"></a> SquidAnalyzer};
1544	$self->{ExcludedMethods} = ();
1545	if ($options{ExcludedMethods}) {
1546		push(@{$self->{ExcludedMethods}}, split(/\s*,\s*/, $options{ExcludedMethods}));
1547	}
1548	$self->{ExcludedCodes} = ();
1549	if ($options{ExcludedCodes}) {
1550		push(@{$self->{ExcludedCodes}}, split(/\s*,\s*/, $options{ExcludedCodes}));
1551	}
1552	$self->{ExcludedMimes} = ();
1553	if ($options{ExcludedMimes}) {
1554		push(@{$self->{ExcludedMimes}}, split(/\s*,\s*/, $options{ExcludedMimes}));
1555	}
1556
1557	if ($self->{Lang}) {
1558		open(IN, "$self->{Lang}") or die "ERROR: can't open translation file $self->{Lang}, $!\n";
1559		while (my $l = <IN>) {
1560			chomp($l);
1561			$l =~ s/\r//gs;
1562			next if ($l =~ /^\s*#/);
1563			next if (!$l);
1564			my ($key, $str) = split(/\t+/, $l);
1565			$Translate{$key} = $str;
1566		}
1567		close(IN);
1568	}
1569	if (!$self->{Output}) {
1570		die "ERROR: 'Output' configuration option must be set.\n";
1571	}
1572	if (! -d $self->{Output}) {
1573		die "ERROR: 'Output' directory $self->{Output} doesn't exists.\n";
1574	}
1575	if (!$self->{rebuild}) {
1576		push(@{$self->{LogFile}}, @{$options{LogFile}});
1577		if ($#{$self->{LogFile}} < 0) {
1578			die "ERROR: 'LogFile' configuration directive must be set or a log file given at command line.\n";
1579		}
1580	}
1581	$self->{OrderUser} = lc($options{OrderUser}) || 'bytes';
1582	$self->{OrderNetwork} = lc($options{OrderNetwork}) || 'bytes';
1583	$self->{OrderUrl} = lc($options{OrderUrl}) || 'bytes';
1584	$self->{OrderMime} = lc($options{OrderMime}) || 'bytes';
1585	if ($self->{OrderUser} !~ /^(hits|bytes|duration)$/) {
1586		die "ERROR: OrderUser must be one of these values: hits, bytes or duration\n";
1587	}
1588	if ($self->{OrderNetwork} !~ /^(hits|bytes|duration)$/) {
1589		die "ERROR: OrderNetwork must be one of these values: hits, bytes or duration\n";
1590	}
1591	if ($self->{OrderUrl} !~ /^(hits|bytes|duration)$/) {
1592		die "ERROR: OrderUrl must be one of these values: hits, bytes or duration\n";
1593	}
1594	if ($self->{OrderMime} !~ /^(hits|bytes)$/) {
1595		die "ERROR: OrderMime must be one of these values: hits or bytes\n";
1596	}
1597	%{$self->{NetworkAlias}} = $self->parse_network_aliases($options{NetworkAlias} || '');
1598	%{$self->{UserAlias}}    = $self->parse_user_aliases($options{UserAlias} || '');
1599	%{$self->{Exclude}}      = $self->parse_exclusion($options{Exclude} || '');
1600	%{$self->{Include}}      = $self->parse_inclusion($options{Include} || '');
1601
1602	$self->{CostPrice} = $options{CostPrice} || 0;
1603	$self->{Currency} = $options{Currency} || '&euro;';
1604	$self->{TopNumber} = $options{TopNumber} || 10;
1605	$self->{TransfertUnit} = $options{TransfertUnit} || 'BYTES';
1606	if (!grep(/^$self->{TransfertUnit}$/i, 'BYTES', 'KB', 'MB', 'GB')) {
1607		die "ERROR: TransfertUnit must be one of these values: KB, MB or GB\n";
1608	} else {
1609		if (uc($self->{TransfertUnit}) eq 'BYTES') {
1610			$self->{TransfertUnitValue} = 1;
1611			$self->{TransfertUnit} = 'Bytes';
1612		} elsif (uc($self->{TransfertUnit}) eq 'KB') {
1613			$self->{TransfertUnitValue} = 1024;
1614		} elsif (uc($self->{TransfertUnit}) eq 'MB') {
1615			$self->{TransfertUnitValue} = 1024*1024;
1616		} elsif (uc($self->{TransfertUnit}) eq 'GB') {
1617			$self->{TransfertUnitValue} = 1024*1024*1024;
1618		}
1619	}
1620
1621	# Init statistics storage hashes
1622	$self->_clear_stats();
1623
1624	# Used to store the first and last date parsed
1625        $self->{last_year} = 0;
1626        $self->{last_month} = ();
1627        $self->{last_day} = ();
1628        $self->{cur_year} = 0;
1629        $self->{cur_month} = 0;
1630        $self->{cur_day} = 0;
1631        $self->{first_year} = 0;
1632        $self->{first_month} = ();
1633	$self->{begin_time} = 0;
1634	$self->{end_time} = 0;
1635	$self->{end_offset} = 0;
1636	$self->{week_parsed} = ();
1637	# Used to stored command line parameters from squid-analyzer
1638	$self->{history_time} = 0;
1639	$self->{preserve} = 0;
1640	$self->{sg_end_time} = 0;
1641	$self->{sg_end_offset} = 0;
1642	$self->{ug_end_time} = 0;
1643	$self->{ug_end_offset} = 0;
1644
1645	# Override verbose mode
1646	$self->{QuietMode} = 0 if ($debug);
1647
1648	# Enable local date format if defined, else strftime will be used. The limitation
1649	# this behavior is that all dates in HTML files will be the same for performences reasons.
1650	if ($self->{Locale}) {
1651		my $lang = 'LANG=' . $self->{Locale};
1652		$self->{start_date} = `$lang date | iconv -t $Translate{CharSet} 2>/dev/null`;
1653		chomp($self->{start_date});
1654	}
1655
1656	# Get the last parsing date for Squid log incremental parsing
1657	if (!$rebuild && -e "$self->{Output}/SquidAnalyzer.current") {
1658		my $current = new IO::File;
1659		unless($current->open("$self->{Output}/SquidAnalyzer.current")) {
1660			print STDERR "ERROR: Can't read file $self->{Output}/SquidAnalyzer.current, $!\n" if (!$self->{QuietMode});
1661			print STDERR "Starting at the first line of Squid access log file.\n" if (!$self->{QuietMode});
1662		} else {
1663			my $tmp = <$current>;
1664			chomp($tmp);
1665			($self->{history_time}, $self->{end_offset}) = split(/[\t]/, $tmp);
1666			$self->{begin_time} = $self->{history_time};
1667			$current->close();
1668			if ($self->{history_time}) {
1669				print STDERR "SQUID LOG HISTORY TIME: ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($self->{history_time})), " - HISTORY OFFSET: $self->{end_offset}\n" if (!$self->{QuietMode});
1670			}
1671		}
1672	}
1673
1674	# Get the last parsing date for SquidGuard log incremental parsing
1675	if (!$rebuild && -e "$self->{Output}/SquidGuard.current") {
1676		my $current = new IO::File;
1677		unless($current->open("$self->{Output}/SquidGuard.current")) {
1678			print STDERR "ERROR: Can't read file $self->{Output}/SquidGuard.current, $!\n" if (!$self->{QuietMode});
1679			print STDERR "Starting at the first line of SquidGuard log file.\n" if (!$self->{QuietMode});
1680		} else {
1681			my $tmp = <$current>;
1682			chomp($tmp);
1683			($self->{sg_history_time}, $self->{sg_end_offset}) = split(/[\t]/, $tmp);
1684			$self->{sg_begin_time} = $self->{sg_history_time};
1685			$current->close();
1686			if ($self->{sg_history_time}) {
1687				print STDERR "SQUIDGUARD LOG HISTORY TIME: ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($self->{sg_history_time})), " - HISTORY OFFSET: $self->{sg_end_offset}\n" if (!$self->{QuietMode});
1688			}
1689		}
1690	}
1691
1692	# Get the last parsing date for ufdbGuard log incremental parsing
1693	if (!$rebuild && -e "$self->{Output}/ufdbGuard.current") {
1694		my $current = new IO::File;
1695		unless($current->open("$self->{Output}/ufdbGuard.current")) {
1696			print STDERR "ERROR: Can't read file $self->{Output}/ufdbGuard.current, $!\n" if (!$self->{QuietMode});
1697			print STDERR "Starting at the first line of ufdbGuard log file.\n" if (!$self->{QuietMode});
1698		} else {
1699			my $tmp = <$current>;
1700			chomp($tmp);
1701			($self->{ug_history_time}, $self->{ug_end_offset}) = split(/[\t]/, $tmp);
1702			$self->{ug_begin_time} = $self->{ug_history_time};
1703			$current->close();
1704			if ($self->{ug_history_time}) {
1705				print STDERR "UFDBGUARD LOG HISTORY TIME: ", strftime("%a %b %e %H:%M:%S %Y", CORE::localtime($self->{ug_history_time})), " - HISTORY OFFSET: $self->{ug_end_offset}\n" if (!$self->{QuietMode});
1706			}
1707		}
1708	}
1709
1710	$self->{menu} = qq{
1711<div id="menu">
1712<ul>
1713<li><a href="../index.html"><span class="iconArrow">$Translate{'Back_link'}</span></a></li>
1714};
1715	if ($self->{UrlReport}) {
1716		$self->{menu} .= qq{
1717<li><a href="domain.html"><span class="iconDomain">$Translate{'Top_domain_link'}</span></a></li>
1718<li><a href="url.html"><span class="iconUrl">$Translate{'Top_url_link'}</span></a></li>
1719<li><a href="denied.html"><span class="iconUrl">$Translate{'Top_denied_link'}</span></a></li>
1720};
1721	}
1722	if ($self->{UserReport}) {
1723		$self->{menu} .= qq{
1724<li><a href="user.html"><span class="iconUser">$Translate{'User_link'}</span></a></li>
1725};
1726	}
1727	$self->{menu} .= qq{
1728<li><a href="network.html"><span class="iconNetwork">$Translate{'Network_link'}</span></a></li>
1729<li><a href="mime_type.html"><span class="iconMime">$Translate{'Mime_link'}</span></a></li>
1730</ul>
1731</div>
1732};
1733
1734	$self->{menu2} = qq{
1735<div id="menu">
1736<ul>
1737<li><a href="../../index.html"><span class="iconArrow">$Translate{'Back_link'}</span></a></li>
1738};
1739	if ($self->{UrlReport}) {
1740		$self->{menu2} .= qq{
1741<li><a href="../../domain.html"><span class="iconDomain">$Translate{'Top_domain_link'}</span></a></li>
1742<li><a href="../../url.html"><span class="iconUrl">$Translate{'Top_url_link'}</span></a></li>A
1743<li><a href="../../denied.html"><span class="iconUrl">$Translate{'Top_denied_link'}</span></a></li>A
1744};
1745	}
1746	if ($self->{UserReport}) {
1747		$self->{menu2} .= qq{
1748<li><a href="../../user.html"><span class="iconUser">$Translate{'User_link'}</span></a></li>
1749};
1750	}
1751	$self->{menu2} .= qq{
1752<li><a href="../../network.html"><span class="iconNetwork">$Translate{'Network_link'}</span></a></li>
1753<li><a href="../../mime_type.html"><span class="iconMime">$Translate{'Mime_link'}</span></a></li>
1754</ul>
1755</div>
1756};
1757
1758	$self->{menu3} = qq{
1759<div id="menu">
1760<ul>
1761<li><a href="../index.html"><span class="iconArrow">$Translate{'Back_link'}</span></a></li>
1762</ul>
1763</div>
1764};
1765
1766}
1767
1768sub _gethostbyaddr
1769{
1770	my ($self, $ip) = @_;
1771
1772	my $host = undef;
1773	unless(exists $CACHE{$ip}) {
1774		eval {
1775			local $SIG{ALRM} = sub { die "DNS lookup timeout.\n"; };
1776			ualarm $self->{DNSLookupTimeout};
1777			$host = gethostbyaddr(inet_aton($ip), AF_INET);
1778			ualarm 0;
1779		};
1780		if ($@) {
1781			$CACHE{$ip} = undef;
1782			#printf "_gethostbyaddr timeout : %s\n", $ip;
1783		}
1784		else {
1785			$CACHE{$ip} = $host;
1786			#printf "_gethostbyaddr success : %s (%s)\n", $ip, $host;
1787		}
1788	}
1789	return $CACHE{$ip} || $ip;
1790}
1791
1792sub _parseData
1793{
1794	my ($self, $time, $elapsed, $client, $code, $bytes, $url, $id, $type, $acl) = @_;
1795
1796	# Save original IP address for dns resolving
1797	my $client_ip_addr = $client;
1798
1799	# Get the current year and month
1800	my ($sec,$min,$hour,$day,$month,$year,$wday,$yday,$isdst) = CORE::localtime($time);
1801	$year += 1900;
1802	$month = sprintf("%02d", $month + 1);
1803	$day = sprintf("%02d", $day);
1804
1805	# Store data when hour change to save memory
1806	if ($self->{cur_year} && ($self->{cur_hour} ne '') && ($hour != $self->{cur_hour}) ) {
1807		# If the day has changed then we want to save stats of the previous one
1808		$self->_append_data($self->{cur_year}, $self->{cur_month}, $self->{cur_day});
1809		# Stats can be cleared
1810		print STDERR "Clearing statistics storage hashes, for $self->{cur_year}-$self->{cur_month}-$self->{cur_day} ", sprintf("%02d", $self->{cur_hour}), ":00:00.\n" if (!$self->{QuietMode});
1811		$self->_clear_stats();
1812	}
1813
1814	# Stores weeks to process
1815	if (!$self->{no_week_stat}) {
1816		if ("$year$month$day" ne "$self->{cur_year}$self->{cur_month}$self->{cur_day}") {
1817			my $wn = &get_week_number($year, $month, $day);
1818			if (!grep(/^$year\/$month\/$wn$/, @{$self->{week_parsed}})) {
1819				push(@{$self->{week_parsed}}, "$year/$month/$wn");
1820			}
1821		}
1822	}
1823
1824	# Extract the domainname part of the URL
1825	$url =~ s/:\d+.*//;
1826	$url =~ m/^[^\/]+\/\/([^\/]+)/;
1827	my $dest = $1 || $url;
1828
1829	# Replace username by his dnsname if there's no username
1830	# (login is equal to ip) and if client is an ip address
1831	if ( ($id eq $client) && $self->{UseClientDNSName}) {
1832		if ($client =~ $ip_regexp) {
1833			my $dnsname = $self->_gethostbyaddr($client);
1834			if ($dnsname) {
1835				$id = $dnsname;
1836			}
1837		}
1838	}
1839
1840	# Replace network by his aliases if any
1841	my $network = '';
1842	foreach my $r (keys %{$self->{NetworkAlias}}) {
1843		if ($r =~ $cidr_regex) {
1844			if (&check_ip($client, $r)) {
1845				$network = $self->{NetworkAlias}->{$r};
1846				last;
1847			}
1848		} elsif ($client =~ /^$r/) {
1849			$network = $self->{NetworkAlias}->{$r};
1850			last;
1851		}
1852	}
1853
1854	# Set default to a class A network
1855	if (!$network) {
1856		$client =~ /^(.*)([:\.]+)\d+$/;
1857		$network = "$1$2". "0";
1858	}
1859
1860	# Replace username by his alias if any
1861	foreach my $u (keys %{$self->{UserAlias}}) {
1862		if ( $id =~ /^$u$/i ) {
1863			$id = $self->{UserAlias}->{$u};
1864			last;
1865		}
1866	}
1867
1868	# Stores last parsed date part
1869	if (!$self->{last_year} || ("$year$month$day" gt "$self->{last_year}$self->{last_month}{$self->{last_year}}$self->{last_day}{$self->{last_year}}")) {
1870		$self->{last_year} = $year;
1871		$self->{last_month}{$self->{last_year}} = $month;
1872		$self->{last_day}{$self->{last_year}} = $day;
1873	}
1874
1875	# Stores first parsed date part
1876	if (!$self->{first_year} || ("$self->{first_year}$self->{first_month}{$self->{first_year}}" gt "$year$month")) {
1877		$self->{first_year} = $year;
1878		$self->{first_month}{$self->{first_year}} = $month;
1879	}
1880
1881	# Stores current processed values
1882	$self->{cur_year} = $year;
1883	$self->{cur_month} = $month;
1884	$self->{cur_day} = $day;
1885	$self->{cur_hour} = $hour;
1886	$hour = sprintf("%02d", $hour);
1887
1888	#### Store access denied statistics
1889	if ($code eq 'DENIED') {
1890		$self->{stat_code_hour}{$code}{$hour}{hits}++;
1891		$self->{stat_code_hour}{$code}{$hour}{bytes} += $bytes;
1892		$self->{stat_code_day}{$code}{$self->{last_day}}{hits}++;
1893		$self->{stat_code_day}{$code}{$self->{last_day}}{bytes} += $bytes;
1894
1895		$self->{stat_throughput_hour}{$code}{$hour}{bytes} += $bytes;
1896		$self->{stat_throughput_day}{$code}{$self->{last_day}}{bytes} += $bytes;
1897		$self->{stat_throughput_hour}{$code}{$hour}{elapsed} += $elapsed;
1898		$self->{stat_throughput_day}{$code}{$self->{last_day}}{elapsed} += $elapsed;
1899
1900		#### Store url statistics
1901		if ($self->{UrlReport}) {
1902			$self->{stat_denied_url_hour}{$id}{$dest}{hits}++;
1903			$self->{stat_denied_url_hour}{$id}{$dest}{firsthit} = $time if (!$self->{stat_denied_url_hour}{$id}{$dest}{firsthit} || ($time < $self->{stat_denied_url_hour}{$id}{$dest}{firsthit}));
1904			$self->{stat_denied_url_hour}{$id}{$dest}{lasthit} = $time if (!$self->{stat_denied_url_hour}{$id}{$dest}{lasthit} || ($time > $self->{stat_denied_url_hour}{$id}{$dest}{lasthit}));
1905			$self->{stat_denied_url_hour}{$id}{$dest}{blacklist}{$acl}++ if ($acl);
1906			$self->{stat_denied_url_day}{$id}{$dest}{hits}++;
1907			$self->{stat_denied_url_day}{$id}{$dest}{firsthit} = $time if (!$self->{stat_denied_url_day}{$id}{$dest}{firsthit} || ($time < $self->{stat_denied_url_day}{$id}{$dest}{firsthit}));
1908			$self->{stat_denied_url_day}{$id}{$dest}{lasthit} = $time if (!$self->{stat_denied_url_day}{$id}{$dest}{lasthit} || ($time > $self->{stat_denied_url_day}{$id}{$dest}{lasthit}));
1909			$self->{stat_denied_url_day}{$id}{$dest}{blacklist}{$acl}++ if ($acl);
1910		}
1911		return;
1912	}
1913	
1914	#### Store client statistics
1915	if ($self->{UserReport}) {
1916		$self->{stat_user_hour}{$id}{$hour}{hits}++;
1917		$self->{stat_user_hour}{$id}{$hour}{bytes} += $bytes;
1918		$self->{stat_user_hour}{$id}{$hour}{duration} += $elapsed;
1919		$self->{stat_user_day}{$id}{$self->{last_day}}{hits}++;
1920		$self->{stat_user_day}{$id}{$self->{last_day}}{bytes} += $bytes;
1921		$self->{stat_user_day}{$id}{$self->{last_day}}{duration} += $elapsed;
1922		if ($bytes > $self->{stat_usermax_hour}{$id}{largest_file_size}) {
1923			$self->{stat_usermax_hour}{$id}{largest_file_size} = $bytes;
1924			$self->{stat_usermax_hour}{$id}{largest_file_url} = $url;
1925		}
1926		if ($bytes > $self->{stat_usermax_day}{$id}{largest_file_size}) {
1927			$self->{stat_usermax_day}{$id}{largest_file_size} = $bytes;
1928			$self->{stat_usermax_day}{$id}{largest_file_url} = $url;
1929		}
1930	}
1931
1932	#### Store networks statistics
1933	$self->{stat_network_hour}{$network}{$hour}{hits}++;
1934	$self->{stat_network_hour}{$network}{$hour}{bytes} += $bytes;
1935	$self->{stat_network_hour}{$network}{$hour}{duration} += $elapsed;
1936	$self->{stat_network_day}{$network}{$self->{last_day}}{hits}++;
1937	$self->{stat_network_day}{$network}{$self->{last_day}}{bytes} += $bytes;
1938	$self->{stat_network_day}{$network}{$self->{last_day}}{duration} += $elapsed;
1939	if ($bytes > $self->{stat_netmax_hour}{$network}{largest_file_size}) {
1940		$self->{stat_netmax_hour}{$network}{largest_file_size} = $bytes;
1941		$self->{stat_netmax_hour}{$network}{largest_file_url} = $url;
1942	}
1943	if ($bytes > $self->{stat_netmax_day}{$network}{largest_file_size}) {
1944		$self->{stat_netmax_day}{$network}{largest_file_size} = $bytes;
1945		$self->{stat_netmax_day}{$network}{largest_file_url} = $url;
1946	}
1947
1948	#### Store HIT/MISS/DENIED statistics
1949	$self->{stat_code_hour}{$code}{$hour}{hits}++;
1950	$self->{stat_code_hour}{$code}{$hour}{bytes} += $bytes;
1951	$self->{stat_code_hour}{$code}{$hour}{elapsed} += $elapsed;
1952	$self->{stat_code_day}{$code}{$self->{last_day}}{hits}++;
1953	$self->{stat_code_day}{$code}{$self->{last_day}}{bytes} += $bytes;
1954	$self->{stat_code_day}{$code}{$self->{last_day}}{elapsed} += $elapsed;
1955
1956	$self->{stat_throughput_hour}{$code}{$hour}{bytes} += $bytes;
1957	$self->{stat_throughput_day}{$code}{$self->{last_day}}{bytes} += $bytes;
1958	$self->{stat_throughput_hour}{$code}{$hour}{elapsed} += $elapsed;
1959	$self->{stat_throughput_day}{$code}{$self->{last_day}}{elapsed} += $elapsed;
1960
1961	#### Store url statistics
1962	if ($self->{UrlReport}) {
1963		$self->{stat_user_url_hour}{$id}{$dest}{duration} += $elapsed;
1964		$self->{stat_user_url_hour}{$id}{$dest}{hits}++;
1965		$self->{stat_user_url_hour}{$id}{$dest}{bytes} += $bytes;
1966		$self->{stat_user_url_hour}{$id}{$dest}{firsthit} = $time if (!$self->{stat_user_url_hour}{$id}{$dest}{firsthit} || ($time < $self->{stat_user_url_hour}{$id}{$dest}{firsthit}));
1967		$self->{stat_user_url_hour}{$id}{$dest}{lasthit} = $time if (!$self->{stat_user_url_hour}{$id}{$dest}{lasthit} || ($time > $self->{stat_user_url_hour}{$id}{$dest}{lasthit}));
1968		$self->{stat_user_url_day}{$id}{$dest}{duration} += $elapsed;
1969		$self->{stat_user_url_day}{$id}{$dest}{hits}++;
1970		$self->{stat_user_url_day}{$id}{$dest}{firsthit} = $time if (!$self->{stat_user_url_day}{$id}{$dest}{firsthit} || ($time < $self->{stat_user_url_day}{$id}{$dest}{firsthit}));
1971		$self->{stat_user_url_day}{$id}{$dest}{lasthit} = $time if (!$self->{stat_user_url_day}{$id}{$dest}{lasthit} || ($time > $self->{stat_user_url_day}{$id}{$dest}{lasthit}));
1972		$self->{stat_user_url_day}{$id}{$dest}{bytes} += $bytes;
1973		if ($code eq 'HIT') {
1974			$self->{stat_user_url_day}{$id}{$dest}{cache_hit}++;
1975			$self->{stat_user_url_day}{$id}{$dest}{cache_bytes} += $bytes;
1976		}
1977	}
1978
1979	#### Store user per networks statistics
1980	if ($self->{UserReport}) {
1981		$self->{stat_netuser_hour}{$network}{$id}{duration} += $elapsed;
1982		$self->{stat_netuser_hour}{$network}{$id}{bytes} += $bytes;
1983		$self->{stat_netuser_hour}{$network}{$id}{hits}++;
1984		if ($bytes > $self->{stat_netuser_hour}{$network}{$id}{largest_file_size}) {
1985			$self->{stat_netuser_hour}{$network}{$id}{largest_file_size} = $bytes;
1986			$self->{stat_netuser_hour}{$network}{$id}{largest_file_url} = $url;
1987		}
1988		$self->{stat_netuser_day}{$network}{$id}{duration} += $elapsed;
1989		$self->{stat_netuser_day}{$network}{$id}{bytes} += $bytes;
1990		$self->{stat_netuser_day}{$network}{$id}{hits}++;
1991		if ($bytes > $self->{stat_netuser_day}{$network}{$id}{largest_file_size}) {
1992			$self->{stat_netuser_day}{$network}{$id}{largest_file_size} = $bytes;
1993			$self->{stat_netuser_day}{$network}{$id}{largest_file_url} = $url;
1994		}
1995	}
1996
1997	#### Store mime type statistics
1998	$self->{stat_mime_type_hour}{"$type"}{hits}++;
1999	$self->{stat_mime_type_hour}{"$type"}{bytes} += $bytes;
2000	$self->{stat_mime_type_day}{"$type"}{hits}++;
2001	$self->{stat_mime_type_day}{"$type"}{bytes} += $bytes;
2002
2003}
2004
2005sub _load_history
2006{
2007	my ($self, $type, $year, $month, $day, $path, $kind, $wn, @wd) = @_;
2008
2009	#### Load history
2010	if ($type eq 'day') {
2011		foreach my $d ("01" .. "31") {
2012			$self->_read_stat($year, $month, $d, 'day', $kind);
2013		}
2014	} elsif ($type eq 'week') {
2015		$path = "$year/week$wn";
2016		foreach my $wdate (@wd) {
2017			$wdate =~ /^(\d+)-(\d+)-(\d+)$/;
2018			$self->_read_stat($1, $2, $3, 'day', $kind, $wn);
2019		}
2020		$type = 'day';
2021	} elsif ($type eq 'month') {
2022		foreach my $m ("01" .. "12") {
2023			$self->_read_stat($year, $m, $day, 'month', $kind);
2024		}
2025	} else {
2026		$self->_read_stat($year, $month, $day, '', $kind);
2027	}
2028
2029}
2030
2031sub _append_stat
2032{
2033	my ($self, $year, $month, $day) = @_;
2034
2035	my $read_type = '';
2036	my $type = 'hour';
2037	if (!$day) {
2038		$type = 'day';
2039	}
2040	if (!$month) {
2041		$type = 'month';
2042	}
2043	$read_type = $type;
2044
2045	my $path = join('/', $year, $month, $day);
2046	$path =~ s/[\/]+$//;
2047
2048	print STDERR "Appending data into $self->{Output}/$path\n" if (!$self->{QuietMode});
2049
2050	#### Save cache statistics
2051	my $dat_file_code = new IO::File;
2052	$dat_file_code->open(">>$self->{Output}/$path/stat_code.dat")
2053		or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_code.dat, $!\n");
2054	flock($dat_file_code, 2) || die "FATAL: can't acquire lock on file, $!\n";
2055	$self->_write_stat_data($dat_file_code, $type, 'stat_code');
2056	$dat_file_code->close();
2057
2058	#### With huge log file we only store global statistics in year and month views
2059	return if ( $self->{no_year_stat} && ($type ne 'hour') );
2060
2061	#### Save url statistics per user
2062	if ($self->{UrlReport}) {
2063		my $dat_file_user_url = new IO::File;
2064		$dat_file_user_url->open(">>$self->{Output}/$path/stat_user_url.dat")
2065			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_user_url.dat, $!\n");
2066		flock($dat_file_user_url, 2) || die "FATAL: can't acquire lock on file, $!\n";
2067		$self->_write_stat_data($dat_file_user_url, $type, 'stat_user_url');
2068		$dat_file_user_url->close();
2069		# Denied URL
2070		my $dat_file_denied_url = new IO::File;
2071		$dat_file_denied_url->open(">>$self->{Output}/$path/stat_denied_url.dat")
2072			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_denied_url.dat, $!\n");
2073		flock($dat_file_denied_url, 2) || die "FATAL: can't acquire lock on file, $!\n";
2074		$self->_write_stat_data($dat_file_denied_url, $type, 'stat_denied_url');
2075		$dat_file_denied_url->close();
2076	}
2077
2078	#### Save user statistics
2079	if ($self->{UserReport}) {
2080		my $dat_file_user = new IO::File;
2081		$dat_file_user->open(">>$self->{Output}/$path/stat_user.dat")
2082			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_user.dat, $!\n");
2083		flock($dat_file_user, 2) || die "FATAL: can't acquire lock on file, $!\n";
2084		$self->_write_stat_data($dat_file_user, $type, 'stat_user');
2085		$dat_file_user->close();
2086	}
2087
2088	#### Save network statistics
2089	my $dat_file_network = new IO::File;
2090	$dat_file_network->open(">>$self->{Output}/$path/stat_network.dat")
2091		or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_network.dat, $!\n");
2092	flock($dat_file_network, 2) || die "FATAL: can't acquire lock on file, $!\n";
2093	$self->_write_stat_data($dat_file_network, $type, 'stat_network');
2094	$dat_file_network->close();
2095
2096	#### Save user per network statistics
2097	if ($self->{UserReport}) {
2098		my $dat_file_netuser = new IO::File;
2099		$dat_file_netuser->open(">>$self->{Output}/$path/stat_netuser.dat")
2100			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_netuser.dat, $!\n");
2101		flock($dat_file_netuser, 2) || die "FATAL: can't acquire lock on file, $!\n";
2102		$self->_write_stat_data($dat_file_netuser, $type, 'stat_netuser');
2103		$dat_file_netuser->close();
2104	}
2105
2106	#### Save mime statistics
2107	my $dat_file_mime_type = new IO::File;
2108	$dat_file_mime_type->open(">>$self->{Output}/$path/stat_mime_type.dat")
2109		or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_mime_type.dat, $!\n");
2110	flock($dat_file_mime_type, 2) || die "FATAL: can't acquire lock on file, $!\n";
2111	$self->_write_stat_data($dat_file_mime_type, $type, 'stat_mime_type');
2112	$dat_file_mime_type->close();
2113	
2114}
2115
2116sub _save_stat
2117{
2118	my ($self, $year, $month, $day, $wn, @wd) = @_;
2119
2120	my $path = join('/', $year, $month, $day);
2121	$path =~ s/[\/]+$//;
2122
2123	my $read_type = '';
2124	my $type = 'hour';
2125	if (!$day) {
2126		$type = 'day';
2127	}
2128	if ($wn) {
2129		$type = 'week';
2130		$path = "$year/week$wn";
2131	}
2132	if (!$month) {
2133		$type = 'month';
2134	}
2135	$read_type = $type;
2136
2137	print STDERR "Saving data into $self->{Output}/$path\n" if (!$self->{QuietMode});
2138
2139	#### Save cache statistics
2140	my $dat_file_code = new IO::File;
2141	$self->_load_history($read_type, $year, $month, $day, $path, 'stat_code', $wn, @wd);
2142	$dat_file_code->open(">$self->{Output}/$path/stat_code.dat")
2143		or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_code.dat, $!\n");
2144	flock($dat_file_code, 2) || die "FATAL: can't acquire lock on file, $!\n";
2145	$self->_write_stat_data($dat_file_code, $type, 'stat_code');
2146	$dat_file_code->close();
2147
2148	#### With huge log file we only store global statistics in year and month views
2149	return if ( $self->{no_year_stat} && (($type ne 'hour') && !$wn) );
2150
2151	#### Save url statistics per user
2152	if ($self->{UrlReport}) {
2153		my $dat_file_user_url = new IO::File;
2154		$self->_load_history($read_type, $year, $month, $day, $path, 'stat_user_url', $wn, @wd);
2155		$dat_file_user_url->open(">$self->{Output}/$path/stat_user_url.dat")
2156			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_user_url.dat, $!\n");
2157		flock($dat_file_user_url, 2) || die "FATAL: can't acquire lock on file, $!\n";
2158		$self->_write_stat_data($dat_file_user_url, $type, 'stat_user_url');
2159		$dat_file_user_url->close();
2160		# Denied URL
2161		my $dat_file_denied_url = new IO::File;
2162		$self->_load_history($read_type, $year, $month, $day, $path, 'stat_denied_url', $wn, @wd);
2163		$dat_file_denied_url->open(">$self->{Output}/$path/stat_denied_url.dat")
2164			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_denied_url.dat, $!\n");
2165		flock($dat_file_denied_url, 2) || die "FATAL: can't acquire lock on file, $!\n";
2166		$self->_write_stat_data($dat_file_denied_url, $type, 'stat_denied_url');
2167		$dat_file_denied_url->close();
2168	}
2169
2170	#### Save user statistics
2171	if ($self->{UserReport}) {
2172		my $dat_file_user = new IO::File;
2173		$self->_load_history($read_type, $year, $month, $day, $path, 'stat_user', $wn, @wd);
2174		$dat_file_user->open(">$self->{Output}/$path/stat_user.dat")
2175			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_user.dat, $!\n");
2176		flock($dat_file_user, 2) || die "FATAL: can't acquire lock on file, $!\n";
2177		$self->_write_stat_data($dat_file_user, $type, 'stat_user');
2178		$dat_file_user->close();
2179	}
2180
2181	#### Save network statistics
2182	my $dat_file_network = new IO::File;
2183	$self->_load_history($read_type, $year, $month, $day, $path, 'stat_network', $wn, @wd);
2184	$dat_file_network->open(">$self->{Output}/$path/stat_network.dat")
2185		or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_network.dat, $!\n");
2186	flock($dat_file_network, 2) || die "FATAL: can't acquire lock on file, $!\n";
2187	$self->_write_stat_data($dat_file_network, $type, 'stat_network');
2188	$dat_file_network->close();
2189
2190	#### Save user per network statistics
2191	if ($self->{UserReport}) {
2192		my $dat_file_netuser = new IO::File;
2193		$self->_load_history($read_type, $year, $month, $day, $path, 'stat_netuser', $wn, @wd);
2194		$dat_file_netuser->open(">$self->{Output}/$path/stat_netuser.dat")
2195			or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_netuser.dat, $!\n");
2196		flock($dat_file_netuser, 2) || die "FATAL: can't acquire lock on file, $!\n";
2197		$self->_write_stat_data($dat_file_netuser, $type, 'stat_netuser');
2198		$dat_file_netuser->close();
2199	}
2200
2201	#### Save mime statistics
2202	my $dat_file_mime_type = new IO::File;
2203	$self->_load_history($read_type, $year, $month, $day, $path, 'stat_mime_type', $wn, @wd);
2204	$dat_file_mime_type->open(">$self->{Output}/$path/stat_mime_type.dat")
2205		or $self->localdie("ERROR: Can't write to file $self->{Output}/$path/stat_mime_type.dat, $!\n");
2206	flock($dat_file_mime_type, 2) || die "FATAL: can't acquire lock on file, $!\n";
2207	$self->_write_stat_data($dat_file_mime_type, $type, 'stat_mime_type');
2208	$dat_file_mime_type->close();
2209
2210}
2211
2212sub _write_stat_data
2213{
2214	my ($self, $fh, $type, $kind) = @_;
2215
2216	$type = 'day' if ($type eq 'week');
2217
2218	#### Save cache statistics
2219	if ($kind eq 'stat_code') {
2220		foreach my $code (sort {$a cmp $b} keys %{$self->{"stat_code_$type"}}) {
2221			$fh->print("$code " .  "hits_$type=");
2222			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_code_$type"}{$code}}) {
2223				$fh->print("$tmp:" . $self->{"stat_code_$type"}{$code}{$tmp}{hits} . ",");
2224			}
2225			$fh->print(";bytes_$type=");
2226			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_code_$type"}{$code}}) {
2227				$fh->print("$tmp:" . $self->{"stat_code_$type"}{$code}{$tmp}{bytes} . ",");
2228			}
2229			$fh->print(";thp_bytes_$type=");
2230			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_throughput_$type"}{$code}}) {
2231				$fh->print("$tmp:" . $self->{"stat_throughput_$type"}{$code}{$tmp}{bytes} . ",");
2232			}
2233			$fh->print(";thp_duration_$type=");
2234			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_throughput_$type"}{$code}}) {
2235				$fh->print("$tmp:" . $self->{"stat_throughput_$type"}{$code}{$tmp}{elapsed} . ",");
2236			}
2237			$fh->print("\n");
2238		}
2239		$self->{"stat_code_$type"} = ();
2240	}
2241
2242	#### Save denied url statistics per user
2243	if ($kind eq 'stat_denied_url') {
2244		foreach my $id (sort {$a cmp $b} keys %{$self->{"stat_denied_url_$type"}}) {
2245			foreach my $dest (keys %{$self->{"stat_denied_url_$type"}{$id}}) {
2246				next if (!$dest);
2247				my $u = $id;
2248				$u = '-' if (!$self->{UserReport});
2249				my $bl = '';
2250				if (exists $self->{"stat_denied_url_$type"}{$id}{$dest}{blacklist}) {
2251					foreach my $b (keys %{$self->{"stat_denied_url_$type"}{$id}{$dest}{blacklist}}) {
2252						$bl .= $b . ',' . $self->{"stat_denied_url_$type"}{$id}{$dest}{blacklist}{$b} . ',';
2253					}
2254					$bl =~ s/,$//;
2255				}
2256				$fh->print(
2257					"$id hits=" . $self->{"stat_denied_url_$type"}{$id}{$dest}{hits} . ";" .
2258					"first=" . $self->{"stat_denied_url_$type"}{$id}{$dest}{firsthit} . ";" .
2259					"last=" . $self->{"stat_denied_url_$type"}{$id}{$dest}{lasthit} . ";" .
2260					"url=$dest" . ";" .
2261					"blacklist=" . $bl .
2262					"\n");
2263			}
2264		}
2265		$self->{"stat_denied_url_$type"} = ();
2266	}
2267
2268	#### Save url statistics per user
2269	if ($kind eq 'stat_user_url') {
2270		foreach my $id (sort {$a cmp $b} keys %{$self->{"stat_user_url_$type"}}) {
2271			foreach my $dest (keys %{$self->{"stat_user_url_$type"}{$id}}) {
2272				my $u = $id;
2273				$u = '-' if (!$self->{UserReport});
2274				$fh->print(
2275					"$id hits=" . $self->{"stat_user_url_$type"}{$id}{$dest}{hits} . ";" .
2276					"bytes=" . $self->{"stat_user_url_$type"}{$id}{$dest}{bytes} . ";" .
2277					"duration=" . $self->{"stat_user_url_$type"}{$id}{$dest}{duration} . ";" .
2278					"first=" . $self->{"stat_user_url_$type"}{$id}{$dest}{firsthit} . ";" .
2279					"last=" . $self->{"stat_user_url_$type"}{$id}{$dest}{lasthit} . ";" .
2280					"url=$dest;" .
2281					"cache_hit=" . ($self->{"stat_user_url_$type"}{$id}{$dest}{cache_hit}||0) . ";" .
2282					"cache_bytes=" . ($self->{"stat_user_url_$type"}{$id}{$dest}{cache_bytes}||0) . "\n");
2283			}
2284		}
2285		$self->{"stat_user_url_$type"} = ();
2286	}
2287
2288	#### Save user statistics
2289	if ($kind eq 'stat_user') {
2290		foreach my $id (sort {$a cmp $b} keys %{$self->{"stat_user_$type"}}) {
2291			my $name = $id;
2292			$name =~ s/\s+//g;
2293			$fh->print("$name hits_$type=");
2294			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_user_$type"}{$id}}) {
2295				$fh->print("$tmp:" . $self->{"stat_user_$type"}{$id}{$tmp}{hits} . ",");
2296			}
2297			$fh->print(";bytes_$type=");
2298			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_user_$type"}{$id}}) {
2299				$fh->print("$tmp:" . $self->{"stat_user_$type"}{$id}{$tmp}{bytes} . ",");
2300			}
2301			$fh->print(";duration_$type=");
2302			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_user_$type"}{$id}}) {
2303				$fh->print("$tmp:" . $self->{"stat_user_$type"}{$id}{$tmp}{duration} . ",");
2304			}
2305			$fh->print(";largest_file_size=" . $self->{"stat_usermax_$type"}{$id}{largest_file_size});
2306			$fh->print(";largest_file_url=" . $self->{"stat_usermax_$type"}{$id}{largest_file_url} . "\n");
2307		}
2308		$self->{"stat_user_$type"} = ();
2309		$self->{"stat_usermax_$type"} = ();
2310	}
2311
2312	#### Save network statistics
2313	if ($kind eq 'stat_network') {
2314		foreach my $net (sort {$a cmp $b} keys %{$self->{"stat_network_$type"}}) {
2315			$fh->print("$net\thits_$type=");
2316			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_network_$type"}{$net}}) {
2317				$fh->print("$tmp:" . $self->{"stat_network_$type"}{$net}{$tmp}{hits} . ",");
2318			}
2319			$fh->print(";bytes_$type=");
2320			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_network_$type"}{$net}}) {
2321				$fh->print("$tmp:" . $self->{"stat_network_$type"}{$net}{$tmp}{bytes} . ",");
2322			}
2323			$fh->print(";duration_$type=");
2324			foreach my $tmp (sort {$a <=> $b} keys %{$self->{"stat_network_$type"}{$net}}) {
2325				$fh->print("$tmp:" . $self->{"stat_network_$type"}{$net}{$tmp}{duration} . ",");
2326			}
2327			$fh->print(";largest_file_size=" . $self->{"stat_netmax_$type"}{$net}{largest_file_size});
2328			$fh->print(";largest_file_url=" . $self->{"stat_netmax_$type"}{$net}{largest_file_url} . "\n");
2329		}
2330		$self->{"stat_network_$type"} = ();
2331		$self->{"stat_netmax_$type"} = ();
2332	}
2333
2334	#### Save user per network statistics
2335	if ($kind eq 'stat_netuser') {
2336		foreach my $net (sort {$a cmp $b} keys %{$self->{"stat_netuser_$type"}}) {
2337			foreach my $id (sort {$a cmp $b} keys %{$self->{"stat_netuser_$type"}{$net}}) {
2338				$fh->print("$net\t$id\thits=" . $self->{"stat_netuser_$type"}{$net}{$id}{hits} . ";" .
2339					"bytes=" . $self->{"stat_netuser_$type"}{$net}{$id}{bytes} . ";" .
2340					"duration=" . $self->{"stat_netuser_$type"}{$net}{$id}{duration} . ";");
2341				$fh->print("largest_file_size=" .
2342					$self->{"stat_netuser_$type"}{$net}{$id}{largest_file_size} . ";" .
2343					"largest_file_url=" . $self->{"stat_netuser_$type"}{$net}{$id}{largest_file_url} . "\n");
2344			}
2345		}
2346		$self->{"stat_netuser_$type"} = ();
2347	}
2348
2349	#### Save mime statistics
2350	if ($kind eq 'stat_mime_type') {
2351		foreach my $mime (sort {$a cmp $b} keys %{$self->{"stat_mime_type_$type"}}) {
2352			$fh->print("$mime hits=" . $self->{"stat_mime_type_$type"}{$mime}{hits} . ";" .
2353				"bytes=" . $self->{"stat_mime_type_$type"}{$mime}{bytes} .  "\n");
2354		}
2355		$self->{"stat_mime_type_$type"} = ();
2356	}
2357	
2358}
2359
2360sub _read_stat
2361{
2362	my ($self, $year, $month, $day, $sum_type, $kind, $wn) = @_;
2363
2364	my $type = 'hour';
2365	if (!$day) {
2366		$type = 'day';
2367	}
2368	if (!$month) {
2369		$type = 'month';
2370	}
2371
2372	my $path = join('/',  $year, $month, $day);
2373	$path =~ s/[\/]+$//;
2374
2375	return if (! -d "$self->{Output}/$path");
2376
2377	#print STDERR "Reading data from previous dat files for $sum_type($type) in $self->{Output}/$path/$kind.dat\n" if (!$self->{QuietMode});
2378
2379	my $k = '';
2380	my $key = '';
2381	$key = $day if ($sum_type eq 'day');
2382	$key = $month if ($sum_type eq 'month');
2383	$sum_type ||= $type;
2384
2385	#### Read previous cache statistics
2386	if (!$kind || ($kind eq 'stat_code')) {
2387		my $dat_file_code = new IO::File;
2388		if ($dat_file_code->open("$self->{Output}/$path/stat_code.dat")) {
2389			my $i = 1;
2390			my $error = 0;
2391			while (my $l = <$dat_file_code>) {
2392				chomp($l);
2393				if ($l =~ s/^([^\s]+)\s+hits_$type=([^;]+);bytes_$type=([^;]+)//) {
2394					my $code = $1;
2395					my $hits = $2 || '';
2396					my $bytes = $3 || '';
2397					$hits =~ s/,$//;
2398					$bytes =~ s/,$//;
2399					my %hits_tmp = split(/[:,]/, $hits);
2400					foreach my $tmp (sort {$a <=> $b} keys %hits_tmp) {
2401						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2402						$self->{"stat_code_$sum_type"}{$code}{$k}{hits} += $hits_tmp{$tmp};
2403					}
2404					my %bytes_tmp = split(/[:,]/, $bytes);
2405					foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
2406						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2407						$self->{"stat_code_$sum_type"}{$code}{$k}{bytes} += $bytes_tmp{$tmp};
2408					}
2409					if ($l =~ s/thp_bytes_$type=([^;]+);thp_duration_$type=([^;]+)//) {
2410						$bytes = $1 || '';
2411						my $elapsed = $2 || '';
2412						$elapsed =~ s/,$//;
2413						my %bytes_tmp = split(/[:,]/, $bytes);
2414						foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
2415							if ($key ne '') { $k = $key; } else { $k = $tmp; }
2416							$self->{"stat_throughput_$sum_type"}{$code}{$k}{bytes} += $bytes_tmp{$tmp};
2417						}
2418						my %elapsed_tmp = split(/[:,]/, $elapsed);
2419						foreach my $tmp (sort {$a <=> $b} keys %elapsed_tmp) {
2420							if ($key ne '') { $k = $key; } else { $k = $tmp; }
2421							$self->{"stat_throughput_$sum_type"}{$code}{$k}{elapsed} += $elapsed_tmp{$tmp};
2422						}
2423					}
2424				} else {
2425					print STDERR "ERROR: bad format at line $i into $self->{Output}/$path/stat_code.dat\n";
2426					print STDERR "$l\n";
2427					if ($error > $self->{MaxFormatError}) {
2428						unlink($self->{pidfile});
2429						exit 0;
2430					}
2431					$error++;
2432				}
2433				$i++;
2434			}
2435			$dat_file_code->close();
2436		}
2437	}
2438
2439	#### With huge log file we only store global statistics in year and month views
2440	return if ($self->{no_year_stat} && ($type ne 'hour'));
2441
2442	#### Read previous client statistics
2443	if (!$kind || ($kind eq 'stat_user')) {
2444		my $dat_file_user = new IO::File;
2445		if ($dat_file_user->open("$self->{Output}/$path/stat_user.dat")) {
2446			my $i = 1;
2447			my $error = 0;
2448			while (my $l = <$dat_file_user>) {
2449				chomp($l);
2450				if ($l =~ s/^([^\s]+)\s+hits_$type=([^;]+);bytes_$type=([^;]+);duration_$type=([^;]+);largest_file_size=([^;]*);largest_file_url=(.*)$//) {
2451					my $id = $1;
2452					my $hits = $2 || '';
2453					my $bytes = $3 || '';
2454					my $duration = $4 || '';
2455					my $lsize = $5 || 0;
2456					my $lurl = $6 || 0;
2457
2458					if ($self->{rebuild}) {
2459						next if (!$self->check_inclusions($id));
2460						next if ($self->check_exclusions($id));
2461					}
2462
2463					# Anonymize all users
2464					if ($self->{AnonymizeLogin} && ($id !~ /^Anon[a-zA-Z0-9]{16}$/)) {
2465						if (!exists $self->{AnonymizedId}{$id}) {
2466							$self->{AnonymizedId}{$id} = &anonymize_id();
2467						}
2468						$id = $self->{AnonymizedId}{$id};
2469					}
2470
2471					if ($lsize > $self->{"stat_usermax_$sum_type"}{$id}{largest_file_size}) {
2472						$self->{"stat_usermax_$sum_type"}{$id}{largest_file_size} = $lsize;
2473						$self->{"stat_usermax_$sum_type"}{$id}{largest_file_url} = $lurl;
2474					}
2475					$hits =~ s/,$//;
2476					$bytes =~ s/,$//;
2477					$duration =~ s/,$//;
2478					my %hits_tmp = split(/[:,]/, $hits);
2479					foreach my $tmp (sort {$a <=> $b} keys %hits_tmp) {
2480						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2481						$self->{"stat_user_$sum_type"}{$id}{$k}{hits} += $hits_tmp{$tmp};
2482					}
2483					my %bytes_tmp = split(/[:,]/, $bytes);
2484					foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
2485						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2486						$self->{"stat_user_$sum_type"}{$id}{$k}{bytes} += $bytes_tmp{$tmp};
2487					}
2488					my %duration_tmp = split(/[:,]/, $duration);
2489					foreach my $tmp (sort {$a <=> $b} keys %duration_tmp) {
2490						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2491						$self->{"stat_user_$sum_type"}{$id}{$k}{duration} += $duration_tmp{$tmp};
2492					}
2493				} else {
2494					print STDERR "ERROR: bad format at line $i into $self->{Output}/$path/stat_user.dat:\n";
2495					print STDERR "$l\n";
2496					if ($error > $self->{MaxFormatError}) {
2497						unlink($self->{pidfile});
2498						exit 0;
2499					}
2500					$error++;
2501				}
2502				$i++;
2503			}
2504			$dat_file_user->close();
2505		}
2506	}
2507
2508	#### Read previous url statistics
2509	if ($self->{UrlReport}) {
2510
2511		if (!$kind || ($kind eq 'stat_user_url')) {
2512			my $dat_file_user_url = new IO::File;
2513			if ($dat_file_user_url->open("$self->{Output}/$path/stat_user_url.dat")) {
2514				my $i = 1;
2515				my $error = 0;
2516				while (my $l = <$dat_file_user_url>) {
2517					chomp($l);
2518					my $id = '';
2519					if ($l =~ /^([^\s]+)\s+hits=/) {
2520						$id = $1;
2521					}
2522					$id = '-' if (!$self->{UserReport});
2523
2524					if ($self->{rebuild}) {
2525						next if (!$self->check_inclusions($id));
2526						next if ($self->check_exclusions($id));
2527					}
2528
2529					# Anonymize all users
2530					if ($self->{AnonymizeLogin} && ($id !~ /^Anon[a-zA-Z0-9]{16}$/)) {
2531						if (!exists $self->{AnonymizedId}{$id}) {
2532							$self->{AnonymizedId}{$id} = &anonymize_id();
2533						}
2534						$id = $self->{AnonymizedId}{$id};
2535					}
2536
2537					if ($l =~ s/^([^\s]+)\s+hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*?);cache_hit=(\d*);cache_bytes=(\d*)//) {
2538						my $url = $7;
2539						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{hits} += $2;
2540						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{bytes} += $3;
2541						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{duration} += abs($4);
2542						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{firsthit} = $5 if (!$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{firsthit} || ($5 < $self->{"stat_user_url_$sum_type"}{$id}{"$url"}{firsthit}));
2543						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{lasthit} = $6 if (!$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{lasthit} || ($6 > $self->{"stat_user_url_$sum_type"}{$id}{"$url"}{lasthit}));
2544						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{cache_hit} += $8;
2545						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{cache_bytes} += $9;
2546						if ($self->{rebuild}) {
2547							if ($self->check_exclusions('', '', $url)) {
2548								delete $self->{"stat_user_url_$sum_type"}{$id}{"$url"};
2549								next;
2550							}
2551						}
2552					} elsif ($l =~ s/^([^\s]+)\s+hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*)$//) {
2553						my $url = $7;
2554						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{hits} += $2;
2555						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{bytes} += $3;
2556						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{duration} += abs($4);
2557						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{firsthit} = $5 if (!$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{firsthit} || ($5 < $self->{"stat_user_url_$sum_type"}{$id}{"$url"}{firsthit}));
2558						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{lasthit} = $6 if (!$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{lasthit} || ($6 > $self->{"stat_user_url_$sum_type"}{$id}{"$url"}{lasthit}));
2559						if ($self->{rebuild}) {
2560							if ($self->check_exclusions('', '', $url)) {
2561								delete $self->{"stat_user_url_$sum_type"}{$id}{"$url"};
2562								next;
2563							}
2564						}
2565					} elsif ($l =~ s/^([^\s]+)\s+hits=(\d+);bytes=(\d+);duration=([\-\d]+);url=(.*)$//) {
2566						my $url = $5;
2567						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{hits} += $2;
2568						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{bytes} += $3;
2569						$self->{"stat_user_url_$sum_type"}{$id}{"$url"}{duration} += abs($4);
2570						if ($self->{rebuild}) {
2571							if ($self->check_exclusions('', '', $url)) {
2572								delete $self->{"stat_user_url_$sum_type"}{$id}{"$url"};
2573								next;
2574							}
2575						}
2576					} else {
2577						print STDERR "ERROR: bad format at line $i into $self->{Output}/$path/stat_user_url.dat\n";
2578						print STDERR "$l\n";
2579						if ($error > $self->{MaxFormatError}) {
2580							unlink($self->{pidfile});
2581							exit 0;
2582						}
2583						$error++;
2584					}
2585					$i++;
2586				}
2587				$dat_file_user_url->close();
2588			}
2589		}
2590
2591		if (!$kind || ($kind eq 'stat_denied_url')) {
2592			my $dat_file_denied_url = new IO::File;
2593			if ($dat_file_denied_url->open("$self->{Output}/$path/stat_denied_url.dat")) {
2594				my $i = 1;
2595				my $error = 0;
2596				while (my $l = <$dat_file_denied_url>) {
2597					chomp($l);
2598					my $id = '';
2599					if ($l =~ /^([^\s]+)\s+hits=/) {
2600						$id = $1;
2601					}
2602					$id = '-' if (!$self->{UserReport});
2603
2604					if ($self->{rebuild}) {
2605						next if (!$self->check_inclusions($id));
2606						next if ($self->check_exclusions($id));
2607					}
2608
2609					# Anonymize all denieds
2610					if ($self->{AnonymizeLogin} && ($id !~ /^Anon[a-zA-Z0-9]{16}$/)) {
2611						if (!exists $self->{AnonymizedId}{$id}) {
2612							$self->{AnonymizedId}{$id} = &anonymize_id();
2613						}
2614						$id = $self->{AnonymizedId}{$id};
2615					}
2616
2617					if ($l =~ s/^([^\s]+)\s+hits=(\d+);first=([^;]*);last=([^;]*);url=(.*);blacklist=(.*)//) {
2618						if ($self->{rebuild}) {
2619							next if ($self->check_exclusions('', '', $5));
2620						}
2621						$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{hits} += $2;
2622						$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{firsthit} = $3 if (!$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{firsthit} || ($3 < $self->{"stat_denied_url_$sum_type"}{$id}{"$7"}{firsthit}));
2623						$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{lasthit} = $4 if (!$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{lasthit} || ($4 > $self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{lasthit}));
2624						if ($6) {
2625							my %tmp = split(/,/, $6);
2626							foreach my $k (keys %tmp) {
2627								$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{blacklist}{$k} += $tmp{$k};
2628							}
2629						}
2630					} elsif ($l =~ s/^([^\s]+)\s+hits=(\d+);first=([^;]*);last=([^;]*);url=(.*)//) {
2631						if ($self->{rebuild}) {
2632							next if ($self->check_exclusions('', '', $5));
2633						}
2634						$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{hits} += $2;
2635						$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{firsthit} = $3 if (!$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{firsthit} || ($3 < $self->{"stat_denied_url_$sum_type"}{$id}{"$7"}{firsthit}));
2636						$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{lasthit} = $4 if (!$self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{lasthit} || ($4 > $self->{"stat_denied_url_$sum_type"}{$id}{"$5"}{lasthit}));
2637					} elsif ($l =~ /^([^\s]+)\s+hits=;first=;last=;url=/) {
2638						# do nothing, this should not appears, but fixes issue #81
2639					} else {
2640						print STDERR "ERROR: bad format at line $i into $self->{Output}/$path/stat_denied_url.dat\n";
2641						print STDERR "$l\n";
2642						if ($error > $self->{MaxFormatError}) {
2643							unlink($self->{pidfile});
2644							exit 0;
2645						}
2646						$error++;
2647					}
2648					$i++;
2649				}
2650				$dat_file_denied_url->close();
2651			}
2652		}
2653
2654	}
2655
2656	#### Read previous network statistics
2657	if (!$kind || ($kind eq 'stat_network')) {
2658		my $dat_file_network = new IO::File;
2659		if ($dat_file_network->open("$self->{Output}/$path/stat_network.dat")) {
2660			my $i = 1;
2661			my $error = 0;
2662			while (my $l = <$dat_file_network>) {
2663				chomp($l);
2664				my ($net, $data) = split(/\t/, $l);
2665				if (!$data) {
2666					# Assume backward compatibility
2667					$l =~ s/^(.*)\shits_$type=/hits_$type=/;
2668					$net = $1;
2669					$data = $l;
2670				}
2671
2672				if ($self->{rebuild} && !exists $self->{NetworkAlias}->{$net}) {
2673					next if (!$self->check_inclusions('', $net));
2674					next if ($self->check_exclusions('', $net));
2675				}
2676
2677				if ($data =~ s/^hits_$type=([^;]+);bytes_$type=([^;]+);duration_$type=([^;]+);largest_file_size=([^;]*);largest_file_url=(.*)$//) {
2678					my $hits = $1 || '';
2679					my $bytes = $2 || '';
2680					my $duration = $3 || '';
2681
2682					if ($4 > $self->{"stat_netmax_$sum_type"}{$net}{largest_file_size}) {
2683						$self->{"stat_netmax_$sum_type"}{$net}{largest_file_size} = $4;
2684						$self->{"stat_netmax_$sum_type"}{$net}{largest_file_url} = $5;
2685					}
2686					$hits =~ s/,$//;
2687					$bytes =~ s/,$//;
2688					$duration =~ s/,$//;
2689					my %hits_tmp = split(/[:,]/, $hits);
2690					foreach my $tmp (sort {$a <=> $b} keys %hits_tmp) {
2691						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2692						$self->{"stat_network_$sum_type"}{$net}{$k}{hits} += $hits_tmp{$tmp};
2693					}
2694					my %bytes_tmp = split(/[:,]/, $bytes);
2695					foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
2696						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2697						$self->{"stat_network_$sum_type"}{$net}{$k}{bytes} += $bytes_tmp{$tmp};
2698					}
2699					my %duration_tmp = split(/[:,]/, $duration);
2700					foreach my $tmp (sort {$a <=> $b} keys %duration_tmp) {
2701						if ($key ne '') { $k = $key; } else { $k = $tmp; }
2702						$self->{"stat_network_$sum_type"}{$net}{$k}{duration} += $duration_tmp{$tmp};
2703					}
2704				} else {
2705					print STDERR "ERROR: bad format at line $i into $self->{Output}/$path/stat_network.dat\n";
2706					print STDERR "$l\n";
2707					if ($error > $self->{MaxFormatError}) {
2708						unlink($self->{pidfile});
2709						exit 0;
2710					}
2711					$error++;
2712				}
2713				$i++;
2714			}
2715			$dat_file_network->close();
2716		}
2717	}
2718
2719	#### Read previous user per network statistics
2720	if ($self->{UserReport}) {
2721		if (!$kind || ($kind eq 'stat_netuser')) {
2722			my $dat_file_netuser = new IO::File;
2723			if ($dat_file_netuser->open("$self->{Output}/$path/stat_netuser.dat")) {
2724				my $i = 1;
2725				my $error = 0;
2726				while (my $l = <$dat_file_netuser>) {
2727					chomp($l);
2728					my ($net, $id, $data) = split(/\t/, $l);
2729					if (!$data) {
2730						# Assume backward compatibility
2731						$l =~ s/^(.*)\s([^\s]+)\shits=/hits=/;
2732						$net = $1;
2733						$id = $2;
2734						$data = $l;
2735					}
2736
2737					if ($self->{rebuild}) {
2738						next if (!$self->check_inclusions($id, $net));
2739						next if ($self->check_exclusions($id, $net));
2740					}
2741
2742					# Anonymize all users
2743					if ($self->{AnonymizeLogin} && ($id !~ /^Anon[a-zA-Z0-9]{16}$/)) {
2744						if (!exists $self->{AnonymizedId}{$id}) {
2745							$self->{AnonymizedId}{$id} = &anonymize_id();
2746						}
2747						$id = $self->{AnonymizedId}{$id};
2748					}
2749
2750					if ($data =~ s/^hits=(\d+);bytes=(\d+);duration=([\-\d]+);largest_file_size=([^;]*);largest_file_url=(.*)$//) {
2751						$self->{"stat_netuser_$sum_type"}{$net}{$id}{hits} += $1;
2752						$self->{"stat_netuser_$sum_type"}{$net}{$id}{bytes} += $2;
2753						$self->{"stat_netuser_$sum_type"}{$net}{$id}{duration} += abs($3);
2754						if ($4 > $self->{"stat_netuser_$sum_type"}{$net}{$id}{largest_file_size}) {
2755							$self->{"stat_netuser_$sum_type"}{$net}{$id}{largest_file_size} = $4;
2756							$self->{"stat_netuser_$sum_type"}{$net}{$id}{largest_file_url} = $5;
2757						}
2758					} else {
2759						print STDERR "ERROR: bad format at line $i into $self->{Output}/$path/stat_netuser.dat\n";
2760						print STDERR "$l\n";
2761						if ($error > $self->{MaxFormatError}) {
2762							unlink($self->{pidfile});
2763							exit 0;
2764						}
2765						$error++;
2766					}
2767					$i++;
2768				}
2769				$dat_file_netuser->close();
2770			}
2771		}
2772	}
2773
2774	#### Read previous mime statistics
2775	if (!$kind || ($kind eq 'stat_mime_type')) {
2776		my $dat_file_mime_type = new IO::File;
2777		if ($dat_file_mime_type->open("$self->{Output}/$path/stat_mime_type.dat")) {
2778			my $i = 1;
2779			my $error = 0;
2780			while (my $l = <$dat_file_mime_type>) {
2781				chomp($l);
2782				if ($l =~ s/^([^\s]+)\s+hits=(\d+);bytes=(\d+)//) {
2783					my $mime = $1;
2784					$self->{"stat_mime_type_$sum_type"}{$mime}{hits} += $2;
2785					$self->{"stat_mime_type_$sum_type"}{$mime}{bytes} += $3;
2786				} else {
2787					print STDERR "ERROR: bad format at line $i into $self->{Output}/$path/stat_mime_type.dat\n";
2788					print STDERR "$l\n";
2789					if ($error > $self->{MaxFormatError}) {
2790						unlink($self->{pidfile});
2791						exit 0;
2792					}
2793					$error++;
2794				}
2795				$i++;
2796			}
2797			$dat_file_mime_type->close();
2798		}
2799	}
2800
2801}
2802
2803sub _save_data
2804{
2805	my ($self, $year, $month, $day, $wn, @wd) = @_;
2806
2807	#### Create directory structure
2808	if (!-d "$self->{Output}/$year") {
2809		mkdir("$self->{Output}/$year", 0755) || $self->localdie("ERROR: can't create directory $self->{Output}/$year, $!\n");
2810	}
2811	if ($month && !-d "$self->{Output}/$year/$month") {
2812		mkdir("$self->{Output}/$year/$month", 0755) || $self->localdie("ERROR: can't create directory $self->{Output}/$year/$month, $!\n");
2813	}
2814	if ($day && !-d "$self->{Output}/$year/$month/$day") {
2815		mkdir("$self->{Output}/$year/$month/$day", 0755) || $self->localdie("ERROR: can't create directory $self->{Output}/$year/$month/$day, $!\n");
2816	}
2817	if (!$self->{no_week_stat}) {
2818		if ($wn && !-d "$self->{Output}/$year/week$wn") {
2819			mkdir("$self->{Output}/$year/week$wn", 0755) || $self->localdie("ERROR: can't create directory $self->{Output}/$year/week$wn, $!\n");
2820		}
2821	}
2822
2823	# Dumping data
2824	$self->_save_stat($year, $month, $day, $wn, @wd);
2825
2826}
2827
2828sub _append_data
2829{
2830	my ($self, $year, $month, $day) = @_;
2831
2832	#### Create directory structure
2833	if (!-d "$self->{Output}/$year") {
2834		mkdir("$self->{Output}/$year", 0755) || $self->localdie("ERROR: can't create directory $self->{Output}/$year, $!\n");
2835	}
2836	if ($month && !-d "$self->{Output}/$year/$month") {
2837		mkdir("$self->{Output}/$year/$month", 0755) || $self->localdie("ERROR: can't create directory $self->{Output}/$year/$month, $!\n");
2838	}
2839	if ($day && !-d "$self->{Output}/$year/$month/$day") {
2840		mkdir("$self->{Output}/$year/$month/$day", 0755) || $self->localdie("ERROR: can't create directory $self->{Output}/$year/$month/$day, $!\n");
2841	}
2842
2843	# Dumping data
2844	$self->_append_stat($year, $month, $day);
2845
2846}
2847
2848
2849sub _print_header
2850{
2851	my ($self, $fileout, $menu, $calendar, $sortpos) = @_;
2852
2853	my $now = $self->{start_date} || strftime("%a %b %e %H:%M:%S %Y", CORE::localtime);
2854	$sortpos ||= 2;
2855	my $sorttable = '';
2856	$sorttable = "var myTH = document.getElementById('contenu').getElementsByTagName('th')[$sortpos]; sorttable.innerSortFunction.apply(myTH, []);";
2857	print $$fileout qq{
2858<html>
2859<head>
2860<meta NAME="robots" CONTENT="noindex,nofollow" />
2861<meta HTTP-EQUIV="Pragma" CONTENT="no-cache" />
2862<meta HTTP-EQUIV="Cache-Control" content="no-cache" />
2863<meta HTTP-EQUIV="Expires" CONTENT="$now" />
2864<meta HTTP-EQUIV="Generator" CONTENT="SquidAnalyzer $VERSION" />
2865<meta HTTP-EQUIV="Date" CONTENT="$now" />
2866<meta HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=$Translate{'CharSet'}" />
2867<title>SquidAnalyzer $VERSION Report</title>
2868<link rel="stylesheet" type="text/css" href="$self->{WebUrl}squidanalyzer.css" media="screen" />
2869<!-- javascript to sort table -->
2870<script type="text/javascript" src="$self->{WebUrl}sorttable.js"></script>
2871<!-- javascript to draw graphics -->
2872<script type="text/javascript" src="$self->{WebUrl}flotr2.js"></script>
2873<script  type="text/javascript" >sortpos = $sortpos;</script>
2874</head>
2875<body onload="$sorttable">
2876<div id="conteneur">
2877<a name="atop"></a>
2878	<div id="header">
2879		<div id="alignLeft">
2880		<h1>
2881		$self->{CustomHeader}
2882		</h1>
2883		<p class="sous-titre">
2884		$Translate{'Generation'} $now.
2885		</p>
2886		</div>
2887		$calendar
2888	</div>
2889	$menu
2890	<div id="contenu">
2891};
2892
2893}
2894
2895sub _print_footer
2896{
2897	my ($self, $fileout) = @_;
2898
2899	print $$fileout qq{
2900	</div>
2901	<div id="footer">
2902		<h4>
2903		$Translate{'File_Generated'} <a href="http://squidanalyzer.darold.net/">SquidAnalyzer v$VERSION</a>
2904		</h4>
2905	</div>
2906</div>
2907</body>
2908</html>
2909};
2910
2911}
2912
2913sub check_build_date
2914{
2915	my ($self, $year, $month, $day) = @_;
2916
2917	return 0 if (!$self->{build_date});
2918
2919	my ($y, $m, $d) = split(/\-/, $self->{build_date});
2920
2921	return 1 if ($year ne $y);
2922	if ($m) {
2923		return 1 if ($month && ($month ne $m));
2924		if ($d) {
2925			return 1 if ($day && ($day ne $d));
2926		}
2927	}
2928
2929	return 0;
2930}
2931
2932sub buildHTML
2933{
2934	my ($self, $outdir) = @_;
2935
2936	# No new log registered, no html buid required
2937	if (!$self->{rebuild}) {
2938		if (!$self->{last_year} && !$self->{last_month} && !$self->{last_day}) {
2939			print STDERR "Skipping HTML build.\n" if (!$self->{QuietMode});
2940			return;
2941		}
2942	}
2943
2944	$outdir ||= $self->{Output};
2945
2946	print STDERR "Building HTML output into $outdir\n" if (!$self->{QuietMode});
2947
2948	# Load history data for incremental scan
2949	my $old_year = 0;
2950	my $old_month = 0;
2951	my $old_day = 0;
2952	my $p_month = 0;
2953	my $p_year = 0;
2954	if ($self->{history_time} || $self->{sg_history_time}) {
2955		my @ltime = CORE::localtime($self->{history_time});
2956		if ($self->{is_squidguard_log}) {
2957			@ltime = CORE::localtime($self->{sg_history_time});
2958		} elsif ($self->{is_ufdbguard_log}) {
2959			@ltime = CORE::localtime($self->{ug_history_time});
2960		}
2961		$old_year = $ltime[5]+1900;
2962		$old_month = $ltime[4]+1;
2963		$old_month = "0$old_month" if ($old_month < 10);
2964		$old_day = $ltime[3];
2965		$old_day = "0$old_day" if ($old_day < 10);
2966	        # Set oldest stat to preserve based on history time, not current time
2967		if ($self->{preserve} > 0) {
2968			if (!$self->{is_squidguard_log} && !$self->{is_ufdbguard_log}) {
2969				@ltime = CORE::localtime($self->{history_time}-($self->{preserve}*2592000));
2970			} elsif (!$self->{is_squidguard_log}) {
2971				@ltime = CORE::localtime($self->{ug_history_time}-($self->{preserve}*2592000));
2972			} else {
2973				@ltime = CORE::localtime($self->{sg_history_time}-($self->{preserve}*2592000));
2974			}
2975			$p_year = $ltime[5]+1900;
2976			$p_month = $ltime[4]+1;
2977			$p_month = sprintf("%02d", $p_month);
2978	        	print STDERR "Obsolete statistics before $p_year-$p_month\n" if (!$self->{QuietMode});
2979		}
2980	}
2981
2982	# Generate all HTML output
2983	opendir(DIR, $outdir) || die "Error: can't opendir $outdir: $!";
2984	my @years = grep { /^\d{4}$/ && -d "$outdir/$_"} readdir(DIR);
2985	closedir DIR;
2986	$self->{child_count} = 0;
2987	my @years_cal = ();
2988	my @months_cal = ();
2989	my @weeks_cal = ();
2990	my @array_count = ();
2991	foreach my $y (sort {$a <=> $b} @years) {
2992		next if (!$y || ($y < $self->{first_year}));
2993		next if ($self->check_build_date($y));
2994		# Remove the full year repository if it is older that the last date to preserve
2995		if ($p_year && ($y < $p_year)) {
2996			print STDERR "Removing obsolete statistics for year $y\n" if (!$self->{QuietMode});
2997			system ($RM_PROG, "-rf", "$outdir/$y");
2998			next;
2999		}
3000		next if (!$p_year && ($y < $old_year));
3001		opendir(DIR, "$outdir/$y") || $self->localdie("FATAL: can't opendir $outdir/$y: $!");
3002		my @months = grep { /^\d{2}$/ && -d "$outdir/$y/$_"} readdir(DIR);
3003		my @weeks  = grep { /^week\d{2}$/ && -d "$outdir/$y/$_"} readdir(DIR);
3004		closedir DIR;
3005		my @weeks_to_build = ();
3006		foreach my $m (sort {$a <=> $b} @months) {
3007			next if (!$m || ($m < $self->{first_month}{$y}));
3008			next if ($self->check_build_date($y, $m));
3009			# Remove the full month repository if it is older that the last date to preserve
3010			if ($p_year && ("$y$m" < "$p_year$p_month")) {
3011				print STDERR "Removing obsolete statistics for month $y-$m\n" if (!$self->{QuietMode});
3012				system ($RM_PROG, "-rf", "$outdir/$y/$m");
3013				next;
3014			}
3015			next if ("$y$m" < "$old_year$old_month");
3016			opendir(DIR, "$outdir/$y/$m") || $self->localdie("FATAL: can't opendir $outdir/$y/$m: $!");
3017			my @days = grep { /^\d{2}$/ && -d "$outdir/$y/$m/$_"} readdir(DIR);
3018			closedir DIR;
3019			foreach my $d (sort {$a <=> $b} @days) {
3020				next if ($self->check_build_date($y, $m, $d));
3021				next if ("$y$m$d" < "$old_year$old_month$old_day");
3022				print STDERR "Generating statistics for day $y-$m-$d\n" if (!$self->{QuietMode});
3023				$self->gen_html_output($outdir, $y, $m, $d);
3024				push(@array_count, "$outdir/$y/$m/$d");
3025				my $wn = &get_week_number($y,$m,$d);
3026				push(@weeks_to_build, $wn) if (!grep(/^$wn$/, @weeks_to_build));
3027			}
3028			print STDERR "Generating statistics for month $y-$m\n" if (!$self->{QuietMode});
3029			push(@months_cal, "$outdir/$y/$m");
3030			$self->gen_html_output($outdir, $y, $m);
3031		}
3032		if (!$self->{no_week_stat}) {
3033			foreach my $w (sort @weeks_to_build) {
3034				$w = sprintf("%02d", $w+1);
3035				push(@array_count, "$outdir/$y/week$w");
3036				print STDERR "Generating statistics for week $w on year $y\n" if (!$self->{QuietMode});
3037				$self->gen_html_output($outdir, $y, '', '', $w);
3038			}
3039		}
3040		print STDERR "Generating statistics for year $y\n" if (!$self->{QuietMode});
3041		$self->gen_html_output($outdir, $y);
3042		push(@years_cal, "$outdir/$y");
3043
3044	}
3045
3046	# Wait for last child stop
3047	$self->wait_all_childs() if ($self->{queue_size} > 1);
3048
3049	# Set calendar in each new files by replacing SA_CALENDAR_SA in the right HTML code
3050	# Same with number of users, urls and domains
3051	foreach my $p (@years_cal) {
3052		$p =~ /\/(\d+)$/;
3053		my $stat_date = $self->set_date($1);
3054		my $cal = $self->_get_calendar($stat_date, $1, $2, 'month', $p);
3055		my $nuser = '-';
3056		my $nurl = '-';
3057		my $ndomain = '-';
3058		# Search for item count
3059		if (-e "$p/stat_count.dat") {
3060			open(IN, "$p/stat_count.dat") or die "FATAL: can't read file $p/stat_count.dat, $!\n";
3061			while (my $l = <IN>) {
3062				chomp($l);
3063				if ($l =~ /^users:(\d+)/) {
3064					$nuser = $1;
3065				} elsif ($l =~ /^urls:(\d+)/) {
3066					$nurl = $1;
3067				} elsif ($l =~ /^domains:(\d+)/) {
3068					$ndomain = $1;
3069				}
3070			}
3071			close(IN);
3072			unlink("$p/stat_count.dat");
3073		}
3074		opendir(DIR, "$p") || $self->localdie("FATAL: can't opendir $p: $!\n");
3075		my @html = grep { /\.html$/ } readdir(DIR);
3076		closedir DIR;
3077		foreach my $f (@html) {
3078			open(IN, "$p/$f") or $self->localdie("FATAL: can't read file $p/$f\n");
3079			my @content = <IN>;
3080			close IN;
3081			map { s/SA_CALENDAR_SA/$cal/ } @content;
3082			map { s/SA_NUSERS_SA/$nuser/ } @content;
3083			map { s/SA_NURLS_SA/$nurl/ } @content;
3084			map { s/SA_NDOMAINS_SA/$ndomain/ } @content;
3085			open(OUT, ">$p/$f") or $self->localdie("FATAL: can't write to file $p/$f\n");
3086			print OUT @content;
3087			close OUT;
3088		}
3089	}
3090
3091	foreach my $p (@months_cal) {
3092		$p =~ /\/(\d+)\/(\d+)$/;
3093		my $stat_date = $self->set_date($1, $2);
3094		my $cal = $self->_get_calendar($stat_date, $1, $2, 'day', $p);
3095		my $nuser = '-';
3096		my $nurl = '-';
3097		my $ndomain = '-';
3098		# Search for item count
3099		if (-e "$p/stat_count.dat") {
3100			open(IN, "$p/stat_count.dat") or die "FATAL: can't read file $p/stat_count.dat, $!\n";
3101			while (my $l = <IN>) {
3102				chomp($l);
3103				if ($l =~ /^users:(\d+)/) {
3104					$nuser = $1;
3105				} elsif ($l =~ /^urls:(\d+)/) {
3106					$nurl = $1;
3107				} elsif ($l =~ /^domains:(\d+)/) {
3108					$ndomain = $1;
3109				}
3110			}
3111			close(IN);
3112			unlink("$p/stat_count.dat");
3113		}
3114		opendir(DIR, "$p") || $self->localdie("FATAL: can't opendir $p: $!\n");
3115		my @html = grep { /\.html$/ } readdir(DIR);
3116		closedir DIR;
3117		foreach my $f (@html) {
3118			open(IN, "$p/$f") or $self->localdie("FATAL: can't read file $p/$f\n");
3119			my @content = <IN>;
3120			close IN;
3121			map { s/SA_CALENDAR_SA/$cal/ } @content;
3122			map { s/SA_NUSERS_SA/$nuser/ } @content;
3123			map { s/SA_NURLS_SA/$nurl/ } @content;
3124			map { s/SA_NDOMAINS_SA/$ndomain/ } @content;
3125			open(OUT, ">$p/$f") or $self->localdie("FATAL: can't write to file $p/$f\n");
3126			print OUT @content;
3127			close OUT;
3128		}
3129	}
3130
3131	foreach my $p (@array_count) {
3132		my $nuser = '-';
3133		my $nurl = '-';
3134		my $ndomain = '-';
3135		my $cal = '';
3136		if ($p =~ /^(.*)\/(\d{4})\/(\d{2})\/\d{2}/) {
3137			my $stat_date = $self->set_date($2, $3);
3138			$cal = $self->_get_calendar($stat_date, $2, $3, 'day', "$1/$2/$3", '../');
3139		}
3140		# Search for item count
3141		if (-e "$p/stat_count.dat") {
3142			open(IN, "$p/stat_count.dat") or die "FATAL: can't read file $p/stat_count.dat, $!\n";
3143			while (my $l = <IN>) {
3144				chomp($l);
3145				if ($l =~ /^users:(\d+)/) {
3146					$nuser = $1;
3147				} elsif ($l =~ /^urls:(\d+)/) {
3148					$nurl = $1;
3149				} elsif ($l =~ /^domains:(\d+)/) {
3150					$ndomain = $1;
3151				}
3152			}
3153			close(IN);
3154			unlink("$p/stat_count.dat");
3155		}
3156		opendir(DIR, "$p") || $self->localdie("FATAL: can't opendir $p: $!\n");
3157		my @html = grep { /\.html$/ } readdir(DIR);
3158		closedir DIR;
3159		foreach my $f (@html) {
3160			open(IN, "$p/$f") or $self->localdie("FATAL: can't read file $p/$f\n");
3161			my @content = <IN>;
3162			close IN;
3163			map { s/SA_CALENDAR_SA/$cal/ } @content;
3164			map { s/SA_NUSERS_SA/$nuser/ } @content;
3165			map { s/SA_NURLS_SA/$nurl/ } @content;
3166			map { s/SA_NDOMAINS_SA/$ndomain/ } @content;
3167			open(OUT, ">$p/$f") or $self->localdie("FATAL: can't write to file $p/$f\n");
3168			print OUT @content;
3169			close OUT;
3170		}
3171	}
3172
3173	$self->_gen_summary($outdir);
3174}
3175
3176sub gen_html_output
3177{
3178	my ($self, $outdir, $year, $month, $day, $week) = @_;
3179
3180	my $dir = "$outdir";
3181	if ($year) {
3182		$dir .= "/$year";
3183	}
3184	if ($month) {
3185		$dir .= "/$month";
3186	}
3187	if ($day) {
3188		$dir .= "/$day";
3189	}
3190	my $stat_date = $self->set_date($year, $month, $day);
3191
3192	if ($week) {
3193		$dir .= "/week$week";
3194		$stat_date = "$Translate{Week} $week - $year";
3195	}
3196
3197	#### With huge log file we do not store detail statistics
3198	if ( !$self->{no_year_stat} || ($self->{no_year_stat} && ($day || $week)) ) {
3199		if ($self->{queue_size} <= 1) {
3200			if ($self->{UserReport}) {
3201				$self->_print_user_stat($dir, $year, $month, $day, $week);
3202			}
3203			$self->_print_mime_stat($dir, $year, $month, $day, $week);
3204			$self->_print_network_stat($dir, $year, $month, $day, $week);
3205			if ($self->{UrlReport}) {
3206				$self->_print_top_url_stat($dir, $year, $month, $day, $week);
3207				$self->_print_top_denied_stat($dir, $year, $month, $day, $week);
3208				$self->_print_top_domain_stat($dir, $year, $month, $day, $week);
3209			}
3210		} else {
3211			if ($self->{UserReport}) {
3212				$self->spawn(sub {
3213					$self->_print_user_stat($dir, $year, $month, $day, $week);
3214				});
3215				$self->{child_count} = $self->manage_queue_size(++$self->{child_count});
3216			}
3217			$self->spawn(sub {
3218				$self->_print_mime_stat($dir, $year, $month, $day, $week);
3219			});
3220			$self->{child_count} = $self->manage_queue_size(++$self->{child_count});
3221			$self->spawn(sub {
3222				$self->_print_network_stat($dir, $year, $month, $day, $week);
3223			});
3224			$self->{child_count} = $self->manage_queue_size(++$self->{child_count});
3225			if ($self->{UrlReport}) {
3226				$self->spawn(sub {
3227					$self->_print_top_url_stat($dir, $year, $month, $day, $week);
3228				});
3229				$self->{child_count} = $self->manage_queue_size(++$self->{child_count});
3230				$self->spawn(sub {
3231					$self->_print_top_denied_stat($dir, $year, $month, $day, $week);
3232				});
3233				$self->{child_count} = $self->manage_queue_size(++$self->{child_count});
3234				$self->spawn(sub {
3235					$self->_print_top_domain_stat($dir, $year, $month, $day, $week);
3236				});
3237				$self->{child_count} = $self->manage_queue_size(++$self->{child_count});
3238			}
3239
3240		}
3241	}
3242	$self->_print_cache_stat($dir, $year, $month, $day, $week);
3243
3244}
3245
3246sub parse_duration
3247{
3248	my ($secondes) = @_;
3249
3250	my $hours = int($secondes/3600);
3251	$hours = "0$hours" if ($hours < 10);
3252	$secondes = $secondes - ($hours*3600);
3253	my $minutes = int($secondes/60);
3254	$minutes = "0$minutes" if ($minutes < 10);
3255	$secondes = $secondes - ($minutes*60);
3256	$secondes = "0$secondes" if ($secondes < 10);
3257
3258	return "$hours:$minutes:$secondes";
3259}
3260
3261sub _print_cache_stat
3262{
3263	my ($self, $outdir, $year, $month, $day, $week) = @_;
3264
3265	print STDERR "\tCache statistics in $outdir...\n" if (!$self->{QuietMode});
3266
3267	$0 = "squid-analyzer: Printing cache statistics in $outdir";
3268
3269	my $stat_date = $self->set_date($year, $month, $day);
3270
3271	my $type = 'hour';
3272	if (!$day) {
3273		$type = 'day';
3274	}
3275	if (!$month) {
3276		$type = 'month';
3277	}
3278	if ($week) {
3279		$type = 'day';
3280	}
3281
3282	# Load code statistics
3283	my %code_stat = ();
3284	my %throughput_stat = ();
3285	my %detail_code_stat = ();
3286	my $infile = new IO::File;
3287	if ($infile->open("$outdir/stat_code.dat")) {
3288		while (my $l = <$infile>) {
3289			chomp($l);
3290			my ($code, $data) = split(/\s/, $l);
3291			$data =~ /hits_$type=([^;]+);bytes_$type=([^;]+)/;
3292			my $hits = $1 || '';
3293			my $bytes = $2 || '';
3294			$hits =~ s/,$//;
3295			$bytes =~ s/,$//;
3296			my %hits_tmp = split(/[:,]/, $hits);
3297			foreach my $tmp (keys %hits_tmp) {
3298				$detail_code_stat{$code}{$tmp}{request} = $hits_tmp{$tmp};
3299				$code_stat{$code}{request} += $hits_tmp{$tmp};
3300			}
3301			my %bytes_tmp = split(/[:,]/, $bytes);
3302			foreach my $tmp (keys %bytes_tmp) {
3303				$detail_code_stat{$code}{$tmp}{bytes} = $bytes_tmp{$tmp};
3304				$code_stat{$code}{bytes} += $bytes_tmp{$tmp};
3305			}
3306			if ($data =~ /thp_bytes_$type=([^;]+);thp_duration_$type=([^;]+)/) {
3307				$bytes = $1 || '';
3308				my $elapsed = $2 || '';
3309				$bytes =~ s/,$//;
3310				$elapsed =~ s/,$//;
3311				my %bytes_tmp = split(/[:,]/, $bytes);
3312				foreach my $tmp (keys %bytes_tmp) {
3313					$detail_code_stat{throughput}{"$tmp"}{bytes} = $bytes_tmp{"$tmp"};
3314					$throughput_stat{$code}{bytes} += $bytes_tmp{$tmp};
3315				}
3316				my %elapsed_tmp = split(/[:,]/, $elapsed);
3317				foreach my $tmp (keys %elapsed_tmp) {
3318					$detail_code_stat{throughput}{"$tmp"}{elapsed} = $elapsed_tmp{"$tmp"};
3319					$throughput_stat{$code}{elapsed} += $elapsed_tmp{$tmp};
3320				}
3321			}
3322		}
3323		$infile->close();
3324	}
3325	my $total_request = ($code_stat{MISS}{request} + $code_stat{HIT}{request}) || 1;
3326	my $total_bytes = ($code_stat{HIT}{bytes} + $code_stat{MISS}{bytes}) || 1;
3327	my $total_elapsed = ($throughput_stat{HIT}{elapsed} + $throughput_stat{MISS}{elapsed}) || 1;
3328	my $total_throughput = int(($throughput_stat{HIT}{bytes} + $throughput_stat{MISS}{bytes}) / (($total_elapsed/1000) || 1));
3329	my $total_all_request = ($code_stat{DENIED}{request} + $code_stat{MISS}{request} + $code_stat{HIT}{request}) || 1;
3330	my $total_all_bytes = ($code_stat{DENIED}{bytes} + $code_stat{HIT}{bytes} + $code_stat{MISS}{bytes}) || 1;
3331
3332	if ($week && !-d "$outdir") {
3333		return;
3334	}
3335	my $file = $outdir . '/index.html';
3336	my $out = new IO::File;
3337	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
3338	# Print the HTML header
3339	my $cal = 'SA_CALENDAR_SA';
3340	$cal = '' if ($week);
3341	if ( !$self->{no_year_stat} || ($type ne 'month') ) {
3342		$self->_print_header(\$out, $self->{menu}, $cal);
3343		print $out $self->_print_title($Translate{'Cache_title'}, $stat_date, $week);
3344	} else {
3345		$self->_print_header(\$out, $self->{menu3}, $cal);
3346		print $out $self->_print_title($Translate{'Cache_title'}, $stat_date, $week);
3347	}
3348
3349	my $total_cost = sprintf("%2.2f", int(($code_stat{HIT}{bytes} + $code_stat{MISS}{bytes})/1000000) * $self->{CostPrice});
3350	my $comma_bytes = $self->format_bytes($total_bytes);
3351	my $comma_throughput = $self->format_bytes($total_throughput);
3352	my $hit_bytes = $self->format_bytes($code_stat{HIT}{bytes});
3353	my $miss_bytes = $self->format_bytes($code_stat{MISS}{bytes});
3354	my $denied_bytes = $self->format_bytes($code_stat{DENIED}{bytes});
3355	my $colspn = 6;
3356	$colspn = 7 if ($self->{CostPrice});
3357
3358	my $title = $Translate{'Hourly'} || 'Hourly';
3359	my $unit = $Translate{'Hours'} || 'Hours';
3360	my @xaxis = ();
3361	my @xstick = ();
3362	if ($type eq 'day') {
3363		if (!$week) {
3364			$title = $Translate{'Daily'} || 'Daily';
3365			for ("01" .. "31") {
3366				push(@xaxis, "$_");
3367			}
3368		} else {
3369			@xaxis = &get_wdays_per_year($week - 1, $year, $month);
3370			foreach my $x (@xaxis) {
3371				push(@xstick, POSIX::strftime("%F", CORE::localtime($x/1000)));
3372			}
3373			map { s/\d{4}-\d{2}-//; } @xstick;
3374			$title = $Translate{'Weekly'} || 'Weekly';
3375			$type = 'week';
3376			$type = '[' . join(',', @xstick) . ']';
3377		}
3378		$unit = $Translate{'Days'} || 'Days';
3379	} elsif ($type eq 'month') {
3380		$title = $Translate{'Monthly'} || 'Monthly';
3381		$unit = $Translate{'Months'} || 'Months';
3382		for ("01" .. "12") {
3383			push(@xaxis, "$_");
3384		}
3385	} else {
3386		for ("00" .. "23") {
3387			push(@xaxis, "$_");
3388		}
3389	}
3390	my @hit = ();
3391	my @miss = ();
3392	my @denied = ();
3393	my @throughput = ();
3394	my @total = ();
3395	for (my $i = 0; $i <= $#xaxis; $i++) {
3396		my $ddate = $xaxis[$i];
3397		$ddate = $xstick[$i] if ($#xstick >= 0);
3398		my $tot = 0;
3399		if (exists $detail_code_stat{HIT}{$ddate}{request}) {
3400			push(@hit, "[ $xaxis[$i], $detail_code_stat{HIT}{$ddate}{request} ]");
3401			$tot += $detail_code_stat{HIT}{$ddate}{request};
3402		} else {
3403			push(@hit, "[ $xaxis[$i], 0 ]");
3404		}
3405		if (exists $detail_code_stat{MISS}{$ddate}{request}) {
3406			push(@miss, "[ $xaxis[$i], $detail_code_stat{MISS}{$ddate}{request} ]");
3407			$tot += $detail_code_stat{MISS}{$ddate}{request};
3408		} else {
3409			push(@miss, "[ $xaxis[$i], 0 ]");
3410		}
3411		if (exists $detail_code_stat{DENIED}{$ddate}{request}) {
3412			push(@denied, "[ $xaxis[$i], $detail_code_stat{DENIED}{$ddate}{request} ]");
3413		} else {
3414			push(@denied, "[ $xaxis[$i], 0 ]");
3415		}
3416		if (exists $detail_code_stat{throughput}{$ddate}{bytes}) {
3417			$detail_code_stat{throughput}{$ddate}{elapsed} ||= 1;
3418			push(@throughput, "[ $xaxis[$i], " . int($detail_code_stat{throughput}{$ddate}{bytes}/($detail_code_stat{throughput}{$ddate}{elapsed}/1000)) . " ]");
3419		} else {
3420			push(@throughput, "[ $xaxis[$i], 0 ]");
3421		}
3422		push(@total, "[ $xaxis[$i], $tot ]");
3423		delete $detail_code_stat{HIT}{$ddate}{request};
3424		delete $detail_code_stat{MISS}{$ddate}{request};
3425		delete $detail_code_stat{DENIED}{$ddate}{request};
3426		delete $detail_code_stat{throughput}{$ddate};
3427	}
3428
3429	my $t1 = $Translate{'Graph_cache_hit_title'};
3430	$t1 =~ s/\%s/$title/;
3431	$t1 = "$t1 $stat_date";
3432	my $xlabel = $unit || '';
3433	my $ylabel = $Translate{'Requests_graph'} || 'Requests';
3434	my $code_requests = $self->flotr2_bargraph(1, 'code_requests', $type, $t1, $xlabel, $ylabel,
3435				join(',', @total), $Translate{'Total_graph'} . " ($Translate{'Hit_graph'}+$Translate{'Miss_graph'})",
3436				join(',', @hit), $Translate{'Hit_graph'},
3437				join(',', @miss), $Translate{'Miss_graph'},
3438				join(',', @denied), $Translate{'Denied_graph'} );
3439	@hit = ();
3440	@miss = ();
3441	@denied = ();
3442	@total = ();
3443
3444	for (my $i = 0; $i <= $#xaxis; $i++) {
3445		my $ddate = $xaxis[$i];
3446		$ddate = $xstick[$i] if ($#xstick >= 0);
3447		my $tot = 0;
3448		if (exists $detail_code_stat{HIT}{$ddate}{bytes}) {
3449			push(@hit, "[ $xaxis[$i], " . int($detail_code_stat{HIT}{$ddate}{bytes}/1000000) . " ]");
3450			$tot += $detail_code_stat{HIT}{$ddate}{bytes};
3451		} else {
3452			push(@hit, "[ $xaxis[$i], 0 ]");
3453		}
3454		if (exists $detail_code_stat{MISS}{$ddate}{bytes}) {
3455			push(@miss, "[ $xaxis[$i], " . int($detail_code_stat{MISS}{$ddate}{bytes}/1000000) . " ]");
3456			$tot += $detail_code_stat{MISS}{$ddate}{bytes};
3457		} else {
3458			push(@miss, "[ $xaxis[$i], 0 ]");
3459		}
3460		if (exists $detail_code_stat{DENIED}{$ddate}{bytes}) {
3461			push(@denied, "[ $xaxis[$i], " . int($detail_code_stat{DENIED}{$ddate}{bytes}/1000000) . " ]");
3462		} else {
3463			push(@denied, "[ $xaxis[$i], 0 ]");
3464		}
3465		push(@total, "[ $xaxis[$i], " . int($tot/1000000) . " ]");
3466	}
3467	%detail_code_stat = ();
3468	$t1 = $Translate{'Graph_cache_byte_title'};
3469	$t1 =~ s/\%s/$title/;
3470	$t1 = "$t1 $stat_date";
3471	$ylabel = $Translate{'Megabytes_graph'} || $Translate{'Megabytes'};
3472	my $code_bytes = $self->flotr2_bargraph(2, 'code_bytes', $type, $t1, $xlabel, $ylabel,
3473				join(',', @total), $Translate{'Total_graph'} . " ($Translate{'Hit_graph'}+$Translate{'Miss_graph'})",
3474				join(',', @hit), $Translate{'Hit_graph'},
3475				join(',', @miss), $Translate{'Miss_graph'},
3476				join(',', @denied), $Translate{'Denied_graph'});
3477	@hit = ();
3478	@miss = ();
3479	@denied = ();
3480	@total = ();
3481
3482	$t1 = $Translate{'Graph_throughput_title'};
3483	$t1 =~ s/\%s/$title/;
3484	$t1 = "$t1 $stat_date";
3485	$ylabel = $Translate{'Bytes_graph'} || $Translate{'Bytes'};
3486	my $throughput_bytes = $self->flotr2_bargraph(3, 'throughput_bytes', $type, $t1, $xlabel, $ylabel,
3487				join(',', @throughput), $Translate{'Throughput_graph'});
3488	@throughput = ();
3489
3490	print $out qq{
3491<table class="stata">
3492<tr>
3493<th colspan="3" class="headerBlack">$Translate{'Requests'}</th>
3494<th colspan="3" class="headerBlack">$Translate{$self->{TransfertUnit}}</th>
3495<th colspan="$colspn" class="headerBlack">$Translate{'Total'}</th>
3496</tr>
3497<tr>
3498<th>$Translate{'Hit'}</th>
3499<th>$Translate{'Miss'}</th>
3500<th>$Translate{'Denied'}</th>
3501<th>$Translate{'Hit'}</th>
3502<th>$Translate{'Miss'}</th>
3503<th>$Translate{'Denied'}</th>
3504<th>$Translate{'Requests'}</th>
3505<th>$Translate{$self->{TransfertUnit}}</th>
3506<th>$Translate{'Throughput'}</th>
3507<th>$Translate{'Users'}</th>
3508<th>$Translate{'Sites'}</th>
3509<th>$Translate{'Domains'}</th>
3510};
3511	print $out qq{
3512<th>$Translate{'Cost'} $self->{Currency}</th>
3513} if ($self->{CostPrice});
3514	my $percent_hit = sprintf("%.2f", ($code_stat{HIT}{request}/$total_all_request)*100);
3515	my $percent_miss = sprintf("%.2f", ($code_stat{MISS}{request}/$total_all_request)*100);
3516	my $percent_denied = sprintf("%.2f", ($code_stat{DENIED}{request}/$total_all_request)*100);
3517	my $percent_bhit = sprintf("%.2f", ($code_stat{HIT}{bytes}/$total_all_bytes)*100);
3518	my $percent_bmiss = sprintf("%.2f", ($code_stat{MISS}{bytes}/$total_all_bytes)*100);
3519	my $percent_bdenied = sprintf("%.2f", ($code_stat{DENIED}{bytes}/$total_all_bytes)*100);
3520	my $trfunit = $self->{TransfertUnit} || 'B';
3521	$trfunit = 'B' if ($trfunit eq 'BYTE');
3522	print $out qq{
3523</tr>
3524<tr>
3525<td title="$percent_hit %">$code_stat{HIT}{request}</td>
3526<td title="$percent_miss %">$code_stat{MISS}{request}</td>
3527<td title="$percent_denied %">$code_stat{DENIED}{request}</td>
3528<td title="$percent_bhit %">$hit_bytes</td>
3529<td title="$percent_bmiss %">$miss_bytes</td>
3530<td title="$percent_bdenied %">$denied_bytes</td>
3531<td>$total_request</td>
3532<td>$comma_bytes</td>
3533<td>$comma_throughput $trfunit/s</td>
3534<td>SA_NUSERS_SA</td>
3535<td>SA_NURLS_SA</td>
3536<td>SA_NDOMAINS_SA</td>
3537};
3538	print $out qq{
3539<td class="cacheValues">$total_cost</td>
3540} if ($self->{CostPrice});
3541	print $out qq{
3542</tr>
3543</table>
3544
3545<style>
3546  #container {
3547    display: table;
3548  }
3549  #row  {
3550    display: table-row;
3551  }
3552  #code_requests, #code_bytes {
3553    display: table-cell;
3554  }
3555  #code_requests { z-index: 999; }
3556</style>
3557
3558<table class="graphs">
3559<tr><td>
3560<div id="container">
3561$code_requests
3562</div>
3563</td><td>
3564<div id="container">
3565$code_bytes
3566</div>
3567</td></tr>
3568<tr><td colspan="2" align="center">$throughput_bytes</td></tr>
3569</table>
3570
3571	<h4>$Translate{'Legend'}</h4>
3572	<div class="line-separator"></div>
3573	<div class="displayLegend">
3574	<span class="legendeTitle">$Translate{'Hit'}:</span> <span class="descLegend">$Translate{'Hit_help'}</span><br/>
3575	<span class="legendeTitle">$Translate{'Miss'}:</span> <span class="descLegend">$Translate{'Miss_help'}</span><br/>
3576	<span class="legendeTitle">$Translate{'Denied'}:</span> <span class="descLegend">$Translate{'Denied_help'}</span><br/>
3577	<span class="legendeTitle">$Translate{'Users'}:</span> <span class="descLegend">$Translate{'Users_help'}</span><br/>
3578	<span class="legendeTitle">$Translate{'Sites'}:</span> <span class="descLegend">$Translate{'Sites_help'}</span><br/>
3579	<span class="legendeTitle">$Translate{'Domains'}:</span> <span class="descLegend">$Translate{'Domains_help'}</span><br/>
3580};
3581	print $out qq{
3582	<span class="legendeTitle">$Translate{'Cost'}:</span> <span class="descLegend">$Translate{'Cost_help'} $self->{CostPrice} $self->{Currency}</span><br/>
3583} if ($self->{CostPrice});
3584	print $out qq{
3585</div>
3586};
3587
3588	%code_stat = ();
3589	$self->_print_footer(\$out);
3590	$out->close();
3591}
3592
3593
3594sub _print_mime_stat
3595{
3596	my ($self, $outdir, $year, $month, $day, $week) = @_;
3597
3598	print STDERR "\tMime type statistics in $outdir...\n" if (!$self->{QuietMode});
3599
3600	$0 = "squid-analyzer: Printing mime statistics in $outdir";
3601
3602	my $stat_date = $self->set_date($year, $month, $day);
3603
3604	my $type = 'hour';
3605	if (!$day) {
3606		$type = 'day';
3607	}
3608	if (!$month) {
3609		$type = 'month';
3610	}
3611	if ($week) {
3612		$type = 'day';
3613	}
3614
3615	# Load code statistics
3616	my $infile = new IO::File;
3617	$infile->open("$outdir/stat_mime_type.dat") || return;
3618	my %mime_stat = ();
3619	my $total_count = 0;
3620	my $total_bytes = 0;
3621	while(my $l = <$infile>) {
3622		chomp($l);
3623		my ($code, $data) = split(/\s/, $l);
3624		$data =~ /hits=(\d+);bytes=(\d+)/;
3625		$mime_stat{$code}{hits} = $1;
3626		$mime_stat{$code}{bytes} = $2;
3627		$total_count += $1;
3628		$total_bytes += $2;
3629	}
3630	$infile->close();
3631
3632	my $ntype = scalar keys %mime_stat;
3633
3634	my $file = $outdir . '/mime_type.html';
3635	my $out = new IO::File;
3636	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
3637
3638	my $sortpos = 1;
3639	$sortpos = 2 if ($self->{OrderMime} eq 'bytes');
3640	$sortpos = 3 if ($self->{OrderMime} eq 'duration');
3641
3642	# Print the HTML header
3643	my $cal = 'SA_CALENDAR_SA';
3644	$cal = '' if ($week);
3645	$self->_print_header(\$out, $self->{menu}, $cal, $sortpos);
3646
3647	# Print title and calendar view
3648	print $out $self->_print_title($Translate{'Mime_title'}, $stat_date, $week);
3649
3650	my %data = ();
3651	$total_count ||= 1;
3652	foreach my $mime (keys %mime_stat) {
3653		if (($mime_stat{$mime}{hits}/$total_count)*100 > $self->{MinPie}) {
3654			$data{$mime} = $mime_stat{$mime}{hits};
3655		} else {
3656			$data{'others'} += $mime_stat{$mime}{hits};
3657		}
3658	}
3659	my $title = "$Translate{'Mime_graph_hits_title'} $stat_date";
3660	my $mime_hits = $self->flotr2_piegraph(1, 'mime_hits', $title, $Translate{'Mime_graph'}, '', %data);
3661	print $out qq{
3662<style>
3663  #container {
3664    display: table;
3665  }
3666  #row  {
3667    display: table-row;
3668  }
3669  #mime_hits, #mime_bytes {
3670    display: table-cell;
3671  }
3672  #mime_hits { z-index: 999; }
3673</style>
3674<table class="graphs"><tr><td>
3675<div id="container">
3676$mime_hits
3677</div>
3678</td><td>
3679};
3680	$mime_hits = '';
3681	%data = ();
3682	$total_bytes ||= 1;
3683	foreach my $mime (keys %mime_stat) {
3684		if (($mime_stat{$mime}{bytes}/$total_bytes)*100 > $self->{MinPie}) {
3685			$data{$mime} = int($mime_stat{$mime}{bytes}/1000000);
3686		} else {
3687			$data{'others'} += $mime_stat{$mime}{bytes};
3688		}
3689	}
3690	$data{'others'} = int($data{'others'}/1000000);
3691	$title = "$Translate{'Mime_graph_bytes_title'} $stat_date";
3692	my $mime_bytes = $self->flotr2_piegraph(1, 'mime_bytes', $title, $Translate{'Mime_graph'}, '', %data);
3693	print $out qq{
3694<div id="container">
3695$mime_bytes
3696</div>
3697</td></tr></table>
3698};
3699	$mime_bytes = '';
3700	%data = ();
3701
3702	print $out "<h3>$Translate{'Mime_number'}: $ntype</h3>\n";
3703	print $out qq{
3704<table class="sortable stata">
3705<thead>
3706<tr>
3707<th>$Translate{'Mime_link'}</th>
3708<th>$Translate{'Requests'} (%)</th>
3709<th>$Translate{$self->{TransfertUnit}} (%)</th>
3710};
3711	print $out qq{
3712<th>$Translate{'Cost'} $self->{Currency}</th>
3713} if ($self->{CostPrice});
3714	print $out qq{
3715</tr>
3716</thead>
3717<tbody>
3718};
3719	foreach (sort { $mime_stat{$b}{"$self->{OrderMime}"} <=> $mime_stat{$a}{"$self->{OrderMime}"} } keys %mime_stat) {
3720		my $c_percent = '0.0';
3721		$c_percent = sprintf("%2.2f", ($mime_stat{$_}{hits}/$total_count) * 100) if ($total_count);
3722		my $b_percent = '0.0';
3723		$b_percent = sprintf("%2.2f", ($mime_stat{$_}{bytes}/$total_bytes) * 100) if ($total_bytes);
3724		my $total_cost = sprintf("%2.2f", int($mime_stat{$_}{bytes}/1000000) * $self->{CostPrice});
3725		my $comma_bytes = $self->format_bytes($mime_stat{$_}{bytes});
3726		print $out qq{
3727<tr>
3728<td>$_</td>
3729<td>$mime_stat{$_}{hits} <span class="italicPercent">($c_percent)</span></td>
3730<td>$comma_bytes <span class="italicPercent">($b_percent)</span></td>
3731};
3732	print $out qq{
3733<td>$total_cost</td>
3734} if ($self->{CostPrice});
3735	print $out qq{
3736</tr>};
3737	}
3738	$sortpos = 1;
3739	$sortpos = 2 if ($self->{OrderMime} eq 'bytes');
3740	print $out qq{
3741</tbody>
3742</table>
3743};
3744
3745	print $out qq{
3746<div class="uplink">
3747    <a href="#atop"><span class="iconUpArrow">$Translate{'Up_link'}</span></a>
3748</div>
3749};
3750	$self->_print_footer(\$out);
3751	$out->close();
3752
3753}
3754
3755sub _print_network_stat
3756{
3757	my ($self, $outdir, $year, $month, $day, $week) = @_;
3758
3759	print STDERR "\tNetwork statistics in $outdir...\n" if (!$self->{QuietMode});
3760
3761	$0 = "squid-analyzer: Printing network statistics in $outdir";
3762
3763	my $stat_date = $self->set_date($year, $month, $day);
3764
3765	my $type = 'hour';
3766	if (!$day) {
3767		$type = 'day';
3768	}
3769	if (!$month) {
3770		$type = 'month';
3771	}
3772	if ($week) {
3773		$type = 'day';
3774	}
3775
3776	# Load code statistics
3777	my $infile = new IO::File;
3778	$infile->open("$outdir/stat_network.dat") || return;
3779	my %network_stat = ();
3780	my %detail_network_stat = ();
3781	my %total_net_detail = ();
3782	my $total_hit = 0;
3783	my $total_bytes = 0;
3784	my $total_duration = 0;
3785	while (my $l = <$infile>) {
3786		chomp($l);
3787		my ($network, $data) = split(/\t/, $l);
3788		if (!$data) {
3789			# Assume backward compatibility
3790			$l =~ s/^(.*)\shits_$type=/hits_$type=/;
3791			$network = $1;
3792			$data = $l;
3793		}
3794		$data =~ /^hits_$type=([^;]+);bytes_$type=([^;]+);duration_$type=([^;]+);largest_file_size=([^;]*);largest_file_url=(.*)/;
3795		if ($self->{rebuild} && !exists $self->{NetworkAlias}->{$network}) {
3796			next if (!$self->check_inclusions('', $network));
3797			next if ($self->check_exclusions('', $network));
3798		}
3799
3800		my $hits = $1 || '';
3801		my $bytes = $2 || '';
3802		my $duration = $3 || '';
3803		$network_stat{$network}{largest_file} = $4;
3804		$network_stat{$network}{url} = $5;
3805		$hits =~ s/,$//;
3806		$bytes =~ s/,$//;
3807		$duration =~ s/,$//;
3808		my %hits_tmp = split(/[:,]/, $hits);
3809		foreach my $tmp (sort {$a <=> $b} keys %hits_tmp) {
3810			$detail_network_stat{$network}{$tmp}{hits} = $hits_tmp{$tmp};
3811			$total_net_detail{$tmp}{hits} += $hits_tmp{$tmp};
3812			$network_stat{$network}{hits} += $hits_tmp{$tmp};
3813			$total_hit += $hits_tmp{$tmp};
3814		}
3815		my %bytes_tmp = split(/[:,]/, $bytes);
3816		foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
3817			$detail_network_stat{$network}{$tmp}{bytes} = $bytes_tmp{$tmp};
3818			$total_net_detail{$tmp}{bytes} += $bytes_tmp{$tmp};
3819			$network_stat{$network}{bytes} += $bytes_tmp{$tmp};
3820			$total_bytes += $bytes_tmp{$tmp};
3821		}
3822		my %duration_tmp = split(/[:,]/, $duration);
3823		foreach my $tmp (sort {$a <=> $b} keys %duration_tmp) {
3824			$detail_network_stat{$network}{$tmp}{duration} = $duration_tmp{$tmp};
3825			$total_net_detail{$tmp}{duration} += $duration_tmp{$tmp};
3826			$network_stat{$network}{duration} += $duration_tmp{$tmp};
3827			$total_duration += $duration_tmp{$tmp};
3828		}
3829	}
3830	$infile->close();
3831	my $nnet = scalar keys %network_stat;
3832
3833	my $sortpos = 1;
3834	$sortpos = 2 if ($self->{OrderNetwork} eq 'bytes');
3835	$sortpos = 3 if ($self->{OrderNetwork} eq 'duration');
3836
3837	my $file = $outdir . '/network.html';
3838	my $out = new IO::File;
3839	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
3840	# Print the HTML header
3841	my $cal = 'SA_CALENDAR_SA';
3842	$cal = '' if ($week);
3843	$self->_print_header(\$out, $self->{menu}, $cal, $sortpos);
3844	print $out $self->_print_title($Translate{'Network_title'}, $stat_date, $week);
3845
3846	my $last = '23';
3847	my $first = '00';
3848	my $title = $Translate{'Hourly'} || 'Hourly';
3849	my $unit = $Translate{'Hours'} || 'Hours';
3850	if ($type eq 'day') {
3851		$last = '31';
3852		$first = '01';
3853		$title = $Translate{'Daily'} || 'Daily';
3854		$unit = $Translate{'Days'} || 'Days';
3855	} elsif ($type eq 'month') {
3856		$last = '12';
3857		$first = '01';
3858		$title = $Translate{'Monthly'} || 'Monthly';
3859		$unit = $Translate{'Months'} || 'Months';
3860	}
3861
3862	print $out "<h3>$Translate{'Network_number'}: $nnet</h3>\n";
3863	print $out qq{
3864<table class="sortable stata">
3865<thead>
3866<tr>
3867<th>$Translate{'Network_link'}</th>
3868<th>$Translate{'Requests'} (%)</th>
3869<th>$Translate{$self->{TransfertUnit}} (%)</th>
3870<th>$Translate{'Duration'} (%)</th>
3871<th>$Translate{'Throughput'} (B/s)</th>
3872};
3873	print $out qq{
3874<th>$Translate{'Cost'} $self->{Currency}</th>
3875} if ($self->{CostPrice});
3876	print $out qq{
3877<th>$Translate{'Users'}</th>
3878} if ($self->{UserReport});
3879	print $out qq{
3880<th>$Translate{'Largest'}</th>
3881<th style="text-align: left;">$Translate{'Url'}</th>
3882</tr>
3883</thead>
3884<tbody>
3885};
3886	if (!-d "$outdir/networks") {
3887		mkdir("$outdir/networks", 0755) || return;
3888	}
3889	$total_duration = abs($total_duration);
3890	foreach my $net (sort { $network_stat{$b}{"$self->{OrderNetwork}"} <=> $network_stat{$a}{"$self->{OrderNetwork}"} } keys %network_stat) {
3891
3892		my $h_percent = '0.0';
3893		$h_percent = sprintf("%2.2f", ($network_stat{$net}{hits}/$total_hit) * 100) if ($total_hit);
3894		my $b_percent = '0.0';
3895		$b_percent = sprintf("%2.2f", ($network_stat{$net}{bytes}/$total_bytes) * 100) if ($total_bytes);
3896		my $d_percent = '0.0';
3897		$d_percent = sprintf("%2.2f", ($network_stat{$net}{duration}/$total_duration) * 100) if ($total_duration);
3898		my $total_cost = sprintf("%2.2f", int($network_stat{$net}{bytes}/1000000) * $self->{CostPrice});
3899		my $total_throughput = int($network_stat{$net}{bytes} / (($network_stat{$net}{duration}/1000) || 1) );
3900		my $comma_throughput = $self->format_bytes($total_throughput);
3901		$network_stat{$net}{duration} = &parse_duration(int($network_stat{$net}{duration}/1000));
3902		my $show = $net;
3903		if ($net =~ /^(\d+\.\d+\.\d+)/) {
3904			$show = "$1.0";
3905			foreach my $r (keys %{$self->{NetworkAlias}}) {
3906
3907				if ($r =~ /^\d+\.\d+\.\d+\.\d+\/\d+$/) {
3908					if (&check_ip($net, $r)) {
3909						$show = $self->{NetworkAlias}->{$r};
3910						last;
3911					}
3912				} elsif ($show =~ /$r/) {
3913					$show = $self->{NetworkAlias}->{$r};
3914					last;
3915				}
3916			}
3917		}
3918		my $comma_bytes = $self->format_bytes($network_stat{$net}{bytes});
3919		print $out qq{
3920<tr>
3921<td><a href="networks/$net/$net.html">$show</a></td>
3922<td>$network_stat{$net}{hits} <span class="italicPercent">($h_percent)</span></td>
3923<td>$comma_bytes <span class="italicPercent">($b_percent)</span></td>
3924<td>$network_stat{$net}{duration} <span class="italicPercent">($d_percent)</span></td>
3925<td>$comma_throughput</td>
3926};
3927	print $out qq{
3928<td>$total_cost</td>
3929} if ($self->{CostPrice});
3930
3931		if (!-d "$outdir/networks/$net") {
3932			mkdir("$outdir/networks/$net", 0755) || return;
3933		}
3934		my $outnet = new IO::File;
3935		$outnet->open(">$outdir/networks/$net/$net.html") || return;
3936		# Print the HTML header
3937		my $cal = 'SA_CALENDAR_SA';
3938		$self->_print_header(\$outnet, $self->{menu2}, $cal, $sortpos);
3939		print $outnet $self->_print_title("$Translate{'Network_title'} $show -", $stat_date, $week);
3940
3941		my @hits = ();
3942		my @bytes = ();
3943		for ("$first" .. "$last") {
3944			if (exists $detail_network_stat{$net}{$_}{hits}) {
3945				push(@hits, "[ $_, " . $detail_network_stat{$net}{$_}{hits} . " ]");
3946			} else {
3947				push(@hits, "[ $_, 0 ]");
3948			}
3949			if (exists $detail_network_stat{$net}{$_}{bytes}) {
3950				push(@bytes, "[ $_, " . int($detail_network_stat{$net}{$_}{bytes}/1000000) . " ]");
3951			} else {
3952				push(@bytes, "[ $_, 0 ]");
3953			}
3954		}
3955		delete $detail_network_stat{$net};
3956
3957		my $t1 = $Translate{'Graph_cache_hit_title'};
3958		$t1 =~ s/\%s/$title $show/;
3959		$t1 = "$t1 $stat_date";
3960		my $xlabel = $unit || '';
3961		my $ylabel = $Translate{'Requests_graph'} || 'Requests';
3962		my $network_hits = $self->flotr2_bargraph(1, 'network_hits', $type, $t1, $xlabel, $ylabel,
3963					join(',', @hits), $Translate{'Hit_graph'} );
3964		@hits = ();
3965		print $outnet qq{
3966<style>
3967  #container {
3968    display: table;
3969  }
3970  #row  {
3971    display: table-row;
3972  }
3973  #network_hits, #network_bytes {
3974    display: table-cell;
3975  }
3976  #network_hits { z-index: 999; }
3977</style>
3978<table class="graphs"><tr><td>
3979<div id="container">
3980$network_hits
3981</div>
3982</td><td>
3983};
3984		$network_hits = '';
3985
3986		$t1 = $Translate{'Graph_cache_byte_title'};
3987		$t1 =~ s/\%s/$title/;
3988		$t1 = "$t1 $stat_date";
3989		$xlabel = $unit || '';
3990		$ylabel = $Translate{'Megabytes_graph'} || $Translate{'Megabytes'};
3991		my $network_bytes = $self->flotr2_bargraph(1, 'network_bytes', $type, $t1, $xlabel, $ylabel,
3992					join(',', @bytes), $Translate{'Bytes'} );
3993		@bytes = ();
3994
3995		print $outnet qq{
3996<div id="container">
3997$network_bytes
3998</div>
3999</td></tr></table>
4000};
4001		$network_bytes = '';
4002		my $retuser = $self->_print_netuser_stat($outdir, \$outnet, $net);
4003		my $comma_largest = $self->format_bytes($network_stat{$net}{largest_file});
4004		print $out qq{
4005<td>$retuser</td>
4006} if ($self->{UserReport});
4007		print $out qq{
4008<td>$comma_largest</td>
4009<td style="text-align: left;">$network_stat{$net}{url}</td>
4010</tr>
4011};
4012		$sortpos = 1;
4013		$sortpos = 2 if ($self->{OrderNetwork} eq 'bytes');
4014		$sortpos = 3 if ($self->{OrderNetwork} eq 'duration');
4015		print $outnet qq{
4016<div class="uplink">
4017    <a href="#atop"><span class="iconUpArrow">$Translate{'Up_link'}</span></a>
4018</div>
4019};
4020		$self->_print_footer(\$outnet);
4021		$outnet->close();
4022	}
4023	print $out "</tbody></table>\n";
4024
4025	$sortpos = 1;
4026	$sortpos = 2 if ($self->{OrderNetwork} eq 'bytes');
4027	$sortpos = 3 if ($self->{OrderNetwork} eq 'duration');
4028	print $out qq{
4029<div class="uplink">
4030    <a href="#atop"><span class="iconUpArrow">$Translate{'Up_link'}</span></a>
4031</div>
4032};
4033	$self->_print_footer(\$out);
4034	$out->close();
4035}
4036
4037sub _print_user_stat
4038{
4039	my ($self, $outdir, $year, $month, $day, $week) = @_;
4040
4041	print STDERR "\tUser statistics in $outdir...\n" if (!$self->{QuietMode});
4042
4043	$0 = "squid-analyzer: Printing user statistics in $outdir";
4044
4045	my $stat_date = $self->set_date($year, $month, $day);
4046
4047	my $type = 'hour';
4048	if (!$day) {
4049		$type = 'day';
4050	}
4051	if (!$month) {
4052		$type = 'month';
4053	}
4054	if ($week) {
4055		$type = 'day';
4056	}
4057
4058	# Load code statistics
4059	my $infile = new IO::File;
4060	$infile->open("$outdir/stat_user.dat") || return;
4061	my %user_stat = ();
4062	my %detail_user_stat = ();
4063	my %total_user_detail = ();
4064        my $total_hit = 0;
4065        my $total_bytes = 0;
4066        my $total_duration = 0;
4067	while (my $l = <$infile>) {
4068		chomp($l);
4069		my ($user, $data) = split(/\s/, $l);
4070
4071		if ($self->{rebuild}) {
4072			next if (!$self->check_inclusions($user));
4073			next if ($self->check_exclusions($user));
4074		}
4075
4076		# Anonymize all users
4077		if ($self->{AnonymizeLogin} && ($user !~ /^Anon[a-zA-Z0-9]{16}$/)) {
4078			if (!exists $self->{AnonymizedId}{$user}) {
4079				$self->{AnonymizedId}{$user} = &anonymize_id();
4080			}
4081			$user = $self->{AnonymizedId}{$user};
4082		}
4083
4084		my ($hits,$bytes,$duration,$largest_file,$url) = ($data =~ /hits_$type=([^;]+);bytes_$type=([^;]+);duration_$type=([^;]+);largest_file_size=([^;]*);largest_file_url=(.*)/);
4085		$user_stat{$user}{largest_file} = $largest_file;
4086		$user_stat{$user}{url} = $url;
4087		$hits =~ s/,$//;
4088		$bytes =~ s/,$//;
4089		$duration =~ s/,$//;
4090		my %hits_tmp = split(/[:,]/, $hits);
4091		foreach my $tmp (sort {$a <=> $b} keys %hits_tmp) {
4092			$detail_user_stat{$user}{$tmp}{hits} = $hits_tmp{$tmp};
4093			$total_user_detail{$tmp}{hits} += $hits_tmp{$tmp};
4094			$user_stat{$user}{hits} += $hits_tmp{$tmp};
4095			$total_hit += $hits_tmp{$tmp};
4096		}
4097		my %bytes_tmp = split(/[:,]/, $bytes);
4098		foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
4099			$detail_user_stat{$user}{$tmp}{bytes} = $bytes_tmp{$tmp};
4100			$total_user_detail{$tmp}{bytes} += $bytes_tmp{$tmp};
4101			$user_stat{$user}{bytes} += $bytes_tmp{$tmp};
4102			$total_bytes += $bytes_tmp{$tmp};
4103		}
4104		my %duration_tmp = split(/[:,]/, $duration);
4105		foreach my $tmp (sort {$a <=> $b} keys %duration_tmp) {
4106			$detail_user_stat{$user}{$tmp}{duration} = $duration_tmp{$tmp};
4107			$total_user_detail{$tmp}{duration} += $duration_tmp{$tmp};
4108			$user_stat{$user}{duration} += $duration_tmp{$tmp};
4109			$total_duration += $duration_tmp{$tmp};
4110		}
4111	}
4112	$infile->close();
4113
4114	# Store number of users
4115	my $nuser = scalar keys %user_stat;
4116	my $outf = new IO::File;
4117	$outf->open(">>$outdir/stat_count.dat") || return;
4118	flock($outf, 2) || die "FATAL: can't acquire lock on file $outdir/stat_count.dat, $!\n";
4119	$outf->print("users:$nuser\n");
4120	$outf->close;
4121
4122	my $file = $outdir . '/user.html';
4123	my $out = new IO::File;
4124	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
4125
4126	my $sortpos = 1;
4127	$sortpos = 2 if ($self->{OrderUser} eq 'bytes');
4128	$sortpos = 3 if ($self->{OrderUser} eq 'duration');
4129
4130	# Print the HTML header
4131	my $cal = 'SA_CALENDAR_SA';
4132	$cal = '' if ($week);
4133	$self->_print_header(\$out, $self->{menu}, $cal, $sortpos);
4134
4135	my $last = '23';
4136	my $first = '00';
4137	my $title = $Translate{'Hourly'} || 'Hourly';
4138	my $unit = $Translate{'Hours'} || 'Hours';
4139	if ($type eq 'day') {
4140		$last = '31';
4141		$first = '01';
4142		$title = $Translate{'Daily'} || 'Daily';
4143		$unit = $Translate{'Days'} || 'Days';
4144	} elsif ($type eq 'month') {
4145		$last = '12';
4146		$first = '01';
4147		$title = $Translate{'Monthly'} || 'Monthly';
4148		$unit = $Translate{'Months'} || 'Months';
4149	}
4150
4151	%total_user_detail = ();
4152
4153	print $out $self->_print_title($Translate{'User_title'}, $stat_date, $week);
4154
4155	print $out "<h3>$Translate{'User_number'}: $nuser</h3>\n";
4156
4157	print $out qq{
4158<table class="sortable stata" >
4159<thead>
4160<tr>
4161<th>$Translate{'Users'}</th>
4162<th>$Translate{'Requests'} (%)</th>
4163<th>$Translate{$self->{TransfertUnit}} (%)</th>
4164<th>$Translate{'Duration'} (%)</th>
4165<th>$Translate{'Throughput'} (B/s)</th>
4166};
4167	print $out qq{
4168<th>$Translate{'Cost'} $self->{Currency}</th>
4169} if ($self->{CostPrice});
4170	print $out qq{
4171<th>$Translate{'Largest'}</th>
4172<th style="text-align: left;">$Translate{'Url'}</th>
4173</tr>
4174</thead>
4175<tbody>
4176};
4177	if (!-d "$outdir/users") {
4178		mkdir("$outdir/users", 0755) || return;
4179	}
4180
4181	$total_duration = abs($total_duration);
4182	foreach my $usr (sort { $user_stat{$b}{"$self->{OrderUser}"} <=> $user_stat{$a}{"$self->{OrderUser}"} } keys %user_stat) {
4183		my $h_percent = '0.0';
4184		$h_percent = sprintf("%2.2f", ($user_stat{$usr}{hits}/$total_hit) * 100) if ($total_hit);
4185		my $b_percent = '0.0';
4186		$b_percent = sprintf("%2.2f", ($user_stat{$usr}{bytes}/$total_bytes) * 100) if ($total_bytes);
4187		my $d_percent = '0.0';
4188		$d_percent = sprintf("%2.2f", ($user_stat{$usr}{duration}/$total_duration) * 100) if ($total_duration);
4189		my $total_cost = sprintf("%2.2f", int($user_stat{$usr}{bytes}/1000000) * $self->{CostPrice});
4190		my $total_throughput = int($user_stat{$usr}{bytes} / (($user_stat{$usr}{duration}/1000) || 1));
4191		my $comma_throughput = $self->format_bytes($total_throughput);
4192		$user_stat{$usr}{duration} = &parse_duration(int($user_stat{$usr}{duration}/1000));
4193		my $show = $usr;
4194		foreach my $u (keys %{$self->{UserAlias}}) {
4195			if ( $usr =~ /^$u$/i ) {
4196				$show = $self->{UserAlias}->{$u};
4197				last;
4198			}
4199		}
4200		$show =~ s/_SPC_/ /g;
4201		my $upath = &escape($usr);
4202		my $comma_bytes = $self->format_bytes($user_stat{$usr}{bytes});
4203		if ($self->{UrlReport}) {
4204			print $out qq{
4205<tr>
4206<td><a href="users/$upath/$upath.html">$show</a></td>
4207};
4208		} else {
4209			print $out qq{
4210<tr>
4211<td>$show</td>
4212};
4213		}
4214		print $out qq{
4215<td>$user_stat{$usr}{hits} <span class="italicPercent">($h_percent)</span></td>
4216<td>$comma_bytes <span class="italicPercent">($b_percent)</span></td>
4217<td>$user_stat{$usr}{duration} <span class="italicPercent">($d_percent)</span></td>
4218<td>$comma_throughput</td>
4219};
4220	print $out qq{
4221<td>$total_cost</td>
4222} if ($self->{CostPrice});
4223	my $comma_largest = $self->format_bytes($user_stat{$usr}{largest_file});
4224	print $out qq{
4225<td>$comma_largest</td>
4226<td style="text-align: left;">$user_stat{$usr}{url}</td>
4227</tr>};
4228
4229		if (!-d "$outdir/users/$upath") {
4230			mkdir("$outdir/users/$upath", 0755) || return;
4231		}
4232		my $outusr = new IO::File;
4233		$outusr->open(">$outdir/users/$upath/$upath.html") || return;
4234		# Print the HTML header
4235		my $cal = 'SA_CALENDAR_SA';
4236		$self->_print_header(\$outusr, $self->{menu2}, $cal, $sortpos);
4237		print $outusr $self->_print_title("$Translate{'User_title'} $usr -", $stat_date, $week);
4238
4239		my @hits = ();
4240		my @bytes = ();
4241		for my $d ("$first" .. "$last") {
4242			if (exists $detail_user_stat{$usr}{$d}{hits}) {
4243				push(@hits, "[ $d, $detail_user_stat{$usr}{$d}{hits} ]");
4244			} else {
4245				push(@hits, "[ $d, 0 ]");
4246			}
4247			if (exists $detail_user_stat{$usr}{$d}{bytes}) {
4248				push(@bytes, "[ $d, " . int($detail_user_stat{$usr}{$d}{bytes}/1000000) . " ]");
4249			} else {
4250				push(@bytes, "[ $d, 0 ]");
4251			}
4252		}
4253		delete $detail_user_stat{$usr};
4254
4255		my $t1 = $Translate{'Graph_cache_hit_title'};
4256		$t1 =~ s/\%s/$title $show/;
4257		$t1 = "$t1 $stat_date";
4258		my $xlabel = $unit || '';
4259		my $ylabel = $Translate{'Requests_graph'} || 'Requests';
4260		my $user_hits = $self->flotr2_bargraph(1, 'user_hits', $type, $t1, $xlabel, $ylabel,
4261					join(',', @hits), $Translate{'Hit_graph'});
4262		@hits = ();
4263		print $outusr qq{
4264<style>
4265  #container {
4266    display: table;
4267  }
4268  #row  {
4269    display: table-row;
4270  }
4271  #user_hits, #user_bytes {
4272    display: table-cell;
4273  }
4274  #user_hits { z-index: 999; }
4275</style>
4276<table class="graphs"><tr><td>
4277<div id="container">
4278$user_hits
4279</div>
4280</td><td>
4281};
4282		$user_hits = '';
4283
4284		$t1 = $Translate{'Graph_cache_byte_title'};
4285		$t1 =~ s/\%s/$title $show/;
4286		$t1 = "$t1 $stat_date";
4287		$xlabel = $unit || '';
4288		$ylabel = $Translate{'Megabytes_graph'} || $Translate{'Megabytes'};
4289		my $user_bytes = $self->flotr2_bargraph(1, 'user_bytes', $type, $t1, $xlabel, $ylabel,
4290					join(',', @bytes), $Translate{'Bytes'});
4291		@bytes = ();
4292		print $outusr qq{
4293<div id="container">
4294$user_bytes
4295</div>
4296</td></tr></table>
4297};
4298		$user_bytes = '';
4299
4300		delete $user_stat{$usr};
4301		if ($self->{UrlReport}) {
4302			$self->_print_user_detail(\$outusr, $outdir, $usr, $type);
4303		}
4304		$self->_print_footer(\$outusr);
4305		$outusr->close();
4306	}
4307	$sortpos = 1;
4308	$sortpos = 2 if ($self->{OrderUser} eq 'bytes');
4309	$sortpos = 3 if ($self->{OrderUser} eq 'duration');
4310	print $out qq{
4311</tbody>
4312</table>
4313};
4314
4315	print $out qq{
4316<div class="uplink">
4317    <a href="#atop"><span class="iconUpArrow">$Translate{'Up_link'}</span></a>
4318</div>
4319};
4320	$self->_print_footer(\$out);
4321	$out->close();
4322
4323	return $nuser;
4324}
4325
4326sub _print_netuser_stat
4327{
4328	my ($self, $outdir, $out, $usrnet) = @_;
4329
4330	$0 = "squid-analyzer: Printing network user statistics in $outdir";
4331
4332	# Load code statistics
4333	my $infile = new IO::File;
4334	$infile->open("$outdir/stat_netuser.dat") || return;
4335	my %netuser_stat = ();
4336	my $total_hit = 0;
4337	my $total_bytes = 0;
4338	my $total_duration = 0;
4339	while(my $l = <$infile>) {
4340		chomp($l);
4341		my ($network, $user, $data) = split(/\t/, $l);
4342		if (!$data) {
4343			# Assume backward compatibility
4344			$l =~ s/^(.*)\s([^\s]+)\shits=/hits=/;
4345			$network = $1;
4346			$user = $2;
4347			$data = $l;
4348		}
4349		next if ($network ne $usrnet);
4350
4351		if ($self->{rebuild}) {
4352			next if (!$self->check_inclusions($user, $network));
4353			next if ($self->check_exclusions($user, $network));
4354		}
4355
4356		# Anonymize all users
4357		if ($self->{AnonymizeLogin} && ($user !~ /^Anon[a-zA-Z0-9]{16}$/)) {
4358			if (!exists $self->{AnonymizedId}{$user}) {
4359				$self->{AnonymizedId}{$user} = &anonymize_id();
4360			}
4361			$user = $self->{AnonymizedId}{$user};
4362		}
4363
4364		$data =~ /^hits=(\d+);bytes=(\d+);duration=([\-\d]+);largest_file_size=([^;]*);largest_file_url=(.*)/;
4365		$netuser_stat{$user}{hits} = $1;
4366		$netuser_stat{$user}{bytes} = $2;
4367		$netuser_stat{$user}{duration} = abs($3);
4368		$netuser_stat{$user}{largest_file} = $4;
4369
4370		$total_hit += $1;
4371		$total_bytes += $2;
4372		$total_duration += $3;
4373		$netuser_stat{$user}{url} = $5;
4374	}
4375	$infile->close();
4376	my $nuser = scalar keys %netuser_stat;
4377
4378	print $$out qq{
4379<h3>$Translate{'User_number'}: $nuser</h3>
4380};
4381	print $$out qq{
4382<table class="sortable stata">
4383<thead>
4384<tr>
4385<th>$Translate{'Users'}</th>
4386<th>$Translate{'Requests'} (%)</th>
4387<th>$Translate{$self->{TransfertUnit}} (%)</th>
4388<th>$Translate{'Duration'} (%)</th>
4389<th>$Translate{'Throughput'} (B/s)</th>
4390};
4391	print $$out qq{
4392<th>$Translate{'Cost'} $self->{Currency}</th>
4393} if ($self->{CostPrice});
4394	print $$out qq{
4395<th>$Translate{'Largest'}</th>
4396<th style="text-align: left;">$Translate{'Url'}</th>
4397</tr>
4398</thead>
4399<tbody>
4400};
4401	$total_duration = abs($total_duration);
4402	foreach my $usr (sort { $netuser_stat{$b}{"$self->{OrderUser}"} <=> $netuser_stat{$a}{"$self->{OrderUser}"} } keys %netuser_stat) {
4403		my $h_percent = '0.0';
4404		$h_percent = sprintf("%2.2f", ($netuser_stat{$usr}{hits}/$total_hit) * 100) if ($total_hit);
4405		my $b_percent = '0.0';
4406		$b_percent = sprintf("%2.2f", ($netuser_stat{$usr}{bytes}/$total_bytes) * 100) if ($total_bytes);
4407		my $d_percent = '0.0';
4408		$d_percent = sprintf("%2.2f", ($netuser_stat{$usr}{duration}/$total_duration) * 100) if ($total_duration);
4409		my $total_cost = sprintf("%2.2f", int($netuser_stat{$usr}{bytes}/1000000) * $self->{CostPrice});
4410		my $total_throughput = int($netuser_stat{$usr}{bytes} / (($netuser_stat{$usr}{duration}/1000) || 1) );
4411		my $comma_throughput = $self->format_bytes($total_throughput);
4412		$netuser_stat{$usr}{duration} = &parse_duration(int($netuser_stat{$usr}{duration}/1000));
4413		my $show = $usr;
4414		foreach my $u (keys %{$self->{UserAlias}}) {
4415			if ( $usr =~ /^$u$/i ) {
4416				$show = $self->{UserAlias}->{$u};
4417				last;
4418			}
4419		}
4420		$show =~ s/_SPC_/ /g;
4421		my $url = &escape($usr);
4422		my $comma_bytes = $self->format_bytes($netuser_stat{$usr}{bytes});
4423		if ($self->{UrlReport}) {
4424			print $$out qq{
4425<tr>
4426<td><a href="../../users/$url/$url.html">$show</a></td>
4427};
4428		} else {
4429			print $$out qq{
4430<tr>
4431<td>$show</td>
4432};
4433		}
4434		print $$out qq{
4435<td>$netuser_stat{$usr}{hits} <span class="italicPercent">($h_percent)</span></td>
4436<td>$comma_bytes <span class="italicPercent">($b_percent)</span></td>
4437<td>$netuser_stat{$usr}{duration} <span class="italicPercent">($d_percent)</span></td>
4438<td>$comma_throughput</td>
4439};
4440		print $$out qq{
4441<td>$total_cost</td>
4442} if ($self->{CostPrice});
4443		my $comma_largest = $self->format_bytes($netuser_stat{$usr}{largest_file});
4444		print $$out qq{
4445<td>$comma_largest</td>
4446<td style="text-align: left;">$netuser_stat{$usr}{url}</td>
4447</tr>};
4448	}
4449	print $$out qq{
4450</tbody>
4451</table>
4452};
4453	return $nuser;
4454}
4455
4456sub _print_user_detail
4457{
4458	my ($self, $out, $outdir, $usr, $type) = @_;
4459
4460	$0 = "squid-analyzer: Printing user details statistics in $outdir";
4461
4462	# Load code statistics
4463	my $infile = new IO::File;
4464	$infile->open("$outdir/stat_user_url.dat") || return;
4465	my %url_stat = ();
4466	my $total_hit = 0;
4467	my $total_bytes = 0;
4468	my $total_duration = 0;
4469	my $total_cache_hit = 0;
4470	my $total_cache_bytes = 0;
4471	my $ok = 0;
4472	while(my $l = <$infile>) {
4473		chomp($l);
4474		my ($user, $data) = split(/\s/, $l);
4475		last if (($user ne $usr) && $ok);
4476		next if ($user ne $usr);
4477		$ok = 1;
4478
4479		if ($self->{rebuild}) {
4480			next if (!$self->check_inclusions($user));
4481			next if ($self->check_exclusions($user));
4482		}
4483
4484		if ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*?);cache_hit=(\d*);cache_bytes=(\d*)/) {
4485			my $url = $6;
4486			$url_stat{$url}{hits} = $1;
4487			$url_stat{$url}{bytes} = $2;
4488			$url_stat{$url}{duration} = abs($3);
4489			$url_stat{$url}{firsthit} = $4 if (!$url_stat{$url}{firsthit} || ($4 < $url_stat{$url}{firsthit}));
4490			$url_stat{$url}{lasthit} = $5 if (!$url_stat{$url}{lasthit} || ($5 > $url_stat{$url}{lasthit}));
4491			$url_stat{$url}{cache_hit} = $7;
4492			$url_stat{$url}{cache_bytes} = $8;
4493			if ($self->check_exclusions('','',$url)) {
4494				delete $url_stat{$url};
4495				next;
4496			}
4497			$total_hit += $url_stat{$url}{hits} || 0;
4498			$total_bytes += $url_stat{$url}{bytes} || 0;
4499			$total_duration += $url_stat{$url}{duration} || 0;
4500			$total_cache_hit += $url_stat{$url}{cache_hit} || 0;
4501			$total_cache_bytes += $url_stat{$url}{cache_bytes} || 0;
4502		} elsif ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*)/) {
4503			my $url = $6;
4504			$url_stat{$6}{hits} = $1;
4505			$url_stat{$6}{bytes} = $2;
4506			$url_stat{$6}{duration} = abs($3);
4507			$url_stat{$6}{firsthit} = $4 if (!$url_stat{$6}{firsthit} || ($4 < $url_stat{$6}{firsthit}));
4508			$url_stat{$6}{lasthit} = $5 if (!$url_stat{$6}{lasthit} || ($5 > $url_stat{$6}{lasthit}));
4509			if ($self->{rebuild}) {
4510				if ($self->check_exclusions('','',$url)) {
4511					delete $url_stat{$url};
4512					next;
4513				}
4514			}
4515			$total_hit += $url_stat{$url}{hits} || 0;
4516			$total_bytes += $url_stat{$url}{bytes} || 0;
4517			$total_duration += $url_stat{$url}{duration} || 0;
4518
4519		} elsif ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);url=(.*)/) {
4520			my $url = $4;
4521			$url_stat{$4}{hits} = $1;
4522			$url_stat{$4}{bytes} = $2;
4523			$url_stat{$4}{duration} = abs($3);
4524			if ($self->{rebuild}) {
4525				if ($self->check_exclusions('','',$url)) {
4526					delete $url_stat{$url};
4527					next;
4528				}
4529			}
4530			$total_hit += $url_stat{$url}{hits} || 0;
4531			$total_bytes += $url_stat{$url}{bytes} || 0;
4532			$total_duration += $url_stat{$url}{duration} || 0;
4533		}
4534	}
4535	$infile->close();
4536
4537	my $nurl = scalar keys %url_stat;
4538	print $$out qq{
4539<h3>$Translate{'Url_number'}: $nurl</h3>
4540<table class="sortable stata">
4541<thead>
4542<tr>
4543<th>$Translate{'Url'}</th>
4544<th>$Translate{'Requests'} (%)</th>
4545<th>$Translate{$self->{TransfertUnit}} (%)</th>
4546<th>$Translate{'Duration'} (%)</th>
4547<th>$Translate{'Throughput'} (B/s)</th>
4548};
4549	print $$out qq{
4550<th>$Translate{'First_visit'}</th>
4551<th>$Translate{'Last_visit'}</th>
4552} if ($type eq 'hour');
4553	print $$out qq{
4554<th>$Translate{'Cost'} $self->{Currency}</th>
4555} if ($self->{CostPrice});
4556	print $$out qq{
4557</tr>
4558</thead>
4559<tbody>
4560};
4561
4562	$total_duration = abs($total_duration);
4563	foreach my $url (sort { $url_stat{$b}{"$self->{OrderUrl}"} <=> $url_stat{$a}{"$self->{OrderUrl}"} } keys %url_stat) {
4564		my $h_percent = '0.0';
4565		$h_percent = sprintf("%2.2f", ($url_stat{$url}{hits}/$total_hit) * 100) if ($total_hit);
4566		my $b_percent = '0.0';
4567		$b_percent = sprintf("%2.2f", ($url_stat{$url}{bytes}/$total_bytes) * 100) if ($total_bytes);
4568		my $d_percent = '0.0';
4569		$d_percent = sprintf("%2.2f", ($url_stat{$url}{duration}/$total_duration) * 100) if ($total_duration);
4570		my $total_cost = sprintf("%2.2f", int($url_stat{$url}{bytes}/1000000) * $self->{CostPrice});
4571		my $comma_bytes = $self->format_bytes($url_stat{$url}{bytes});
4572		my $total_throughput = int($url_stat{$url}{bytes} / (($url_stat{$url}{duration}/1000) || 1) );
4573		my $comma_throughput = $self->format_bytes($total_throughput);
4574		$url_stat{$url}{duration} = &parse_duration(int($url_stat{$url}{duration}/1000));
4575		my $firsthit = '-';
4576		if ($url_stat{$url}{firsthit}) {
4577			$firsthit = ucfirst(strftime("%b %d %T", CORE::localtime($url_stat{$url}{firsthit})));
4578		}
4579		my $lasthit = '-';
4580		if ($url_stat{$url}{lasthit}) {
4581			$lasthit = ucfirst(strftime("%b %d %T", CORE::localtime($url_stat{$url}{lasthit})));
4582		}
4583		if ($type eq 'hour') {
4584			if ($url_stat{$url}{firsthit}) {
4585				$firsthit = ucfirst(strftime("%T", CORE::localtime($url_stat{$url}{firsthit})));
4586			} else {
4587				$firsthit = '-';
4588			}
4589			if ($url_stat{$url}{lasthit}) {
4590				$lasthit = ucfirst(strftime("%T", CORE::localtime($url_stat{$url}{lasthit})));
4591			} else {
4592				$firsthit = '-';
4593			}
4594		}
4595		print $$out qq{
4596<tr>
4597<td><a href="http://$url/" target="_blank" class="domainLink">$url</a></td>
4598<td>$url_stat{$url}{hits} <span class="italicPercent">($h_percent)</span></td>
4599<td>$comma_bytes <span class="italicPercent">($b_percent)</span></td>
4600<td>$url_stat{$url}{duration} <span class="italicPercent">($d_percent)</span></td>
4601<td>$comma_throughput</td>
4602};
4603		print $$out qq{
4604<td>$firsthit</td>
4605<td>$lasthit</td>
4606} if ($type eq 'hour');
4607		print $$out qq{
4608<td>$total_cost</td>
4609} if ($self->{CostPrice});
4610		print $$out qq{
4611</tr>};
4612	}
4613	my $sortpos = 1;
4614	$sortpos = 2 if ($self->{OrderUrl} eq 'bytes');
4615	$sortpos = 3 if ($self->{OrderUrl} eq 'duration');
4616	print $$out qq{
4617</tbody>
4618</table>
4619};
4620
4621}
4622
4623sub _print_top_url_stat
4624{
4625	my ($self, $outdir, $year, $month, $day, $week) = @_;
4626
4627	print STDERR "\tTop URL statistics in $outdir...\n" if (!$self->{QuietMode});
4628
4629	$0 = "squid-analyzer: Printing top urls statistics in $outdir";
4630
4631	my $stat_date = $self->set_date($year, $month, $day);
4632
4633	my $type = 'hour';
4634	if (!$day) {
4635		$type = 'day';
4636	}
4637	if (!$month) {
4638		$type = 'month';
4639	}
4640	if ($week) {
4641		$type = 'day';
4642	}
4643
4644	# Load user URL statistics
4645	my $infile = new IO::File;
4646	$infile->open("$outdir/stat_user_url.dat") || return;
4647	my %url_stat = ();
4648	my $total_hits = 0;
4649	my $total_bytes = 0;
4650	my $total_duration = 0;
4651	my $total_cache_hit = 0;
4652	my $total_cache_bytes = 0;
4653	while (my $l = <$infile>) {
4654		chomp($l);
4655		my ($user, $data) = split(/\s/, $l);
4656
4657		if ($self->{rebuild}) {
4658			next if (!$self->check_inclusions($user));
4659			next if ($self->check_exclusions($user));
4660		}
4661		# Anonymize all users
4662		if ($self->{UserReport}) {
4663			if ($self->{AnonymizeLogin} && ($user !~ /^Anon[a-zA-Z0-9]{16}$/)) {
4664				if (!exists $self->{AnonymizedId}{$user}) {
4665					$self->{AnonymizedId}{$user} = &anonymize_id();
4666				}
4667				$user = $self->{AnonymizedId}{$user};
4668			}
4669		} else {
4670			$user = '-';
4671		}
4672
4673		if ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*?);cache_hit=(\d*);cache_bytes=(\d*)/) {
4674			my $url = $6;
4675			$url_stat{$url}{hits} = $1;
4676			$url_stat{$url}{bytes} = $2;
4677			$url_stat{$url}{duration} = abs($3);
4678			$url_stat{$url}{firsthit} = $4 if (!$url_stat{$url}{firsthit} || ($4 < $url_stat{$url}{firsthit}));
4679			$url_stat{$url}{lasthit} = $5 if (!$url_stat{$url}{lasthit} || ($5 > $url_stat{$url}{lasthit}));
4680			$url_stat{$url}{cache_hit} = $7;
4681			$url_stat{$url}{cache_bytes} = $8;
4682	
4683			if ($self->{rebuild}) {
4684				if ($self->check_exclusions('','',$url)) {
4685					delete $url_stat{$url};
4686					next;
4687				}
4688			}
4689			$total_hits += $url_stat{$url}{hits} || 0;
4690			$total_bytes += $url_stat{$url}{bytes} || 0;
4691			$total_duration += $url_stat{$url}{duration} || 0;
4692			$total_cache_hit += $url_stat{$url}{cache_hit} || 0;
4693			$total_cache_bytes += $url_stat{$url}{cache_bytes} || 0;
4694		} elsif ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*)/) {
4695			my $url = $6;
4696			$url_stat{$url}{hits} = $1;
4697			$url_stat{$url}{bytes} = $2;
4698			$url_stat{$url}{duration} = abs($3);
4699			$url_stat{$url}{firsthit} = $4 if (!$url_stat{$url}{firsthit} || ($4 < $url_stat{$url}{firsthit}));
4700			$url_stat{$url}{lasthit} = $5 if (!$url_stat{$url}{lasthit} || ($5 > $url_stat{$url}{lasthit}));
4701			$url_stat{$url}{users}{$user}++ if ($self->{TopUrlUser} && $self->{UserReport});
4702			if ($self->{rebuild}) {
4703				if ($self->check_exclusions('','',$url)) {
4704					delete $url_stat{$url};
4705					next;
4706				}
4707			}
4708			$total_hits += $url_stat{$url}{hits} || 0;
4709			$total_bytes += $url_stat{$url}{bytes} || 0;
4710			$total_duration += $url_stat{$url}{duration} || 0;
4711		} elsif ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);url=(.*)/) {
4712			my $url = $4;
4713			$url_stat{$url}{hits} = $1;
4714			$url_stat{$url}{bytes} = $2;
4715			$url_stat{$url}{duration} = abs($3);
4716			$url_stat{$url}{users}{$user}++ if ($self->{TopUrlUser} && $self->{UserReport});
4717			if ($self->{rebuild}) {
4718				if ($self->check_exclusions('','',$url)) {
4719					delete $url_stat{$url};
4720					next;
4721				}
4722			}
4723			$total_hits += $url_stat{$url}{hits} || 0;
4724			$total_bytes += $url_stat{$url}{bytes} || 0;
4725			$total_duration += $url_stat{$url}{duration} || 0;
4726		}
4727	}
4728	$infile->close();
4729
4730	# Store number of urls
4731	my $nurl = scalar keys %url_stat;
4732	my $outf = new IO::File;
4733	$outf->open(">>$outdir/stat_count.dat") || return;
4734	flock($outf, 2) || die "FATAL: can't acquire lock on file $outdir/stat_count.dat, $!\n";
4735	$outf->print("urls:$nurl\n");
4736	$outf->close;
4737
4738	my $file = $outdir . '/url.html';
4739	my $out = new IO::File;
4740	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
4741
4742	my $sortpos = 1;
4743	$sortpos = 2 if ($self->{OrderUrl} eq 'bytes');
4744	$sortpos = 3 if ($self->{OrderUrl} eq 'duration');
4745
4746	# Print the HTML header
4747	my $cal = 'SA_CALENDAR_SA';
4748	$cal = '' if ($week);
4749	$self->_print_header(\$out, $self->{menu}, $cal, $sortpos);
4750	print $out "<h3>$Translate{'Url_number'}: $nurl</h3>\n";
4751	for my $tpe ('Hits', 'Bytes', 'Duration') {
4752		my $t1 = $Translate{"Url_${tpe}_title"};
4753		$t1 =~ s/\%d/$self->{TopNumber}/;
4754		if ($tpe eq 'Hits') {
4755			print $out $self->_print_title($t1, $stat_date, $week);
4756		} else {
4757			print $out "<h4>$t1 $stat_date</h4><div class=\"line-separator\"></div>\n";
4758		}
4759		print $out qq{
4760<table class="sortable stata">
4761<thead>
4762<tr>
4763<th>$Translate{'Url'}</th>
4764<th>$Translate{'Requests'} (%)</th>
4765<th>$Translate{$self->{TransfertUnit}} (%)</th>
4766<th>$Translate{'Duration'} (%)</th>
4767<th>$Translate{'Throughput'} (B/s)</th>
4768};
4769	print $out qq{
4770<th>$Translate{'First_visit'}</th>
4771<th>$Translate{'Last_visit'}</th>
4772} if ($type eq 'hour');
4773	print $out qq{
4774<th>$Translate{'Cost'} $self->{Currency}</th>
4775} if ($self->{CostPrice});
4776	print $out qq{
4777</tr>
4778</thead>
4779<tbody>
4780};
4781		$total_duration = abs($total_duration);
4782		my $i = 0;
4783		foreach my $u (sort { $url_stat{$b}{"\L$tpe\E"} <=> $url_stat{$a}{"\L$tpe\E"} } keys %url_stat) {
4784			my $h_percent = '0.0';
4785			$h_percent = sprintf("%2.2f", ($url_stat{$u}{hits}/$total_hits) * 100) if ($total_hits);
4786			my $b_percent = '0.0';
4787			$b_percent = sprintf("%2.2f", ($url_stat{$u}{bytes}/$total_bytes) * 100) if ($total_bytes);
4788			my $d_percent = '0.0';
4789			$d_percent = sprintf("%2.2f", ($url_stat{$u}{duration}/$total_duration) * 100) if ($total_duration);
4790			my $total_cost = sprintf("%2.2f", int($url_stat{$u}{bytes}/1000000) * $self->{CostPrice});
4791			my $duration = &parse_duration(int($url_stat{$u}{duration}/1000));
4792			my $comma_bytes = $self->format_bytes($url_stat{$u}{bytes});
4793			my $total_throughput = int($url_stat{$u}{bytes} / (($url_stat{$u}{duration}/1000) || 1));
4794			my $comma_throughput = $self->format_bytes($total_throughput);
4795			my $firsthit = '-';
4796			if ($url_stat{$u}{firsthit}) {
4797				$firsthit = ucfirst(strftime("%b %d %T", CORE::localtime($url_stat{$u}{firsthit})));
4798			}
4799			my $lasthit = '-';
4800			if ($url_stat{$u}{lasthit}) {
4801				$lasthit = ucfirst(strftime("%b %d %T", CORE::localtime($url_stat{$u}{lasthit})));
4802			}
4803			if ($type eq 'hour') {
4804				if ($url_stat{$u}{firsthit}) {
4805					$firsthit = ucfirst(strftime("%T", CORE::localtime($url_stat{$u}{firsthit})));
4806				} else {
4807					$firsthit = '-';
4808				}
4809				if ($url_stat{$u}{lasthit}) {
4810					$lasthit = ucfirst(strftime("%T", CORE::localtime($url_stat{$u}{lasthit})));
4811				} else {
4812					$firsthit = '-';
4813				}
4814			}
4815			print $out "<tr><td>\n";
4816			if (exists $url_stat{$u}{users} && $self->{UserReport}) {
4817				print $out qq{
4818<div class="tooltipLink"><span class="information"><a href="http://$u/" target="_blank" class="domainLink">$u</a></span><div class="tooltip">
4819<table><tr><th>$Translate{'User'}</th><th>$Translate{'Count'}</th></tr>
4820};
4821				my $k = 1;
4822				foreach my $user (sort { $url_stat{$u}{users}{$b} <=> $url_stat{$u}{users}{$a} } keys %{$url_stat{$u}{users}}) {
4823					print $out "<tr><td>$user</td><td>$url_stat{$u}{users}{$user}</td></tr>\n";
4824					$k++;
4825					last if ($k > $self->{TopUrlUser});
4826				}
4827				print $out "</table>\n</div></div>\n";
4828			} else {
4829				print $out "<a href=\"http://$u/\" target=\"_blank\" class=\"domainLink\">$u</a>\n";
4830			}
4831			print $out qq{
4832</td>
4833<td>$url_stat{$u}{hits} <span class="italicPercent">($h_percent)</span></td>
4834<td>$comma_bytes <span class="italicPercent">($b_percent)</span></td>
4835<td>$duration <span class="italicPercent">($d_percent)</span></td>
4836<td>$comma_throughput</span></td>
4837};
4838	print $out qq{
4839<td>$firsthit</td>
4840<td>$lasthit</td>
4841} if ($type eq 'hour');
4842	print $out qq{
4843<td>$total_cost</td>
4844} if ($self->{CostPrice});
4845	print $out qq{
4846</tr>};
4847			$i++;
4848			last if ($i > $self->{TopNumber});
4849		}
4850		print $out qq{</tbody></table>};
4851		# Do not show other tables if required
4852		last if ($self->{UrlHitsOnly});
4853	}
4854
4855	print $out qq{
4856<div class="uplink">
4857    <a href="#atop"><span class="iconUpArrow">$Translate{'Up_link'}</span></a>
4858</div>
4859};
4860	$self->_print_footer(\$out);
4861	$out->close();
4862
4863	return $nurl;
4864}
4865
4866sub _print_top_denied_stat
4867{
4868	my ($self, $outdir, $year, $month, $day, $week) = @_;
4869
4870	print STDERR "\tTop denied URL statistics in $outdir...\n" if (!$self->{QuietMode});
4871
4872	$0 = "squid-analyzer: Printing top denied url statistics in $outdir";
4873
4874	my $stat_date = $self->set_date($year, $month, $day);
4875
4876	my $type = 'hour';
4877	if (!$day) {
4878		$type = 'day';
4879	}
4880	if (!$month) {
4881		$type = 'month';
4882	}
4883	if ($week) {
4884		$type = 'day';
4885	}
4886
4887	# Load user URL statistics
4888	my $infile = new IO::File;
4889	$infile->open("$outdir/stat_denied_url.dat") || return;
4890	my %denied_stat = ();
4891	my $total_hits = 0;
4892	my $total_acl = 0;
4893	while (my $l = <$infile>) {
4894		chomp($l);
4895		my ($user, $data) = split(/\s/, $l);
4896
4897		if ($self->{rebuild}) {
4898			next if (!$self->check_inclusions($user));
4899			next if ($self->check_exclusions($user));
4900		}
4901
4902		# Anonymize all users
4903		if ($self->{UserReport}) {
4904			if ($self->{AnonymizeLogin} && ($user !~ /^Anon[a-zA-Z0-9]{16}$/)) {
4905				if (!exists $self->{AnonymizedId}{$user}) {
4906					$self->{AnonymizedId}{$user} = &anonymize_id();
4907				}
4908				$user = $self->{AnonymizedId}{$user};
4909			}
4910		} else {
4911			$user = '-';
4912		}
4913
4914		if ($data =~ /hits=(\d+);first=([^;]*);last=([^;]*);url=(.*);blacklist=(.*)/) {
4915			if ($self->{rebuild}) {
4916				next if ($self->check_exclusions('','',$4));
4917			}
4918			$denied_stat{$4}{hits} = $1;
4919			$denied_stat{$4}{firsthit} = $2 if (!$denied_stat{$4}{firsthit} || ($2 < $denied_stat{$4}{firsthit}));
4920			$denied_stat{$4}{lasthit} = $3 if (!$denied_stat{$4}{lasthit} || ($3 > $denied_stat{$4}{lasthit}));
4921			$total_hits += $1;
4922			$denied_stat{$4}{users}{$user}++ if ($self->{TopUrlUser} && $self->{UserReport});
4923			if ($5) {
4924				my %tmp = split(/,/, $5);
4925				foreach my $k (keys %tmp) {
4926					$denied_stat{$4}{blacklist}{$k} += $tmp{$k};
4927					$denied_stat{$4}{users}{$user}{blacklist}{$k} += $tmp{$k} if ($self->{TopUrlUser} && $self->{UserReport});
4928					$total_acl += $tmp{$k};
4929				}
4930			}
4931		}
4932	}
4933	$infile->close();
4934
4935	# Store number of denieds
4936	my $ndenied = scalar keys %denied_stat;
4937	my $outf = new IO::File;
4938	$outf->open(">>$outdir/stat_count.dat") || return;
4939	flock($outf, 2) || die "FATAL: can't acquire lock on file $outdir/stat_count.dat, $!\n";
4940	$outf->print("denied:$ndenied\n");
4941	$outf->close;
4942
4943	my $file = $outdir . '/denied.html';
4944	my $out = new IO::File;
4945	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
4946
4947	my $sortpos = 1;
4948	# Print the HTML header
4949	my $cal = 'SA_CALENDAR_SA';
4950	$cal = '' if ($week);
4951	$self->_print_header(\$out, $self->{menu}, $cal, $sortpos);
4952	print $out "<h3>$Translate{'Url_number'}: $ndenied</h3>\n";
4953
4954	my %data_acl = ();
4955	$total_acl ||= 1;
4956	foreach my $u (sort { $denied_stat{$b}{hits} <=> $denied_stat{$a}{hits} } keys %denied_stat) {
4957		next if (!exists $denied_stat{$u}{blacklist});
4958		foreach my $k (sort keys %{$denied_stat{$u}{blacklist}}) {
4959			$data_acl{$k} += $denied_stat{$u}{blacklist}{$k};
4960		}
4961	}
4962	foreach my $k (keys %data_acl) {
4963		if (($data_acl{$k}/$total_acl)*100 < $self->{MinPie}) {
4964			$data_acl{'others'} += $data_acl{$k};
4965			delete $data_acl{$k};
4966		}
4967	} 
4968	if (scalar keys %data_acl) {
4969		print $out $self->_print_title($Translate{"Blocklist_acl_title"}, $stat_date, $week);
4970		my $squidguard_acl = $self->flotr2_piegraph(1, 'squidguard_acl', $Translate{"Blocklist_acl_title"}, $Translate{'Blocklist_acl_graph'}, '', %data_acl);
4971		print $out qq{<table class="graphs"><tr><td>$squidguard_acl</td></tr></table>};
4972	}
4973
4974	my $t1 = $Translate{"Url_Hits_title"};
4975	$t1 =~ s/\%d/$self->{TopNumber}/;
4976	print $out $self->_print_title($t1, $stat_date, $week);
4977	print $out qq{
4978<table class="sortable stata">
4979<thead>
4980<tr>
4981<th>$Translate{'Url'}</th>
4982<th>$Translate{'Requests'} (%)</th>
4983};
4984	print $out qq{
4985<th>$Translate{'First_visit'}</th>
4986<th>$Translate{'Last_visit'}</th>
4987} if ($type eq 'hour');
4988	print $out qq{<th>Blocklist ACLs</th>};
4989	print $out qq{
4990</tr>
4991</thead>
4992<tbody>
4993};
4994	my $i = 0;
4995	foreach my $u (sort { $denied_stat{$b}{hits} <=> $denied_stat{$a}{hits} } keys %denied_stat) {
4996		my $h_percent = '0.0';
4997		$h_percent = sprintf("%2.2f", ($denied_stat{$u}{hits}/$total_hits) * 100) if ($total_hits);
4998		my $firsthit = '-';
4999		if ($denied_stat{$u}{firsthit}) {
5000			$firsthit = ucfirst(strftime("%b %d %T", localtime($denied_stat{$u}{firsthit})));
5001		}
5002		my $lasthit = '-';
5003		if ($denied_stat{$u}{lasthit}) {
5004			$lasthit = ucfirst(strftime("%b %d %T", localtime($denied_stat{$u}{lasthit})));
5005		}
5006		if ($type eq 'hour') {
5007			if ($denied_stat{$u}{firsthit}) {
5008				$firsthit = ucfirst(strftime("%T", localtime($denied_stat{$u}{firsthit})));
5009			} else {
5010				$firsthit = '-';
5011			}
5012			if ($denied_stat{$u}{lasthit}) {
5013				$lasthit = ucfirst(strftime("%T", localtime($denied_stat{$u}{lasthit})));
5014			} else {
5015				$firsthit = '-';
5016			}
5017		}
5018		print $out "<tr><td>\n";
5019
5020		if (exists $denied_stat{$u}{users} && $self->{UserReport}) {
5021			print $out qq{
5022<div class="tooltipLink"><span class="information"><a href="http://$u/" target="_blank" class="domainLink">$u</a></span><div class="tooltip">
5023<table><tr><th>$Translate{'User'}</th><th>$Translate{'Count'}</th></tr>
5024};
5025			my $k = 1;
5026			foreach my $user (sort { $denied_stat{$u}{users}{$b} <=> $denied_stat{$u}{users}{$a} } keys %{$denied_stat{$u}{users}}) {
5027				print $out "<tr><td>$user</td><td>$denied_stat{$u}{users}{$user}</td></tr>\n";
5028				$k++;
5029				last if ($k > $self->{TopUrlUser});
5030			}
5031			print $out "</table>\n</div></div>";
5032		} else {
5033			print $out "<a href=\"http://$u/\" target=\"_blank\" class=\"domainLink\">$u</a>\n";
5034		}
5035		print $out qq{
5036</td>
5037<td>$denied_stat{$u}{hits} <span class="italicPercent">($h_percent)</span></td>
5038};
5039		print $out qq{
5040<td>$firsthit</td>
5041<td>$lasthit</td>
5042} if ($type eq 'hour');
5043		my $bl = '-';
5044		if (exists $denied_stat{$u}{blacklist}) {
5045			$bl = '';
5046			foreach my $k (sort keys %{$denied_stat{$u}{blacklist}}) {
5047				$bl .=  $k . '=' . $denied_stat{$u}{blacklist}{$k} . ' ';
5048			}
5049		}
5050		print $out qq{
5051	<td>$bl</td>
5052</tr>};
5053		$i++;
5054		last if ($i > $self->{TopNumber});
5055	}
5056	print $out qq{</tbody></table>};
5057
5058	print $out qq{
5059<div class="uplink">
5060    <a href="#atop"><span class="iconUpArrow">$Translate{'Up_link'}</span></a>
5061</div>
5062};
5063	$self->_print_footer(\$out);
5064	$out->close();
5065
5066	return $ndenied;
5067}
5068
5069sub _print_top_domain_stat
5070{
5071	my ($self, $outdir, $year, $month, $day, $week) = @_;
5072
5073	print STDERR "\tTop domain statistics in $outdir...\n" if (!$self->{QuietMode});
5074
5075	$0 = "squid-analyzer: Printing top domain statistics in $outdir";
5076
5077	my $stat_date = $self->set_date($year, $month, $day);
5078
5079	my $type = 'hour';
5080	if (!$day) {
5081		$type = 'day';
5082	}
5083	if (!$month) {
5084		$type = 'month';
5085	}
5086	if ($week) {
5087		$type = 'day';
5088	}
5089
5090	# Load code statistics
5091	my $infile = new IO::File;
5092	$infile->open("$outdir/stat_user_url.dat") || return;
5093	my %url_stat = ();
5094	my %domain_stat = ();
5095	my $total_hits = 0;
5096	my $total_bytes = 0;
5097	my $total_duration = 0;
5098	my %perdomain = ();
5099	my $url = '';
5100	my $hits = 0;
5101	my $bytes = 0;
5102	my $cache_hit = 0;
5103	my $cache_bytes = 0;
5104	my $total_cache_hit = 0;
5105	my $total_cache_bytes = 0;
5106	my $duration = 0;
5107	my $first = 0;
5108	my $last = 0;
5109	my $tld_pattern1 = '(' . join('|', @TLD1) . ')';
5110	$tld_pattern1 = qr/([^\.]+)$tld_pattern1$/;
5111	my $tld_pattern2 = '(' . join('|', @TLD2) . ')';
5112	$tld_pattern2 = qr/([^\.]+)$tld_pattern2$/;
5113	while (my $l = <$infile>) {
5114		chomp($l);
5115		my ($user, $data) = split(/\s/, $l);
5116		$user = '-' if (!$self->{UserReport});
5117
5118		if ($self->{rebuild}) {
5119			next if (!$self->check_inclusions($user));
5120			next if ($self->check_exclusions($user));
5121		}
5122
5123		if ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*?);cache_hit=(\d*);cache_bytes=(\d*)/) {
5124			$url = lc($6);
5125			$hits = $1;
5126			$bytes = $2;
5127			$duration = abs($3);
5128			$first = $4;
5129			$last = $5;
5130			$cache_hit = $7;
5131			$cache_bytes= $8;
5132		} elsif ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);first=([^;]*);last=([^;]*);url=(.*)/) {
5133			$url = lc($6);
5134			$hits = $1;
5135			$bytes = $2;
5136			$duration = abs($3);
5137			$first = $4;
5138			$last = $5;
5139		} elsif ($data =~ /hits=(\d+);bytes=(\d+);duration=([\-\d]+);url=(.*)/) {
5140			$url = $4;
5141			$hits = $1;
5142			$bytes = $2;
5143			$duration = abs($3);
5144		}
5145
5146		if ($self->{rebuild}) {
5147			next if ($self->check_exclusions('','',$url));
5148		}
5149
5150		my $done = 0;
5151		if ($url !~ /\.\d+$/) {
5152			if ($url =~ $tld_pattern1) {
5153				$domain_stat{"$1$2"}{hits} += $hits;
5154				$domain_stat{"$1$2"}{bytes} += $bytes;
5155				$domain_stat{"$1$2"}{duration} += $duration;
5156				$domain_stat{"$1$2"}{firsthit} = $first if (!$domain_stat{"$1$2"}{firsthit} || ($first < $domain_stat{"$1$2"}{firsthit}));
5157				$domain_stat{"$1$2"}{lasthit} = $last if (!$domain_stat{"$1$2"}{lasthit} || ($last > $domain_stat{"$1$2"}{lasthit}));
5158				$domain_stat{"$1$2"}{users}{$user}++ if ($self->{TopUrlUser} && $self->{UserReport});
5159				$domain_stat{"$1$2"}{cache_hit} += $cache_hit;
5160				$domain_stat{"$1$2"}{cache_bytes} += $cache_bytes;
5161				$perdomain{"$2"}{hits} += $hits;
5162				$perdomain{"$2"}{bytes} += $bytes;
5163				$perdomain{"$2"}{cache_hit} += $cache_hit;
5164				$perdomain{"$2"}{cache_bytes} += $cache_bytes;
5165				$done = 1;
5166			} elsif ($url =~ $tld_pattern2) {
5167				$domain_stat{"$1$2"}{hits} += $hits;
5168				$domain_stat{"$1$2"}{bytes} += $bytes;
5169				$domain_stat{"$1$2"}{duration} += $duration;
5170				$domain_stat{"$1$2"}{firsthit} = $first if (!$domain_stat{"$1$2"}{firsthit} || ($first < $domain_stat{"$1$2"}{firsthit}));
5171				$domain_stat{"$1$2"}{lasthit} = $last if (!$domain_stat{"$1$2"}{lasthit} || ($last > $domain_stat{"$1$2"}{lasthit}));
5172				$domain_stat{"$1$2"}{users}{$user}++ if ($self->{TopUrlUser} && $self->{UserReport});
5173				$domain_stat{"$1$2"}{cache_hit} += $cache_hit;
5174				$domain_stat{"$1$2"}{cache_bytes} += $cache_bytes;
5175				$perdomain{"$2"}{hits} += $hits;
5176				$perdomain{"$2"}{bytes} += $bytes;
5177				$perdomain{"$2"}{cache_hit} += $cache_hit;
5178				$perdomain{"$2"}{cache_bytes} += $cache_bytes;
5179				$done = 1;
5180			}
5181		}
5182		if (!$done) {
5183			$perdomain{'others'}{hits} += $hits;
5184			$perdomain{'others'}{bytes} += $bytes;
5185			$domain_stat{'unknown'}{hits} += $hits;
5186			$domain_stat{'unknown'}{bytes} += $bytes;
5187			$domain_stat{'unknown'}{duration} += $duration;
5188			$domain_stat{'unknown'}{firsthit} = $first if (!$domain_stat{'unknown'}{firsthit} || ($first < $domain_stat{'unknown'}{firsthit}));
5189			$domain_stat{'unknown'}{lasthit} = $last if (!$domain_stat{'unknown'}{lasthit} || ($last > $domain_stat{'unknown'}{lasthit}));
5190			$domain_stat{'unknown'}{users}{$user}++ if ($self->{TopUrlUser} && $self->{UserReport});
5191			$domain_stat{'unknown'}{cache_hit} += $cache_hit;
5192			$domain_stat{'unknown'}{cache_bytes} += $cache_bytes;
5193			$perdomain{'others'}{cache_hit} += $cache_hit;
5194			$perdomain{'others'}{cache_bytes} += $cache_bytes;
5195		}
5196		$total_hits += $hits;
5197		$total_bytes += $bytes;
5198		$total_duration += $duration;
5199		$total_cache_hit += $cache_hit;
5200		$total_cache_bytes += $cache_bytes;
5201	}
5202	$infile->close();
5203
5204	# Store number of urls
5205	my $ndom = scalar keys %domain_stat;
5206	my $outf = new IO::File;
5207	$outf->open(">>$outdir/stat_count.dat") || return;
5208	flock($outf, 2) || die "FATAL: can't acquire lock on file $outdir/stat_count.dat, $!\n";
5209	$outf->print("domains:$ndom\n");
5210	$outf->close;
5211
5212	my $file = $outdir . '/domain.html';
5213	my $out = new IO::File;
5214	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
5215
5216	my $sortpos = 1;
5217	$sortpos = 2 if ($self->{OrderUrl} eq 'bytes');
5218	$sortpos = 3 if ($self->{OrderUrl} eq 'duration');
5219
5220	# Print the HTML header
5221	my $cal = 'SA_CALENDAR_SA';
5222	$cal = '' if ($week);
5223	$self->_print_header(\$out, $self->{menu}, $cal, $sortpos);
5224	print $out "<h3>$Translate{'Domain_number'}: $ndom</h3>\n";
5225
5226	$total_hits ||= 1;
5227	$total_bytes ||= 1;
5228	for my $tpe ('Hits', 'Bytes', 'Duration') {
5229		my $t1 = $Translate{"Domain_${tpe}_title"};
5230		$t1 =~ s/\%d/$self->{TopNumber}/;
5231
5232		if ($tpe eq 'Hits') {
5233
5234			print $out $self->_print_title($t1, $stat_date, $week);
5235
5236			my %data = ();
5237			foreach my $dom (keys %perdomain) {
5238				if (($perdomain{$dom}{hits}/$total_hits)*100 > $self->{MinPie}) {
5239					$data{$dom} = $perdomain{$dom}{hits};
5240				} else {
5241					$data{'others'} += $perdomain{$dom}{hits};
5242				}
5243			}
5244			my $title = "$Translate{'Domain_graph_hits_title'} $stat_date";
5245			my $domain_hits = $self->flotr2_piegraph(1, 'domain_hits', $title, $Translate{'Domains_graph'}, '', %data);
5246			%data = ();
5247			foreach my $dom (keys %domain_stat) {
5248				if (($domain_stat{$dom}{hits}/$total_hits)*100 > $self->{MinPie}) {
5249					$data{$dom} = $domain_stat{$dom}{hits};
5250				} else {
5251					$data{'others'} += $domain_stat{$dom}{hits};
5252				}
5253			}
5254			my $title2 = "$Translate{'Second_domain_graph_hits_title'} $stat_date";
5255			my $domain2_hits = $self->flotr2_piegraph(1, 'second_domain_hits', $title2, $Translate{'Domains_graph'}, '', %data);
5256			print $out qq{
5257<style>
5258  #container {
5259    display: table;
5260  }
5261  #row  {
5262    display: table-row;
5263  }
5264  #domain_hits, #second_domain_hits {
5265    display: table-cell;
5266  }
5267  #domain_hits { z-index: 999; }
5268</style>
5269<table class="graphs"><tr><td>
5270<div id="container">
5271$domain_hits
5272</div>
5273</td><td>
5274<div id="container">
5275$domain2_hits
5276</div>
5277</td></tr>
5278};
5279			$domain_hits = '';
5280			$domain2_hits = '';
5281			%data = ();
5282			foreach my $dom (keys %perdomain) {
5283				if (($perdomain{$dom}{bytes}/$total_bytes)*100 > $self->{MinPie}) {
5284					$data{$dom} = $perdomain{$dom}{bytes};
5285				} else {
5286					$data{'others'} += $perdomain{$dom}{bytes};
5287				}
5288			}
5289			$title = "$Translate{'Domain_graph_bytes_title'} $stat_date";
5290			my $domain_bytes = $self->flotr2_piegraph(1, 'domain_bytes', $title, $Translate{'Domains_graph'}, '', %data);
5291			%data = ();
5292			foreach my $dom (keys %domain_stat) {
5293				if (($domain_stat{$dom}{bytes}/$total_bytes)*100 > $self->{MinPie}) {
5294					$data{$dom} = $domain_stat{$dom}{bytes};
5295				} else {
5296					$data{'others'} += $domain_stat{$dom}{bytes};
5297				}
5298			}
5299			$title2 = "$Translate{'Second_domain_graph_bytes_title'} $stat_date";
5300			my $domain2_bytes = $self->flotr2_piegraph(1, 'second_domain_bytes', $title2, $Translate{'Domains_graph'}, '', %data);
5301			print $out qq{<tr><td>
5302<style>
5303  #container {
5304    display: table;
5305  }
5306  #row  {
5307    display: table-row;
5308  }
5309  #domain_bytes, #second_domain_bytes {
5310    display: table-cell;
5311  }
5312  #domain_bytes { z-index: 999; }
5313</style>
5314<div id="container">
5315$domain_bytes
5316</div>
5317</td><td>
5318<div id="container">
5319$domain2_bytes
5320</div>
5321</td></tr></table>};
5322			$domain_bytes = '';
5323			$domain2_bytes = '';
5324			%data = ();
5325		} else {
5326			print $out "<h4>$t1 $stat_date</h4><div class=\"line-separator\"></div>\n";
5327		}
5328		print $out qq{
5329<table class="sortable stata">
5330<thead>
5331<tr>
5332<th>$Translate{'Url'}</th>
5333<th>$Translate{'Requests'} (%)</th>
5334<th>$Translate{$self->{TransfertUnit}} (%)</th>
5335<th>$Translate{'Duration'} (%)</th>
5336<th>$Translate{'Throughput'} (B/s)</th>
5337};
5338	print $out qq{
5339<th>$Translate{'First_visit'}</th>
5340<th>$Translate{'Last_visit'}</th>
5341} if ($type eq 'hour');
5342	print $out qq{
5343<th>$Translate{'Cost'} $self->{Currency}</th>
5344} if ($self->{CostPrice});
5345	print $out qq{
5346</tr>
5347</thead>
5348<tbody>
5349};
5350		$total_duration = abs($total_duration);
5351		my $i = 0;
5352		foreach my $u (sort { $domain_stat{$b}{"\L$tpe\E"} <=> $domain_stat{$a}{"\L$tpe\E"} } keys %domain_stat) {
5353			my $h_percent = '0.0';
5354			$h_percent = sprintf("%2.2f", ($domain_stat{$u}{hits}/$total_hits) * 100) if ($total_hits);
5355			my $b_percent = '0.0';
5356			$b_percent = sprintf("%2.2f", ($domain_stat{$u}{bytes}/$total_bytes) * 100) if ($total_bytes);
5357			my $d_percent = '0.0';
5358			$d_percent = sprintf("%2.2f", ($domain_stat{$u}{duration}/$total_duration) * 100) if ($total_duration);
5359			my $total_cost = sprintf("%2.2f", int($domain_stat{$u}{bytes}/1000000) * $self->{CostPrice});
5360			my $duration = &parse_duration(int($domain_stat{$u}{duration}/1000));
5361			my $comma_bytes = $self->format_bytes($domain_stat{$u}{bytes});
5362			my $total_throughput = int($domain_stat{$u}{bytes} / (($domain_stat{$u}{duration}/1000) || 1));
5363			my $comma_throughput = $self->format_bytes($total_throughput);
5364			my $firsthit = '-';
5365			if ($domain_stat{$u}{firsthit}) {
5366				$firsthit = ucfirst(strftime("%b %d %T", CORE::localtime($domain_stat{$u}{firsthit})));
5367			}
5368			my $lasthit = '-';
5369			if ($domain_stat{$u}{lasthit}) {
5370				$lasthit = ucfirst(strftime("%b %d %T", CORE::localtime($domain_stat{$u}{lasthit})));
5371			}
5372			if ($type eq 'hour') {
5373				if ($domain_stat{$u}{firsthit}) {
5374					$firsthit = ucfirst(strftime("%T", CORE::localtime($domain_stat{$u}{firsthit})));
5375				} else {
5376					$firsthit = '-';
5377				}
5378				if ($domain_stat{$u}{lasthit}) {
5379					$lasthit = ucfirst(strftime("%T", CORE::localtime($domain_stat{$u}{lasthit})));
5380				} else {
5381					$lasthit = '-';
5382				}
5383			}
5384			print $out "<tr><td>\n";
5385			if (exists $domain_stat{$u}{users} && $self->{UserReport}) {
5386				my $dname = "*.$u";
5387				$dname = $u if (grep(/^$u$/i, 'localhost', 'unknown'));
5388				print $out qq{
5389<div class="tooltipLink"><span class="information">$dname</span><div class="tooltip">
5390<table><tr><th>$Translate{'User'}</th><th>$Translate{'Count'}</th></tr>
5391};
5392				my $k = 1;
5393				foreach my $user (sort { $domain_stat{$u}{users}{$b} <=> $domain_stat{$u}{users}{$a} } keys %{$domain_stat{$u}{users}}) {
5394					print $out "<tr><td>$user</td><td>$domain_stat{$u}{users}{$user}</td></tr>\n";
5395					$k++;
5396					last if ($k > $self->{TopUrlUser});
5397				}
5398				print $out "</table>\n";
5399			} else {
5400				print $out "*.$u\n";
5401			}
5402			print $out qq{
5403</div></div>
5404</td>
5405<td>$domain_stat{$u}{hits} <span class="italicPercent">($h_percent)</span></td>
5406<td>$comma_bytes <span class="italicPercent">($b_percent)</span></td>
5407<td>$duration <span class="italicPercent">($d_percent)</span></td>
5408<td>$comma_throughput</td>
5409};
5410	print $out qq{
5411<td>$firsthit</td>
5412<td>$lasthit</td>
5413}  if ($type eq 'hour');
5414	print $out qq{
5415<td>$total_cost</td>
5416} if ($self->{CostPrice});
5417	print $out qq{
5418</tr>};
5419			$i++;
5420			last if ($i > $self->{TopNumber});
5421		}
5422		print $out qq{</tbody></table>};
5423		# Do not show other tables if required
5424		last if ($self->{UrlHitsOnly});
5425	}
5426
5427	print $out qq{
5428<div class="uplink">
5429<a href="#atop"><span class="iconUpArrow">$Translate{'Up_link'}</span></a>
5430</div>
5431};
5432	$self->_print_footer(\$out);
5433	$out->close();
5434
5435	return $ndom;
5436}
5437
5438sub _gen_summary
5439{
5440	my ($self, $outdir) = @_;
5441
5442	# Get all day subdirectory
5443        opendir(DIR, "$outdir") or $self->localdie("ERROR: Can't read directory $outdir, $!\n");
5444        my @dirs = grep { /^\d{4}$/ && -d "$outdir/$_" } readdir(DIR);
5445        closedir DIR;
5446
5447	my %code_stat = ();
5448	my %throughput_stat = ();
5449	my %total_request =  ();
5450	my %total_bytes = ();
5451	my %total_elapsed = ();
5452	my %total_throughput = ();
5453	foreach my $d (@dirs) {
5454		# Load code statistics
5455		my $infile = new IO::File;
5456		$infile->open("$outdir/$d/stat_code.dat") || return;
5457		while (my $l = <$infile>) {
5458			chomp($l);
5459			my ($code, $data) = split(/\s/, $l);
5460			$data =~ /hits_month=([^;]+);bytes_month=([^;]+)/;
5461			my $hits = $1 || '';
5462			my $bytes = $2 || '';
5463			$hits =~ s/,$//;
5464			$bytes =~ s/,$//;
5465			my %hits_tmp = split(/[:,]/, $hits);
5466			foreach my $tmp (sort {$a <=> $b} keys %hits_tmp) {
5467				$code_stat{$d}{$code}{request} += $hits_tmp{$tmp};
5468			}
5469			my %bytes_tmp = split(/[:,]/, $bytes);
5470			foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
5471				$code_stat{$d}{$code}{bytes} += $bytes_tmp{$tmp};
5472			}
5473			if ($data =~ /thp_bytes_month=([^;]+);thp_duration_month=([^;]+)/) {
5474				$bytes = $1 || '';
5475				my $elapsed = $2 || '';
5476				$elapsed =~ s/,$//;
5477				my %bytes_tmp = split(/[:,]/, $bytes);
5478				foreach my $tmp (sort {$a <=> $b} keys %bytes_tmp) {
5479					$throughput_stat{$d}{$code}{bytes} += $bytes_tmp{$tmp};
5480				}
5481				my %elapsed_tmp = split(/[:,]/, $elapsed);
5482				foreach my $tmp (sort {$a <=> $b} keys %elapsed_tmp) {
5483					$throughput_stat{$d}{$code}{elapsed} += $elapsed_tmp{$tmp};
5484				}
5485			}
5486		}
5487		$infile->close();
5488		$total_request{$d} =  $code_stat{$d}{HIT}{request} + $code_stat{$d}{MISS}{request};
5489		$total_bytes{$d} = $code_stat{$d}{HIT}{bytes} + $code_stat{$d}{MISS}{bytes};
5490		$total_bytes{$d} = $code_stat{$d}{HIT}{bytes} + $code_stat{$d}{MISS}{bytes};
5491		$total_throughput{$d} = $throughput_stat{$d}{HIT}{bytes} + $throughput_stat{$d}{MISS}{bytes};
5492		$total_elapsed{$d} = $code_stat{$d}{HIT}{elapsed} + $code_stat{$d}{MISS}{elapsed};
5493	}
5494	my $file = $outdir . '/index.html';
5495	my $out = new IO::File;
5496	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
5497	# Print the HTML header
5498	$self->_print_header(\$out);
5499	my $colspn = 3;
5500	$colspn = 4 if ($self->{CostPrice});
5501	print $out qq{
5502    <h4>$Translate{'Globals_Statistics'}</h4>
5503    <div class="line-separator"></div>
5504	<table class="stata">
5505	<thead>
5506	<tr>
5507	<th class="nobg"></th>
5508	<th colspan="3" scope="col" class="headerBlack">$Translate{'Requests'}</th>
5509	<th colspan="3" scope="col" class="headerBlack">$Translate{$self->{TransfertUnit}}</th>
5510	<th colspan="$colspn" scope="col" class="headerBlack">$Translate{'Total'}</th>
5511	</tr>
5512	<tr>
5513	<th scope="col">$Translate{'Years'}</th>
5514	<th scope="col">$Translate{'Hit'}</th>
5515	<th scope="col">$Translate{'Miss'}</th>
5516	<th scope="col">$Translate{'Denied'}</th>
5517	<th scope="col">$Translate{'Hit'}</th>
5518	<th scope="col">$Translate{'Miss'}</th>
5519	<th scope="col">$Translate{'Denied'}</th>
5520	<th scope="col">$Translate{'Requests'}</th>
5521	<th scope="col">$Translate{$self->{TransfertUnit}}</th>
5522	<th scope="col">$Translate{'Throughput'}</th>
5523};
5524	print $out qq{
5525	<th scope="col">$Translate{'Cost'} $self->{Currency}</th>
5526} if ($self->{CostPrice});
5527	print $out qq{
5528	</tr>
5529	</thead>
5530	<tbody>
5531};
5532	foreach my $year (sort {$b <=> $a} keys %code_stat) {
5533		my $comma_bytes = $self->format_bytes($total_bytes{$year});
5534		my $hit_bytes = $self->format_bytes($code_stat{$year}{HIT}{bytes});
5535		my $miss_bytes = $self->format_bytes($code_stat{$year}{MISS}{bytes});
5536		my $denied_bytes = $self->format_bytes($code_stat{$year}{DENIED}{bytes});
5537		my $total_cost = sprintf("%2.2f", int($total_bytes{$year}/1000000) * $self->{CostPrice});
5538		my $subtotal = ($throughput_stat{$year}{MISS}{elapsed}+$throughput_stat{$year}{HIT}{elapsed}) || 1;
5539		my $total_throughputs = int($total_throughput{$year}/(($subtotal/1000) || 1));
5540		my $comma_throughput = $self->format_bytes($total_throughputs);
5541		my $trfunit = $self->{TransfertUnit} || 'B';
5542		$trfunit = 'B' if ($trfunit eq 'BYTE');
5543		print $out qq{
5544	<tr>
5545	<td><a href="$year/index.html">$Translate{'Stat_label'} $year *</a></td>
5546	<td>$code_stat{$year}{HIT}{request}</td>
5547	<td>$code_stat{$year}{MISS}{request}</td>
5548	<td>$code_stat{$year}{DENIED}{request}</td>
5549	<td>$hit_bytes</td>
5550	<td>$miss_bytes</td>
5551	<td>$denied_bytes</td>
5552	<td>$total_request{$year}</td>
5553	<td>$comma_bytes</td>
5554	<td>$comma_throughput $trfunit/s</td>
5555};
5556		print $out qq{<td>$total_cost</td>} if ($self->{CostPrice});
5557		print $out qq{</tr>};
5558	}
5559	print $out qq{
5560	</tbody>
5561	</table>
5562	<blockquote class="notification">(*) $Translate{'Click_year_stat'}</blockquote>
5563
5564	<h4>$Translate{'Legend'}</h4>
5565	<div class="line-separator"></div>
5566	<div class="displayLegend">
5567		<span class="legendeTitle">$Translate{'Hit'}</span>: <span class="descLegend">$Translate{'Hit_help'}</span><br/>
5568		<span class="legendeTitle">$Translate{'Miss'}</span>: <span class="descLegend">$Translate{'Miss_help'}</span><br/>
5569};
5570	print $out qq{<span class="legendeTitle">$Translate{'Cost'}</span>: <span class="descLegend">$Translate{'Cost_help'} $self->{CostPrice} $self->{Currency}</span><br/>} if ($self->{CostPrice});
5571	print $out qq{
5572	</div>
5573};
5574	$self->_print_footer(\$out);
5575	$out->close();
5576
5577}
5578
5579sub parse_config
5580{
5581	my ($self, $file, $log_file, $rebuild) = @_;
5582
5583	$self->localdie("FATAL: no configuration file!\n") if (!-e $file);
5584
5585	my %opt = ();
5586	open(CONF, $file) or $self->localdie("ERROR: can't open file $file, $!\n");
5587	while (my $l = <CONF>) {
5588		chomp($l);
5589		$l =~ s/\r//;
5590		next if (!$l || ($l =~ /^\s*#/));
5591		my ($key, $val) = split(/\s+/, $l, 2);
5592		if ($key ne 'LogFile') {
5593			$opt{$key} = $val;
5594		} else {
5595			push(@{$opt{LogFile}}, split(/[,]+/, $val));
5596		}
5597	}
5598	close(CONF);
5599
5600	# Set logfile from command line if any.
5601	@{$opt{LogFile}} = split(/[,]+/, $log_file) if ($log_file);
5602
5603	# Check config
5604	if (!exists $opt{Output} || !-d $opt{Output}) {
5605		$self->localdie("ERROR: you must give a valid output directory. See option: Output\n");
5606	}
5607	if ( ($#{$opt{LogFile}} < 0) && !$rebuild) {
5608		$self->localdie("ERROR: you must give a Squid log file to parse. See LogFile or option -l\n");
5609	} elsif (!$rebuild) {
5610		foreach my $f (@{$opt{LogFile}}) {
5611			if (!-f $f) {
5612				$self->localdie("ERROR: you must give a valid path to the Squid log file.\n");
5613			}
5614		}
5615	}
5616	if (exists $opt{DateFormat}) {
5617		if ( ($opt{DateFormat} !~ m#\%y#) || (($opt{DateFormat} !~ m#\%m#) && ($opt{DateFormat} !~ m#\%M#) )|| ($opt{DateFormat} !~ m#\%d#) ) {
5618			$self->localdie("ERROR: bad date format: $opt{DateFormat}, must have \%y, \%m or \%M, \%d. See DateFormat option.\n");
5619		}
5620	}
5621	if ($opt{Lang} && !-e $opt{Lang}) {
5622		$self->localdie("ERROR: can't find translation file $opt{Lang}. See option: Lang\n");
5623	}
5624	if ($opt{ImgFormat} && !grep(/^$opt{ImgFormat}$/, 'png','jpg')) {
5625		$self->localdie("ERROR: unknown image format. See option: ImgFormat\n");
5626	}
5627
5628	if ($opt{TimeZone} && $opt{TimeZone} !~ /^[+\-]\d{1,2}$/) {
5629		$self->localdie("ERROR: timezone format: +/-HH, ex: +01. See option: TimeZone\n");
5630	}
5631
5632	return %opt;
5633}
5634
5635sub parse_network_aliases
5636{
5637	my ($self, $file) = @_;
5638
5639	return if (!$file || !-f $file);
5640
5641	my %alias = ();
5642	open(ALIAS, $file) or $self->localdie("ERROR: can't open network aliases file $file, $!\n");
5643	my $i = 0;
5644	while (my $l = <ALIAS>) {
5645		chomp($l);
5646		$i++;
5647		next if (!$l || ($l =~ /^\s*#/));
5648		$l =~ s/\s*#.*//;
5649		my @data = split(/\t+/, $l, 2);
5650		if ($#data == 1) {
5651			my @rg = split(/(?<!\{\d)[\s,;](?!\d+\})/, $data[1]);
5652			foreach my $r (@rg) {
5653				$r =~ s/^\^//;
5654				# If this is not a cidr notation
5655				if ($r !~ /^\d+\.\d+\.\d+\.\d+\/\d+$/) {
5656					$self->check_regex($r, "$file at line $i");
5657				}
5658				$alias{"$r"} = $data[0];
5659			}
5660		} else {
5661			$self->localdie("ERROR: wrong format in network aliases file $file, line $i\n");
5662		}
5663	}
5664	close(ALIAS);
5665
5666	return %alias;
5667}
5668
5669sub parse_user_aliases
5670{
5671	my ($self, $file) = @_;
5672
5673	return if (!$file || !-f $file);
5674
5675	my %alias = ();
5676	open(ALIAS, $file) or $self->localdie("ERROR: can't open user aliases file $file, $!\n");
5677	my $i = 0;
5678	while (my $l = <ALIAS>) {
5679		chomp($l);
5680		$i++;
5681		next if (!$l || ($l =~ /^\s*#/));
5682		my @data = split(/\t+/, $l, 2);
5683		$data[0] =~ s/\s+/_SPC_/g; # Replace space, they are not allowed
5684		if ($#data == 1) {
5685			my @rg = split(/(?<!\{\d)[\s,;](?!\d+\})/, $data[1]);
5686			foreach my $r (@rg) {
5687				$r =~ s/^\^//;
5688				$r =~ s/([^\\])\$$/$1/;
5689				$self->check_regex($r, "$file at line $i");
5690				$alias{"$r"} = $data[0];
5691			}
5692		} else {
5693			$self->localdie("ERROR: wrong format in user aliases file $file, line $i\n");
5694		}
5695	}
5696	close(ALIAS);
5697
5698	return %alias;
5699}
5700
5701sub parse_exclusion
5702{
5703	my ($self, $file) = @_;
5704
5705	return if (!$file || !-f $file);
5706
5707	my %exclusion = ();
5708	open(EXCLUDED, $file) or $self->localdie("ERROR: can't open exclusion file $file, $!\n");
5709	my $i = 0;
5710	while (my $l = <EXCLUDED>) {
5711		chomp($l);
5712		$i++;
5713		next if (!$l || ($l =~ /^\s*#/));
5714		# remove comments at end of line
5715		$l =~ s/\s*#.*//;
5716		if ($l =~ m#^(USER|CLIENT|URI|NETWORK)\s+(.*)#) {
5717			my $lbl = lc($1) . 's';
5718			my @rg =  split(m#\s+#, $2);
5719			foreach my $r (@rg) {
5720				next if ($lbl eq 'networks');
5721				$self->check_regex($r, "$file at line $i");
5722			}
5723			push(@{$exclusion{$lbl}}, @rg);
5724		} else {
5725			# backward compatibility is not more supported
5726			$self->localdie("ERROR: wrong line format in file $file at line $i => $l\n");
5727		}
5728	}
5729	close(EXCLUDED);
5730
5731	return %exclusion;
5732}
5733
5734sub parse_inclusion
5735{
5736	my ($self, $file) = @_;
5737
5738	return if (!$file || !-f $file);
5739
5740	my %inclusion = ();
5741	open(INCLUDED, $file) or $self->localdie("ERROR: can't open inclusion file $file, $!\n");
5742	my $i = 0;
5743	while (my $l = <INCLUDED>) {
5744		chomp($l);
5745		$i++;
5746		next if (!$l || ($l =~ /^\s*#/));
5747		# remove comments at end of line
5748		$l =~ s/\s*#.*//;
5749		if ($l =~ m#^(USER|CLIENT|NETWORK)\s+(.*)#) {
5750			my $lbl = lc($1) . 's';
5751			my @rg =  split(m#\s+#, $2);
5752			foreach my $r (@rg) {
5753				next if ($lbl eq 'networks');
5754				$self->check_regex($r, "$file at line $i");
5755			}
5756			push(@{$inclusion{$lbl}}, @rg);
5757		} else {
5758			# backward compatibility is not more supported
5759			$self->localdie("ERROR: wrong line format in file $file at line $i\n");
5760		}
5761	}
5762	close(INCLUDED);
5763
5764	return %inclusion;
5765}
5766
5767# User URL-encode
5768sub escape
5769{
5770	my ($toencode) = @_;
5771
5772	return undef unless defined($toencode);
5773
5774	$toencode =~ s/[^a-zA-Z0-9_.-]/_/g;
5775
5776	return $toencode;
5777}
5778
5779# Set date to user format
5780sub set_date
5781{
5782	my ($self, $year, $month, $day) = @_;
5783
5784	my $date_format = $self->{DateFormat};
5785
5786        $date_format =~ s/\%y/$year/;
5787        $date_format =~ s/\%m/$month/;
5788        $date_format =~ s/\%d/$day/;
5789        $date_format =~ s/\%M/$Translate{$month}/;
5790
5791	$date_format =~ s/([^\p{Letter}\p{Digit}]){2,3}/$1/;
5792	$date_format =~ s/^[^\p{Letter}\p{Digit}]+//;
5793	$date_format =~ s/[^\p{Letter}\p{Digit}]+$//;
5794
5795        return $date_format;
5796}
5797
5798# Format bytes with comma for better reading
5799sub format_bytes
5800{
5801	my ($self, $text) = @_;
5802
5803	if ($self->{TransfertUnitValue} > 1) {
5804		$text = sprintf("%.2f", $text / $self->{TransfertUnitValue});
5805	}
5806	$text = reverse $text;
5807
5808	$text =~ s/(\d\d\d)(?=\d)(?!\d*\.)/$1,/g;
5809
5810	return scalar reverse $text;
5811}
5812
5813sub _print_title
5814{
5815	my ($self, $title, $stat_date, $week) = @_;
5816
5817	my $week_title = '';
5818	$week_title = " $Translate{Week} $week" if ($week);
5819
5820	my $para = qq{
5821<h4>$title $stat_date$week_title</h4>
5822<div class="line-separator"></div>
5823};
5824
5825	return $para;
5826}
5827
5828sub _get_calendar
5829{
5830	my ($self, $stat_date, $year, $month, $type, $outdir, $rewind) = @_;
5831
5832	my $para = "<div id=\"calendar\">\n";
5833	if ($type eq 'day') {
5834		$para .= "<table><tr><th colspan=\"8\">$stat_date</th></tr>\n";
5835		my @wday = qw(Mo Tu We Th Fr Sa Su);
5836		my @std_day = qw(Su Mo Tu We Th Fr Sa);
5837		my %day_lbl = ();
5838		if (exists $Translate{WeekDay}) {
5839			my @tmpwday = split(/\s+/, $Translate{WeekDay});
5840			for (my $i = 0; $i <= $#std_day; $i++) {
5841				$day_lbl{$std_day[$i]} = $tmpwday[$i];
5842			}
5843		} else {
5844			for (my $i = 0; $i <= $#wday; $i++) {
5845				$day_lbl{$wday[$i]} = $wday[$i];
5846			}
5847		}
5848		$para .= "<tr><td>&nbsp;</td>";
5849		map { $para .= '<td align="center">' . $day_lbl{$_} . '</td>'; } @wday;
5850		$para .= "</tr>\n";
5851
5852		my @currow = ('','','','','','','');
5853		my %weeks_num = ();
5854		my $wn = '';
5855		my $wn_ok = '';
5856		my $wd = '';
5857		for my $d ("01" .. "31") {
5858			$wn =  &get_week_number($year,$month,$d);
5859			next if ($wn == -1);
5860			$wd = &get_day_of_week($year,$month,$d);
5861			next if ($wd == -1);
5862			$wn_ok = $wn;
5863			if (-f "$outdir/$d/index.html") {
5864				$currow[$wd-1] = "<td><a href=\"$rewind$d/index.html\">$d</a></td>";
5865			} else {
5866				$currow[$wd-1] = "<td>$d</td>";
5867			}
5868			if ($wd == 7) {
5869				map { $_ = "<td>&nbsp;</td>" if ($_ eq ''); } @currow;
5870				@{$weeks_num{$wn_ok}} = @currow;
5871				@currow = ('','','','','','','');
5872			}
5873		}
5874		if ( ($wd < 7) && ($wd != -1) && ($wn_ok != -1) ) {
5875			map { $_ = "<td>&nbsp;</td>" if ($_ eq ''); } @currow;
5876			@{$weeks_num{$wn_ok}} = @currow;
5877		}
5878		my $path = $outdir;
5879		$path =~ s/(\/\d{4})\/\d{2}.*/$1/;
5880		my $prefix = $self->{WebUrl} || '';
5881		$prefix .= '/' if ( $self->{WebUrl} && ($self->{WebUrl} !~ m#\/$#) );
5882		foreach my $w (sort { $a <=> $b } keys %weeks_num) {
5883			my $ww = sprintf("%02d", $w+1);
5884			my $week = "<tr><th>$ww</th>";
5885			if (-d "$path/week$ww") {
5886				$week = "<tr><th><a href=\"$prefix$year/week$ww\">$ww</a></th>";
5887			}
5888			$para .= $week . join('', @{$weeks_num{$w}}) . "</tr>\n";
5889		}
5890		$para .= "</table>\n";
5891
5892	} elsif ($type eq 'month') {
5893
5894		$para .= "<table><tr><th colspan=\"4\">$stat_date</th></tr>\n";
5895		for my $i ('01' .. '12') {
5896			$para .= "<tr>" if (grep(/^$i$/, '01', '04', '07','10'));
5897			if (-f "$outdir/$i/index.html") {
5898				$para .= "<td><a href=\"$i/index.html\">$Translate{$i}</a></td>";
5899			} else {
5900				$para .= "<td>$Translate{$i}</td>";
5901			}
5902			$para .= "</tr>\n" if (grep(/^$i$/, '03', '06', '09', '12'));
5903		}
5904		$para .= "</table>\n";
5905
5906	}
5907	$para .= "</div>\n";
5908
5909	return $para;
5910}
5911
5912
5913sub anonymize_id
5914{
5915        my $u_id = '';
5916        while (length($u_id) < 16) {
5917                my $c = chr(int(rand(127)));
5918                if ($c =~ /[a-zA-Z0-9]/) {
5919                        $u_id .= $c;
5920                }
5921        }
5922
5923        return 'Anon' . $u_id;
5924
5925}
5926
5927
5928sub flotr2_bargraph
5929{
5930	my ($self, $buttonid, $divid, $xtype, $title, $xtitle, $ytitle, $data1, $legend1, $data2, $legend2, $data3, $legend3, $data4, $legend4) = @_;
5931
5932        my @legend = ();
5933        my @data = ();
5934
5935	my $i = 0;
5936	push(@data, "var d1 = [$data1];\n") if ($data1);
5937	push(@legend, "{ data: d1, label: \"$legend1\", color: \"$GRAPH_COLORS[$i++]\", mouse:{track:true} },\n") if ($data1);
5938	push(@data, "var d2 = [$data2];\n") if ($data2);
5939	push(@legend, "{ data: d2, label: \"$legend2\", color: \"$GRAPH_COLORS[$i++]\", mouse:{track:true} },\n") if ($data2);
5940	push(@data, "var d3 = [$data3];\n") if ($data3);
5941	push(@legend, "{ data: d3, label: \"$legend3\", color: \"$GRAPH_COLORS[$i++]\", mouse:{track:true} },\n") if ($data3);
5942	push(@data, "var d4 = [$data4];\n") if ($data4);
5943	push(@legend, "{ data: d4, label: \"$legend4\", color: \"$GRAPH_COLORS[$i++]\", mouse:{track:true} },\n") if ($data4);
5944
5945
5946	my $month_array = '';
5947	my $xlabel = '';
5948	my $numticks = 0;
5949	my $xmode = 'normal';
5950	if ($xtype eq 'month') {
5951		$month_array = qq{var months = [ '$Translate{"01"}', '$Translate{"02"}', '$Translate{"03"}', '$Translate{"04"}', '$Translate{"05"}', '$Translate{"06"}', '$Translate{"07"}', '$Translate{"08"}', '$Translate{"09"}', '$Translate{"10"}', '$Translate{"11"}', '$Translate{"12"}' ];
5952};
5953		$xlabel = qq{return months[(pos -1) % 12];};
5954		$numticks = 12;
5955	} elsif ($xtype eq 'day') {
5956		$xlabel = qq{var days = [01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31];
5957		return days[(pos - 1) % 31];
5958};
5959		$numticks = 31;
5960	} elsif ($xtype =~ /\[.*\]/) {
5961		$xmode = 'time';
5962		$xlabel = qq{var days = $xtype;
5963		return days[(pos - 1) % 7];
5964};
5965		$numticks = 7;
5966	} else  {
5967		$xlabel = qq{var hours = [00,01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23];
5968		return hours[pos % 24];
5969};
5970		$numticks = 24;
5971	}
5972
5973	my $tickFormatter = qq{
5974            tickFormatter: function(x) {
5975                var pos = parseInt(x);
5976		$xlabel
5977            },
5978};
5979	$tickFormatter = '' if ($xmode eq 'time');
5980
5981	my $dateTracker_lblopts = '';
5982	map { if (/label: "([^"]+)"/) { $dateTracker_lblopts .= "'$1',"; } } @legend;
5983	$dateTracker_lblopts =~ s/,$//;
5984	$dateTracker_lblopts = "[$dateTracker_lblopts]";
5985
5986	my $dateTracker_dataopts = '';
5987	map { if (/var (d\d+) =/) { $dateTracker_dataopts .= "$1,"; } } @data;
5988	$dateTracker_dataopts =~ s/,$//;
5989	$dateTracker_dataopts = "[$dateTracker_dataopts]";
5990
5991	return <<EOF;
5992<div id="$divid"></div>
5993<script type="text/javascript">
5994$month_array
5995(function mouse_zoom(container) {
5996
5997//document.writeln('<table class="tbbutton"><tr><td><input type="button" class="dldButton" value="To Image" id="toimage$buttonid" onclick="return false;">'+
5998//	'<input type="button" class="dldButton" value="Download" id="download$buttonid" onclick="return false;">' +
5999//	'<input type="button" class="dldButton" value="Reset" id="reset$buttonid" onclick="return false;"></td></tr><tr><td>&nbsp;</td></tr></table>'
6000//	);
6001    @data
6002    var options = {
6003        mouse: {
6004            track: true,
6005            relative: true,
6006	    trackFormatter: function(obj){ return dateTracker(obj,'$xtype',$dateTracker_lblopts,$dateTracker_dataopts) },
6007        },
6008        yaxis: {
6009            min: 0,
6010            mode: "normal",
6011            autoscaleMargin: 1,
6012            title: "$ytitle",
6013        },
6014        xaxis: {
6015            mode: "$xmode",
6016            noTicks: $numticks,
6017	    $tickFormatter
6018            title: "$xtitle",
6019        },
6020        title: "$title",
6021        legend: {
6022            position: "nw",
6023            backgroundColor: "#D2E8FF",
6024	    backgroundOpacity: 0.4
6025        },
6026        HtmlText: false,
6027    };
6028
6029    function drawGraph(opts) {
6030        var o = Flotr._.extend(Flotr._.clone(options), opts );
6031        return Flotr.draw(
6032        	container,
6033        	[
6034@legend
6035    		],
6036    		o
6037    	);
6038    }
6039
6040    var graph = drawGraph();
6041    Flotr.EventAdapter.observe(container, "flotr:select", function(area) {
6042        f = drawGraph({
6043            xaxis: {
6044                min: area.x1,
6045                max: area.x2
6046            },
6047            yaxis: {
6048                min: area.y1,
6049                max: area.y2
6050            }
6051        });
6052    });
6053    Flotr.EventAdapter.observe(container, "flotr:click", function() {
6054        drawGraph();
6055    });
6056
6057/*
6058    document.getElementById('reset$buttonid').onclick = function() {
6059      graph.download.restoreCanvas();
6060    };
6061    document.getElementById('download$buttonid').onclick = function(){
6062	if (Flotr.isIE && Flotr.isIE < 9) {
6063		alert(
6064		"Your browser doesn't allow you to get a bitmap image from the plot, " +
6065		"you can only get a VML image that you can use in Microsoft Office.<br />"
6066		);
6067	}
6068      graph.download.saveImage('$self->{ImgFormat}');
6069    };
6070    document.getElementById('toimage$buttonid').onclick = function() {
6071	if (Flotr.isIE && Flotr.isIE < 9) {
6072		alert(
6073		"Your browser doesn't allow you to get a bitmap image from the plot, " +
6074		"you can only get a VML image that you can use in Microsoft Office.<br />"
6075		);
6076	}
6077      graph.download.saveImage('$self->{ImgFormat}', null, null, true);
6078    };
6079*/
6080})(document.getElementById("$divid"));
6081</script>
6082EOF
6083
6084}
6085
6086
6087sub flotr2_piegraph
6088{
6089	my ($self, $buttonid, $divid, $title, $xlabel, $ylabel, %data) = @_;
6090
6091	my @datadef = ();
6092	my @contdef = ();
6093	my $i       = 1;
6094	foreach my $k (sort {$data{$b} <=> $data{$a} } keys %data) {
6095		push(@datadef, "var d$i = [ [0,$data{$k}] ];\n");
6096		push(@contdef, "{ data: d$i, label: \"$k\" },\n");
6097		$i++;
6098	}
6099	return <<EOF;
6100<div id="$divid"></div>
6101<script type="text/javascript">
6102(function basic_pie(container) {
6103
6104
6105//document.writeln('<input type="button" class="dldButton" value="To Image" id="toimage$buttonid" onclick="return false;">'+
6106//	'<input type="button" class="dldButton" value="Download" id="download$buttonid" onclick="return false;">' +
6107//	'<input type="button" class="dldButton" value="Reset" id="reset$buttonid" onclick="return false;">'
6108//	);
6109
6110    @datadef
6111    var graph = Flotr.draw(container, [
6112    @contdef
6113    ], {
6114        title: "$title",
6115        HtmlText: false,
6116        grid: {
6117            verticalLines: false,
6118            horizontalLines: false,
6119	    outline: '',
6120        },
6121        xaxis: {
6122            showLabels: false,
6123	    title: "$xlabel"
6124        },
6125        yaxis: {
6126            showLabels: false,
6127	    title: "$ylabel"
6128        },
6129        pie: {
6130            show: true,
6131	    explode: 6
6132        },
6133        mouse: {
6134            track: true,
6135	    trackFormatter: function(obj){ return pieTracker(obj) },
6136	    relative: true
6137        },
6138        legend: {
6139            position: "sw",
6140            backgroundColor: "#D2E8FF",
6141	    backgroundOpacity: 0.4
6142        }
6143    });
6144/*
6145    document.getElementById('reset$buttonid').onclick = function() {
6146      graph.download.restoreCanvas();
6147    };
6148    document.getElementById('download$buttonid').onclick = function(){
6149	if (Flotr.isIE && Flotr.isIE < 9) {
6150		alert(
6151		"Your browser doesn't allow you to get a bitmap image from the plot, " +
6152		"you can only get a VML image that you can use in Microsoft Office.<br />"
6153		);
6154	}
6155      graph.download.saveImage('$self->{ImgFormat}');
6156    };
6157    document.getElementById('toimage$buttonid').onclick = function() {
6158	if (Flotr.isIE && Flotr.isIE < 9) {
6159		alert(
6160		"Your browser doesn't allow you to get a bitmap image from the plot, " +
6161		"you can only get a VML image that you can use in Microsoft Office.<br />"
6162		);
6163	}
6164      graph.download.saveImage('$self->{ImgFormat}', null, null, true);
6165    };
6166i*/
6167
6168})(document.getElementById("$divid"));
6169</script>
6170EOF
6171
6172}
6173
6174sub check_regex
6175{
6176	my ($self, $pattern, $label) = @_;
6177
6178	eval { $pattern =~ m/^$pattern$/i;};
6179	if ($@) {
6180		$self->localdie("FATAL: $label invalid regex '$pattern', $!\n");
6181	}
6182}
6183
6184sub check_ip
6185{
6186	my ($ip, $block) = @_;
6187
6188	# When $client_ip is not an ip address proceed to regex search
6189	if ($ip !~ /^\d+\.\d+\.\d+\.\d+$/) {
6190		if ( $ip =~ /$block/) {
6191			return 1;
6192		} else {
6193			return 0;
6194		}
6195	}
6196
6197	my @ip = split(/\./, $ip);
6198	my $ip1 = $ip[0] * 2**24 + $ip[1] * 2**16 + $ip[2] * 2**8 + $ip[3];
6199	my @submask = split(/\//, $block);
6200
6201	my $ip2 = $submask[0];
6202	my $netmask = $submask[1];
6203
6204	my @ip2 = split(/\./, $ip2);
6205	$ip2 = $ip2[0] * 2**24 + $ip2[1] * 2**16 + $ip2[2] * 2**8 + $ip2[3];
6206	if ( $ip1 >> (32-$netmask) == $ip2 >> (32-$netmask)) {
6207		return 1;
6208	}
6209	return 0;
6210}
6211
6212sub _gen_year_summary
6213{
6214	my ($self, $outdir) = @_;
6215
6216	# Get all day subdirectory
6217        opendir(DIR, "$outdir") or $self->localdie("ERROR: Can't read directory $outdir, $!\n");
6218        my @dirs = grep { /^\d{4}$/ && -d "$outdir/$_" } readdir(DIR);
6219        closedir DIR;
6220
6221	my %code_stat = ();
6222	my %total_request =  ();
6223	my %total_bytes = ();
6224	foreach my $d (@dirs) {
6225		$code_stat{$d} = 1;
6226	}
6227	my $file = $outdir . '/index.html';
6228	my $out = new IO::File;
6229	$out->open(">$file") || $self->localdie("ERROR: Unable to open $file. $!\n");
6230	# Print the HTML header
6231	$self->_print_header(\$out);
6232	my $colspn = 2;
6233	$colspn = 3 if ($self->{CostPrice});
6234	print $out qq{
6235    <h4>$Translate{'Globals_Statistics'}</h4>
6236    <div class="line-separator"></div>
6237	<table class="stata">
6238	<thead>
6239	<tr>
6240	<th scope="col">$Translate{'Years'}</th>
6241	</tr>
6242	</thead>
6243	<tbody>
6244};
6245	foreach my $year (sort {$b <=> $a} keys %code_stat) {
6246		print $out qq{
6247	<tr>
6248	<td><a href="$year/index.html">$Translate{'Stat_label'} $year *</a></td>
6249	</tr>
6250};
6251	}
6252	print $out qq{
6253	</tbody>
6254	</table>
6255	<blockquote class="notification">(*) $Translate{'Click_year_stat'}</blockquote>
6256
6257	<h4>$Translate{'Legend'}</h4>
6258	<div class="line-separator"></div>
6259	<div class="displayLegend">
6260		<span class="legendeTitle">$Translate{'Hit'}</span>: <span class="descLegend">$Translate{'Hit_help'}</span><br/>
6261		<span class="legendeTitle">$Translate{'Miss'}</span>: <span class="descLegend">$Translate{'Miss_help'}</span><br/>
6262};
6263	print $out qq{<span class="legendeTitle">$Translate{'Cost'}</span>: <span class="descLegend">$Translate{'Cost_help'} $self->{CostPrice} $self->{Currency}</span><br/>} if ($self->{CostPrice});
6264	print $out qq{
6265	</div>
6266};
6267	$self->_print_footer(\$out);
6268	$out->close();
6269
6270}
6271
6272####
6273# Get the week day of a date
6274####
6275sub get_day_of_week
6276{
6277	my ($year, $month, $day) = @_;
6278
6279#       %u     The day of the week as a decimal, range 1 to 7, Monday being 1.
6280#       %w     The day of the week as a decimal, range 0 to 6, Sunday being 0.
6281
6282	#my $weekDay = POSIX::strftime("%u", gmtime timelocal_nocheck(0,0,0,$day,--$month,$year));
6283	my $weekDay = POSIX::strftime("%u", 1,1,1,$day,--$month,$year-1900);
6284
6285	return $weekDay;
6286}
6287
6288####
6289# Get week number
6290####
6291sub get_week_number
6292{
6293	my ($year, $month, $day) = @_;
6294
6295#       %U     The week number of the current year as a decimal number, range 00 to 53, starting with the first
6296#              Sunday as the first day of week 01.
6297#       %V     The  ISO 8601  week  number (see NOTES) of the current year as a decimal number, range 01 to 53,
6298#              where week 1 is the first week that has at least 4 days in the new year.
6299#       %W     The week number of the current year as a decimal number, range 00 to 53, starting with the first
6300#              Monday as the first day of week 01.
6301
6302	# Check if the date is valide first
6303	my $datefmt = POSIX::strftime("%F", 1, 1, 1, $day, $month - 1, $year - 1900);
6304	if ($datefmt ne "$year-$month-$day") {
6305		return -1;
6306	}
6307	my $weekNumber = POSIX::strftime("%W", 1, 1, 1, $day, $month - 1, $year - 1900);
6308
6309	return $weekNumber;
6310}
6311
6312# Returns all days following the week number
6313sub get_wdays_per_month
6314{
6315	my $wn = shift;
6316	my ($year, $month) = split(/\-/, shift);
6317	my @months = ();
6318	my @retdays = ();
6319
6320	$month ||= '01';
6321	push(@months, "$year$month");
6322	if ($month eq '01') {
6323		unshift(@months, ($year - 1) . "12");
6324	} else {
6325		unshift(@months, $year . sprintf("%02d", $month - 1));
6326	}
6327	if ($month == 12) {
6328		push(@months, ($year+1) . "01");
6329	} else {
6330		push(@months, $year . sprintf("%02d", $month + 1));
6331	}
6332
6333	foreach my $d (@months) {
6334		$d =~ /^(\d{4})(\d{2})$/;
6335		my $y = $1;
6336		my $m = $2;
6337		foreach my $day ("01" .. "31") {
6338			# Check if the date is valide first
6339			my $datefmt = POSIX::strftime("%F", 1, 1, 1, $day, $m - 1, $y - 1900);
6340			if ($datefmt ne "$y-$m-$day") {
6341				next;
6342			}
6343			my $weekNumber = POSIX::strftime("%W", 1, 1, 1, $day, $m - 1, $y - 1900);
6344			if ( ($weekNumber == $wn) || ( ($weekNumber eq '00') && (($wn == 1) || ($wn >= 52)) ) ) {
6345				push(@retdays, "$year-$m-$day");
6346				return @retdays if ($#retdays == 6);
6347			}
6348			next if ($weekNumber > $wn);
6349		}
6350	}
6351
6352	return @retdays;
6353}
6354
6355# Returns all days following the week number
6356sub get_wdays_per_year
6357{
6358	my ($wn, $year, $mon) = @_;
6359
6360	$mon ||= '01';
6361
6362	my @months = ("$year$mon");
6363	my @retdays = ();
6364	foreach my $a ($months[0] .. "${year}12") {
6365		push(@months, $a) if (!grep(/^$a$/, @months));
6366	}
6367
6368	if ($mon == 1) {
6369		unshift(@months, ($year - 1) . "12");
6370	} else {
6371		my $d = $year . sprintf("%02d", $mon - 1);
6372		unshift(@months, $d) if (!grep(/^$d$/, @months));
6373	}
6374	if ($mon == 12) {
6375		push(@months, ($year+1) . "01");
6376	} else {
6377		my $d = $year . sprintf("%02d", $mon + 1);
6378		push(@months, $d) if (!grep(/^$d$/, @months));
6379	}
6380
6381	foreach my $d (@months) {
6382		$d =~ /^(\d{4})(\d{2})$/;
6383		my $y = $1;
6384		my $m = $2;
6385		foreach my $day ("01" .. "31") {
6386			# Check if the date is valide first
6387			my $datefmt = POSIX::strftime("%F", 1, 1, 1, $day, $m - 1, $y - 1900);
6388			if ($datefmt ne "$y-$m-$day") {
6389				next;
6390			}
6391			my $weekNumber = POSIX::strftime("%W", 1, 1, 1, $day, $m - 1, $y - 1900);
6392			#if ( ($weekNumber == $wn) || ( (($weekNumber eq '00') || ($weekNumber == 53) ) && (($wn == 1) || ($wn >= 52)) ) ) {
6393			if ( ($weekNumber == $wn) || (($weekNumber == 0) && ($wn == 52)) || (($weekNumber == 52) && ($wn == 0)) ) {
6394				my $time = timelocal_nocheck(0, 0, 0, $day, $m - 1, $y - 1900);
6395				push(@retdays, $time*1000);
6396				return @retdays if ($#retdays == 6);
6397			}
6398		}
6399	}
6400	return @retdays;
6401}
6402
64031;
6404
6405__END__