· 8 years ago · Jan 24, 2018, 06:36 PM
1#0 0x0000000000415482 in _mm_madd_epi16(long long __vector(2), long long __vector(2)) (__B=<error reading variable: Cannot access memory at address 0x7f8de56f8440>,
2 __A=...) at /usr/lib/gcc/x86_64-redhat-linux/7/include/emmintrin.h:1113
3#1 dot_product (t=t@entry=0x25318d0, w=w@entry=0x7f8de56f83d0, n=<optimized out>)
4 at fp8_v6_v99.cpp:1215
5#2 0x000000000041578c in Mixer::p (this=this@entry=0x63aec0 <contextModel2()::m>)
6 at fp8_v6_v99.cpp:1309
7#3 0x0000000000413f1c in contextModel2 () at fp8_v6_v99.cpp:3540
8#4 0x0000000000414095 in Predictor::update (this=this@entry=0x7ffd4cda3de0)
9 at fp8_v6_v99.cpp:3576
10#5 0x00000000004163f1 in Encoder::code (this=this@entry=0x7ffd4cda3de0,
11 i=<optimized out>) at fp8_v6_v99.cpp:3624
12#6 0x00000000004164d2 in Encoder::compress (this=0x7ffd4cda3de0, c=48)
13 at fp8_v6_v99.cpp:3648
14#7 0x0000000000414bd2 in direct_encode_block (type=<optimized out>, in=0x2534100,
15 len=6432, en=..., info=48) at fp8_v6_v99.cpp:5129
16#8 0x00000000004150f2 in transform_encode_block (type=type@entry=IMAGE24,
17 in=in@entry=0x2531e80, len=len@entry=6432, en=..., info=<optimized out>,
18 blstr=blstr@entry=0x7ffd4cda3cb0 "36", it=0, p1=p1@entry=0.137200713,
19 p2=p2@entry=0.137745962, begin=1618486) at fp8_v6_v99.cpp:5204
20#9 0x0000000000415390 in compressRecursive (in=in@entry=0x2531e80, n=10177994,
21 n@entry=11796480, en=..., blstr=blstr@entry=0x7ffd4cda3cb0 "36",
22 it=it@entry=0, p1=0.137200713, p1@entry=0, p2=0.137745962, p2@entry=1)
23 at fp8_v6_v99.cpp:5250
24#10 0x000000000041543c in compress (filename=<optimized out>, filesize=11796480,
25 en=...) at fp8_v6_v99.cpp:5271
26#11 0x000000000040207e in main (argc=2, argv=<optimized out>)
27 at fp8_v6_v99.cpp:5675
28
29
30==============================
31
32/* fp8 file compressor/archiver. Released on August 18, 2017
33
34 Copyright (C) 2008 Matt Mahoney, Serge Osnach, Alexander Ratushnyak,
35 Bill Pettis, Przemyslaw Skibinski, Matthew Fite, wowtiger, Andrew Paterson,
36 Jan Ondrus, Andreas Morphis, Pavel L. Holoborodko, Kaido Orav, Simon Berger,
37 Neill Corlett, Mбrcio Pais
38 LICENSE
39
40 This program is free software; you can redistribute it and/or
41 modify it under the terms of the GNU General Public License as
42 published by the Free Software Foundation; either version 2 of
43 the License, or (at your option) any later version.
44
45 This program is distributed in the hope that it will be useful, but
46 WITHOUT ANY WARRANTY; without even the implied warranty of
47 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
48 General Public License for more details at
49 Visit <http://www.gnu.org/copyleft/gpl.html>.
50
51To install and use in Windows:
52
53- To install, put fp8.exe or a shortcut to it on your desktop.
54- To compress a file or folder, drop it on the fp8 icon.
55- To decompress, drop a .fp8 file on the icon.
56
57A .fp8 extension is added for compression, removed for decompression.
58The output will go in the same folder as the input.
59
60While fp8 is working, a command window will appear and report
61progress. When it is done you can close the window by pressing
62ENTER or clicking [X].
63
64
65COMMAND LINE INTERFACE
66
67- To install, put fp8.exe somewhere in your PATH.
68- To compress: fp8 [-N] file1 [file2...]
69- To decompress: fp8 [-d] file1.fp8 [dir2]
70- To view contents: more < file1.fp8
71
72The compressed output file is named by adding ".fp8" extension to
73the first named file (file1.fp8). Each file that exists will be
74added to the archive and its name will be stored without a path.
75The option -N specifies a compression level ranging from -0
76(fastest) to -8 (smallest). The default is -5. If there is
77no option and only one file, then the program will pause when
78finished until you press the ENTER key (to support drag and drop).
79If file1.fp8 exists then it is overwritten.
80
81If the first named file ends in ".fp8" then it is assumed to be
82an archive and the files within are extracted to the same directory
83as the archive unless a different directory (dir2) is specified.
84The -d option forces extraction even if there is not a ".fp8"
85extension. If any output file already exists, then it is compared
86with the archive content and the first byte that differs is reported.
87No files are overwritten or deleted. If there is only one argument
88(no -d or dir2) then the program will pause when finished until
89you press ENTER.
90
91For compression, if any named file is actually a directory, then all
92files and subdirectories are compressed, preserving the directory
93structure, except that empty directories are not stored, and file
94attributes (timestamps, permissions, etc.) are not preserved.
95During extraction, directories are created as needed. For example:
96
97 fp8 -4 c:\tmp\foo bar
98
99compresses foo and bar (if they exist) to c:\tmp\foo.fp8 at level 4.
100
101 fp8 -d c:\tmp\foo.fp8 .
102
103extracts foo and compares bar in the current directory. If foo and bar
104are directories then their contents are extracted/compared.
105
106There are no commands to update an existing archive or to extract
107part of an archive. Files and archives larger than 2GB are not
108supported (but might work on 64-bit machines, not tested).
109File names with nonprintable characters are not supported (spaces
110are OK).
111
112
113TO COMPILE
114
115There are 2 files: fp8.cpp (C++) and paq7asm.asm (NASM/YASM).
116paq7asm.asm is the same as in paq7 and paq8x. fp8.cpp recognizes the
117following compiler options:
118
119 -DWINDOWS (to compile in Windows)
120 -DUNIX (to compile in Unix, Linux, Solairs, MacOS/Darwin, etc)
121 -DNOASM (to replace paq7asm.asm with equivalent C++)
122 -DDEFAULT_OPTION=N (to change the default compression level from 5 to N).
123
124If you compile without -DWINDOWS or -DUNIX, you can still compress files,
125but you cannot compress directories or create them during extraction.
126You can extract directories if you manually create the empty directories
127first.
128
129Use -DEFAULT_OPTION=N to change the default compression level to support
130drag and drop on machines with less than 256 MB of memory. Use
131-DDEFAULT_OPTION=4 for 128 MB, 3 for 64 MB, 2 for 32 MB, etc.
132
133Use -DNOASM for non x86-32 machines, or older than a Pentium-MMX (about
1341997), or if you don't have NASM or YASM to assemble paq7asm.asm. The
135program will still work but it will be slower. For NASM in Windows,
136use the options "--prefix _" and either "-f win32" or "-f obj" depending
137on your C++ compiler. In Linux, use "-f elf".
138
139Recommended compiler commands and optimizations:
140
141 MINGW g++:
142 g++ fp8.cpp -DWINDOWS -lz -Wall -Wextra -O3 -static -static-libgcc -ofp8.exe
143
144
145MinGW produces faster executables than Borland or Mars, but Intel 9
146is about 4% faster than MinGW).
147
148
149ARCHIVE FILE FORMAT
150
151An archive has the following format. It is intended to be both
152human and machine readable. The header ends with CTRL-Z (Windows EOF)
153so that the binary compressed data is not displayed on the screen.
154
155 fp8 -N CR LF
156 size TAB filename CR LF
157 size TAB filename CR LF
158 ...
159 CTRL-Z
160 compressed binary data
161
162-N is the option (-0 to -9), even if a default was used.
163Plain file names are stored without a path. Files in compressed
164directories are stored with path relative to the compressed directory
165(using UNIX style forward slashes "/"). For example, given these files:
166
167 123 C:\dir1\file1.txt
168 456 C:\dir2\file2.txt
169
170Then
171
172 fp8 archive \dir1\file1.txt \dir2
173
174will create archive.fp8 with the header:
175
176 fp8 -5
177 123 file1.txt
178 456 dir2/file2.txt
179
180The command:
181
182 fp8 archive.fp8 C:\dir3
183
184will create the files:
185
186 C:\dir3\file1.txt
187 C:\dir3\dir2\file2.txt
188
189Decompression will fail if the first 7 bytes are not "fp8 -". Sizes
190are stored as decimal numbers. CR, LF, TAB, CTRL-Z are ASCII codes
19113, 10, 9, 26 respectively.
192
193
194ARITHMETIC CODING
195
196The binary data is arithmetic coded as the shortest base 256 fixed point
197number x = SUM_i x_i 256^-1-i such that p(<y) <= x < p(<=y), where y is the
198input string, x_i is the i'th coded byte, p(<y) (and p(<=y)) means the
199probability that a string is lexicographcally less than (less than
200or equal to) y according to the model, _ denotes subscript, and ^ denotes
201exponentiation.
202
203The model p(y) for y is a conditional bit stream,
204p(y) = PROD_j p(y_j | y_0..j-1) where y_0..j-1 denotes the first j
205bits of y, and y_j is the next bit. Compression depends almost entirely
206on the ability to predict the next bit accurately.
207
208
209MODEL MIXING
210
211fp8 uses a neural network to combine a large number of models. The
212i'th model independently predicts
213p1_i = p(y_j = 1 | y_0..j-1), p0_i = 1 - p1_i.
214The network computes the next bit probabilty
215
216 p1 = squash(SUM_i w_i t_i), p0 = 1 - p1 (1)
217
218where t_i = stretch(p1_i) is the i'th input, p1_i is the prediction of
219the i'th model, p1 is the output prediction, stretch(p) = ln(p/(1-p)),
220and squash(s) = 1/(1+exp(-s)). Note that squash() and stretch() are
221inverses of each other.
222
223After bit y_j (0 or 1) is received, the network is trained:
224
225 w_i := w_i + eta t_i (y_j - p1) (2)
226
227where eta is an ad-hoc learning rate, t_i is the i'th input, (y_j - p1)
228is the prediction error for the j'th input but, and w_i is the i'th
229weight. Note that this differs from back propagation:
230
231 w_i := w_i + eta t_i (y_j - p1) p0 p1 (3)
232
233which is a gradient descent in weight space to minimize root mean square
234error. Rather, the goal in compression is to minimize coding cost,
235which is -log(p0) if y = 1 or -log(p1) if y = 0. Taking
236the partial derivative of cost with respect to w_i yields (2).
237
238
239MODELS
240
241Most models are context models. A function of the context (last few
242bytes) is mapped by a lookup table or hash table to a state which depends
243on the bit history (prior sequence of 0 and 1 bits seen in this context).
244The bit history is then mapped to p1_i by a fixed or adaptive function.
245There are several types of bit history states:
246
247- Run Map. The state is (b,n) where b is the last bit seen (0 or 1) and
248 n is the number of consecutive times this value was seen. The initial
249 state is (0,0). The output is computed directly:
250
251 t_i = (2b - 1)K log(n + 1).
252
253 where K is ad-hoc, around 4 to 10. When bit y_j is seen, the state
254 is updated:
255
256 (b,n) := (b,n+1) if y_j = b, else (y_j,1).
257
258- Stationary Map. The state is p, initially 1/2. The output is
259 t_i = stretch(p). The state is updated at ad-hoc rate K (around 0.01):
260
261 p := p + K(y_j - p)
262
263- Nonstationary Map. This is a compromise between a stationary map, which
264 assumes uniform statistics, and a run map, which adapts quickly by
265 discarding old statistics. An 8 bit state represents (n0,n1,h), initially
266 (0,0,0) where:
267
268 n0 is the number of 0 bits seen "recently".
269 n1 is the number of 1 bits seen "recently".
270 n = n0 + n1.
271 h is the full bit history for 0 <= n <= 4,
272 the last bit seen (0 or 1) if 5 <= n <= 15,
273 0 for n >= 16.
274
275 The primaty output is t_i := stretch(sm(n0,n1,h)), where sm(.) is
276 a stationary map with K = 1/256, initialized to
277 sm(n0,n1,h) = (n1+(1/64))/(n+2/64). Four additional inputs are also
278 be computed to improve compression slightly:
279
280 p1_i = sm(n0,n1,h)
281 p0_i = 1 - p1_i
282 t_i := stretch(p_1)
283 t_i+1 := K1 (p1_i - p0_i)
284 t_i+2 := K2 stretch(p1) if n0 = 0, -K2 stretch(p1) if n1 = 0, else 0
285 t_i+3 := K3 (-p0_i if n1 = 0, p1_i if n0 = 0, else 0)
286 t_i+4 := K3 (-p0_i if n0 = 0, p1_i if n1 = 0, else 0)
287
288 where K1..K4 are ad-hoc constants.
289
290 h is updated as follows:
291 If n < 4, append y_j to h.
292 Else if n <= 16, set h := y_j.
293 Else h = 0.
294
295 The update rule is biased toward newer data in a way that allows
296 n0 or n1, but not both, to grow large by discarding counts of the
297 opposite bit. Large counts are incremented probabilistically.
298 Specifically, when y_j = 0 then the update rule is:
299
300 n0 := n0 + 1, n < 29
301 n0 + 1 with probability 2^(27-n0)/2 else n0, 29 <= n0 < 41
302 n0, n = 41.
303 n1 := n1, n1 <= 5
304 round(8/3 lg n1), if n1 > 5
305
306 swapping (n0,n1) when y_j = 1.
307
308 Furthermore, to allow an 8 bit representation for (n0,n1,h), states
309 exceeding the following values of n0 or n1 are replaced with the
310 state with the closest ratio n0:n1 obtained by decrementing the
311 smaller count: (41,0,h), (40,1,h), (12,2,h), (5,3,h), (4,4,h),
312 (3,5,h), (2,12,h), (1,40,h), (0,41,h). For example:
313 (12,2,1) 0-> (7,1,0) because there is no state (13,2,0).
314
315- Match Model. The state is (c,b), initially (0,0), where c is 1 if
316 the context was previously seen, else 0, and b is the next bit in
317 this context. The prediction is:
318
319 t_i := (2b - 1)Kc log(m + 1)
320
321 where m is the length of the context. The update rule is c := 1,
322 b := y_j. A match model can be implemented efficiently by storing
323 input in a buffer and storing pointers into the buffer into a hash
324 table indexed by context. Then c is indicated by a hash table entry
325 and b can be retrieved from the buffer.
326
327
328CONTEXTS
329
330High compression is achieved by combining a large number of contexts.
331Most (not all) contexts start on a byte boundary and end on the bit
332immediately preceding the predicted bit. The contexts below are
333modeled with both a run map and a nonstationary map unless indicated.
334
335- Order n. The last n bytes, up to about 16. For general purpose data.
336 Most of the compression occurs here for orders up to about 6.
337 An order 0 context includes only the 0-7 bits of the partially coded
338 byte and the number of these bits (255 possible values).
339
340- Sparse. Usually 1 or 2 of the last 8 bytes preceding the byte containing
341 the predicted bit, e.g (2), (3),..., (8), (1,3), (1,4), (1,5), (1,6),
342 (2,3), (2,4), (3,6), (4,8). The ordinary order 1 and 2 context, (1)
343 or (1,2) are included above. Useful for binary data.
344
345- Text. Contexts consists of whole words (a-z, converted to lower case
346 and skipping other values). Contexts may be sparse, e.g (0,2) meaning
347 the current (partially coded) word and the second word preceding the
348 current one. Useful contexts are (0), (0,1), (0,1,2), (0,2), (0,3),
349 (0,4). The preceding byte may or may not be included as context in the
350 current word.
351
352- Formatted text. The column number (determined by the position of
353 the last linefeed) is combined with other contexts: the charater to
354 the left and the character above it.
355
356- Fixed record length. The record length is determined by searching for
357 byte sequences with a uniform stride length. Once this is found, then
358 the record length is combined with the context of the bytes immediately
359 preceding it and the corresponding byte locations in the previous
360 one or two records (as with formatted text).
361
362- Context gap. The distance to the previous occurrence of the order 1
363 or order 2 context is combined with other low order (1-2) contexts.
364
365- FAX. For 2-level bitmapped images. Contexts are the surrounding
366 pixels already seen. Image width is assumed to be 1728 bits (as
367 in calgary/pic).
368
369- Image. For uncompressed 24-bit color BMP, TIFF and TGA images. Contexts
370 are the high order bits of the surrounding pixels and linear
371 combinations of those pixels, including other color planes. The
372 image width is detected from the file header. When an image is
373 detected, other models are turned off to improve speed.
374
375- JPEG. Files are further compressed by partially uncompressing back
376 to the DCT coefficients to provide context for the next Huffman code.
377 Only baseline DCT-Huffman coded files are modeled. (This ia about
378 90% of images, the others are usually progresssive coded). JPEG images
379 embedded in other files (quite common) are detected by headers. The
380 baseline JPEG coding process is:
381 - Convert to grayscale and 2 chroma colorspace.
382 - Sometimes downsample the chroma images 2:1 or 4:1 in X and/or Y.
383 - Divide each of the 3 images into 8x8 blocks.
384 - Convert using 2-D discrete cosine transform (DCT) to 64 12-bit signed
385 coefficients.
386 - Quantize the coefficients by integer division (lossy).
387 - Split the image into horizontal slices coded independently, separated
388 by restart codes.
389 - Scan each block starting with the DC (0,0) coefficient in zigzag order
390 to the (7,7) coefficient, interleaving the 3 color components in
391 order to scan the whole image left to right starting at the top.
392 - Subtract the previous DC component from the current in each color.
393 - Code the coefficients using RS codes, where R is a run of R zeros (0-15)
394 and S indicates 0-11 bits of a signed value to follow. (There is a
395 special RS code (EOB) to indicate the rest of the 64 coefficients are 0).
396 - Huffman code the RS symbol, followed by S literal bits.
397 The most useful contexts are the current partially coded Huffman code
398 (including S following bits) combined with the coefficient position
399 (0-63), color (0-2), and last few RS codes.
400
401- Match. When a context match of 400 bytes or longer is detected,
402 the next bit of the match is predicted and other models are turned
403 off to improve speed.
404
405- Exe. When a x86 file (.exe, .obj, .dll) is detected, sparse contexts
406 with gaps of 1-12 selecting only the prefix, opcode, and the bits
407 of the modR/M byte that are relevant to parsing are selected.
408 This model is turned off otherwise.
409
410- Indirect. The history of the last 1-3 bytes in the context of the
411 last 1-2 bytes is combined with this 1-2 byte context.
412
413- DMC. A bitwise n-th order context is built from a state machine using
414 DMC, described in http://plg.uwaterloo.ca/~ftp/dmc/dmc.c
415 The effect is to extend a single context, one bit at a time and predict
416 the next bit based on the history in this context. The model here differs
417 in that two predictors are used. One is a pair of counts as in the original
418 DMC. The second predictor is a bit history state mapped adaptively to
419 a probability as as in a Nonstationary Map.
420
421ARCHITECTURE
422
423The context models are mixed by several of several hundred neural networks
424selected by a low-order context. The outputs of these networks are
425combined using a second neural network, then fed through several stages of
426adaptive probability maps (APM) before arithmetic coding.
427
428For images, only one neural network is used and its context is fixed.
429
430An APM is a stationary map combining a context and an input probability.
431The input probability is stretched and divided into 32 segments to
432combine with other contexts. The output is interpolated between two
433adjacent quantized values of stretch(p1). There are 2 APM stages in series:
434
435 p1 := (p1 + 3 APM(order 0, p1)) / 4.
436 p1 := (APM(order 1, p1) + 2 APM(order 2, p1) + APM(order 3, p1)) / 4.
437
438PREPROCESSING
439
440fp8 uses preprocessing transforms on certain data types to improve
441compression. To improve reliability, the decoding transform is
442tested during compression to ensure that the input file can be
443restored. If the decoder output is not identical to the input file
444due to a bug, then the transform is abandoned and the data is compressed
445without a transform so that it will still decompress correctly.
446
447The input is split into blocks with the format <type> <decoded size> <data>
448where <type> is 1 byte (0 = no transform), <decoded size> is the size
449of the data after decoding, which may be different than the size of <data>.
450Blocks do not span file boundaries, and have a maximum size of 4MB to
4512GB depending on compression level. Large files are split into blocks
452of this size. The preprocessor has 3 parts:
453
454- Detector. Splits the input into smaller blocks depending on data type.
455
456- Coder. Input is a block to be compressed. Output is a temporary
457 file. The coder determines whether a transform is to be applied
458 based on file type, and if so, which one. A coder may use lots
459 of resources (memory, time) and make multiple passes through the
460 input file. The file type is stored (as one byte) during compression.
461
462- Decoder. Performs the inverse transform of the coder. It uses few
463 resorces (fast, low memory) and runs in a single pass (stream oriented).
464 It takes input either from a file or the arithmetic decoder. Each call
465 to the decoder returns a single decoded byte.
466
467The following transforms are used:
468
469- EXE: CALL (0xE8) and JMP (0xE9) address operands are converted from
470 relative to absolute address. The transform is to replace the sequence
471 E8/E9 xx xx xx 00/FF by adding file offset modulo 2^25 (signed range,
472 little-endian format). Data to transform is identified by trying the
473 transform and applying a crude compression test: testing whether the
474 byte following the E8/E8 (LSB of the address) occurred more recently
475 in the transformed data than the original and within 4KB 4 times in
476 a row. The block ends when this does not happen for 4KB.
477
478- JPEG: detected by SOI and SOF and ending with EOI or any nondecodable
479 data. No transform is applied. The purpose is to separate images
480 embedded in execuables to block the EXE transform, and for a future
481 place to insert a transform.
482
483
484IMPLEMENTATION
485
486Hash tables are designed to minimize cache misses, which consume most
487of the CPU time.
488
489Most of the memory is used by the nonstationary context models.
490Contexts are represented by 32 bits, possibly a hash. These are
491mapped to a bit history, represented by 1 byte. The hash table is
492organized into 64-byte buckets on cache line boundaries. Each bucket
493contains 7 x 7 bit histories, 7 16-bit checksums, and a 2 element LRU
494queue packed into one byte. Each 7 byte element represents 7 histories
495for a context ending on a 3-bit boundary plus 0-2 more bits. One
496element (for bits 0-1, which have 4 unused bytes) also contains a run model
497consisting of the last byte seen and a count (as 1 byte each).
498
499Run models use 4 byte hash elements consisting of a 2 byte checksum, a
500repeat count (0-255) and the byte value. The count also serves as
501a priority.
502
503Stationary models are most appropriate for small contexts, so the
504context is used as a direct table lookup without hashing.
505
506The match model maintains a pointer to the last match until a mismatching
507bit is found. At the start of the next byte, the hash table is referenced
508to find another match. The hash table of pointers is updated after each
509whole byte. There is no checksum. Collisions are detected by comparing
510the current and matched context in a rotating buffer.
511
512The inner loops of the neural network prediction (1) and training (2)
513algorithms are implemented in MMX assembler, which computes 4 elements
514at a time. Using assembler is 8 times faster than C++ for this code
515and 1/3 faster overall. (However I found that SSE2 code on an AMD-64,
516which computes 8 elements at a time, is not any faster).
517
518
519DIFFERENCES FROM PAQ7
520
521An .exe model and filter are added. Context maps are improved using 16-bit
522checksums to reduce collisions. The state table uses probabilistic updates
523for large counts, more states that remember the last bit, and decreased
524discounting of the opposite count. It is implemented as a fixed table.
525There are also many minor changes.
526
527DIFFERENCES FROM PAQ8A
528
529The user interface supports directory compression and drag and drop.
530The preprocessor segments the input into blocks and uses more robust
531EXE detection. An indirect context model was added. There is no
532dictionary preprocesor like PAQ8B/C/D/E.
533
534DIFFERENCES FROM PAQ8F
535
536Different models, usually from paq8hp*. Also changed rate from 8 to 7. A bug
537in Array was fixed that caused the program to silently crash upon exit.
538
539DIFFERENCES FROM PAQ8J
540
5411) Slightly improved sparse model.
5422) Added new family of sparse contexts. Each byte mapped to 3-bit value, where
543different values corresponds to different byte classes. For example, input
544byte 0x00 transformed into 0, all bytes that less then 16 -- into 5, all
545punctuation marks (ispunct(c)!=0) -- into 2 etc. Then this flags from 11
546previous bytes combined into 32-bit pseudo-context.
547
548All this improvements gives only 62 byte on BOOK1, but on binaries archive size
549reduced on 1-2%.
550
551DIFFERENCES FROM PAQ8JA
552
553Introduced distance model. Distance model uses distance to last occurence
554of some anchor char (0x00, space, newline, 0xff), combined with previous
555charactes as context. This slightly improves compression of files with
556variable-width record data.
557
558DIFFERENCES FROM PAQ8JB
559
560Restored recordModel(), broken in paq8hp*. Slightly tuned indirectModel().
561
562DIFFERENCES FROM PAQ8JC
563
564Changed the APMs in the Predictor. Up to a 0.2% improvement for some files.
565
566DIFFERENCES FROM PAQ8JD
567
568Added DMCModel. Removed some redundant models from SparseModel and other
569minor tuneups. Changes introduced in PAQ8K were not carried over.
570
571PAQ8L v.2
572
573Changed Mixer::p() to p() to fix a compiler error in Linux
574(patched by Indrek Kruusa, Apr. 15, 2007).
575
576DIFFERENCES FROM PAQ8L, PAQ8M
577
578Modified JPEG model by Jan Ondrus (paq8fthis2). The new model improves
579compression by using decoded pixel values of current and adjacent blocks
580as context. PAQ8M was an earlier version of the new JPEG model
581(from paq8fthis).
582
583DIFFERENCES FROM PAQ8N
584
585Improved bmp model. Slightly faster.
586
587DIFFERENCES FROM PAQ8O
588
589Modified JPEG model by Jan Ondrus (paq8fthis4).
590Added PGM (grayscale image) model form PAQ8I.
591Added grayscale BMP model to PGM model.
592Ver. 2 can be compiled using either old or new "for" loop scoping rules.
593Added APM and StateMap from LPAQ1
594Code optimizations from Enrico Zeidler
595Detection of BMP 4,8,24 bit and PGM 8 bit images before compress
596On non BMP,PGM,JPEG data mem is lower
597Fixed bug in BMP 8-bit detection in other files like .exe
59815. oct 2007
599Updates JPEG model by Jan Ondrus
600PGM detection bug fix
60122. oct 2007
602improved JPEG model by Jan Ondrus
60316. feb 2008
604fixed bmp detection bug
605added .rgb file support (uncompressed grayscale)
606
607DIFFERENCES FROM PAQ8O9
608
609Added wav Model. Slightly improved bmp model.
610
611DIFFERENCES FROM PAQ8P
612
613Added nestModel from paq8p3
614Modified wordModel from paq8p3
615Modified .pbm, .pgm, .ppm, .bmp, .rgb detection (from paq8p3)
616Modified WAV model (from paq8p_)
617Modified JPEG model (from paq8p2)
618Added im1bitModel (1-bit) (from paq8p3)
619Added compression of PPM, PBM images (from paq8p3)
620Removed pic model
621Modified EXE transformation (e8/e9)
622Changed image and audio data handling (separated in blocks)
623Added compression of (8-bit, 24-bit) TGA image data
624Improved TIFF image detection
625Added zlib stream recompression
626Added base64 transform (from paq8pxd)
627Added gif recompression
628
629DIFFERENCES FROM PAQ8PX
630
631Reduced number of models
632Only 2 APM stages
633Reduced nubmer of prediction added to mixer per context (from 6 to 2)
634
635*/
636
637#define PROGNAME "fp8" // Please change this if you change the program.
638
639#include <stdio.h>
640#include <stdlib.h>
641#include <string.h>
642#include <time.h>
643#include <math.h>
644#include <ctype.h>
645#include <zlib.h>
646#define NDEBUG // remove for debugging (turns on Array bound checks)
647#include <assert.h>
648
649#ifdef UNIX
650#include <sys/types.h>
651#include <sys/stat.h>
652#include <dirent.h>
653#include <errno.h>
654#endif
655
656#ifndef DEFAULT_OPTION
657#define DEFAULT_OPTION 5
658#endif
659
660// 8, 16, 32 bit unsigned types (adjust as appropriate)
661typedef unsigned char U8;
662typedef unsigned short U16;
663typedef unsigned int U32;
664
665// min, max functions
666inline int min(int a, int b) {return a<b?a:b;}
667inline int max(int a, int b) {return a<b?b:a;}
668
669// Error handler: print message if any, and exit
670void quit(const char* message=0) {
671 throw message;
672}
673
674// strings are equal ignoring case?
675int equals(const char* a, const char* b) {
676 assert(a && b);
677 while (*a && *b) {
678 int c1=*a;
679 if (c1>='A'&&c1<='Z') c1+='a'-'A';
680 int c2=*b;
681 if (c2>='A'&&c2<='Z') c2+='a'-'A';
682 if (c1!=c2) return 0;
683 ++a;
684 ++b;
685 }
686 return *a==*b;
687}
688
689//////////////////////// Program Checker /////////////////////
690
691// Track time and memory used
692class ProgramChecker {
693 int memused; // bytes allocated by Array<T> now
694 int maxmem; // most bytes allocated ever
695 clock_t start_time; // in ticks
696public:
697 void alloc(int n) { // report memory allocated, may be negative
698 memused+=n;
699 if (memused>maxmem) maxmem=memused;
700 }
701 ProgramChecker(): memused(0), maxmem(0) {
702 start_time=clock();
703 assert(sizeof(U8)==1);
704 assert(sizeof(U16)==2);
705 assert(sizeof(U32)==4);
706 assert(sizeof(short)==2);
707 assert(sizeof(int)==4);
708 }
709 void print() const { // print time and memory used
710 printf("Time %1.2f sec, used %d bytes of memory\n",
711 double(clock()-start_time)/CLOCKS_PER_SEC, maxmem);
712 }
713} programChecker;
714
715//////////////////////////// Array ////////////////////////////
716
717// Array<T> a(n); creates n elements of T initialized to 0 bits.
718// Constructors for T are not called.
719// Indexing is bounds checked if assertions are on.
720// a.size() returns n.
721// a.resize(n) changes size to n, padding with 0 bits or truncating.
722// a.push_back(x) appends x and increases size by 1, reserving up to size*2.
723// a.pop_back() decreases size by 1, does not free memory.
724// Copy and assignment are not supported.
725
726template <class T, const int Align=16> class Array {
727private:
728 int n; // user size
729 int reserved; // actual size
730 char *ptr; // allocated memory, zeroed
731 T* data; // start of n elements of aligned data
732 void create(int i); // create with size i
733public:
734 explicit Array(int i=0) {create(i);}
735 ~Array();
736 T& operator[](int i) {
737#ifndef NDEBUG
738 if (i<0 || i>=n) fprintf(stderr, "%d out of bounds %d\n", i, n), quit();
739#endif
740 return data[i];
741 }
742 const T& operator[](int i) const {
743#ifndef NDEBUG
744 if (i<0 || i>=n) fprintf(stderr, "%d out of bounds %d\n", i, n), quit();
745#endif
746 return data[i];
747 }
748 int size() const {return n;}
749 void resize(int i); // change size to i
750 void pop_back() {if (n>0) --n;} // decrement size
751 void push_back(const T& x); // increment size, append x
752private:
753 Array(const Array&); // no copy or assignment
754 Array& operator=(const Array&);
755};
756
757template<class T, const int Align> void Array<T,Align>::resize(int i) {
758 if (i<=reserved) {
759 n=i;
760 return;
761 }
762 char *saveptr=ptr;
763 T *savedata=data;
764 int saven=n;
765 create(i);
766 if (saveptr) {
767 if (savedata) {
768 memcpy(data, savedata, sizeof(T)*min(i, saven));
769 programChecker.alloc(-n*sizeof(T));
770 }
771 free(saveptr);
772 }
773}
774
775template<class T, const int Align> void Array<T,Align>::create(int i) {
776 n=reserved=i;
777 if (i<=0 || Align&(Align-1)) {
778 data=0;
779 ptr=0;
780 return;
781 }
782 const int sz=n*sizeof(T);
783 programChecker.alloc(sz);
784 ptr = (char*)calloc(sz, 1);
785 if (!ptr) quit("Out of memory");
786 data = (T*)ptr;
787}
788
789template<class T, const int Align> Array<T,Align>::~Array() {
790 programChecker.alloc(-n*sizeof(T));
791 free(ptr);
792}
793
794template<class T, const int Align> void Array<T,Align>::push_back(const T& x) {
795 if (n==reserved) {
796 int saven=n;
797 resize(max(1, n*2));
798 n=saven;
799 }
800 data[n++]=x;
801}
802
803/////////////////////////// String /////////////////////////////
804
805// A tiny subset of std::string
806// size() includes NUL terminator.
807
808class String: public Array<char> {
809public:
810 const char* c_str() const {return &(*this)[0];}
811 void operator=(const char* s) {
812 resize(strlen(s)+1);
813 strcpy(&(*this)[0], s);
814 }
815 void operator+=(const char* s) {
816 assert(s);
817 pop_back();
818 while (*s) push_back(*s++);
819 push_back(0);
820 }
821 String(const char* s=""): Array<char>(1) {
822 (*this)+=s;
823 }
824};
825
826
827//////////////////////////// rnd ///////////////////////////////
828
829// 32-bit pseudo random number generator
830class Random{
831 Array<U32> table;
832 int i;
833public:
834 Random(): table(64) {
835 table[0]=123456789;
836 table[1]=987654321;
837 for (int j=0; j<62; j++) table[j+2]=table[j+1]*11+table[j]*23/16;
838 i=0;
839 }
840 U32 operator()() {
841 return ++i, table[i&63]=table[(i-24)&63]^table[(i-55)&63];
842 }
843} rnd;
844
845////////////////////////////// Buf /////////////////////////////
846
847// Buf(n) buf; creates an array of n bytes (must be a power of 2).
848// buf[i] returns a reference to the i'th byte with wrap (no out of bounds).
849// buf(i) returns i'th byte back from pos (i > 0)
850// buf.size() returns n.
851
852int pos; // Number of input bytes in buf (not wrapped)
853
854class Buf {
855 Array<U8> b;
856public:
857 Buf(int i=0): b(i) {}
858 void setsize(int i) {
859 if (!i) return;
860 assert(i>0 && (i&(i-1))==0);
861 b.resize(i);
862 }
863 U8& operator[](int i) {
864 return b[i&(b.size()-1)];
865 }
866 int operator()(int i) const {
867 assert(i>0);
868 return b[(pos-i)&(b.size()-1)];
869 }
870 int size() const {
871 return b.size();
872 }
873};
874
875// IntBuf(n) is a buffer of n int (must be a power of 2).
876// intBuf[i] returns a reference to i'th element with wrap.
877
878class IntBuf {
879 Array<int> b;
880public:
881 IntBuf(int i=0): b(i) {}
882 int& operator[](int i) {
883 return b[i&(b.size()-1)];
884 }
885};
886
887/////////////////////// Global context /////////////////////////
888
889int level=DEFAULT_OPTION; // Compression level 0 to 8
890#define MEM (0x10000<<level)
891int y=0; // Last bit, 0 or 1, set by encoder
892
893// Global context set by Predictor and available to all models.
894int c0=1; // Last 0-7 bits of the partial byte with a leading 1 bit (1-255)
895U32 c4=0; // Last 4 whole bytes, packed. Last byte is bits 0-7.
896int bpos=0; // bits in c0 (0 to 7)
897Buf buf; // Rotating input queue set by Predictor
898int blpos=0; // Relative position in block
899
900#define PNGFlag (1<<31)
901#define GrayFlag (1<<30)
902
903///////////////////////////// ilog //////////////////////////////
904
905// ilog(x) = round(log2(x) * 16), 0 <= x < 64K
906class Ilog {
907 Array<U8> t;
908public:
909 int operator()(U16 x) const {return t[x];}
910 Ilog();
911} ilog;
912
913// Compute lookup table by numerical integration of 1/x
914Ilog::Ilog(): t(65536) {
915 U32 x=14155776;
916 for (int i=2; i<65536; ++i) {
917 x+=774541002/(i*2-1); // numerator is 2^29/ln 2
918 t[i]=x>>24;
919 }
920}
921
922// llog(x) accepts 32 bits
923inline int llog(U32 x) {
924 if (x>=0x1000000)
925 return 256+ilog(x>>16);
926 else if (x>=0x10000)
927 return 128+ilog(x>>8);
928 else
929 return ilog(x);
930}
931
932inline unsigned BitCount(unsigned v){
933 v -= ((v>>1)&0x55555555);
934 v = ((v>>2)&0x33333333) + (v&0x33333333);
935 v = ((v>>4)+v)&0x0f0f0f0f;
936 v = ((v>>8)+v)&0x00ff00ff;
937 v = ((v>>16)+v)&0x0000ffff;
938 return v;
939}
940
941inline unsigned ilog2(unsigned x) {
942 x = x | (x >> 1);
943 x = x | (x >> 2);
944 x = x | (x >> 4);
945 x = x | (x >> 8);
946 x = x | (x >>16);
947 return BitCount(x >> 1);
948}
949
950///////////////////////// state table ////////////////////////
951
952// State table:
953// nex(state, 0) = next state if bit y is 0, 0 <= state < 256
954// nex(state, 1) = next state if bit y is 1
955// nex(state, 2) = number of zeros in bit history represented by state
956// nex(state, 3) = number of ones represented
957//
958// States represent a bit history within some context.
959// State 0 is the starting state (no bits seen).
960// States 1-30 represent all possible sequences of 1-4 bits.
961// States 31-252 represent a pair of counts, (n0,n1), the number
962// of 0 and 1 bits respectively. If n0+n1 < 16 then there are
963// two states for each pair, depending on if a 0 or 1 was the last
964// bit seen.
965// If n0 and n1 are too large, then there is no state to represent this
966// pair, so another state with about the same ratio of n0/n1 is substituted.
967// Also, when a bit is observed and the count of the opposite bit is large,
968// then part of this count is discarded to favor newer data over old.
969
970#if 1 // change to #if 0 to generate this table at run time (4% slower)
971static const U8 State_table[256][4]={
972 { 1, 2, 0, 0},{ 3, 5, 1, 0},{ 4, 6, 0, 1},{ 7, 10, 2, 0}, // 0-3
973 { 8, 12, 1, 1},{ 9, 13, 1, 1},{ 11, 14, 0, 2},{ 15, 19, 3, 0}, // 4-7
974 { 16, 23, 2, 1},{ 17, 24, 2, 1},{ 18, 25, 2, 1},{ 20, 27, 1, 2}, // 8-11
975 { 21, 28, 1, 2},{ 22, 29, 1, 2},{ 26, 30, 0, 3},{ 31, 33, 4, 0}, // 12-15
976 { 32, 35, 3, 1},{ 32, 35, 3, 1},{ 32, 35, 3, 1},{ 32, 35, 3, 1}, // 16-19
977 { 34, 37, 2, 2},{ 34, 37, 2, 2},{ 34, 37, 2, 2},{ 34, 37, 2, 2}, // 20-23
978 { 34, 37, 2, 2},{ 34, 37, 2, 2},{ 36, 39, 1, 3},{ 36, 39, 1, 3}, // 24-27
979 { 36, 39, 1, 3},{ 36, 39, 1, 3},{ 38, 40, 0, 4},{ 41, 43, 5, 0}, // 28-31
980 { 42, 45, 4, 1},{ 42, 45, 4, 1},{ 44, 47, 3, 2},{ 44, 47, 3, 2}, // 32-35
981 { 46, 49, 2, 3},{ 46, 49, 2, 3},{ 48, 51, 1, 4},{ 48, 51, 1, 4}, // 36-39
982 { 50, 52, 0, 5},{ 53, 43, 6, 0},{ 54, 57, 5, 1},{ 54, 57, 5, 1}, // 40-43
983 { 56, 59, 4, 2},{ 56, 59, 4, 2},{ 58, 61, 3, 3},{ 58, 61, 3, 3}, // 44-47
984 { 60, 63, 2, 4},{ 60, 63, 2, 4},{ 62, 65, 1, 5},{ 62, 65, 1, 5}, // 48-51
985 { 50, 66, 0, 6},{ 67, 55, 7, 0},{ 68, 57, 6, 1},{ 68, 57, 6, 1}, // 52-55
986 { 70, 73, 5, 2},{ 70, 73, 5, 2},{ 72, 75, 4, 3},{ 72, 75, 4, 3}, // 56-59
987 { 74, 77, 3, 4},{ 74, 77, 3, 4},{ 76, 79, 2, 5},{ 76, 79, 2, 5}, // 60-63
988 { 62, 81, 1, 6},{ 62, 81, 1, 6},{ 64, 82, 0, 7},{ 83, 69, 8, 0}, // 64-67
989 { 84, 71, 7, 1},{ 84, 71, 7, 1},{ 86, 73, 6, 2},{ 86, 73, 6, 2}, // 68-71
990 { 44, 59, 5, 3},{ 44, 59, 5, 3},{ 58, 61, 4, 4},{ 58, 61, 4, 4}, // 72-75
991 { 60, 49, 3, 5},{ 60, 49, 3, 5},{ 76, 89, 2, 6},{ 76, 89, 2, 6}, // 76-79
992 { 78, 91, 1, 7},{ 78, 91, 1, 7},{ 80, 92, 0, 8},{ 93, 69, 9, 0}, // 80-83
993 { 94, 87, 8, 1},{ 94, 87, 8, 1},{ 96, 45, 7, 2},{ 96, 45, 7, 2}, // 84-87
994 { 48, 99, 2, 7},{ 48, 99, 2, 7},{ 88,101, 1, 8},{ 88,101, 1, 8}, // 88-91
995 { 80,102, 0, 9},{103, 69,10, 0},{104, 87, 9, 1},{104, 87, 9, 1}, // 92-95
996 {106, 57, 8, 2},{106, 57, 8, 2},{ 62,109, 2, 8},{ 62,109, 2, 8}, // 96-99
997 { 88,111, 1, 9},{ 88,111, 1, 9},{ 80,112, 0,10},{113, 85,11, 0}, // 100-103
998 {114, 87,10, 1},{114, 87,10, 1},{116, 57, 9, 2},{116, 57, 9, 2}, // 104-107
999 { 62,119, 2, 9},{ 62,119, 2, 9},{ 88,121, 1,10},{ 88,121, 1,10}, // 108-111
1000 { 90,122, 0,11},{123, 85,12, 0},{124, 97,11, 1},{124, 97,11, 1}, // 112-115
1001 {126, 57,10, 2},{126, 57,10, 2},{ 62,129, 2,10},{ 62,129, 2,10}, // 116-119
1002 { 98,131, 1,11},{ 98,131, 1,11},{ 90,132, 0,12},{133, 85,13, 0}, // 120-123
1003 {134, 97,12, 1},{134, 97,12, 1},{136, 57,11, 2},{136, 57,11, 2}, // 124-127
1004 { 62,139, 2,11},{ 62,139, 2,11},{ 98,141, 1,12},{ 98,141, 1,12}, // 128-131
1005 { 90,142, 0,13},{143, 95,14, 0},{144, 97,13, 1},{144, 97,13, 1}, // 132-135
1006 { 68, 57,12, 2},{ 68, 57,12, 2},{ 62, 81, 2,12},{ 62, 81, 2,12}, // 136-139
1007 { 98,147, 1,13},{ 98,147, 1,13},{100,148, 0,14},{149, 95,15, 0}, // 140-143
1008 {150,107,14, 1},{150,107,14, 1},{108,151, 1,14},{108,151, 1,14}, // 144-147
1009 {100,152, 0,15},{153, 95,16, 0},{154,107,15, 1},{108,155, 1,15}, // 148-151
1010 {100,156, 0,16},{157, 95,17, 0},{158,107,16, 1},{108,159, 1,16}, // 152-155
1011 {100,160, 0,17},{161,105,18, 0},{162,107,17, 1},{108,163, 1,17}, // 156-159
1012 {110,164, 0,18},{165,105,19, 0},{166,117,18, 1},{118,167, 1,18}, // 160-163
1013 {110,168, 0,19},{169,105,20, 0},{170,117,19, 1},{118,171, 1,19}, // 164-167
1014 {110,172, 0,20},{173,105,21, 0},{174,117,20, 1},{118,175, 1,20}, // 168-171
1015 {110,176, 0,21},{177,105,22, 0},{178,117,21, 1},{118,179, 1,21}, // 172-175
1016 {110,180, 0,22},{181,115,23, 0},{182,117,22, 1},{118,183, 1,22}, // 176-179
1017 {120,184, 0,23},{185,115,24, 0},{186,127,23, 1},{128,187, 1,23}, // 180-183
1018 {120,188, 0,24},{189,115,25, 0},{190,127,24, 1},{128,191, 1,24}, // 184-187
1019 {120,192, 0,25},{193,115,26, 0},{194,127,25, 1},{128,195, 1,25}, // 188-191
1020 {120,196, 0,26},{197,115,27, 0},{198,127,26, 1},{128,199, 1,26}, // 192-195
1021 {120,200, 0,27},{201,115,28, 0},{202,127,27, 1},{128,203, 1,27}, // 196-199
1022 {120,204, 0,28},{205,115,29, 0},{206,127,28, 1},{128,207, 1,28}, // 200-203
1023 {120,208, 0,29},{209,125,30, 0},{210,127,29, 1},{128,211, 1,29}, // 204-207
1024 {130,212, 0,30},{213,125,31, 0},{214,137,30, 1},{138,215, 1,30}, // 208-211
1025 {130,216, 0,31},{217,125,32, 0},{218,137,31, 1},{138,219, 1,31}, // 212-215
1026 {130,220, 0,32},{221,125,33, 0},{222,137,32, 1},{138,223, 1,32}, // 216-219
1027 {130,224, 0,33},{225,125,34, 0},{226,137,33, 1},{138,227, 1,33}, // 220-223
1028 {130,228, 0,34},{229,125,35, 0},{230,137,34, 1},{138,231, 1,34}, // 224-227
1029 {130,232, 0,35},{233,125,36, 0},{234,137,35, 1},{138,235, 1,35}, // 228-231
1030 {130,236, 0,36},{237,125,37, 0},{238,137,36, 1},{138,239, 1,36}, // 232-235
1031 {130,240, 0,37},{241,125,38, 0},{242,137,37, 1},{138,243, 1,37}, // 236-239
1032 {130,244, 0,38},{245,135,39, 0},{246,137,38, 1},{138,247, 1,38}, // 240-243
1033 {140,248, 0,39},{249,135,40, 0},{250, 69,39, 1},{ 80,251, 1,39}, // 244-247
1034 {140,252, 0,40},{249,135,41, 0},{250, 69,40, 1},{ 80,251, 1,40}, // 248-251
1035 {140,252, 0,41}}; // 252, 253-255 are reserved
1036
1037#define nex(state,sel) State_table[state][sel]
1038
1039// The code used to generate the above table at run time (4% slower).
1040// To print the table, uncomment the 4 lines of print statements below.
1041// In this code x,y = n0,n1 is the number of 0,1 bits represented by a state.
1042#else
1043
1044class StateTable {
1045 Array<U8> ns; // state*4 -> next state if 0, if 1, n0, n1
1046 enum {B=5, N=64}; // sizes of b, t
1047 static const int b[B]; // x -> max y, y -> max x
1048 static U8 t[N][N][2]; // x,y -> state number, number of states
1049 int num_states(int x, int y); // compute t[x][y][1]
1050 void discount(int& x); // set new value of x after 1 or y after 0
1051 void next_state(int& x, int& y, int b); // new (x,y) after bit b
1052public:
1053 int operator()(int state, int sel) {return ns[state*4+sel];}
1054 StateTable();
1055} nex;
1056
1057const int StateTable::b[B]={42,41,13,6,5}; // x -> max y, y -> max x
1058U8 StateTable::t[N][N][2];
1059
1060int StateTable::num_states(int x, int y) {
1061 if (x<y) return num_states(y, x);
1062 if (x<0 || y<0 || x>=N || y>=N || y>=B || x>=b[y]) return 0;
1063
1064 // States 0-30 are a history of the last 0-4 bits
1065 if (x+y<=4) { // x+y choose x = (x+y)!/x!y!
1066 int r=1;
1067 for (int i=x+1; i<=x+y; ++i) r*=i;
1068 for (int i=2; i<=y; ++i) r/=i;
1069 return r;
1070 }
1071
1072 // States 31-255 represent a 0,1 count and possibly the last bit
1073 // if the state is reachable by either a 0 or 1.
1074 else
1075 return 1+(y>0 && x+y<16);
1076}
1077
1078// New value of count x if the opposite bit is observed
1079void StateTable::discount(int& x) {
1080 if (x>2) x=ilog(x)/6-1;
1081}
1082
1083// compute next x,y (0 to N) given input b (0 or 1)
1084void StateTable::next_state(int& x, int& y, int b) {
1085 if (x<y)
1086 next_state(y, x, 1-b);
1087 else {
1088 if (b) {
1089 ++y;
1090 discount(x);
1091 }
1092 else {
1093 ++x;
1094 discount(y);
1095 }
1096 while (!t[x][y][1]) {
1097 if (y<2) --x;
1098 else {
1099 x=(x*(y-1)+(y/2))/y;
1100 --y;
1101 }
1102 }
1103 }
1104}
1105
1106// Initialize next state table ns[state*4] -> next if 0, next if 1, x, y
1107StateTable::StateTable(): ns(1024) {
1108
1109 // Assign states
1110 int state=0;
1111 for (int i=0; i<256; ++i) {
1112 for (int y=0; y<=i; ++y) {
1113 int x=i-y;
1114 int n=num_states(x, y);
1115 if (n) {
1116 t[x][y][0]=state;
1117 t[x][y][1]=n;
1118 state+=n;
1119 }
1120 }
1121 }
1122
1123 // Print/generate next state table
1124 state=0;
1125 for (int i=0; i<N; ++i) {
1126 for (int y=0; y<=i; ++y) {
1127 int x=i-y;
1128 for (int k=0; k<t[x][y][1]; ++k) {
1129 int x0=x, y0=y, x1=x, y1=y; // next x,y for input 0,1
1130 int ns0=0, ns1=0;
1131 if (state<15) {
1132 ++x0;
1133 ++y1;
1134 ns0=t[x0][y0][0]+state-t[x][y][0];
1135 ns1=t[x1][y1][0]+state-t[x][y][0];
1136 if (x>0) ns1+=t[x-1][y+1][1];
1137 ns[state*4]=ns0;
1138 ns[state*4+1]=ns1;
1139 ns[state*4+2]=x;
1140 ns[state*4+3]=y;
1141 }
1142 else if (t[x][y][1]) {
1143 next_state(x0, y0, 0);
1144 next_state(x1, y1, 1);
1145 ns[state*4]=ns0=t[x0][y0][0];
1146 ns[state*4+1]=ns1=t[x1][y1][0]+(t[x1][y1][1]>1);
1147 ns[state*4+2]=x;
1148 ns[state*4+3]=y;
1149 }
1150 // uncomment to print table above
1151// printf("{%3d,%3d,%2d,%2d},", ns[state*4], ns[state*4+1],
1152// ns[state*4+2], ns[state*4+3]);
1153// if (state%4==3) printf(" // %d-%d\n ", state-3, state);
1154 assert(state>=0 && state<256);
1155 assert(t[x][y][1]>0);
1156 assert(t[x][y][0]<=state);
1157 assert(t[x][y][0]+t[x][y][1]>state);
1158 assert(t[x][y][1]<=6);
1159 assert(t[x0][y0][1]>0);
1160 assert(t[x1][y1][1]>0);
1161 assert(ns0-t[x0][y0][0]<t[x0][y0][1]);
1162 assert(ns0-t[x0][y0][0]>=0);
1163 assert(ns1-t[x1][y1][0]<t[x1][y1][1]);
1164 assert(ns1-t[x1][y1][0]>=0);
1165 ++state;
1166 }
1167 }
1168 }
1169// printf("%d states\n", state); exit(0); // uncomment to print table above
1170}
1171
1172#endif
1173
1174///////////////////////////// Squash //////////////////////////////
1175
1176// return p = 1/(1 + exp(-d)), d scaled by 8 bits, p scaled by 12 bits
1177int squash(int d) {
1178 static const int t[33]={
1179 1,2,3,6,10,16,27,45,73,120,194,310,488,747,1101,
1180 1546,2047,2549,2994,3348,3607,3785,3901,3975,4022,
1181 4050,4068,4079,4085,4089,4092,4093,4094};
1182 if (d>2047) return 4095;
1183 if (d<-2047) return 0;
1184 int w=d&127;
1185 d=(d>>7)+16;
1186 return (t[d]*(128-w)+t[(d+1)]*w+64) >> 7;
1187}
1188
1189//////////////////////////// Stretch ///////////////////////////////
1190
1191// Inverse of squash. d = ln(p/(1-p)), d scaled by 8 bits, p by 12 bits.
1192// d has range -2047 to 2047 representing -8 to 8. p has range 0 to 4095.
1193
1194class Stretch {
1195 Array<short> t;
1196public:
1197 Stretch();
1198 int operator()(int p) const {
1199 assert(p>=0 && p<4096);
1200 return t[p];
1201 }
1202} stretch;
1203
1204Stretch::Stretch(): t(4096) {
1205 int pi=0;
1206 for (int x=-2047; x<=2047; ++x) { // invert squash()
1207 int i=squash(x);
1208 for (int j=pi; j<=i; ++j)
1209 t[j]=x;
1210 pi=i+1;
1211 }
1212 t[4095]=2047;
1213}
1214
1215//////////////////////////// Mixer /////////////////////////////
1216
1217// Mixer m(N, M, S=1, w=0) combines models using M neural networks with
1218// N inputs each, of which up to S may be selected. If S > 1 then
1219// the outputs of these neural networks are combined using another
1220// neural network (with parameters S, 1, 1). If S = 1 then the
1221// output is direct. The weights are initially w (+-32K).
1222// It is used as follows:
1223// m.update() trains the network where the expected output is the
1224// last bit (in the global variable y).
1225// m.add(stretch(p)) inputs prediction from one of N models. The
1226// prediction should be positive to predict a 1 bit, negative for 0,
1227// nominally +-256 to +-2K. The maximum allowed value is +-32K but
1228// using such large values may cause overflow if N is large.
1229// m.set(cxt, range) selects cxt as one of 'range' neural networks to
1230// use. 0 <= cxt < range. Should be called up to S times such
1231// that the total of the ranges is <= M.
1232// m.p() returns the output prediction that the next bit is 1 as a
1233// 12 bit number (0 to 4095).
1234
1235#if !defined(__GNUC__)
1236#if (2 == _M_IX86_FP)
1237# define __SSE2__
1238#endif
1239#endif
1240#if defined(__SSE2__)
1241#include <emmintrin.h>
1242
1243static int dot_product (const short* const t, const short* const w, int n) {
1244 __m128i sum = _mm_setzero_si128 ();
1245 while ((n -= 8) >= 0) {
1246 __m128i tmp = _mm_madd_epi16 (*(__m128i *) &t[n], *(__m128i *) &w[n]);
1247 tmp = _mm_srai_epi32 (tmp, 8);
1248 sum = _mm_add_epi32 (sum, tmp);
1249 }
1250 sum = _mm_add_epi32 (sum, _mm_srli_si128 (sum, 8));
1251 sum = _mm_add_epi32 (sum, _mm_srli_si128 (sum, 4));
1252 return _mm_cvtsi128_si32 (sum);
1253}
1254
1255static void train (const short* const t, short* const w, int n, const int e) {
1256 if (e) {
1257 const __m128i one = _mm_set1_epi16 (1);
1258 const __m128i err = _mm_set1_epi16 (short(e));
1259 while ((n -= 8) >= 0) {
1260 __m128i tmp = _mm_adds_epi16 (*(__m128i *) &t[n], *(__m128i *) &t[n]);
1261 tmp = _mm_mulhi_epi16 (tmp, err);
1262 tmp = _mm_adds_epi16 (tmp, one);
1263 tmp = _mm_srai_epi16 (tmp, 1);
1264 tmp = _mm_adds_epi16 (tmp, *(__m128i *) &w[n]);
1265 *(__m128i *) &w[n] = tmp;
1266 }
1267 }
1268}
1269#else
1270
1271static int dot_product (const short* const t, const short* const w, int n) {
1272 int sum = 0;
1273 while ((n -= 2) >= 0) {
1274 sum += (t[n] * w[n] + t[n + 1] * w[n + 1]) >> 8;
1275 }
1276 return sum;
1277}
1278
1279static void train (const short* const t, short* const w, int n, const int err) {
1280 if (err) {
1281 while ((n -= 1) >= 0) {
1282 int wt = w[n] + ((((t[n] * err * 2) >> 16) + 1) >> 1);
1283 if (wt < -32768) {
1284 w[n] = -32768;
1285 } else if (wt > 32767) {
1286 w[n] = 32767;
1287 } else {
1288 w[n] = wt;
1289 }
1290 }
1291 }
1292}
1293#endif
1294
1295class Mixer {
1296 const int N, M, S; // max inputs, max contexts, max context sets
1297 Array<short> tx; // N inputs from add()
1298 Array<short> wx; // N*M weights
1299 Array<int> cxt; // S contexts
1300 int ncxt; // number of contexts (0 to S)
1301 int base; // offset of next context
1302 int nx; // Number of inputs in tx, 0 to N
1303 Array<int> pr; // last result (scaled 12 bits)
1304 Mixer* mp; // points to a Mixer to combine results
1305public:
1306 Mixer(int n, int m, int s=1, int w=0);
1307
1308 // Adjust weights to minimize coding cost of last prediction
1309 void update() {
1310 for (int i=0; i<ncxt; ++i) {
1311 int err=((y<<12)-pr[i])*7;
1312 assert(err>=-32768 && err<32768);
1313 if (err) train(&tx[0], &wx[cxt[i]*N], nx, err);
1314 }
1315 nx=base=ncxt=0;
1316 }
1317
1318 // Input x (call up to N times)
1319 void add(int x) {
1320 assert(nx<N);
1321 tx[nx++]=x;
1322 }
1323
1324 // Set a context (call S times, sum of ranges <= M)
1325 void set(int cx, int range) {
1326 assert(range>=0);
1327 assert(ncxt<S);
1328 assert(cx>=0);
1329 assert(base+cx<M);
1330 cxt[ncxt++]=base+cx;
1331 base+=range;
1332 }
1333
1334 // predict next bit
1335 int p() {
1336 while (nx&7) tx[nx++]=0; // pad
1337 if (mp) { // combine outputs
1338 mp->update();
1339 for (int i=0; i<ncxt; ++i) {
1340 pr[i]=squash(dot_product(&tx[0], &wx[cxt[i]*N], nx)>>5);
1341 mp->add(stretch(pr[i]));
1342 }
1343 mp->set(0, 1);
1344 return mp->p();
1345 }
1346 else { // S=1 context
1347 return pr[0]=squash(dot_product(&tx[0], &wx[0], nx)>>8);
1348 }
1349 }
1350 ~Mixer();
1351};
1352
1353Mixer::~Mixer() {
1354 delete mp;
1355}
1356
1357
1358Mixer::Mixer(int n, int m, int s, int w):
1359 N((n+7)&-8), M(m), S(s), tx(N), wx(N*M),
1360 cxt(S), ncxt(0), base(0), nx(0), pr(S), mp(0) {
1361 assert(n>0 && N>0 && (N&7)==0 && M>0);
1362 int i;
1363 for (i=0; i<S; ++i)
1364 pr[i]=2048;
1365 for (i=0; i<N*M; ++i)
1366 wx[i]=w;
1367 if (S>1) mp=new Mixer(S, 1, 1);
1368}
1369
1370//////////////////////////// APM1 //////////////////////////////
1371
1372// APM1 maps a probability and a context into a new probability
1373// that bit y will next be 1. After each guess it updates
1374// its state to improve future guesses. Methods:
1375//
1376// APM1 a(N) creates with N contexts, uses 66*N bytes memory.
1377// a.p(pr, cx, rate=7) returned adjusted probability in context cx (0 to
1378// N-1). rate determines the learning rate (smaller = faster, default 7).
1379// Probabilities are scaled 12 bits (0-4095).
1380
1381class APM1 {
1382 int index; // last p, context
1383 const int N; // number of contexts
1384 Array<U16> t; // [N][33]: p, context -> p
1385public:
1386 APM1(int n);
1387 int p(int pr=2048, int cxt=0, int rate=7) {
1388 assert(pr>=0 && pr<4096 && cxt>=0 && cxt<N && rate>0 && rate<32);
1389 pr=stretch(pr);
1390 int g=(y<<16)+(y<<rate)-y-y;
1391 t[index] += (g-t[index]) >> rate;
1392 t[index+1] += (g-t[index+1]) >> rate;
1393 const int w=pr&127; // interpolation weight (33 points)
1394 index=((pr+2048)>>7)+cxt*33;
1395 return (t[index]*(128-w)+t[index+1]*w) >> 11;
1396 }
1397};
1398
1399// maps p, cxt -> p initially
1400APM1::APM1(int n): index(0), N(n), t(n*33) {
1401 for (int i=0; i<N; ++i)
1402 for (int j=0; j<33; ++j)
1403 t[i*33+j] = i==0 ? squash((j-16)*128)*16 : t[j];
1404}
1405
1406//////////////////////////// StateMap, APM //////////////////////////
1407
1408// A StateMap maps a context to a probability. Methods:
1409//
1410// Statemap sm(n) creates a StateMap with n contexts using 4*n bytes memory.
1411// sm.p(y, cx, limit) converts state cx (0..n-1) to a probability (0..4095).
1412// that the next y=1, updating the previous prediction with y (0..1).
1413// limit (1..1023, default 1023) is the maximum count for computing a
1414// prediction. Larger values are better for stationary sources.
1415
1416static int dt[1024]; // i -> 16K/(i+3)
1417
1418class StateMap {
1419protected:
1420 const int N; // Number of contexts
1421 int cxt; // Context of last prediction
1422 Array<U32> t; // cxt -> prediction in high 22 bits, count in low 10 bits
1423 inline void update(int limit) {
1424 assert(cxt>=0 && cxt<N);
1425 U32 *p=&t[cxt], p0=p[0];
1426 int n=p0&1023, pr=p0>>10; // count, prediction
1427 if (n<limit) ++p0;
1428 else p0=(p0&0xfffffc00)|limit;
1429 p0+=(((y<<22)-pr)>>3)*dt[n]&0xfffffc00;
1430 p[0]=p0;
1431 }
1432
1433public:
1434 StateMap(int n=256);
1435
1436 // update bit y (0..1), predict next bit in context cx
1437 int p(int cx, int limit=1023) {
1438 assert(cx>=0 && cx<N);
1439 assert(limit>0 && limit<1024);
1440 update(limit);
1441 return t[cxt=cx]>>20;
1442 }
1443};
1444
1445StateMap::StateMap(int n): N(n), cxt(0), t(n) {
1446 for (int i=0; i<N; ++i)
1447 t[i]=1<<31;
1448}
1449
1450// An APM maps a probability and a context to a new probability. Methods:
1451//
1452// APM a(n) creates with n contexts using 96*n bytes memory.
1453// a.pp(y, pr, cx, limit) updates and returns a new probability (0..4095)
1454// like with StateMap. pr (0..4095) is considered part of the context.
1455// The output is computed by interpolating pr into 24 ranges nonlinearly
1456// with smaller ranges near the ends. The initial output is pr.
1457// y=(0..1) is the last bit. cx=(0..n-1) is the other context.
1458// limit=(0..1023) defaults to 255.
1459
1460class APM: public StateMap {
1461public:
1462 APM(int n);
1463 int p(int pr, int cx, int limit=255) {
1464 // assert(y>>1==0);
1465 assert(pr>=0 && pr<4096);
1466 assert(cx>=0 && cx<N/24);
1467 assert(limit>0 && limit<1024);
1468 update(limit);
1469 pr=(stretch(pr)+2048)*23;
1470 int wt=pr&0xfff; // interpolation weight of next element
1471 cx=cx*24+(pr>>12);
1472 assert(cx>=0 && cx<N-1);
1473 cxt=cx+(wt>>11);
1474 pr=((t[cx]>>13)*(0x1000-wt)+(t[cx+1]>>13)*wt)>>19;
1475 return pr;
1476 }
1477};
1478
1479APM::APM(int n): StateMap(n*24) {
1480 for (int i=0; i<N; ++i) {
1481 int p=((i%24*2+1)*4096)/48-2048;
1482 t[i]=(U32(squash(p))<<20)+6;
1483 }
1484}
1485
1486
1487//////////////////////////// hash //////////////////////////////
1488
1489// Hash 2-5 ints.
1490inline U32 hash(U32 a, U32 b, U32 c=0xffffffff, U32 d=0xffffffff,
1491 U32 e=0xffffffff) {
1492 U32 h=a*200002979u+b*30005491u+c*50004239u+d*70004807u+e*110002499u;
1493 return h^h>>9^a>>2^b>>3^c>>4^d>>5^e>>6;
1494}
1495
1496///////////////////////////// BH ////////////////////////////////
1497
1498// A BH maps a 32 bit hash to an array of B bytes (checksum and B-2 values)
1499//
1500// BH bh(N); creates N element table with B bytes each.
1501// N must be a power of 2. The first byte of each element is
1502// reserved for a checksum to detect collisions. The remaining
1503// B-1 bytes are values, prioritized by the first value. This
1504// byte is 0 to mark an unused element.
1505//
1506// bh[i] returns a pointer to the i'th element, such that
1507// bh[i][0] is a checksum of i, bh[i][1] is the priority, and
1508// bh[i][2..B-1] are other values (0-255).
1509// The low lg(n) bits as an index into the table.
1510// If a collision is detected, up to M nearby locations in the same
1511// cache line are tested and the first matching checksum or
1512// empty element is returned.
1513// If no match or empty element is found, then the lowest priority
1514// element is replaced.
1515
1516// 2 byte checksum with LRU replacement (except last 2 by priority)
1517template <int B> class BH {
1518 enum {M=8}; // search limit
1519 Array<U8> t; // elements
1520 U32 n; // size-1
1521public:
1522 BH(int i): t(i*B), n(i-1) {
1523 assert(B>=2 && i>0 && (i&(i-1))==0); // size a power of 2?
1524 }
1525 U8* operator[](U32 i);
1526};
1527
1528template <int B>
1529inline U8* BH<B>::operator[](U32 i) {
1530 U16 chk=(i>>16^i)&0xffff;
1531 i=i*M&n;
1532 U8 *p;
1533 U16 *cp;
1534 int j;
1535 for (j=0; j<M; ++j) {
1536 p=&t[(i+j)*B];
1537 cp=(U16*)p;
1538 if (p[2]==0) {*cp=chk;break;}
1539 if (*cp==chk) break; // found
1540 }
1541 if (j==0) return p+1; // front
1542 static U8 tmp[B]; // element to move to front
1543 if (j==M) {
1544 --j;
1545 memset(tmp, 0, B);
1546 memmove(tmp, &chk, 2);
1547 if (M>2 && t[(i+j)*B+2]>t[(i+j-1)*B+2]) --j;
1548 }
1549 else memcpy(tmp, cp, B);
1550 memmove(&t[(i+1)*B], &t[i*B], j*B);
1551 memcpy(&t[i*B], tmp, B);
1552 return &t[i*B+1];
1553}
1554
1555
1556//////////////////////////// HashTable /////////////////////////
1557
1558// A HashTable maps a 32-bit index to an array of B bytes.
1559// The first byte is a checksum using the upper 8 bits of the
1560// index. The second byte is a priority (0 = empty) for hash
1561// replacement. The index need not be a hash.
1562
1563// HashTable<B> h(n) - create using n bytes n and B must be
1564// powers of 2 with n >= B*4, and B >= 2.
1565// h[i] returns array [1..B-1] of bytes indexed by i, creating and
1566// replacing another element if needed. Element 0 is the
1567// checksum and should not be modified.
1568
1569template <int B>
1570class HashTable {
1571 Array<U8,64> t; // table: 1 element = B bytes: checksum priority data data
1572 const int N; // size in bytes
1573public:
1574 HashTable(int n): t(n), N(n) {
1575 assert(B>=2 && (B&B-1)==0);
1576 assert(N>=B*4 && (N&N-1)==0);
1577 }
1578 U8* operator[](U32 i);
1579};
1580
1581template <int B>
1582inline U8* HashTable<B>::operator[](U32 i) {
1583 i*=123456791;
1584 i=i<<16|i>>16;
1585 i*=234567891;
1586 int chk=i>>24;
1587 i=i*B&(N-B);
1588 U8 *p = &t[0];
1589 if (p[i]==chk) return p+i;
1590 if (p[i^B]==chk) return p+(i^B);
1591 if (p[i^B*2]==chk) return p+(i^B*2);
1592 if (p[i+1]>p[(i+1)^B] || p[i+1]>p[(i+1)^B*2]) i^=B;
1593 if (p[i+1]>p[(i+1)^B^B*2]) i^=B^B*2;
1594 memset(p+i, 0, B);
1595 p[i]=chk;
1596 return p+i;
1597}
1598
1599/////////////////////////// ContextMap /////////////////////////
1600//
1601// A ContextMap maps contexts to a bit histories and makes predictions
1602// to a Mixer. Methods common to all classes:
1603//
1604// ContextMap cm(M, C); creates using about M bytes of memory (a power
1605// of 2) for C contexts.
1606// cm.set(cx); sets the next context to cx, called up to C times
1607// cx is an arbitrary 32 bit value that identifies the context.
1608// It should be called before predicting the first bit of each byte.
1609// cm.mix(m) updates Mixer m with the next prediction. Returns 1
1610// if context cx is found, else 0. Then it extends all the contexts with
1611// global bit y. It should be called for every bit:
1612//
1613// if (bpos==0)
1614// for (int i=0; i<C; ++i) cm.set(cxt[i]);
1615// cm.mix(m);
1616//
1617// The different types are as follows:
1618//
1619// - SmallStationaryContextMap. 0 <= cx < M/512.
1620// The state is a 16-bit probability that is adjusted after each
1621// prediction. C=1.
1622// - ContextMap. For large contexts, C >= 1. Context need not be hashed.
1623
1624
1625
1626
1627// Context is looked up directly. m=size is power of 2 in bytes.
1628// Context should be < m/512. High bits are discarded.
1629class SmallStationaryContextMap {
1630 Array<U16> t;
1631 int cxt;
1632 U16 *cp;
1633public:
1634 SmallStationaryContextMap(int m): t(m/2), cxt(0) {
1635 assert((m/2&m/2-1)==0); // power of 2?
1636 for (int i=0; i<t.size(); ++i)
1637 t[i]=32768;
1638 cp=&t[0];
1639 }
1640 void set(U32 cx) {
1641 cxt=cx*256&(t.size()-256);
1642 }
1643 void mix(Mixer& m, int rate=7) {
1644 *cp += ((y<<16)-(*cp)+(1<<(rate-1))) >> rate;
1645 cp=&t[cxt+c0];
1646 m.add(stretch((*cp)>>4));
1647 }
1648};
1649
1650// Context map for large contexts. Most modeling uses this type of context
1651// map. It includes a built in RunContextMap to predict the last byte seen
1652// in the same context, and also bit-level contexts that map to a bit
1653// history state.
1654//
1655// Bit histories are stored in a hash table. The table is organized into
1656// 64-byte buckets alinged on cache page boundaries. Each bucket contains
1657// a hash chain of 7 elements, plus a 2 element queue (packed into 1 byte)
1658// of the last 2 elements accessed for LRU replacement. Each element has
1659// a 2 byte checksum for detecting collisions, and an array of 7 bit history
1660// states indexed by the last 0 to 2 bits of context. The buckets are indexed
1661// by a context ending after 0, 2, or 5 bits of the current byte. Thus, each
1662// byte modeled results in 3 main memory accesses per context, with all other
1663// accesses to cache.
1664//
1665// On bits 0, 2 and 5, the context is updated and a new bucket is selected.
1666// The most recently accessed element is tried first, by comparing the
1667// 16 bit checksum, then the 7 elements are searched linearly. If no match
1668// is found, then the element with the lowest priority among the 5 elements
1669// not in the LRU queue is replaced. After a replacement, the queue is
1670// emptied (so that consecutive misses favor a LFU replacement policy).
1671// In all cases, the found/replaced element is put in the front of the queue.
1672//
1673// The priority is the state number of the first element (the one with 0
1674// additional bits of context). The states are sorted by increasing n0+n1
1675// (number of bits seen), implementing a LFU replacement policy.
1676//
1677// When the context ends on a byte boundary (bit 0), only 3 of the 7 bit
1678// history states are used. The remaining 4 bytes implement a run model
1679// as follows: <count:7,d:1> <b1> <unused> <unused> where <b1> is the last byte
1680// seen, possibly repeated. <count:7,d:1> is a 7 bit count and a 1 bit
1681// flag (represented by count * 2 + d). If d=0 then <count> = 1..127 is the
1682// number of repeats of <b1> and no other bytes have been seen. If d is 1 then
1683// other byte values have been seen in this context prior to the last <count>
1684// copies of <b1>.
1685//
1686// As an optimization, the last two hash elements of each byte (representing
1687// contexts with 2-7 bits) are not updated until a context is seen for
1688// a second time. This is indicated by <count,d> = <1,0> (2). After update,
1689// <count,d> is updated to <2,0> or <1,1> (4 or 3).
1690
1691class ContextMap {
1692 const int C; // max number of contexts
1693 class E { // hash element, 64 bytes
1694 U16 chk[7]; // byte context checksums
1695 U8 last; // last 2 accesses (0-6) in low, high nibble
1696 public:
1697 U8 bh[7][7]; // byte context, 3-bit context -> bit history state
1698 // bh[][0] = 1st bit, bh[][1,2] = 2nd bit, bh[][3..6] = 3rd bit
1699 // bh[][0] is also a replacement priority, 0 = empty
1700 U8* get(U16 chk); // Find element (0-6) matching checksum.
1701 // If not found, insert or replace lowest priority (not last).
1702 };
1703 Array<E> t; // bit histories for bits 0-1, 2-4, 5-7
1704 // For 0-1, also contains a run count in bh[][4] and value in bh[][5]
1705 // and pending update count in bh[7]
1706 Array<U8*> cp; // C pointers to current bit history
1707 Array<U8*> cp0; // First element of 7 element array containing cp[i]
1708 Array<U32> cxt; // C whole byte contexts (hashes)
1709 Array<U8*> runp; // C [0..3] = count, value, unused, unused
1710 StateMap *sm; // C maps of state -> p
1711 int cn; // Next context to set by set()
1712 bool fast;
1713 void update(U32 cx, int c); // train model that context cx predicts c
1714 int mix1(Mixer& m, int cc, int bp, int c1, int y1);
1715 // mix() with global context passed as arguments to improve speed.
1716public:
1717 ContextMap(int m, int c=1, bool f=false); // m = memory in bytes, a power of 2, C = c
1718 ~ContextMap();
1719 void set(U32 cx, int next=-1); // set next whole byte context to cx
1720 // if next is 0 then set order does not matter
1721 int mix(Mixer& m) {return mix1(m, c0, bpos, buf(1), y);}
1722};
1723
1724// Find or create hash element matching checksum ch
1725inline U8* ContextMap::E::get(U16 ch) {
1726 if (chk[last&15]==ch) return &bh[last&15][0];
1727 int b=0xffff, bi=0;
1728 for (int i=0; i<7; ++i) {
1729 if (chk[i]==ch) return last=last<<4|i, (U8*)&bh[i][0];
1730 int pri=bh[i][0];
1731 if (pri<b && (last&15)!=i && last>>4!=i) b=pri, bi=i;
1732 }
1733 return last=0xf0|bi, chk[bi]=ch, (U8*)memset(&bh[bi][0], 0, 7);
1734}
1735
1736// Construct using m bytes of memory for c contexts
1737ContextMap::ContextMap(int m, int c, bool f): C(c), t(m>>6), cp(c), cp0(c),
1738 cxt(c), runp(c), cn(0), fast(f) {
1739 assert(m>=64 && (m&m-1)==0); // power of 2?
1740 assert(sizeof(E)==64);
1741 sm=new StateMap[C];
1742 for (int i=0; i<C; ++i) {
1743 cp0[i]=cp[i]=&t[0].bh[0][0];
1744 runp[i]=cp[i]+3;
1745 }
1746}
1747
1748ContextMap::~ContextMap() {
1749 delete[] sm;
1750}
1751
1752// Set the i'th context to cx
1753inline void ContextMap::set(U32 cx, int next) {
1754 int i=cn++;
1755 i&=next;
1756 assert(i>=0 && i<C);
1757 cx=cx*987654323+i; // permute (don't hash) cx to spread the distribution
1758 cx=cx<<16|cx>>16;
1759 cxt[i]=cx*123456791+i;
1760}
1761
1762// Update the model with bit y1, and predict next bit to mixer m.
1763// Context: cc=c0, bp=bpos, c1=buf(1), y1=y.
1764int ContextMap::mix1(Mixer& m, int cc, int bp, int c1, int y1) {
1765
1766 // Update model with y
1767 int result=0;
1768 for (int i=0; i<cn; ++i) {
1769 if (cp[i]) {
1770 assert(cp[i]>=&t[0].bh[0][0] && cp[i]<=&t[t.size()-1].bh[6][6]);
1771 assert((long(cp[i])&63)>=15);
1772 int ns=nex(*cp[i], y1);
1773 if (ns>=204 && rnd() << ((452-ns)>>3)) ns-=4; // probabilistic increment
1774 *cp[i]=ns;
1775 }
1776
1777 // Update context pointers
1778 if (bpos>1 && runp[i][0]==0)
1779 {
1780 cp[i]=0;
1781 }
1782 else
1783 {
1784 switch(bpos)
1785 {
1786 case 1: case 3: case 6: cp[i]=cp0[i]+1+(cc&1); break;
1787 case 4: case 7: cp[i]=cp0[i]+3+(cc&3); break;
1788 case 2: case 5: cp0[i]=cp[i]=t[(cxt[i]+cc)&(t.size()-1)].get(cxt[i]>>16); break;
1789 default:
1790 {
1791 cp0[i]=cp[i]=t[(cxt[i]+cc)&(t.size()-1)].get(cxt[i]>>16);
1792 // Update pending bit histories for bits 2-7
1793 if (cp0[i][3]==2) {
1794 const int c=cp0[i][4]+256;
1795 U8 *p=t[(cxt[i]+(c>>6))&(t.size()-1)].get(cxt[i]>>16);
1796 p[0]=1+((c>>5)&1);
1797 p[1+((c>>5)&1)]=1+((c>>4)&1);
1798 p[3+((c>>4)&3)]=1+((c>>3)&1);
1799 p=t[(cxt[i]+(c>>3))&(t.size()-1)].get(cxt[i]>>16);
1800 p[0]=1+((c>>2)&1);
1801 p[1+((c>>2)&1)]=1+((c>>1)&1);
1802 p[3+((c>>1)&3)]=1+(c&1);
1803 cp0[i][6]=0;
1804 }
1805 // Update run count of previous context
1806 if (runp[i][0]==0) // new context
1807 runp[i][0]=2, runp[i][1]=c1;
1808 else if (runp[i][1]!=c1) // different byte in context
1809 runp[i][0]=1, runp[i][1]=c1;
1810 else if (runp[i][0]<254) // same byte in context
1811 runp[i][0]+=2;
1812 runp[i]=cp0[i]+3;
1813 } break;
1814 }
1815 }
1816
1817 if (!fast) {
1818 // predict from last byte in context
1819 if ((runp[i][1]+256)>>(8-bp)==cc) {
1820 int rc=runp[i][0]; // count*2, +1 if 2 different bytes seen
1821 int b=(runp[i][1]>>(7-bp)&1)*2-1; // predicted bit + for 1, - for 0
1822 int c=ilog(rc+1)<<(2+(~rc&1));
1823 m.add(b*c);
1824 }
1825 else
1826 m.add(0);
1827 }
1828
1829
1830 // predict from bit context
1831 int p;
1832 if (cp[i])
1833 {
1834 result+=(*cp[i]>0);
1835 p = sm[i].p(*cp[i]);
1836 }
1837 else
1838 {
1839 p = sm[i].p(0);
1840 }
1841 m.add(stretch(p));
1842
1843
1844 }
1845 if (bp==7) cn=0;
1846 return result;
1847}
1848
1849//////////////////////////// Models //////////////////////////////
1850
1851// All of the models below take a Mixer as a parameter and write
1852// predictions to it.
1853
1854//////////////////////////// matchModel ///////////////////////////
1855
1856// matchModel() finds the longest matching context and returns its length
1857
1858int matchModel(Mixer& m) {
1859 const int MAXLEN=65534; // longest allowed match + 1
1860 static Array<int> t(MEM); // hash table of pointers to contexts
1861 static int h=0; // hash of last 7 bytes
1862 static int ptr=0; // points to next byte of match if any
1863 static int len=0; // length of match, or 0 if no match
1864 static int result=0;
1865
1866 static SmallStationaryContextMap scm1(0x20000), scm2(0x20000);
1867 static int posnl=0;
1868
1869 if (!bpos) {
1870 h=(h*997*8+buf(1)+1)&(t.size()-1); // update context hash
1871 if (len) ++len, ++ptr;
1872 else { // find match
1873 ptr=t[h];
1874 if (ptr && pos-ptr<buf.size())
1875 while (buf(len+1)==buf[ptr-len-1] && len<MAXLEN) ++len;
1876 }
1877 t[h]=pos; // update hash table
1878 result=len;
1879// if (result>0 && !(result&0xfff)) printf("pos=%d len=%d ptr=%d\n", pos, len, ptr);
1880 scm1.set(pos);
1881
1882 if (buf(1)==0xff||buf(1)=='\r'||buf(1)=='\n') posnl=pos;
1883 scm2.set(min(pos-posnl,255));
1884 }
1885
1886 // predict
1887 if (len)
1888 {
1889 if (buf(1)==buf[ptr-1] && c0==(buf[ptr]+256)>>(8-bpos))
1890 {
1891 if (len>MAXLEN) len=MAXLEN;
1892 if (buf[ptr]>>(7-bpos)&1)
1893 {
1894 m.add(ilog(len)<<2);
1895 m.add(min(len, 32)<<6);
1896 }
1897 else
1898 {
1899 m.add(-(ilog(len)<<2));
1900 m.add(-(min(len, 32)<<6));
1901 }
1902 }
1903 else
1904 {
1905 len=0;
1906 m.add(0);
1907 m.add(0);
1908 }
1909 }
1910 else
1911 {
1912 m.add(0);
1913 m.add(0);
1914 }
1915
1916 scm1.mix(m);
1917 scm2.mix(m);
1918
1919 return result;
1920}
1921
1922
1923
1924//////////////////////////// jpegModel /////////////////////////
1925
1926// Model JPEG. Return 1 if a JPEG file is detected or else 0.
1927// Only the baseline and 8 bit extended Huffman coded DCT modes are
1928// supported. The model partially decodes the JPEG image to provide
1929// context for the Huffman coded symbols.
1930
1931// Print a JPEG segment at buf[p...] for debugging
1932/*
1933void dump(const char* msg, int p) {
1934 printf("%s:", msg);
1935 int len=buf[p+2]*256+buf[p+3];
1936 for (int i=0; i<len+2; ++i)
1937 printf(" %02X", buf[p+i]);
1938 printf("\n");
1939}
1940*/
1941
1942#define finish(success){ \
1943 int length = pos - images[idx].offset; \
1944 /*if (success && idx && pos-lastPos==1)*/ \
1945 /*printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\bEmbedded JPEG at offset %d, size: %d bytes, level %d\nCompressing... ", images[idx].offset-pos+blpos, length, idx), fflush(stdout);*/ \
1946 memset(&images[idx], 0, sizeof(JPEGImage)); \
1947 mcusize=0,dqt_state=-1; \
1948 idx-=(idx>0); \
1949 images[idx].app-=length; \
1950 if (images[idx].app < 0) \
1951 images[idx].app = 0; \
1952}
1953
1954// Detect invalid JPEG data. The proper response is to silently
1955// fall back to a non-JPEG model.
1956#define jassert(x) if (!(x)) { \
1957/* printf("JPEG error at %d, line %d: %s\n", pos, __LINE__, #x); */ \
1958 if (idx>0) \
1959 finish(false) \
1960 else \
1961 images[idx].jpeg=0; \
1962 return images[idx].next_jpeg;}
1963
1964struct HUF {U32 min, max; int val;}; // Huffman decode tables
1965 // huf[Tc][Th][m] is the minimum, maximum+1, and pointer to codes for
1966 // coefficient type Tc (0=DC, 1=AC), table Th (0-3), length m+1 (m=0-15)
1967
1968struct JPEGImage{
1969 int offset, // offset of SOI marker
1970 jpeg, // 1 if JPEG is header detected, 2 if image data
1971 next_jpeg, // updated with jpeg on next byte boundary
1972 app, // Bytes remaining to skip in this marker
1973 sof, sos, data, // pointers to buf
1974 htsize; // number of pointers in ht
1975 int ht[8]; // pointers to Huffman table headers
1976 U8 qtab[256]; // table
1977 int qmap[10]; // block -> table number
1978};
1979
1980int jpegModel(Mixer& m) {
1981
1982 // State of parser
1983 enum {SOF0=0xc0, SOF1, SOF2, SOF3, DHT, RST0=0xd0, SOI=0xd8, EOI, SOS, DQT,
1984 DNL, DRI, APP0=0xe0, COM=0xfe, FF}; // Second byte of 2 byte codes
1985 const static int MaxEmbeddedLevel = 3;
1986 static JPEGImage images[MaxEmbeddedLevel];
1987 static int idx=-1;
1988 static int lastPos=0;
1989
1990 // Huffman decode state
1991 static U32 huffcode=0; // Current Huffman code including extra bits
1992 static int huffbits=0; // Number of valid bits in huffcode
1993 static int huffsize=0; // Number of bits without extra bits
1994 static int rs=-1; // Decoded huffcode without extra bits. It represents
1995 // 2 packed 4-bit numbers, r=run of zeros, s=number of extra bits for
1996 // first nonzero code. huffcode is complete when rs >= 0.
1997 // rs is -1 prior to decoding incomplete huffcode.
1998
1999 static int mcupos=0; // position in MCU (0-639). The low 6 bits mark
2000 // the coefficient in zigzag scan order (0=DC, 1-63=AC). The high
2001 // bits mark the block within the MCU, used to select Huffman tables.
2002
2003 // Decoding tables
2004 static Array<HUF> huf(128); // Tc*64+Th*16+m -> min, max, val
2005 static int mcusize=0; // number of coefficients in an MCU
2006 static int hufsel[2][10]; // DC/AC, mcupos/64 -> huf decode table
2007 static Array<U8> hbuf(2048); // Tc*1024+Th*256+hufcode -> RS
2008
2009 // Image state
2010 static Array<int> color(10); // block -> component (0-3)
2011 static Array<int> pred(4); // component -> last DC value
2012 static int dc=0; // DC value of the current block
2013 static int width=0; // Image width in MCU
2014 static int row=0, column=0; // in MCU (column 0 to width-1)
2015 static Buf cbuf(0x20000); // Rotating buffer of coefficients, coded as:
2016 // DC: level shifted absolute value, low 4 bits discarded, i.e.
2017 // [-1023...1024] -> [0...255].
2018 // AC: as an RS code: a run of R (0-15) zeros followed by an S (0-15)
2019 // bit number, or 00 for end of block (in zigzag order).
2020 // However if R=0, then the format is ssss11xx where ssss is S,
2021 // xx is the first 2 extra bits, and the last 2 bits are 1 (since
2022 // this never occurs in a valid RS code).
2023 static int cpos=0; // position in cbuf
2024 static int rs1; // last RS code
2025 static int rstpos=0,rstlen=0; // reset position
2026 static int ssum=0, ssum1=0, ssum2=0, ssum3=0;
2027 // sum of S in RS codes in block and sum of S in first component
2028
2029 static IntBuf cbuf2(0x20000);
2030 static Array<int> adv_pred(4), sumu(8), sumv(8), run_pred(6);
2031 static int prev_coef=0, prev_coef2=0, prev_coef_rs=0;
2032 static Array<int> ls(10); // block -> distance to previous block
2033 static Array<int> blockW(10), blockN(10), SamplingFactors(4);
2034 static Array<int> lcp(7), zpos(64);
2035
2036 //for parsing Quantization tables
2037 static int dqt_state = -1, dqt_end = 0, qnum = 0;
2038
2039 const static U8 zzu[64]={ // zigzag coef -> u,v
2040 0,1,0,0,1,2,3,2,1,0,0,1,2,3,4,5,4,3,2,1,0,0,1,2,3,4,5,6,7,6,5,4,
2041 3,2,1,0,1,2,3,4,5,6,7,7,6,5,4,3,2,3,4,5,6,7,7,6,5,4,5,6,7,7,6,7};
2042 const static U8 zzv[64]={
2043 0,0,1,2,1,0,0,1,2,3,4,3,2,1,0,0,1,2,3,4,5,6,5,4,3,2,1,0,0,1,2,3,
2044 4,5,6,7,7,6,5,4,3,2,1,2,3,4,5,6,7,7,6,5,4,3,4,5,6,7,7,6,5,6,7,7};
2045
2046 // Standard Huffman tables (cf. JPEG standard section K.3)
2047 // IMPORTANT: these are only valid for 8-bit data precision
2048 const static U8 bits_dc_luminance[16] = {
2049 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0
2050 };
2051 const static U8 values_dc_luminance[12] = {
2052 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
2053 };
2054
2055 const static U8 bits_dc_chrominance[16] = {
2056 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0
2057 };
2058 const static U8 values_dc_chrominance[12] = {
2059 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
2060 };
2061
2062 const static U8 bits_ac_luminance[16] = {
2063 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d
2064 };
2065 const static U8 values_ac_luminance[162] = {
2066 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12,
2067 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07,
2068 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08,
2069 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0,
2070 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16,
2071 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28,
2072 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39,
2073 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49,
2074 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
2075 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69,
2076 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79,
2077 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89,
2078 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
2079 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
2080 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
2081 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5,
2082 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4,
2083 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2,
2084 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea,
2085 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,
2086 0xf9, 0xfa
2087 };
2088
2089 const static U8 bits_ac_chrominance[16] = {
2090 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77
2091 };
2092 const static U8 values_ac_chrominance[162] = {
2093 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21,
2094 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
2095 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
2096 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0,
2097 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34,
2098 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26,
2099 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38,
2100 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48,
2101 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
2102 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
2103 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
2104 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
2105 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96,
2106 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5,
2107 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4,
2108 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3,
2109 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2,
2110 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda,
2111 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9,
2112 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,
2113 0xf9, 0xfa
2114 };
2115
2116 if (idx < 0){
2117 memset(&images[0], 0, sizeof(images));
2118 idx = 0;
2119 lastPos = pos;
2120 }
2121
2122 // Be sure to quit on a byte boundary
2123 if (!bpos) images[idx].next_jpeg=images[idx].jpeg>1;
2124 if (bpos && !images[idx].jpeg) return images[idx].next_jpeg;
2125 if (!bpos && images[idx].app>0){
2126 --images[idx].app;
2127 if (idx<MaxEmbeddedLevel && buf(4)==FF && buf(3)==SOI && buf(2)==FF && ((buf(1)&0xFE)==0xC0 || buf(1)==0xC4 || (buf(1)>=0xDB && buf(1)<=0xFE)) )
2128 memset(&images[++idx], 0, sizeof(JPEGImage));
2129 }
2130 if (images[idx].app>0) return images[idx].next_jpeg;
2131 if (!bpos) {
2132
2133 // Parse. Baseline DCT-Huffman JPEG syntax is:
2134 // SOI APPx... misc... SOF0 DHT... SOS data EOI
2135 // SOI (= FF D8) start of image.
2136 // APPx (= FF Ex) len ... where len is always a 2 byte big-endian length
2137 // including the length itself but not the 2 byte preceding code.
2138 // Application data is ignored. There may be more than one APPx.
2139 // misc codes are DQT, DNL, DRI, COM (ignored).
2140 // SOF0 (= FF C0) len 08 height width Nf [C HV Tq]...
2141 // where len, height, width (in pixels) are 2 bytes, Nf is the repeat
2142 // count (1 byte) of [C HV Tq], where C is a component identifier
2143 // (color, 0-3), HV is the horizontal and vertical dimensions
2144 // of the MCU (high, low bits, packed), and Tq is the quantization
2145 // table ID (not used). An MCU (minimum compression unit) consists
2146 // of 64*H*V DCT coefficients for each color.
2147 // DHT (= FF C4) len [TcTh L1...L16 V1,1..V1,L1 ... V16,1..V16,L16]...
2148 // defines Huffman table Th (1-4) for Tc (0=DC (first coefficient)
2149 // 1=AC (next 63 coefficients)). L1..L16 are the number of codes
2150 // of length 1-16 (in ascending order) and Vx,y are the 8-bit values.
2151 // A V code of RS means a run of R (0-15) zeros followed by S (0-15)
2152 // additional bits to specify the next nonzero value, negative if
2153 // the first additional bit is 0 (e.g. code x63 followed by the
2154 // 3 bits 1,0,1 specify 7 coefficients: 0, 0, 0, 0, 0, 0, 5.
2155 // Code 00 means end of block (remainder of 63 AC coefficients is 0).
2156 // SOS (= FF DA) len Ns [Cs TdTa]... 0 3F 00
2157 // Start of scan. TdTa specifies DC/AC Huffman tables (0-3, packed
2158 // into one byte) for component Cs matching C in SOF0, repeated
2159 // Ns (1-4) times.
2160 // EOI (= FF D9) is end of image.
2161 // Huffman coded data is between SOI and EOI. Codes may be embedded:
2162 // RST0-RST7 (= FF D0 to FF D7) mark the start of an independently
2163 // compressed region.
2164 // DNL (= FF DC) 04 00 height
2165 // might appear at the end of the scan (ignored).
2166 // FF 00 is interpreted as FF (to distinguish from RSTx, DNL, EOI).
2167
2168 // Detect JPEG (SOI followed by a valid marker)
2169 if (!images[idx].jpeg && buf(4)==FF && buf(3)==SOI && buf(2)==FF && ((buf(1)&0xFE)==0xC0 || buf(1)==0xC4 || (buf(1)>=0xDB && buf(1)<=0xFE)) ){
2170 images[idx].jpeg=1;
2171 images[idx].offset = pos-4;
2172 images[idx].sos=images[idx].sof=images[idx].htsize=images[idx].data=0, images[idx].app=(buf(1)>>4==0xE)*2;
2173 mcusize=huffcode=huffbits=huffsize=mcupos=cpos=0, rs=-1;
2174 memset(&huf[0], 0, sizeof(huf));
2175 memset(&pred[0], 0, pred.size()*sizeof(int));
2176 rstpos=rstlen=0;
2177 }
2178
2179 // Detect end of JPEG when data contains a marker other than RSTx
2180 // or byte stuff (00), or if we jumped in position since the last byte seen
2181 if (!images[idx].data && !images[idx].app && buf(4)==FF && (((buf(3)>0xC1) && (buf(3)<=0xCF) && (buf(3)!=DHT)) || ((buf(3)>=0xDC) && (buf(3)<=0xFE)))){
2182 jassert((buf(1)==EOI) || (pos-lastPos>1));
2183 finish(true);
2184 }
2185 lastPos = pos;
2186 if (!images[idx].jpeg) return images[idx].next_jpeg;
2187
2188 // Detect APPx, COM or other markers, so we can skip them
2189 if (!images[idx].data && !images[idx].app && buf(4)==FF && (((buf(3)>=0xC1) && (buf(3)<=0xCF) && (buf(3)!=DHT)) || ((buf(3)>=0xDC) && (buf(3)<=0xFE)))){
2190 images[idx].app=buf(2)*256+buf(1)+2;
2191 if (idx>0)
2192 jassert( pos + images[idx].app < images[idx].offset + images[idx-1].app );
2193 }
2194
2195 // Save pointers to sof, ht, sos, data,
2196 if (buf(5)==FF && buf(4)==SOS) {
2197 int len=buf(3)*256+buf(2);
2198 if (len==6+2*buf(1) && buf(1) && buf(1)<=4) // buf(1) is Ns
2199 images[idx].sos=pos-5, images[idx].data=images[idx].sos+len+2, images[idx].jpeg=2;
2200 }
2201 if (buf(4)==FF && buf(3)==DHT && images[idx].htsize<8) images[idx].ht[images[idx].htsize++]=pos-4;
2202 if (buf(4)==FF && (buf(3)&0xFE)==SOF0) images[idx].sof=pos-4;
2203
2204 // Parse Quantizazion tables
2205 if (buf(4)==FF && buf(3)==DQT)
2206 dqt_end=pos+buf(2)*256+buf(1)-1, dqt_state=0;
2207 else if (dqt_state>=0) {
2208 if (pos>=dqt_end)
2209 dqt_state = -1;
2210 else {
2211 if (dqt_state%65==0)
2212 qnum = buf(1);
2213 else {
2214 jassert(buf(1)>0);
2215 jassert(qnum>=0 && qnum<4);
2216 images[idx].qtab[qnum*64+((dqt_state%65)-1)]=buf(1)-1;
2217 }
2218 dqt_state++;
2219 }
2220 }
2221
2222 // Restart
2223 if (buf(2)==FF && (buf(1)&0xf8)==RST0) {
2224 huffcode=huffbits=huffsize=mcupos=0, rs=-1;
2225 memset(&pred[0], 0, pred.size()*sizeof(int));
2226 rstlen=column+row*width-rstpos;
2227 rstpos=column+row*width;
2228 }
2229 }
2230
2231 {
2232 // Build Huffman tables
2233 // huf[Tc][Th][m] = min, max+1 codes of length m, pointer to byte values
2234 if (pos==images[idx].data && bpos==1) {
2235 int i;
2236 for (i=0; i<images[idx].htsize; ++i) {
2237 int p=images[idx].ht[i]+4; // pointer to current table after length field
2238 int end=p+buf[p-2]*256+buf[p-1]-2; // end of Huffman table
2239 int count=0; // sanity check
2240 while (p<end && end<pos && end<p+2100 && ++count<10) {
2241 int tc=buf[p]>>4, th=buf[p]&15;
2242 if (tc>=2 || th>=4) break;
2243 jassert(tc>=0 && tc<2 && th>=0 && th<4);
2244 HUF* h=&huf[tc*64+th*16]; // [tc][th][0];
2245 int val=p+17; // pointer to values
2246 int hval=tc*1024+th*256; // pointer to RS values in hbuf
2247 int j;
2248 for (j=0; j<256; ++j) // copy RS codes
2249 hbuf[hval+j]=buf[val+j];
2250 int code=0;
2251 for (j=0; j<16; ++j) {
2252 h[j].min=code;
2253 h[j].max=code+=buf[p+j+1];
2254 h[j].val=hval;
2255 val+=buf[p+j+1];
2256 hval+=buf[p+j+1];
2257 code*=2;
2258 }
2259 p=val;
2260 jassert(hval>=0 && hval<2048);
2261 }
2262 jassert(p==end);
2263 }
2264 huffcode=huffbits=huffsize=0, rs=-1;
2265
2266 // load default tables
2267 if (!images[idx].htsize){
2268 for (int tc = 0; tc < 2; tc++) {
2269 for (int th = 0; th < 2; th++) {
2270 HUF* h = &huf[tc*64+th*16];
2271 int hval = tc*1024 + th*256;
2272 int code = 0, c = 0, x = 0;
2273
2274 for (int i = 0; i < 16; i++) {
2275 switch (tc*2+th) {
2276 case 0: x = bits_dc_luminance[i]; break;
2277 case 1: x = bits_dc_chrominance[i]; break;
2278 case 2: x = bits_ac_luminance[i]; break;
2279 case 3: x = bits_ac_chrominance[i];
2280 }
2281
2282 h[i].min = code;
2283 h[i].max = (code+=x);
2284 h[i].val = hval;
2285 hval+=x;
2286 code+=code;
2287 c+=x;
2288 }
2289
2290 hval = tc*1024 + th*256;
2291 c--;
2292
2293 while (c >= 0){
2294 switch (tc*2+th) {
2295 case 0: x = values_dc_luminance[c]; break;
2296 case 1: x = values_dc_chrominance[c]; break;
2297 case 2: x = values_ac_luminance[c]; break;
2298 case 3: x = values_ac_chrominance[c];
2299 }
2300
2301 hbuf[hval+c] = x;
2302 c--;
2303 }
2304 }
2305 }
2306 images[idx].htsize = 4;
2307 }
2308
2309 // Build Huffman table selection table (indexed by mcupos).
2310 // Get image width.
2311 if (!images[idx].sof && images[idx].sos) return images[idx].next_jpeg;
2312 int ns=buf[images[idx].sos+4];
2313 int nf=buf[images[idx].sof+9];
2314 jassert(ns<=4 && nf<=4);
2315 mcusize=0; // blocks per MCU
2316 int hmax=0; // MCU horizontal dimension
2317 for (i=0; i<ns; ++i) {
2318 for (int j=0; j<nf; ++j) {
2319 if (buf[images[idx].sos+2*i+5]==buf[images[idx].sof+3*j+10]) { // Cs == C ?
2320 int hv=buf[images[idx].sof+3*j+11]; // packed dimensions H x V
2321 SamplingFactors[j] = hv;
2322 if (hv>>4>hmax) hmax=hv>>4;
2323 hv=(hv&15)*(hv>>4); // number of blocks in component C
2324 jassert(hv>=1 && hv+mcusize<=10);
2325 while (hv) {
2326 jassert(mcusize<10);
2327 hufsel[0][mcusize]=buf[images[idx].sos+2*i+6]>>4&15;
2328 hufsel[1][mcusize]=buf[images[idx].sos+2*i+6]&15;
2329 jassert (hufsel[0][mcusize]<4 && hufsel[1][mcusize]<4);
2330 color[mcusize]=i;
2331 int tq=buf[images[idx].sof+3*j+12]; // quantization table index (0..3)
2332 jassert(tq>=0 && tq<4);
2333 images[idx].qmap[mcusize]=tq; // quantizazion table mapping
2334 --hv;
2335 ++mcusize;
2336 }
2337 }
2338 }
2339 }
2340 jassert(hmax>=1 && hmax<=10);
2341 int j;
2342 for (j=0; j<mcusize; ++j) {
2343 ls[j]=0;
2344 for (int i=1; i<mcusize; ++i) if (color[(j+i)%mcusize]==color[j]) ls[j]=i;
2345 ls[j]=(mcusize-ls[j])<<6;
2346 }
2347 for (j=0; j<64; ++j) zpos[zzu[j]+8*zzv[j]]=j;
2348 width=buf[images[idx].sof+7]*256+buf[images[idx].sof+8]; // in pixels
2349 width=(width-1)/(hmax*8)+1; // in MCU
2350 jassert(width>0);
2351 mcusize*=64; // coefficients per MCU
2352 row=column=0;
2353
2354 // we can have more blocks than components then we have subsampling
2355 int x=0, y=0;
2356 for (j = 0; j<(mcusize>>6); j++) {
2357 int i = color[j];
2358 int w = SamplingFactors[i]>>4, h = SamplingFactors[i]&0xf;
2359 blockW[j] = x==0?mcusize-64*(w-1):64;
2360 blockN[j] = y==0?mcusize*width-64*w*(h-1):w*64;
2361 x++;
2362 if (x>=w) { x=0; y++; }
2363 if (y>=h) { x=0; y=0; }
2364 }
2365 }
2366 }
2367
2368
2369 // Decode Huffman
2370 {
2371 if (mcusize && buf(1+(!bpos))!=FF) { // skip stuffed byte
2372 jassert(huffbits<=32);
2373 huffcode+=huffcode+y;
2374 ++huffbits;
2375 if (rs<0) {
2376 jassert(huffbits>=1 && huffbits<=16);
2377 const int ac=(mcupos&63)>0;
2378 jassert(mcupos>=0 && (mcupos>>6)<10);
2379 jassert(ac==0 || ac==1);
2380 const int sel=hufsel[ac][mcupos>>6];
2381 jassert(sel>=0 && sel<4);
2382 const int i=huffbits-1;
2383 jassert(i>=0 && i<16);
2384 const HUF *h=&huf[ac*64+sel*16]; // [ac][sel];
2385 jassert(h[i].min<=h[i].max && h[i].val<2048 && huffbits>0);
2386 if (huffcode<h[i].max) {
2387 jassert(huffcode>=h[i].min);
2388 int k=h[i].val+huffcode-h[i].min;
2389 jassert(k>=0 && k<2048);
2390 rs=hbuf[k];
2391 huffsize=huffbits;
2392 }
2393 }
2394 if (rs>=0) {
2395 if (huffsize+(rs&15)==huffbits) { // done decoding
2396 rs1=rs;
2397 int x=0; // decoded extra bits
2398 if (mcupos&63) { // AC
2399 if (rs==0) { // EOB
2400 mcupos=(mcupos+63)&-64;
2401 jassert(mcupos>=0 && mcupos<=mcusize && mcupos<=640);
2402 while (cpos&63) {
2403 cbuf2[cpos]=0;
2404 cbuf[cpos]=(!rs)?0:(63-(cpos&63))<<4; cpos++; rs++;
2405 }
2406 }
2407 else { // rs = r zeros + s extra bits for the next nonzero value
2408 // If first extra bit is 0 then value is negative.
2409 jassert((rs&15)<=10);
2410 const int r=rs>>4;
2411 const int s=rs&15;
2412 jassert(mcupos>>6==(mcupos+r)>>6);
2413 mcupos+=r+1;
2414 x=huffcode&((1<<s)-1);
2415 if (s && !(x>>(s-1))) x-=(1<<s)-1;
2416 for (int i=r; i>=1; --i) {
2417 cbuf2[cpos]=0;
2418 cbuf[cpos++]=i<<4|s;
2419 }
2420 cbuf2[cpos]=x;
2421 cbuf[cpos++]=(s<<4)|(huffcode<<2>>s&3)|12;
2422 ssum+=s;
2423 }
2424 }
2425 else { // DC: rs = 0S, s<12
2426 jassert(rs<12);
2427 ++mcupos;
2428 x=huffcode&((1<<rs)-1);
2429 if (rs && !(x>>(rs-1))) x-=(1<<rs)-1;
2430 jassert(mcupos>=0 && mcupos>>6<10);
2431 const int comp=color[mcupos>>6];
2432 jassert(comp>=0 && comp<4);
2433 dc=pred[comp]+=x;
2434 jassert((cpos&63)==0);
2435 cbuf2[cpos]=dc;
2436 cbuf[cpos++]=(dc+1023)>>3;
2437 if ((mcupos>>6)==0) {
2438 ssum1=0;
2439 ssum2=ssum3;
2440 } else {
2441 if (color[(mcupos>>6)-1]==color[0]) ssum1+=(ssum3=ssum);
2442 ssum2=ssum1;
2443 }
2444 ssum=rs;
2445 }
2446 jassert(mcupos>=0 && mcupos<=mcusize);
2447 if (mcupos>=mcusize) {
2448 mcupos=0;
2449 if (++column==width) column=0, ++row;
2450 }
2451 huffcode=huffsize=huffbits=0, rs=-1;
2452
2453 // UPDATE_ADV_PRED !!!!
2454 {
2455 const int acomp=mcupos>>6, q=64*images[idx].qmap[acomp];
2456 const int zz=mcupos&63, cpos_dc=cpos-zz;
2457 const bool norst=rstpos!=column+row*width;
2458 if (zz==0) {
2459 for (int i=0; i<8; ++i) sumu[i]=sumv[i]=0;
2460 // position in the buffer of first (DC) coefficient of the block
2461 // of this same component that is to the west of this one, not
2462 // necessarily in this MCU
2463 int offset_DC_W = cpos_dc - blockW[acomp];
2464 // position in the buffer of first (DC) coefficient of the block
2465 // of this same component that is to the north of this one, not
2466 // necessarily in this MCU
2467 int offset_DC_N = cpos_dc - blockN[acomp];
2468 for (int i=0; i<64; ++i) {
2469 sumu[zzu[i]]+=(zzv[i]&1?-1:1)*(zzv[i]?16*(16+zzv[i]):185)*(images[idx].qtab[q+i]+1)*cbuf2[offset_DC_N+i];
2470 sumv[zzv[i]]+=(zzu[i]&1?-1:1)*(zzu[i]?16*(16+zzu[i]):185)*(images[idx].qtab[q+i]+1)*cbuf2[offset_DC_W+i];
2471 }
2472 }
2473 else {
2474 sumu[zzu[zz-1]]-=(zzv[zz-1]?16*(16+zzv[zz-1]):185)*(images[idx].qtab[q+zz-1]+1)*cbuf2[cpos-1];
2475 sumv[zzv[zz-1]]-=(zzu[zz-1]?16*(16+zzu[zz-1]):185)*(images[idx].qtab[q+zz-1]+1)*cbuf2[cpos-1];
2476 }
2477
2478 for (int i=0; i<3; ++i)
2479 {
2480 run_pred[i]=run_pred[i+3]=0;
2481 for (int st=0; st<10 && zz+st<64; ++st) {
2482 const int zz2=zz+st;
2483 int p=sumu[zzu[zz2]]*i+sumv[zzv[zz2]]*(2-i);
2484 p/=(images[idx].qtab[q+zz2]+1)*185*(16+zzv[zz2])*(16+zzu[zz2])/128;
2485 if (zz2==0 && (norst || ls[acomp]==64)) p-=cbuf2[cpos_dc-ls[acomp]];
2486 p=(p<0?-1:+1)*ilog(abs(p)+1);
2487 if (st==0) {
2488 adv_pred[i]=p;
2489 }
2490 else if (abs(p)>abs(adv_pred[i])+2 && abs(adv_pred[i]) < 210) {
2491 if (run_pred[i]==0) run_pred[i]=st*2+(p>0);
2492 if (abs(p)>abs(adv_pred[i])+21 && run_pred[i+3]==0) run_pred[i+3]=st*2+(p>0);
2493 }
2494 }
2495 }
2496 x=0;
2497 for (int i=0; i<8; ++i) x+=(zzu[zz]<i)*sumu[i]+(zzv[zz]<i)*sumv[i];
2498 x=(sumu[zzu[zz]]*(2+zzu[zz])+sumv[zzv[zz]]*(2+zzv[zz])-x*2)*4/(zzu[zz]+zzv[zz]+16);
2499 x/=(images[idx].qtab[q+zz]+1)*185;
2500 if (zz==0 && (norst || ls[acomp]==64)) x-=cbuf2[cpos_dc-ls[acomp]];
2501 adv_pred[3]=(x<0?-1:+1)*ilog(abs(x)+1);
2502
2503 for (int i=0; i<4; ++i) {
2504 const int a=(i&1?zzv[zz]:zzu[zz]), b=(i&2?2:1);
2505 if (a<b) x=65535;
2506 else {
2507 const int zz2=zpos[zzu[zz]+8*zzv[zz]-(i&1?8:1)*b];
2508 x=(images[idx].qtab[q+zz2]+1)*cbuf2[cpos_dc+zz2]/(images[idx].qtab[q+zz]+1);
2509 x=(x<0?-1:+1)*(ilog(abs(x)+1)+(x!=0?17:0));
2510 }
2511 lcp[i]=x;
2512 }
2513 if ((zzu[zz]*zzv[zz])!=0){
2514 const int zz2=zpos[zzu[zz]+8*zzv[zz]-9];
2515 x=(images[idx].qtab[q+zz2]+1)*cbuf2[cpos_dc+zz2]/(images[idx].qtab[q+zz]+1);
2516 lcp[4]=(x<0?-1:+1)*(ilog(abs(x)+1)+(x!=0?17:0));
2517
2518 x=(images[idx].qtab[q+zpos[8*zzv[zz]]]+1)*cbuf2[cpos_dc+zpos[8*zzv[zz]]]/(images[idx].qtab[q+zz]+1);
2519 lcp[5]=(x<0?-1:+1)*(ilog(abs(x)+1)+(x!=0?17:0));
2520
2521 x=(images[idx].qtab[q+zpos[zzu[zz]]]+1)*cbuf2[cpos_dc+zpos[zzu[zz]]]/(images[idx].qtab[q+zz]+1);
2522 lcp[6]=(x<0?-1:+1)*(ilog(abs(x)+1)+(x!=0?17:0));
2523 }
2524 else
2525 lcp[4]=lcp[5]=lcp[6]=65535;
2526
2527 int prev1=0,prev2=0,cnt1=0,cnt2=0,r=0,s=0;
2528 prev_coef_rs = cbuf[cpos-64];
2529 for (int i=0; i<acomp; i++) {
2530 x=0;
2531 x+=cbuf2[cpos-(acomp-i)*64];
2532 if (zz==0 && (norst || ls[i]==64)) x-=cbuf2[cpos_dc-(acomp-i)*64-ls[i]];
2533 if (color[i]==color[acomp]-1) { prev1+=x; cnt1++; r+=cbuf[cpos-(acomp-i)*64]>>4; s+=cbuf[cpos-(acomp-i)*64]&0xF; }
2534 if (color[acomp]>1 && color[i]==color[0]) { prev2+=x; cnt2++; }
2535 }
2536 if (cnt1>0) prev1/=cnt1, r/=cnt1, s/=cnt1, prev_coef_rs=(r<<4)|s;
2537 if (cnt2>0) prev2/=cnt2;
2538 prev_coef=(prev1<0?-1:+1)*ilog(11*abs(prev1)+1)+(cnt1<<20);
2539 prev_coef2=(prev2<0?-1:+1)*ilog(11*abs(prev2)+1);
2540
2541 if (column==0 && blockW[acomp]>64*acomp) run_pred[1]=run_pred[2], run_pred[0]=0, adv_pred[1]=adv_pred[2], adv_pred[0]=0;
2542 if (row==0 && blockN[acomp]>64*acomp) run_pred[1]=run_pred[0], run_pred[2]=0, adv_pred[1]=adv_pred[0], adv_pred[2]=0;
2543 } // !!!!
2544
2545 }
2546 }
2547 }
2548 }
2549
2550 // Estimate next bit probability
2551 if (!images[idx].jpeg || !images[idx].data) return images[idx].next_jpeg;
2552 if (buf(1+(!bpos))==FF) {
2553 m.add(128);
2554 m.set(2, 264);
2555 m.set(0, 256);
2556 m.set(buf(1), 256);
2557 return 1;
2558 }
2559 if (rstlen>0 && rstlen==column+row*width-rstpos && mcupos==0 && (int)huffcode==(1<<huffbits)-1) {
2560 m.add(4095);
2561 m.set(2, 264);
2562 m.set(0, 256);
2563 m.set(buf(1), 256);
2564 return 1;
2565 }
2566
2567 // Context model
2568 const int N=32; // size of t, number of contexts
2569 static BH<9> t(MEM); // context hash -> bit history
2570 // As a cache optimization, the context does not include the last 1-2
2571 // bits of huffcode if the length (huffbits) is not a multiple of 3.
2572 // The 7 mapped values are for context+{"", 0, 00, 01, 1, 10, 11}.
2573 static Array<U32> cxt(N); // context hashes
2574 static Array<U8*> cp(N); // context pointers
2575 static StateMap sm[N];
2576 static Mixer m1(N+1, 2050, 3);
2577 static APM a1(0x8000), a2(0x20000);
2578
2579
2580 // Update model
2581 if (cp[N-1]) {
2582 for (int i=0; i<N; ++i)
2583 *cp[i]=nex(*cp[i],y);
2584 }
2585 m1.update();
2586
2587 // Update context
2588 const int comp=color[mcupos>>6];
2589 const int coef=(mcupos&63)|comp<<6;
2590 const int hc=(huffcode*4+((mcupos&63)==0)*2+(comp==0))|1<<(huffbits+2);
2591 const bool firstcol=column==0 && blockW[mcupos>>6]>mcupos;
2592 static int hbcount=2;
2593 if (++hbcount>2 || huffbits==0) hbcount=0;
2594 jassert(coef>=0 && coef<256);
2595 const int zu=zzu[mcupos&63], zv=zzv[mcupos&63];
2596 if (hbcount==0) {
2597 int n=hc*32;
2598 cxt[0]=hash(++n, coef, adv_pred[2]/12+(run_pred[2]<<8), ssum2>>6, prev_coef/72);
2599 cxt[1]=hash(++n, coef, adv_pred[0]/12+(run_pred[0]<<8), ssum2>>6, prev_coef/72);
2600 cxt[2]=hash(++n, coef, adv_pred[1]/11+(run_pred[1]<<8), ssum2>>6);
2601 cxt[3]=hash(++n, rs1, adv_pred[2]/7, run_pred[5]/2, prev_coef/10);
2602 cxt[4]=hash(++n, rs1, adv_pred[0]/7, run_pred[3]/2, prev_coef/10);
2603 cxt[5]=hash(++n, rs1, adv_pred[1]/11, run_pred[4]);
2604 cxt[6]=hash(++n, adv_pred[2]/14, run_pred[2], adv_pred[0]/14, run_pred[0]);
2605 cxt[7]=hash(++n, cbuf[cpos-blockN[mcupos>>6]]>>4, adv_pred[3]/17, run_pred[1], run_pred[5]);
2606 cxt[8]=hash(++n, cbuf[cpos-blockW[mcupos>>6]]>>4, adv_pred[3]/17, run_pred[1], run_pred[3]);
2607 cxt[9]=hash(++n, lcp[0]/22, lcp[1]/22, adv_pred[1]/7, run_pred[1]);
2608 cxt[10]=hash(++n, lcp[0]/22, lcp[1]/22, mcupos&63, lcp[4]/30);
2609 cxt[11]=hash(++n, zu/2, lcp[0]/13, lcp[2]/30, prev_coef/40+((prev_coef2/28)<<20));
2610 cxt[12]=hash(++n, zv/2, lcp[1]/13, lcp[3]/30, prev_coef/40+((prev_coef2/28)<<20));
2611 cxt[13]=hash(++n, rs1, prev_coef/42, prev_coef2/34, hash(lcp[0]/60,lcp[2]/14,lcp[1]/60,lcp[3]/14));
2612 cxt[14]=hash(++n, mcupos&63, column>>1);
2613 cxt[15]=hash(++n, column>>3, min(5+2*(!comp),zu+zv), hash(lcp[0]/10,lcp[2]/40,lcp[1]/10,lcp[3]/40));
2614 cxt[16]=hash(++n, ssum>>3, mcupos&63);
2615 cxt[17]=hash(++n, rs1, mcupos&63, run_pred[1]);
2616 cxt[18]=hash(++n, coef, ssum2>>5, adv_pred[3]/30, (comp)?hash(prev_coef/22,prev_coef2/50):ssum/((mcupos&0x3F)+1));
2617 cxt[19]=hash(++n, lcp[0]/40, lcp[1]/40, adv_pred[1]/28, hash( (comp)?prev_coef/40+((prev_coef2/40)<<20):lcp[4]/22, min(7,zu+zv), ssum/(2*(zu+zv)+1) ) );
2618 cxt[20]=hash(++n, zv, cbuf[cpos-blockN[mcupos>>6]], adv_pred[2]/28, run_pred[2]);
2619 cxt[21]=hash(++n, zu, cbuf[cpos-blockW[mcupos>>6]], adv_pred[0]/28, run_pred[0]);
2620 cxt[22]=hash(++n, adv_pred[2]/7, run_pred[2]);
2621 cxt[23]=hash(n, adv_pred[0]/7, run_pred[0]);
2622 cxt[24]=hash(n, adv_pred[1]/7, run_pred[1]);
2623 cxt[25]=hash(++n, zv, lcp[1]/14, adv_pred[2]/16, run_pred[5]);
2624 cxt[26]=hash(++n, zu, lcp[0]/14, adv_pred[0]/16, run_pred[3]);
2625 cxt[27]=hash(++n, lcp[0]/14, lcp[1]/14, adv_pred[3]/16);
2626 cxt[28]=hash(++n, coef, prev_coef/10, prev_coef2/20);
2627 cxt[29]=hash(++n, coef, ssum>>2, prev_coef_rs);
2628 cxt[30]=hash(++n, coef, adv_pred[1]/17, hash(lcp[(zu<zv)]/24,lcp[2]/20,lcp[3]/24));
2629 cxt[31]=hash(++n, coef, adv_pred[3]/11, hash(lcp[(zu<zv)]/50,lcp[2+3*(zu*zv>1)]/50,lcp[3+3*(zu*zv>1)]/50));
2630 }
2631
2632 // Predict next bit
2633 assert(hbcount<=2);
2634 int p;
2635 switch(hbcount)
2636 {
2637 case 0: for (int i=0; i<N; ++i){ cp[i]=t[cxt[i]]+1, m1.add(p=stretch(sm[i].p(*cp[i]))); } break;
2638 case 1: { int hc=1+(huffcode&1)*3; for (int i=0; i<N; ++i){ cp[i]+=hc, m1.add(p=stretch(sm[i].p(*cp[i]))); }} break;
2639 default: { int hc=1+(huffcode&1); for (int i=0; i<N; ++i){ cp[i]+=hc, m1.add(p=stretch(sm[i].p(*cp[i]))); }} break;
2640 }
2641
2642 m1.set(firstcol, 2);
2643 m1.set(coef+256*min(3,huffbits), 1024);
2644 m1.set((hc&0x1FE)*2+min(3,ilog2(zu+zv)), 1024);
2645 int pr=m1.p();
2646 m.add(stretch(pr)>>2);
2647 m.add((pr>>4)-(255-((pr>>4))));
2648 pr=a1.p(pr, (hc&511)|(((adv_pred[1]/16)&63)<<9), 1023);
2649 m.add(stretch(pr)>>2);
2650 m.add((pr>>4)-(255-((pr>>4))));
2651 pr=a2.p(pr, (hc&511)|(coef<<9), 1023);
2652 m.add(stretch(pr)>>2);
2653 m.add((pr>>4)-(255-((pr>>4))));
2654 m.set(1, 264);
2655 m.set(hc&255, 256);
2656 m.set(buf(1), 256);
2657 return 1;
2658}
2659
2660
2661//////////////////////////// wavModel /////////////////////////////////
2662
2663// Model a 16/8-bit stereo/mono uncompressed .wav file.
2664// Based on 'An asymptotically Optimal Predictor for Stereo Lossless Audio Compression'
2665// by Florin Ghido.
2666
2667static int S,D;
2668static int wmode;
2669
2670inline int s2(int i) { return int(short(buf(i)+256*buf(i-1))); }
2671inline int t2(int i) { return int(short(buf(i-1)+256*buf(i))); }
2672
2673inline int X1(int i) {
2674 switch (wmode) {
2675 case 0: return buf(i)-128;
2676 case 1: return buf(i<<1)-128;
2677 case 2: return s2(i<<1);
2678 case 3: return s2(i<<2);
2679 case 4: return (buf(i)^128)-128;
2680 case 5: return (buf(i<<1)^128)-128;
2681 case 6: return t2(i<<1);
2682 case 7: return t2(i<<2);
2683 default: return 0;
2684 }
2685}
2686
2687inline int X2(int i) {
2688 switch (wmode) {
2689 case 0: return buf(i+S)-128;
2690 case 1: return buf((i<<1)-1)-128;
2691 case 2: return s2((i+S)<<1);
2692 case 3: return s2((i<<2)-2);
2693 case 4: return (buf(i+S)^128)-128;
2694 case 5: return (buf((i<<1)-1)^128)-128;
2695 case 6: return t2((i+S)<<1);
2696 case 7: return t2((i<<2)-2);
2697 default: return 0;
2698 }
2699}
2700
2701void wavModel(Mixer& m, int info) {
2702 static int pr[3][2], n[2], counter[2];
2703 static double F[49][49][2],L[49][49];
2704 int j,k,l,i=0;
2705 long double sum;
2706 const double a=0.996,a2=1/a;
2707 const int SC=0x20000;
2708 static SmallStationaryContextMap scm1(SC), scm2(SC), scm3(SC), scm4(SC), scm5(SC), scm6(SC), scm7(SC);
2709 static ContextMap cm(MEM*4, 10+1);
2710 static int bits, channels, w, ch;
2711 static int z1, z2, z3, z4, z5, z6, z7;
2712
2713 if (!bpos && !blpos) {
2714 bits=((info%4)/2)*8+8;
2715 channels=info%2+1;
2716 w=channels*(bits>>3);
2717 wmode=info;
2718 if (channels==1) S=48,D=0; else S=36,D=12;
2719 for (int j=0; j<channels; j++) {
2720 for (k=0; k<=S+D; k++) for (l=0; l<=S+D; l++) F[k][l][j]=0, L[k][l]=0;
2721 F[1][0][j]=1;
2722 n[j]=counter[j]=pr[2][j]=pr[1][j]=pr[0][j]=0;
2723 z1=z2=z3=z4=z5=z6=z7=0;
2724 }
2725 }
2726 // Select previous samples and predicted sample as context
2727 if (!bpos && blpos>=w) {
2728 /*const int*/ ch=blpos%w;
2729 const int msb=ch%(bits>>3);
2730 const int chn=ch/(bits>>3);
2731 if (!msb) {
2732 z1=X1(1), z2=X1(2), z3=X1(3), z4=X1(4), z5=X1(5);
2733 k=X1(1);
2734 for (l=0; l<=min(S,counter[chn]-1); l++) { F[0][l][chn]*=a; F[0][l][chn]+=X1(l+1)*k; }
2735 for (l=1; l<=min(D,counter[chn]); l++) { F[0][l+S][chn]*=a; F[0][l+S][chn]+=X2(l+1)*k; }
2736 if (channels==2) {
2737 k=X2(2);
2738 for (l=1; l<=min(D,counter[chn]); l++) { F[S+1][l+S][chn]*=a; F[S+1][l+S][chn]+=X2(l+1)*k; }
2739 for (l=1; l<=min(S,counter[chn]-1); l++) { F[l][S+1][chn]*=a; F[l][S+1][chn]+=X1(l+1)*k; }
2740 z6=X2(1)+X1(1)-X2(2), z7=X2(1);
2741 } else z6=2*X1(1)-X1(2), z7=X1(1);
2742 if (++n[chn]==(256>>level)) {
2743 if (channels==1) for (k=1; k<=S+D; k++) for (l=k; l<=S+D; l++) F[k][l][chn]=(F[k-1][l-1][chn]-X1(k)*X1(l))*a2;
2744 else for (k=1; k<=S+D; k++) if (k!=S+1) for (l=k; l<=S+D; l++) if (l!=S+1) F[k][l][chn]=(F[k-1][l-1][chn]-(k-1<=S?X1(k):X2(k-S))*(l-1<=S?X1(l):X2(l-S)))*a2;
2745 for (i=1; i<=S+D; i++) {
2746 sum=F[i][i][chn];
2747 for (k=1; k<i; k++) sum-=L[i][k]*L[i][k];
2748 sum=floor(sum+0.5);
2749 sum=1/sum;
2750 if (sum>0) {
2751 L[i][i]=sqrt(sum);
2752 for (j=(i+1); j<=S+D; j++) {
2753 sum=F[i][j][chn];
2754 for (k=1; k<i; k++) sum-=L[j][k]*L[i][k];
2755 sum=floor(sum+0.5);
2756 L[j][i]=sum*L[i][i];
2757 }
2758 } else break;
2759 }
2760 if (i>S+D && counter[chn]>S+1) {
2761 for (k=1; k<=S+D; k++) {
2762 F[k][0][chn]=F[0][k][chn];
2763 for (j=1; j<k; j++) F[k][0][chn]-=L[k][j]*F[j][0][chn];
2764 F[k][0][chn]*=L[k][k];
2765 }
2766 for (k=S+D; k>0; k--) {
2767 for (j=k+1; j<=S+D; j++) F[k][0][chn]-=L[j][k]*F[j][0][chn];
2768 F[k][0][chn]*=L[k][k];
2769 }
2770 }
2771 n[chn]=0;
2772 }
2773 sum=0;
2774 for (l=1; l<=S+D; l++) sum+=F[l][0][chn]*(l<=S?X1(l):X2(l-S));
2775 pr[2][chn]=pr[1][chn];
2776 pr[1][chn]=pr[0][chn];
2777 pr[0][chn]=int(floor(sum));
2778 counter[chn]++;
2779 }
2780 const int y1=pr[0][chn], y2=pr[1][chn], y3=pr[2][chn];
2781 int x1=buf(1), x2=buf(2), x3=buf(3);
2782 if (wmode==4 || wmode==5) x1^=128, x2^=128;
2783 if (bits==8) x1-=128, x2-=128;
2784 const int t=((bits==8) || ((!msb)^(wmode<6)));
2785 i=ch<<4;
2786 if ((msb)^(wmode<6)) {
2787 cm.set(hash(++i, y1&0xff));
2788 cm.set(hash(++i, y1&0xff, ((z1-y2+z2-y3)>>1)&0xff));
2789 cm.set(hash(++i, x1, y1&0xff));
2790 cm.set(hash(++i, x1, x2>>3, x3));
2791 if (bits==8)
2792 cm.set(hash(++i, y1&0xFE, ilog2(abs((int)(z1-y2)))*2+(z1>y2) ));
2793 else
2794 cm.set(hash(++i, (y1+z1-y2)&0xff));
2795 cm.set(hash(++i, x1));
2796 cm.set(hash(++i, x1, x2));
2797 cm.set(hash(++i, z1&0xff));
2798 cm.set(hash(++i, (z1*2-z2)&0xff));
2799 cm.set(hash(++i, z6&0xff));
2800 cm.set(hash( ++i, y1&0xFF, ((z1-y2+z2-y3)/(bits>>3))&0xFF ));
2801 } else {
2802 cm.set(hash(++i, (y1-x1+z1-y2)>>8));
2803 cm.set(hash(++i, (y1-x1)>>8));
2804 cm.set(hash(++i, (y1-x1+z1*2-y2*2-z2+y3)>>8));
2805 cm.set(hash(++i, (y1-x1)>>8, (z1-y2+z2-y3)>>9));
2806 cm.set(hash(++i, z1>>12));
2807 cm.set(hash(++i, x1));
2808 cm.set(hash(++i, x1>>7, x2, x3>>7));
2809 cm.set(hash(++i, z1>>8));
2810 cm.set(hash(++i, (z1*2-z2)>>8));
2811 cm.set(hash(++i, y1>>8));
2812 cm.set(hash( ++i, (y1-x1)>>6 ));
2813 }
2814 scm1.set(t*ch);
2815 scm2.set(t*((z1-x1+y1)>>9)&0xff);
2816 scm3.set(t*((z1*2-z2-x1+y1)>>8)&0xff);
2817 scm4.set(t*((z1*3-z2*3+z3-x1)>>7)&0xff);
2818 scm5.set(t*((z1+z7-x1+y1*2)>>10)&0xff);
2819 scm6.set(t*((z1*4-z2*6+z3*4-z4-x1)>>7)&0xff);
2820 scm7.set(t*((z1*5-z2*10+z3*10-z4*5+z5-x1+y1)>>9)&0xff);
2821 }
2822
2823 // Predict next bit
2824 scm1.mix(m);
2825 scm2.mix(m);
2826 scm3.mix(m);
2827 scm4.mix(m);
2828 scm5.mix(m);
2829 scm6.mix(m);
2830 scm7.mix(m);
2831 cm.mix(m);
2832 static int col=0;
2833 if (++col>=w*8) col=0;
2834 m.set(5, 264);
2835 m.set(col%bits<8, 2);
2836 m.set(col%bits, bits);
2837 m.set(col, w*8);
2838 m.set(c0, 256);
2839}
2840
2841
2842//////////////////////////// im24bitModel /////////////////////////////////
2843
2844inline U8 Clip(int Px){
2845 return min(0xFF,max(0,Px));
2846}
2847inline U8 Clamp4( int Px, U8 n1, U8 n2, U8 n3, U8 n4){
2848 return min( max(n1,max(n2,max(n3,n4))), max( min(n1,min(n2,min(n3,n4))), Px ));
2849}
2850
2851inline U8 LogMeanDiffQt(U8 a, U8 b){
2852 return (a!=b)?((a>b)<<3)|ilog2((a+b)/max(2,abs(a-b)*2)+1):0;
2853}
2854
2855// Square buf(i)
2856inline int sqrbuf(int i) {
2857 assert(i>0);
2858 return buf(i)*buf(i);
2859}
2860
2861class RingBuffer {
2862 Array<U8> b;
2863 U32 offset;
2864public:
2865 RingBuffer(int i=0): b(i), offset(0) {}
2866 void Add(U8 B){
2867 b[offset&(b.size()-1)] = B;
2868 offset++;
2869 }
2870 int operator()(int i) const {
2871 return b[(offset-i)&(b.size()-1)];
2872 }
2873};
2874
2875inline U8 Paeth(U8 W, U8 N, U8 NW){
2876 int p = W+N-NW;
2877 int pW=abs(p-(int)W), pN=abs(p-(int)N), pNW=abs(p-(int)NW);
2878 if (pW<=pN && pW<=pNW) return W;
2879 else if (pN<=pNW) return N;
2880 return NW;
2881}
2882
2883// Model for filtered (PNG) or unfiltered 24/32-bit image data
2884
2885void im24bitModel(Mixer& m, int info, int alpha=0) {
2886 const int SC=0x20000;
2887 static SmallStationaryContextMap scm1(SC), scm2(SC),
2888 scm3(SC), scm4(SC), scm5(SC), scm6(SC), scm7(SC), scm8(SC), scm9(SC*2), scm10(512);
2889 static ContextMap cm(MEM*4, 13+11);
2890 static RingBuffer buffer(0x100000); // internal rotating buffer for PNG unfiltered pixel data
2891 static U8 WWW, WW, W, NWW, NW, N, NE, NEE, NNWW, NNW, NN, NNE, NNEE, NNN; //pixel neighborhood
2892 static U8 px = 0; // current PNG filter prediction
2893 static int color = -1;
2894 static int stride = 3;
2895 static int ctx, padding, lastPos, filter=0, filterOn=0, x=0, w=0, line=0, isPNG=0;
2896
2897 if (!bpos) {
2898 if ((color < 0) || (pos-lastPos != 1)){
2899 stride = 3+alpha;
2900 w = info&0xFFFFFF;
2901 isPNG = (info&PNGFlag)!=0;
2902 padding = w%stride;
2903 x = color = line = 0;
2904 }
2905 else{
2906 x*=(++x)<w+isPNG;
2907 line+=(!x);
2908 }
2909 lastPos = pos;
2910
2911 if (x==1 && isPNG)
2912 filter = (U8)c4;
2913 else{
2914 U8 B = (U8)c4;
2915 if (x+padding<w)
2916 color*=(++color)<stride;
2917 else
2918 color=(padding)*(stride+1);
2919
2920 if (isPNG){
2921 switch (filter){
2922 case 1: {
2923 buffer.Add((U8)( B + buffer(stride)*(x>stride+1) ) );
2924 filterOn = x>stride;
2925 px = buffer(stride);
2926 break;
2927 }
2928 case 2: {
2929 buffer.Add((U8)( B + buffer(w)*(filterOn=(line>0)) ) );
2930 px = buffer(w);
2931 break;
2932 }
2933 case 3: {
2934 buffer.Add((U8)( B + (buffer(w)*(line>0) + buffer(stride)*(x>stride+1))/2 ) );
2935 filterOn = (x>stride || line>0);
2936 px = (buffer(stride)+buffer(w))/2;
2937 break;
2938 }
2939 case 4: {
2940 buffer.Add((U8)( B + Paeth(buffer(stride)*(x>stride+1), buffer(w)*(line>0), buffer(w+stride)*(line>0 && x>stride+1)) ) );
2941 filterOn = (x>stride || line>0);
2942 px = Paeth(buffer(stride),buffer(w),buffer(w+stride));
2943 break;
2944 }
2945 default: buffer.Add(B);
2946 filterOn = false;
2947 px = buffer(w);
2948 }
2949 px*=filterOn;
2950 }
2951 }
2952
2953 if (x || !isPNG){
2954 int i=color<<5;
2955
2956 if (!isPNG){
2957 WWW=buf(3*stride), WW=buf(2*stride), W=buf(stride), NWW=buf(w+2*stride), NW=buf(w+stride), N=buf(w), NE=buf(w-stride), NEE=buf(w-2*stride), NNWW=buf((w+stride)*2), NNW=buf(w*2+stride), NN=buf(w*2), NNE=buf(w*2-stride), NNEE=buf((w-stride)*2), NNN=buf(w*3);
2958 int mean=W+NW+N+NE;
2959 const int var=(W*W+NW*NW+N*N+NE*NE-mean*mean/4)>>2;
2960 mean>>=2;
2961 const int logvar=ilog(var);
2962
2963 cm.set(hash( (N+1)>>1, LogMeanDiffQt(N,Clip(NN*2-NNN)) ));
2964 cm.set(hash( (W+1)>>1, LogMeanDiffQt(W,Clip(WW*2-WWW)) ));
2965 cm.set(hash( Clamp4(W+N-NW,W,NW,N,NE), LogMeanDiffQt(Clip(N+NE-NNE), Clip(N+NW-NNW))));
2966 cm.set(hash( (NNN+N+4)/8, Clip(N*3-NN*3+NNN)>>1 ));
2967 cm.set(hash( (WWW+W+4)/8, Clip(W*3-WW*3+WWW)>>1 ));
2968 cm.set(hash(++i, (W+Clip(NE*3-NNE*3+buf(w*3-stride)))/4 ));
2969 cm.set(hash(++i, Clip((-buf(4*stride)+5*WWW-10*WW+10*W+Clamp4(NE*4-NNE*6+buf(w*3-stride)*4-buf(w*4-stride),N,NE,buf(w-2*stride),buf(w-3*stride)))/5)/4 ));
2970 cm.set( Clip(NEE+N-NNEE) );
2971 cm.set( Clip(NN+W-NNW) );
2972 cm.set(hash(++i, buf(1)));
2973 cm.set(hash(++i, buf(2)));
2974
2975 cm.set(hash(++i, W));
2976 cm.set(hash(++i, W, buf(1)));
2977 cm.set(hash(++i, W, buf(1), buf(2)));
2978 cm.set(hash(++i, N));
2979 cm.set(hash(++i, N, buf(1)));
2980 cm.set(hash(++i, N, buf(1), buf(2)));
2981 cm.set(hash(++i, (W+N)>>3, buf(1)>>4, buf(2)>>4));
2982 cm.set(hash(++i, buf(1), buf(2)));
2983 cm.set(hash(++i, W, buf(1)-buf(stride+1)));
2984 cm.set(hash(++i, W+buf(1)-buf(stride+1)));
2985 cm.set(hash(++i, N, buf(1)-buf(w+1)));
2986 cm.set(hash(++i, N+buf(1)-buf(w+1)));
2987 cm.set(hash(++i, mean, logvar>>4));
2988 scm1.set(W+N-NW);
2989 scm2.set(W+NE-N);
2990 scm3.set(W*2-WW);
2991 scm4.set(N*2-NN);
2992 scm5.set(NW*2-NNWW);
2993 scm6.set(NE*2-NNEE);
2994 scm7.set(NE+buf(1)-buf(w-stride+1));
2995 scm8.set(N+NE-NNE);
2996 scm9.set(mean>>1|(logvar<<1&0x180));
2997 }
2998 else{
2999 i|=(filterOn)?((0x100|filter)<<8):0;
3000 WWW=buffer(3*stride), WW=buffer(2*stride), W=buffer(stride), NWW=buffer(w+2*stride), NW=buffer(w+stride), N=buffer(w), NE=buffer(w-stride), NEE=buffer(w-2*stride), NNWW=buffer((w+stride)*2), NNW=buffer(w*2+stride), NN=buffer(w*2), NNE=buffer(w*2-stride), NNEE=buffer((w-stride)*2), NNN=buffer(w*3);
3001
3002 cm.set(hash(++i, Clip(W+N-NW)-px, Clip(W+buffer(1)-buffer(stride+1))-px));
3003 cm.set(hash(++i, N-px, Clip(N+buffer(1)-buffer(w+1))-px));
3004 cm.set(hash(++i, Clip(NE+N-NNE)-px, Clip(NE+buffer(1)-buffer(w-stride+1))-px));
3005 cm.set(hash(++i, Clip(NW+N-NNW)-px, Clip(NW+buffer(1)-buffer(w+stride+1))-px));
3006 cm.set(hash(++i, Clip(N*2-NN)-px, LogMeanDiffQt(N,Clip(NN*2-NNN))));
3007 cm.set(hash(++i, Clip(W*2-WW)-px, LogMeanDiffQt(W,Clip(WW*2-WWW))));
3008 cm.set(hash(++i, Clip(N+buffer(1)-buffer(w+1))-px, Clip(N+buffer(2)-buffer(w+2))-px));
3009 cm.set(hash(++i, Clip(W+buffer(1)-buffer(stride+1))-px, Clip(W+buffer(2)-buffer(stride+2))-px));
3010 cm.set(hash(++i, Clip(W+N-NW)-px, Clip(N+NE-NNE)-Clip(N+NW-NNW)));
3011 cm.set(hash(++i, buf(stride+(x<=stride)), buf(1+(x<2)), buf(2+(x<3))));
3012 cm.set(hash(++i, buf(1+(x<2)), px));
3013 cm.set(hash(++i, Clip(N*3-NN*3+NNN)-px));
3014 cm.set(hash(++i, Clip(W*3-WW*3+WWW)-px));
3015 cm.set(hash(++i, (W+Clip(NE*3-NNE*3+buffer(w*3-stride)))/2-px));
3016 }
3017 ctx = (min(color,stride)<<9)|((abs(W-N)>8)<<8)|((W>N)<<7)|((W>NW)<<6)|((abs(N-NW)>8)<<5)|((N>NW)<<4)|((abs(N-NE)>8)<<3)|((N>NE)<<2)|((W>WW)<<1)|(N>NN);
3018 }
3019 }
3020
3021 // Predict next bit
3022 if (x || !isPNG){
3023 cm.mix(m);
3024 if (!isPNG){
3025 scm1.mix(m);
3026 scm2.mix(m);
3027 scm3.mix(m);
3028 scm4.mix(m);
3029 scm5.mix(m);
3030 scm6.mix(m);
3031 scm7.mix(m);
3032 scm8.mix(m);
3033 scm9.mix(m);
3034 scm10.mix(m);
3035 }
3036 static int col=0;
3037 if (++col>=stride*8) col=0;
3038 m.set(5+ctx, 2048+5 );
3039 m.set(col, 32);
3040 m.set((buffer(1+(alpha && !color))>>4)*stride+(x%stride), 64);
3041 m.set(c0, 256);
3042 }
3043 else{
3044 m.add( -2048+((filter>>(7-bpos))&1)*4096 );
3045 m.set(min(4,filter),5);
3046 }
3047}
3048
3049//////////////////////////// im8bitModel /////////////////////////////////
3050
3051// Model for 8-bit image data
3052void im8bitModel(Mixer& m, int w, int gray = 0) {
3053 const int SC=0x20000;
3054 static SmallStationaryContextMap scm1(SC), scm2(SC),
3055 scm3(SC), scm4(SC), scm5(SC), scm6(SC*2);
3056 static ContextMap cm(MEM*4, 62, true);
3057 static int itype=0, id8=1, id9=1;
3058 static int col=0;
3059 // Select nearby pixels as context
3060 if (!bpos) {
3061 assert(w>3);
3062 int mean=buf(1)+buf(w-1)+buf(w)+buf(w+1);
3063 const int var=(sqrbuf(1)+sqrbuf(w-1)+sqrbuf(w)+sqrbuf(w+1)-mean*mean/4)>>2;
3064 mean>>=2;
3065 const int logvar=ilog(var);
3066 int i=0;
3067
3068 const int errr=(buf(2)+buf(w+1)-buf(w));
3069 if(abs(errr-buf(w-1)+buf(1)-buf(w))>255) id8++; else id9++;
3070 if (blpos==0) id8=id9=1,itype=0; // reset on new block
3071 if(blpos%w==0 && blpos>w) itype=(id9/id8)<4; // select model
3072
3073 if (itype==0) { //faster, for smooth images
3074 cm.set(hash(++i,buf(1),0));
3075 cm.set(hash(++i,buf(w-1),0));
3076 cm.set(hash(++i,buf(w-2),0));
3077 cm.set(hash(++i,buf(2),0));
3078 cm.set(hash(++i,buf(w*2-1),0));
3079 cm.set(hash(++i,buf(w-1)+buf(1)-buf(w),buf(1)));
3080 cm.set(hash(++i,buf(w+1),0));
3081 cm.set(hash(++i,buf(w*2-2),0));
3082 cm.set(hash(++i,2*buf(w-1)-buf(w*2-1),buf(1)));
3083 cm.set(hash(++i,2*buf(1)-buf(2),buf(1)));
3084 cm.set(hash(++i,(abs(buf(1)-buf(2))+abs(buf(w-1)-buf(w))+abs(buf(w-1)-buf(w-2))),buf(1)));
3085 cm.set(hash(++i,(abs(buf(1)-buf(w))+abs(buf(w-1)-buf(w*2-1))+abs(buf(w-2)-buf(w*2-2))),buf(1)));
3086 cm.set(hash(++i,abs(errr-buf(w-1)+buf(1)-buf(w)),buf(1)));
3087 cm.set(hash(++i,mean,logvar));
3088 cm.set(hash(++i,2*buf(1)-buf(2),2*buf(w-1)-buf(w*2-1)));
3089 cm.set(hash(++i,(abs(buf(1)-buf(2))+abs(buf(w-1)-buf(w))+abs(buf(w-1)-buf(w-2))), (abs(buf(1)-buf(w))+abs(buf(w-1)-buf(w*2-1))+abs(buf(w-2)-buf(w*2-2)))));
3090 cm.set(hash(++i,buf(1)>>2, buf(w)>>2));
3091 cm.set(hash(++i,buf(1)>>2, buf(2)>>2));
3092 cm.set(hash(++i,buf(w)>>2, buf(w*2)>>2));
3093 cm.set(hash(++i,buf(1)>>2, buf(w-1)>>2));
3094 cm.set(hash(++i,buf(w)>>2, buf(w+1)>>2));
3095 cm.set(hash(++i,buf(w+1)>>2, buf(w+2)>>2));
3096 cm.set(hash(++i,(buf(w+1)+buf(w*2+2))>>1));
3097 cm.set(hash(++i,(buf(w-1)+buf(w*2-2))>>1));
3098 cm.set(hash(++i,2*buf(w-1)-buf(w*2-1),buf(w-1)));
3099 cm.set(hash(++i,2*buf(1)-buf(2),buf(w-1)));
3100 cm.set(hash(++i,buf(w*2-1),buf(w-2),buf(1)));
3101 cm.set(hash(++i,(buf(1)+buf(w))>>1));
3102 cm.set(hash(++i,(buf(1)+buf(2))>>1));
3103 cm.set(hash(++i,(buf(w)+buf(w*2))>>1));
3104 cm.set(hash(++i,(buf(1)+buf(w-1))>>1));
3105 cm.set(hash(++i,(buf(w)+buf(w+1))>>1));
3106 cm.set(hash(++i,(buf(w+1)+buf(w+2))>>1));
3107 cm.set(hash(++i,(buf(w+1)+buf(w*2+2))>>1));
3108 cm.set(hash(++i,(buf(w-1)+buf(w*2-2))>>1));
3109 cm.set(hash(++i,buf(w*2-2),buf(w-1),buf(1)));
3110 cm.set(hash(++i,buf(w+1),buf(w-1),buf(w-2),buf(1)));
3111 cm.set(hash(++i,2*buf(1)-buf(2),buf(w-2),buf(w*2-2)));
3112 cm.set(hash(++i,buf(2),buf(w+1),buf(w),buf(w-1)));
3113 cm.set(hash(++i,buf(w*3), buf(w),buf(1)));
3114 cm.set(hash(++i,buf(w)>>2, buf(3)>>2, buf(w-1)>>2));
3115 cm.set(hash(++i,buf(3)>>2, buf(w-2)>>2, buf(w*2-2)>>2));
3116 cm.set(hash(++i,buf(w)>>2, buf(1)>>2, buf(w-1)>>2));
3117 cm.set(hash(++i,buf(w-1)>>2, buf(w)>>2, buf(w+1)>>2));
3118 cm.set(hash(++i,buf(1)>>2, buf(w-1)>>2, buf(w*2-1)>>2));
3119 } else {
3120 i=512;
3121 cm.set(hash(++i,buf(1),0));
3122 cm.set(hash(++i,buf(2), 0));
3123 cm.set(hash(++i,buf(w), 0));
3124 cm.set(hash(++i,buf(w+1), 0));
3125 cm.set(hash(++i,buf(w-1), 0));
3126 cm.set(hash(++i,(buf(2)+buf(w)-buf(w+1)), 0));
3127 cm.set(hash(++i,(buf(w)+buf(2)-buf(w+1))>>1, 0));
3128 cm.set(hash(++i,(buf(2)+buf(w+1))>>1, 0));
3129 cm.set(hash(++i,(buf(w-1)-buf(w)), buf(1)>>1));
3130 cm.set(hash(++i,(buf(w)-buf(w+1)), buf(1)>>1));
3131 cm.set(hash(++i,(buf(w+1)+buf(2)), buf(1)>>1));
3132 cm.set(hash(++i,buf(1)>>2, buf(w)>>2));
3133 cm.set(hash(++i,buf(1)>>2, buf(2)>>2));
3134 cm.set(hash(++i,buf(w)>>2, buf(w*2)>>2));
3135 cm.set(hash(++i,buf(1)>>2, buf(w-1)>>2));
3136 cm.set(hash(++i,buf(w)>>2, buf(w+1)>>2));
3137 cm.set(hash(++i,buf(w+1)>>2, buf(w+2)>>2));
3138 cm.set(hash(++i,buf(w+1)>>2, buf(w*2+2)>>2));
3139 cm.set(hash(++i,buf(w-1)>>2, buf(w*2-2)>>2));
3140 cm.set(hash(++i,(buf(1)+buf(w))>>1));
3141 cm.set(hash(++i,(buf(1)+buf(2))>>1));
3142 cm.set(hash(++i,(buf(w)+buf(w*2))>>1));
3143 cm.set(hash(++i,(buf(1)+buf(w-1))>>1));
3144 cm.set(hash(++i,(buf(w)+buf(w+1))>>1));
3145 cm.set(hash(++i,(buf(w+1)+buf(w+2))>>1));
3146 cm.set(hash(++i,(buf(w+1)+buf(w*2+2))>>1));
3147 cm.set(hash(++i,(buf(w-1)+buf(w*2-2))>>1));
3148 cm.set(hash(++i,buf(w)>>2, buf(1)>>2, buf(w-1)>>2));
3149 cm.set(hash(++i,buf(w-1)>>2, buf(w)>>2, buf(w+1)>>2));
3150 cm.set(hash(++i,buf(1)>>2, buf(w-1)>>2, buf(w*2-1)>>2));
3151 cm.set(hash(++i,(buf(3)+buf(w))>>1, buf(1)>>2, buf(2)>>2));
3152 cm.set(hash(++i,(buf(2)+buf(1))>>1,(buf(w)+buf(w*2))>>1,buf(w-1)>>2));
3153 cm.set(hash(++i,(buf(2)+buf(1))>>2,(buf(w-1)+buf(w))>>2));
3154 cm.set(hash(++i,(buf(2)+buf(1))>>1,(buf(w)+buf(w*2))>>1));
3155 cm.set(hash(++i,(buf(2)+buf(1))>>1,(buf(w-1)+buf(w*2-2))>>1));
3156 cm.set(hash(++i,(buf(2)+buf(1))>>1,(buf(w+1)+buf(w*2+2))>>1));
3157 cm.set(hash(++i,(buf(w)+buf(w*2))>>1,(buf(w-1)+buf(w*2+2))>>1));
3158 cm.set(hash(++i,(buf(w-1)+buf(w))>>1,(buf(w)+buf(w+1))>>1));
3159 cm.set(hash(++i,(buf(1)+buf(w-1))>>1,(buf(w)+buf(w*2))>>1));
3160 cm.set(hash(++i,(buf(1)+buf(w-1))>>2,(buf(w)+buf(w+1))>>2));
3161 cm.set(hash(++i,(((buf(1)-buf(w-1))>>1)+buf(w))>>2));
3162 cm.set(hash(++i,(((buf(w-1)-buf(w))>>1)+buf(1))>>2));
3163 cm.set(hash(++i,(-buf(1)+buf(w-1)+buf(w))>>2));
3164 cm.set(hash(++i,(buf(1)*2-buf(2))>>1));
3165 cm.set(hash(++i,mean,logvar));
3166 cm.set(hash(++i,(buf(w)*2-buf(w*2))>>1));
3167 cm.set(hash(++i,(buf(1)+buf(w)-buf(w+1))>>1));
3168 cm.set(hash(++i,(buf(4)+buf(3))>>2,(buf(w-1)+buf(w))>>2));
3169 cm.set(hash(++i,(buf(4)+buf(3))>>1,(buf(w)+buf(w*2))>>1));
3170 cm.set(hash(++i,(buf(4)+buf(3))>>1,(buf(w-1)+buf(w*2-2))>>1));
3171 cm.set(hash(++i,(buf(4)+buf(3))>>1,(buf(w+1)+buf(w*2+2))>>1));
3172 cm.set(hash(++i,(buf(4)+buf(1))>>2,(buf(w-3)+buf(w))>>2));
3173 cm.set(hash(++i,(buf(4)+buf(1))>>1,(buf(w)+buf(w*2))>>1));
3174 cm.set(hash(++i,(buf(4)+buf(1))>>1,(buf(w-3)+buf(w*2-3))>>1));
3175 cm.set(hash(++i,(buf(4)+buf(1))>>1,(buf(w+3)+buf(w*2+3))>>1));
3176 cm.set(hash(++i,buf(w)>>2, buf(3)>>2, buf(w-1)>>2));
3177 cm.set(hash(++i,buf(3)>>2, buf(w-2)>>2, buf(w*2-2)>>2));
3178 }
3179
3180 int WWW=buf(3), WW=buf(2), W=buf(1), NW=buf(w+1), N=buf(w), NE=buf(w-1), NEE=buf(w-2), NNW=buf(w*2+1), NN=buf(w*2), NNE=buf(w*2-1), NNN=buf(w*3);
3181
3182 if (gray){
3183
3184 cm.set(hash( ++i, (N+1)>>1, LogMeanDiffQt(N,Clip(NN*2-NNN)) ));
3185 cm.set(hash( ++i, (W+1)>>1, LogMeanDiffQt(W,Clip(WW*2-WWW)) ));
3186 cm.set(hash( ++i, Clamp4(W+N-NW,W,NW,N,NE), LogMeanDiffQt(Clip(N+NE-NNE), Clip(N+NW-NNW))));
3187 cm.set(hash( ++i, (NNN+N+4)/8, Clip(N*3-NN*3+NNN)>>1 ));
3188 cm.set(hash( ++i, (WWW+W+4)/8, Clip(W*3-WW*3+WWW)>>1 ));
3189 }
3190 else{
3191
3192 cm.set(hash( ++i, W, NEE ));
3193 cm.set(hash( ++i, WW, NN ));
3194 cm.set(hash( ++i, W, WWW ));
3195 cm.set(hash( ++i, N, NNN ));
3196 cm.set(hash( ++i, NNW, NN ));
3197 }
3198
3199 scm1.set((buf(1)+buf(w))>>1);
3200 scm2.set((buf(1)+buf(w)-buf(w+1))>>1);
3201 scm3.set((buf(1)*2-buf(2))>>1);
3202 scm4.set((buf(w)*2-buf(w*2))>>1);
3203 scm5.set((buf(1)+buf(w)-buf(w-1))>>1);
3204 scm6.set(mean>>1|(logvar<<1&0x180));
3205 }
3206
3207 // Predict next bit
3208 scm1.mix(m);
3209 scm2.mix(m);
3210 scm3.mix(m);
3211 scm4.mix(m);
3212 scm5.mix(m);
3213 scm6.mix(m);
3214 cm.mix(m);
3215 if (++col>=8) col=0; // reset after every 24 columns?
3216 m.set(4, 264);
3217 m.set(col, 256);
3218 m.set((buf(w)+buf(1))>>4, 256);
3219 m.set(c0, 256);
3220}
3221
3222//////////////////////////// im4bitModel /////////////////////////////////
3223/*
3224 Model for 4-bit image data
3225
3226 Changelog:
3227 (31/08/2017) v103: Initial release by Marcio Pais
3228*/
3229
3230// Model for 4-bit image data
3231void im4bitModel(Mixer& m, int w) {
3232 static HashTable<16> t(MEM/2);
3233 const int S=11; // number of contexts
3234 static U8* cp[S];
3235 static StateMap sm[S];
3236 static U8 WW=0, W=0, NWW=0, NW=0, N=0, NE=0, NEE=0, NNWW = 0, NNW=0, NN=0, NNE=0, NNEE=0;
3237 static int col=0, line=0, run=0, prevColor=0, px=0;
3238 int i;
3239 if (!cp[0]){
3240 for (i=0;i<S;i++)
3241 cp[i]=t[263*i]+1;
3242 }
3243 for (i=0;i<S;i++)
3244 *cp[i]=nex(*cp[i],y);
3245
3246 if (!bpos || bpos==4){
3247 WW=W, NWW=NW, NW=N, N=NE, NE=NEE, NNWW=NWW, NNW=NN, NN=NNE, NNE=NNEE;
3248 if (!bpos)
3249 W=c4&0xF, NEE=buf(w-1)>>4, NNEE=buf(w*2-1)>>4;
3250 else
3251 W=c0&0xF, NEE=buf(w-1)&0xF, NNEE=buf(w*2-1)&0xF;
3252 run=(W!=WW || !col)?(prevColor=WW,0):min(0xFFF,run+1);
3253 px=1, i=0;
3254
3255 cp[i++]=t[hash(W,NW,N)]+1;
3256 cp[i++]=t[hash(N, min(0xFFF, col/8))]+1;
3257 cp[i++]=t[hash(W,NW,N,NN,NE)]+1;
3258 cp[i++]=t[hash(W, N, NE+NNE*16, NEE+NNEE*16)]+1;
3259 cp[i++]=t[hash(W, N, NW+NNW*16, NWW+NNWW*16)]+1;
3260 cp[i++]=t[hash(W, ilog2(run+1), prevColor, col/max(1,w/2) )]+1;
3261 cp[i++]=t[hash(NE, min(0x3FF, (col+line)/max(1,w*8)))]+1;
3262 cp[i++]=t[hash(NW, (col-line)/max(1,w*8))]+1;
3263 cp[i++]=t[hash(WW*16+W,NN*16+N,NNWW*16+NW)]+1;
3264 cp[i++]=t[N+NN*16]+1;
3265 cp[i++]=t[-1]+1;
3266
3267 col*=(++col)<w*2;
3268 line+=(!col);
3269 }
3270 else{
3271 px+=px+y;
3272 int j=(y+1)<<(bpos&3);
3273 for (i=0;i<S;i++)
3274 cp[i]+=j;
3275 }
3276
3277 // predict
3278 for (i=0; i<S; i++)
3279 m.add(stretch(sm[i].p(*cp[i])));
3280
3281 m.set(W*16+px, 256);
3282 m.set(min(31,col/max(1,w/16))+N*32, 512);
3283 m.set((bpos&3)+4*W+64*min(7,ilog2(run+1)), 512);
3284 m.set(W+NE*16+(bpos&3)*256, 1024);
3285 m.set(px, 16);
3286 m.set(0,1);
3287}
3288
3289//////////////////////////// im1bitModel /////////////////////////////////
3290
3291// Model for 1-bit image data
3292
3293void im1bitModel(Mixer& m, int w) {
3294 static U32 r0, r1, r2, r3; // last 4 rows, bit 8 is over current pixel
3295 static Array<U8> t(0x23000); // model: cxt -> state
3296 const int N=11; // number of contexts
3297 static int cxt[N]; // contexts
3298 static StateMap sm[N];
3299
3300 // update the model
3301 int i;
3302 for (i=0; i<N; ++i)
3303 t[cxt[i]]=nex(t[cxt[i]],y);
3304
3305 // update the contexts (pixels surrounding the predicted one)
3306 r0+=r0+y;
3307 r1+=r1+((buf(w-1)>>(7-bpos))&1);
3308 r2+=r2+((buf(w+w-1)>>(7-bpos))&1);
3309 r3+=r3+((buf(w+w+w-1)>>(7-bpos))&1);
3310 cxt[0]=(r0&0x7)|(r1>>4&0x38)|(r2>>3&0xc0);
3311 cxt[1]=0x100+((r0&1)|(r1>>4&0x3e)|(r2>>2&0x40)|(r3>>1&0x80));
3312 cxt[2]=0x200+((r0&1)|(r1>>4&0x1d)|(r2>>1&0x60)|(r3&0xC0));
3313 cxt[3]=0x300+(y|((r0<<1)&4)|((r1>>1)&0xF0)|((r2>>3)&0xA));
3314 cxt[4]=0x400+((r0>>4&0x2AC)|(r1&0xA4)|(r2&0x349)|(!(r3&0x14D)));
3315 cxt[5]=0x800+(y|((r1>>4)&0xE)|((r2>>1)&0x70)|((r3<<2)&0x380));
3316 cxt[6]=0xC00+(((r1&0x30)^(r3&0x0c0c))|(r0&3));
3317 cxt[7]=0x1000+((!(r0&0x444))|(r1&0xC0C)|(r2&0xAE3)|(r3&0x51C));
3318 cxt[8]=0x2000+((r0&7)|((r1>>1)&0x3F8)|((r2<<5)&0xC00));
3319 cxt[9]=0x3000+((r0&0x3f)^(r1&0x3ffe)^(r2<<2&0x7f00)^(r3<<5&0xf800));
3320 cxt[10]=0x13000+((r0&0x3e)^(r1&0x0c0c)^(r2&0xc800));
3321
3322 // predict
3323 for (i=0; i<N; ++i) m.add(stretch(sm[i].p(t[cxt[i]])));
3324}
3325
3326
3327
3328//////////////////////////// dmcModel //////////////////////////
3329
3330// Model using DMC. The bitwise context is represented by a state graph,
3331// initilaized to a bytewise order 1 model as in
3332// http://plg.uwaterloo.ca/~ftp/dmc/dmc.c but with the following difference:
3333// - It uses integer arithmetic.
3334// - The threshold for cloning a state increases as memory is used up.
3335// - Each state maintains both a 0,1 count and a bit history (as in a
3336// context model). The 0,1 count is best for stationary data, and the
3337// bit history for nonstationary data. The bit history is mapped to
3338// a probability adaptively using a StateMap. The two computed probabilities
3339// are combined.
3340// - When memory is used up the state graph is reinitialized to a bytewise
3341// order 1 context as in the original DMC. However, the bit histories
3342// are not cleared.
3343
3344struct DMCNode { // 12 bytes
3345 unsigned int nx[2]; // next pointers
3346 U8 state; // bit history
3347 unsigned int c0:12, c1:12; // counts * 256
3348};
3349
3350void dmcModel(Mixer& m) {
3351 static int top=0, curr=0; // allocated, current node
3352 static Array<DMCNode> t(MEM*2); // state graph
3353 static StateMap sm;
3354 static int threshold=256;
3355
3356 // clone next state
3357 if (top>0 && top<t.size()) {
3358 int next=t[curr].nx[y];
3359 int n=y?t[curr].c1:t[curr].c0;
3360 int nn=t[next].c0+t[next].c1;
3361 if (n>=threshold*2 && nn-n>=threshold*3) {
3362 int r=n*4096/nn;
3363 assert(r>=0 && r<=4096);
3364 t[next].c0 -= t[top].c0 = t[next].c0*r>>12;
3365 t[next].c1 -= t[top].c1 = t[next].c1*r>>12;
3366 t[top].nx[0]=t[next].nx[0];
3367 t[top].nx[1]=t[next].nx[1];
3368 t[top].state=t[next].state;
3369 t[curr].nx[y]=top;
3370 ++top;
3371 if (top==MEM*2) threshold=512;
3372 if (top==MEM*3) threshold=768;
3373 }
3374 }
3375
3376 // Initialize to a bytewise order 1 model at startup or when flushing memory
3377 if (top==t.size() && bpos==1) top=0;
3378 if (top==0) {
3379 assert(t.size()>=65536);
3380 for (int i=0; i<256; ++i) {
3381 for (int j=0; j<256; ++j) {
3382 if (i<127) {
3383 t[j*256+i].nx[0]=j*256+i*2+1;
3384 t[j*256+i].nx[1]=j*256+i*2+2;
3385 }
3386 else {
3387 t[j*256+i].nx[0]=(i-127)*256;
3388 t[j*256+i].nx[1]=(i+1)*256;
3389 }
3390 t[j*256+i].c0=192;
3391 t[j*256+i].c1=192;
3392 }
3393 }
3394 top=65536;
3395 curr=0;
3396 threshold=256;
3397 }
3398
3399 // update count, state
3400 if (y) {
3401 if (t[curr].c1<3800) t[curr].c1+=256;
3402 }
3403 else if (t[curr].c0<3800) t[curr].c0+=256;
3404 t[curr].state=nex(t[curr].state, y);
3405 curr=t[curr].nx[y];
3406
3407 // predict
3408 const int pr1=sm.p(t[curr].state);
3409 const int n1=t[curr].c1;
3410 const int n0=t[curr].c0;
3411 const int pr2=(n1+5)*4096/(n0+n1+10);
3412 m.add(stretch(pr1));
3413 m.add(stretch(pr2));
3414}
3415
3416
3417//////////////////////////// exeModel /////////////////////////
3418
3419// Model x86 code. The contexts are sparse containing only those
3420// bits relevant to parsing (2 prefixes, opcode, and mod and r/m fields
3421// of modR/M byte).
3422
3423inline int pref(int i) { return (buf(i)==0x0f)+2*(buf(i)==0x66)+3*(buf(i)==0x67); }
3424
3425// Get context at buf(i) relevant to parsing 32-bit x86 code
3426U32 execxt(int i) {
3427 int prefix=0, opcode=0, modrm=0, sib=0;
3428 if (i) prefix+=4*pref(i--);
3429 if (i) prefix+=pref(i--);
3430 if (i) opcode+=buf(i--);
3431 if (i) modrm+=buf(i--)&0xc7;
3432 if (i&&((modrm&0x07)==4)&&(modrm<0xc0)) sib=buf(i)&0xc0;
3433 return hash(prefix,opcode,modrm,sib);
3434}
3435//////////////////////////// contextModel //////////////////////
3436
3437
3438typedef enum {DEFAULT, JPEG, HDR, IMAGE1, IMAGE4, IMAGE8, IMAGE8GRAY, IMAGE24, IMAGE32, AUDIO, EXE, CD, ZLIB, BASE64, GIF} Filetype;
3439
3440
3441// This combines all the context models with a Mixer.
3442
3443int contextModel2() {
3444 static ContextMap cm(MEM*32, 22);
3445 static Mixer m(76, 1288, 5);
3446 static U32 cxt[16]; // order 0-11 contexts
3447 static Filetype ft2,filetype=DEFAULT;
3448 static int size=0; // bytes remaining in block
3449 static int info=0; // image width or audio type
3450
3451 // Parse filetype and size
3452 if (bpos==0) {
3453 --size;
3454 ++blpos;
3455 if (size==-1) ft2=(Filetype)buf(1);
3456 if (size==-5 && ft2!=IMAGE1 && ft2!=IMAGE4 && ft2!=IMAGE8 && ft2!=IMAGE8GRAY && ft2!=IMAGE24 && ft2!=IMAGE32 && ft2!=AUDIO) {
3457 size=buf(4)<<24|buf(3)<<16|buf(2)<<8|buf(1);
3458 if (ft2==CD || ft2==ZLIB || ft2==BASE64 || ft2==GIF) size=0;
3459 blpos=0;
3460 }
3461 if (size==-9) {
3462 size=buf(8)<<24|buf(7)<<16|buf(6)<<8|buf(5);
3463 info=buf(4)<<24|buf(3)<<16|buf(2)<<8|buf(1);
3464 blpos=0;
3465 }
3466 if (!blpos) filetype=ft2;
3467 if (size==0) filetype=DEFAULT;
3468 }
3469
3470 m.update();
3471
3472 // Test for special file types
3473 int ismatch=ilog(matchModel(m)); // Length of longest matching context +4
3474
3475 if (filetype==IMAGE1) im1bitModel(m, info); // +9
3476 if (filetype==IMAGE4) return im4bitModel(m, info), m.p();
3477 if (filetype==IMAGE8) return im8bitModel(m, info), m.p();
3478 if (filetype==IMAGE8GRAY) return im8bitModel(m, info, 1), m.p();
3479 if (filetype==IMAGE24) return im24bitModel(m, info), m.p();
3480 if (filetype==IMAGE32) return im24bitModel(m, info, 1), m.p();
3481 if (filetype==AUDIO) return wavModel(m, info), m.p();
3482 if (filetype==JPEG) if (jpegModel(m)) return m.p();
3483
3484
3485 dmcModel(m); // +2
3486
3487 // Normal model
3488 static U32 t1[256];
3489 static U16 t2[0x10000];
3490 static U32 word0=0, word1=0;
3491 static U32 mask = 0, mask2 = 0;
3492
3493 if (bpos==0) {
3494 int i;
3495 for (i=14; i>0; --i) // update order 0-11 context hashes
3496 cxt[i]=hash(cxt[i-1], (c4&255));
3497
3498 cm.set(0);
3499 cm.set(c4&0x000000ff);
3500 cm.set(c4&0x0000ffff);
3501 cm.set(c4&0x00ffffff);
3502 cm.set(c4);
3503 cm.set(cxt[5]);
3504 cm.set(cxt[6]);
3505 cm.set(cxt[14]);
3506
3507 // sparse model
3508 cm.set(c4&0xf8f8c0ff);
3509 cm.set(c4&0x00e0e0e0);
3510 cm.set(c4&0xffc0ff80);
3511 cm.set(ismatch|(c4&0xffff0000));
3512 cm.set(ismatch|(c4&0x0000ff00));
3513 cm.set(ismatch|(c4&0x00ff0000));
3514
3515 int fl = 0;
3516 if ((c4&0xff) != 0) {
3517 if (isalpha(c4&0xff)) fl = 1;
3518 else if (ispunct(c4&0xff)) fl = 2;
3519 else if (isspace(c4&0xff)) fl = 3;
3520 else if ((c4&0xff) == 0xff) fl = 4;
3521 else if ((c4&0xff) < 16) fl = 5;
3522 else if ((c4&0xff) < 64) fl = 6;
3523 else fl = 7;
3524 }
3525 mask = (mask<<3)|fl;
3526 cm.set(mask);
3527 mask2 = (mask2<<3)|((mask>>27) & 7);
3528 cm.set(hash(mask<<5,mask2<<2));
3529
3530
3531 // indirect model
3532 U32& ic1r=t1[buf(2)];
3533 ic1r=ic1r<<8|buf(1);
3534 U16& ic2r=t2[(buf(3)<<8)|buf(2)];
3535 ic2r=ic2r<<8|buf(1);
3536 const U32 ic1=(buf(1))|t1[buf(1)]<<8;
3537 const U32 ic2=((buf(2)<<8)|buf(1))|t2[(buf(2)<<8)|buf(1)]<<16;
3538
3539 cm.set((ic1>>8)&((1<<16)-1));
3540 cm.set((ic2>>16)&((1<<8)-1));
3541 cm.set(ic1&((1<<16)-1));
3542 cm.set(ic2&((1<<24)-1));
3543
3544 // word/exe model
3545 if (filetype==EXE)
3546 {
3547 cm.set(execxt(4));
3548 cm.set(execxt(5));
3549 }
3550 else
3551 {
3552 int c = buf(1);
3553 if (c>='A' && c<='Z') c+='a'-'A';
3554 if ((c>='a' && c<='z') || c>=128) word0=hash(word0, c);
3555 else if (word0) word1=word0,word0=0;
3556 cm.set(word0);
3557 cm.set(hash(word0, word1));
3558 }
3559 }
3560
3561
3562 int order=cm.mix(m); // +44
3563
3564 U32 c1=buf(1), c2=buf(2);
3565
3566 m.set(c1+8, 264);
3567 m.set(c0, 256);
3568 m.set(order+16*(c1>32)+32*(bpos==0)+64*(c1==c2)+128*(filetype==EXE), 256);
3569 m.set(c2, 256);
3570 m.set(ismatch, 256);
3571 int pr=m.p();
3572
3573 return pr;
3574}
3575
3576
3577//////////////////////////// Predictor /////////////////////////
3578
3579// A Predictor estimates the probability that the next bit of
3580// uncompressed data is 1. Methods:
3581// p() returns P(1) as a 12 bit number (0-4095).
3582// update(y) trains the predictor with the actual bit (0 or 1).
3583
3584class Predictor {
3585 int pr; // next prediction
3586public:
3587 Predictor();
3588 int p() const {assert(pr>=0 && pr<4096); return pr;}
3589 void update();
3590};
3591
3592Predictor::Predictor(): pr(2048) {}
3593
3594void Predictor::update() {
3595 static APM1 a(256), a1(0x10000), a2(0x10000);
3596
3597 // Update global context: pos, bpos, c0, c4, buf
3598 c0+=c0+y;
3599 if (c0>=256) {
3600 buf[pos++]=c0;
3601 c4=(c4<<8)+c0-256;
3602 c0=1;
3603 }
3604 bpos=(bpos+1)&7;
3605
3606 // Filter the context model with APMs
3607 int pr0=contextModel2();
3608 pr=a.p(pr0, c0);
3609
3610 int pr1=a1.p(pr0, c0+256*buf(1));
3611 int pr2=a2.p(pr0, (hash(bpos, buf(1), buf(2)))&0xffff);
3612
3613 pr=(pr*5+pr1*15+pr2*12+16)>>5;
3614}
3615
3616//////////////////////////// Encoder ////////////////////////////
3617
3618// An Encoder does arithmetic encoding. Methods:
3619// Encoder(COMPRESS, f) creates encoder for compression to archive f, which
3620// must be open past any header for writing in binary mode.
3621// Encoder(DECOMPRESS, f) creates encoder for decompression from archive f,
3622// which must be open past any header for reading in binary mode.
3623// code(i) in COMPRESS mode compresses bit i (0 or 1) to file f.
3624// code() in DECOMPRESS mode returns the next decompressed bit from file f.
3625// Global y is set to the last bit coded or decoded by code().
3626// compress(c) in COMPRESS mode compresses one byte.
3627// decompress() in DECOMPRESS mode decompresses and returns one byte.
3628// flush() should be called exactly once after compression is done and
3629// before closing f. It does nothing in DECOMPRESS mode.
3630// size() returns current length of archive
3631// setFile(f) sets alternate source to FILE* f for decompress() in COMPRESS
3632// mode (for testing transforms).
3633// If level (global) is 0, then data is stored without arithmetic coding.
3634
3635typedef enum {COMPRESS, DECOMPRESS} Mode;
3636class Encoder {
3637private:
3638 Predictor predictor;
3639 const Mode mode; // Compress or decompress?
3640 FILE* archive; // Compressed data file
3641 U32 x1, x2; // Range, initially [0, 1), scaled by 2^32
3642 U32 x; // Decompress mode: last 4 input bytes of archive
3643 FILE *alt; // decompress() source in COMPRESS mode
3644 float p1, p2;
3645
3646 // Compress bit y or return decompressed bit
3647 int code(int i=0) {
3648 int p=predictor.p();
3649 assert(p>=0 && p<4096);
3650 p+=p<2048;
3651 U32 xmid=x1 + ((x2-x1)>>12)*p + (((x2-x1)&0xfff)*p>>12);
3652 assert(xmid>=x1 && xmid<x2);
3653 if (mode==DECOMPRESS) y=x<=xmid; else y=i;
3654 y ? (x2=xmid) : (x1=xmid+1);
3655 predictor.update();
3656 while (((x1^x2)&0xff000000)==0) { // pass equal leading bytes of range
3657 if (mode==COMPRESS) putc(x2>>24, archive);
3658 x1<<=8;
3659 x2=(x2<<8)+255;
3660 if (mode==DECOMPRESS) x=(x<<8)+(getc(archive)&255); // EOF is OK
3661 }
3662 return y;
3663 }
3664
3665public:
3666 Encoder(Mode m, FILE* f);
3667 Mode getMode() const {return mode;}
3668 long size() const {return ftell(archive);} // length of archive so far
3669 void flush(); // call this when compression is finished
3670 void setFile(FILE* f) {alt=f;}
3671
3672 // Compress one byte
3673 void compress(int c) {
3674 assert(mode==COMPRESS);
3675 if (level==0)
3676 putc(c, archive);
3677 else
3678 for (int i=7; i>=0; --i)
3679 code((c>>i)&1);
3680 }
3681
3682 // Decompress and return one byte
3683 int decompress() {
3684 if (mode==COMPRESS) {
3685 assert(alt);
3686 return getc(alt);
3687 }
3688 else if (level==0)
3689 return getc(archive);
3690 else {
3691 int c=0;
3692 for (int i=0; i<8; ++i)
3693 c+=c+code();
3694 return c;
3695 }
3696 }
3697
3698 void set_status_range(float perc1, float perc2) { p1=perc1; p2=perc2; }
3699 void print_status(int n, int size) {
3700 printf("%6.2f%%\b\b\b\b\b\b\b", (p1+(p2-p1)*n/(size+1))*100), fflush(stdout);
3701 }
3702 void print_status() {
3703 printf("%6.2f%%\b\b\b\b\b\b\b", float(size())/(p2+1)*100), fflush(stdout);
3704 }
3705};
3706
3707Encoder::Encoder(Mode m, FILE* f):
3708 mode(m), archive(f), x1(0), x2(0xffffffff), x(0), alt(0) {
3709 if (mode==DECOMPRESS) {
3710 long start=size();
3711 fseek(f, 0, SEEK_END);
3712 set_status_range(0, size());
3713 fseek(archive, start, SEEK_SET);
3714 }
3715 if (level>0 && mode==DECOMPRESS) { // x = first 4 bytes of archive
3716 for (int i=0; i<4; ++i)
3717 x=(x<<8)+(getc(archive)&255);
3718 }
3719 for (int i=0; i<1024; ++i)
3720 dt[i]=16384/(i+i+3);
3721
3722}
3723
3724void Encoder::flush() {
3725 if (mode==COMPRESS && level>0)
3726 putc(x1>>24, archive); // Flush first unequal byte of range
3727}
3728
3729/////////////////////////// Filters /////////////////////////////////
3730//
3731// Before compression, data is encoded in blocks with the following format:
3732//
3733// <type> <size> <encoded-data>
3734//
3735// Type is 1 byte (type Filetype): DEFAULT=0, JPEG, EXE
3736// Size is 4 bytes in big-endian format.
3737// Encoded-data decodes to <size> bytes. The encoded size might be
3738// different. Encoded data is designed to be more compressible.
3739//
3740// void encode(FILE* in, FILE* out, int n);
3741//
3742// Reads n bytes of in (open in "rb" mode) and encodes one or
3743// more blocks to temporary file out (open in "wb+" mode).
3744// The file pointer of in is advanced n bytes. The file pointer of
3745// out is positioned after the last byte written.
3746//
3747// en.setFile(FILE* out);
3748// int decode(Encoder& en);
3749//
3750// Decodes and returns one byte. Input is from en.decompress(), which
3751// reads from out if in COMPRESS mode. During compression, n calls
3752// to decode() must exactly match n bytes of in, or else it is compressed
3753// as type 0 without encoding.
3754//
3755// Filetype detect(FILE* in, int n, Filetype type);
3756//
3757// Reads n bytes of in, and detects when the type changes to
3758// something else. If it does, then the file pointer is repositioned
3759// to the start of the change and the new type is returned. If the type
3760// does not change, then it repositions the file pointer n bytes ahead
3761// and returns the old type.
3762//
3763// For each type X there are the following 2 functions:
3764//
3765// void encode_X(FILE* in, FILE* out, int n, ...);
3766//
3767// encodes n bytes from in to out.
3768//
3769// int decode_X(Encoder& en);
3770//
3771// decodes one byte from en and returns it. decode() and decode_X()
3772// maintain state information using static variables.
3773#define bswap(x) \
3774+ ((((x) & 0xff000000) >> 24) | \
3775+ (((x) & 0x00ff0000) >> 8) | \
3776+ (((x) & 0x0000ff00) << 8) | \
3777+ (((x) & 0x000000ff) << 24))
3778
3779#define IMG_DET(type,start_pos,header_len,width,height) return dett=(type),\
3780deth=(header_len),detd=(width)*(height),info=(width),\
3781fseek(in, start+(start_pos), SEEK_SET),HDR
3782
3783#define AUD_DET(type,start_pos,header_len,data_len,wmode) return dett=(type),\
3784deth=(header_len),detd=(data_len),info=(wmode),\
3785fseek(in, start+(start_pos), SEEK_SET),HDR
3786
3787
3788// Function ecc_compute(), edc_compute() and eccedc_init() taken from
3789// ** UNECM - Decoder for ECM (Error Code Modeler) format.
3790// ** Version 1.0
3791// ** Copyright (C) 2002 Neill Corlett
3792
3793/* LUTs used for computing ECC/EDC */
3794static U8 ecc_f_lut[256];
3795static U8 ecc_b_lut[256];
3796static U32 edc_lut[256];
3797static int luts_init=0;
3798
3799void eccedc_init(void) {
3800 if (luts_init) return;
3801 U32 i, j, edc;
3802 for(i = 0; i < 256; i++) {
3803 j = (i << 1) ^ (i & 0x80 ? 0x11D : 0);
3804 ecc_f_lut[i] = j;
3805 ecc_b_lut[i ^ j] = i;
3806 edc = i;
3807 for(j = 0; j < 8; j++) edc = (edc >> 1) ^ (edc & 1 ? 0xD8018001 : 0);
3808 edc_lut[i] = edc;
3809 }
3810 luts_init=1;
3811}
3812
3813void ecc_compute(U8 *src, U32 major_count, U32 minor_count, U32 major_mult, U32 minor_inc, U8 *dest) {
3814 U32 size = major_count * minor_count;
3815 U32 major, minor;
3816 for(major = 0; major < major_count; major++) {
3817 U32 index = (major >> 1) * major_mult + (major & 1);
3818 U8 ecc_a = 0;
3819 U8 ecc_b = 0;
3820 for(minor = 0; minor < minor_count; minor++) {
3821 U8 temp = src[index];
3822 index += minor_inc;
3823 if(index >= size) index -= size;
3824 ecc_a ^= temp;
3825 ecc_b ^= temp;
3826 ecc_a = ecc_f_lut[ecc_a];
3827 }
3828 ecc_a = ecc_b_lut[ecc_f_lut[ecc_a] ^ ecc_b];
3829 dest[major ] = ecc_a;
3830 dest[major + major_count] = ecc_a ^ ecc_b;
3831 }
3832}
3833
3834U32 edc_compute(const U8 *src, int size) {
3835 U32 edc = 0;
3836 while(size--) edc = (edc >> 8) ^ edc_lut[(edc ^ (*src++)) & 0xFF];
3837 return edc;
3838}
3839
3840int expand_cd_sector(U8 *data, int a, int test) {
3841 U8 d2[2352];
3842 eccedc_init();
3843 d2[0]=d2[11]=0;
3844 for (int i=1; i<11; i++) d2[i]=255;
3845 int mode=(data[15]!=1?2:1);
3846 int form=(data[15]==3?2:1);
3847 if (a==-1) for (int i=12; i<15; i++) d2[i]=data[i]; else {
3848 int c1=(a&15)+((a>>4)&15)*10;
3849 int c2=((a>>8)&15)+((a>>12)&15)*10;
3850 int c3=((a>>16)&15)+((a>>20)&15)*10;
3851 c1=(c1+1)%75;
3852 if (c1==0) {
3853 c2=(c2+1)%60;
3854 if (c2==0) c3++;
3855 }
3856 d2[12]=(c3%10)+16*(c3/10);
3857 d2[13]=(c2%10)+16*(c2/10);
3858 d2[14]=(c1%10)+16*(c1/10);
3859 }
3860 d2[15]=mode;
3861 if (mode==2) for (int i=16; i<24; i++) d2[i]=data[i-4*(i>=20)];
3862 if (form==1) {
3863 if (mode==2) {
3864 d2[1]=d2[12],d2[2]=d2[13],d2[3]=d2[14];
3865 d2[12]=d2[13]=d2[14]=d2[15]=0;
3866 } else {
3867 for(int i=2068; i<2076; i++) d2[i]=0;
3868 }
3869 for (int i=16+8*(mode==2); i<2064+8*(mode==2); i++) d2[i]=data[i];
3870 U32 edc=edc_compute(d2+16*(mode==2), 2064-8*(mode==2));
3871 for (int i=0; i<4; i++) d2[2064+8*(mode==2)+i]=(edc>>(8*i))&0xff;
3872 ecc_compute(d2+12, 86, 24, 2, 86, d2+2076);
3873 ecc_compute(d2+12, 52, 43, 86, 88, d2+2248);
3874 if (mode==2) {
3875 d2[12]=d2[1],d2[13]=d2[2],d2[14]=d2[3],d2[15]=2;
3876 d2[1]=d2[2]=d2[3]=255;
3877 }
3878 }
3879 for (int i=0; i<2352; i++) if (d2[i]!=data[i] && test) form=2;
3880 if (form==2) {
3881 for (int i=24; i<2348; i++) d2[i]=data[i];
3882 U32 edc=edc_compute(d2+16, 2332);
3883 for (int i=0; i<4; i++) d2[2348+i]=(edc>>(8*i))&0xff;
3884 }
3885 for (int i=0; i<2352; i++) if (d2[i]!=data[i] && test) return 0; else data[i]=d2[i];
3886 return mode+form-1;
3887}
3888
3889int parse_zlib_header(int header) {
3890 switch (header) {
3891 case 0x2815 : return 0; case 0x2853 : return 1; case 0x2891 : return 2; case 0x28cf : return 3;
3892 case 0x3811 : return 4; case 0x384f : return 5; case 0x388d : return 6; case 0x38cb : return 7;
3893 case 0x480d : return 8; case 0x484b : return 9; case 0x4889 : return 10; case 0x48c7 : return 11;
3894 case 0x5809 : return 12; case 0x5847 : return 13; case 0x5885 : return 14; case 0x58c3 : return 15;
3895 case 0x6805 : return 16; case 0x6843 : return 17; case 0x6881 : return 18; case 0x68de : return 19;
3896 case 0x7801 : return 20; case 0x785e : return 21; case 0x789c : return 22; case 0x78da : return 23;
3897 }
3898 return -1;
3899}
3900int zlib_inflateInit(z_streamp strm, int zh) {
3901 if (zh==-1) return inflateInit2(strm, -MAX_WBITS); else return inflateInit(strm);
3902}
3903
3904bool IsGrayscalePalette(FILE* in, int n = 256, int isRGBA = 0){
3905 long offset = ftell(in);
3906 int stride = 3+isRGBA, res = (n>0)<<8, order=1;
3907 for (int i = 0; (i < n*stride) && (res>>8); i++) {
3908 int b = getc(in);
3909 if (b==EOF){
3910 res = 0;
3911 break;
3912 }
3913 if (!i) {
3914 res = 0x100|b;
3915 order = 1-2*(b>0);
3916 continue;
3917 }
3918
3919 //"j" is the index of the current byte in this color entry
3920 int j = i%stride;
3921 if (!j)
3922 res = (res&((b-(res&0xFF)==order)<<8))|b; // load first component of this entry
3923 else if (j==3)
3924 res&=((!b || (b==0xFF))*0x1FF); // alpha/attribute component must be zero or 0xFF
3925 else
3926 res&=((b==(res&0xFF))<<9)-1;
3927 }
3928 fseek(in, offset, SEEK_SET);
3929 return res>>8;
3930}
3931
3932// Detect blocks
3933Filetype detect(FILE* in, int n, Filetype type, int &info) {
3934 U32 buf3=0, buf2=0, buf1=0, buf0=0; // last 16 bytes
3935 long start=ftell(in);
3936
3937 // For EXE detection
3938 Array<int> abspos(256), // CALL/JMP abs. addr. low byte -> last offset
3939 relpos(256); // CALL/JMP relative addr. low byte -> last offset
3940 int e8e9count=0; // number of consecutive CALL/JMPs
3941 int e8e9pos=0; // offset of first CALL or JMP instruction
3942 int e8e9last=0; // offset of most recent CALL or JMP
3943
3944 int soi=0, sof=0, sos=0, app=0; // For JPEG detection - position where found
3945 int wavi=0,wavsize=0,wavch=0,wavbps=0,wavm=0,wavtype=0,wavlen=0,wavlist=0; // For WAVE detection
3946 int aiff=0,aiffm=0,aiffs=0; // For AIFF detection
3947 int s3mi=0,s3mno=0,s3mni=0; // For S3M detection
3948 int bmp=0,imgbpp=0,bmpx=0,bmpy=0,bmpof=0,bmps=0,hdrless=0; // For BMP detection
3949 int rgbi=0,rgbx=0,rgby=0; // For RGB detection
3950 int tga=0,tgax=0,tgay=0,tgaz=0,tgat=0; // For TGA detection
3951 int pgm=0,pgmcomment=0,pgmw=0,pgmh=0,pgm_ptr=0,pgmc=0,pgmn=0,pamatr=0,pamd=0; // For PBM, PGM, PPM, PAM detection
3952 char pgm_buf[32];
3953 int cdi=0,cda=0,cdm=0; // For CD sectors detection
3954 U32 cdf=0;
3955 unsigned char zbuf[32], zin[1<<16], zout[1<<16]; // For ZLIB stream detection
3956 int zbufpos=0,zzippos=-1;
3957 int pdfim=0,pdfimw=0,pdfimh=0,pdfimb=0,pdfgray=0,pdfimp=0;
3958 int b64s=0,b64i=0,b64line=0,b64nl=0; // For base64 detection
3959 int gif=0,gifa=0,gifi=0,gifw=0,gifc=0,gifb=0; // For GIF detection
3960 int png=0, pngw=0, pngh=0, pngbps=0, pngtype=0, lastchunk=0, nextchunk=0; // For PNG detection
3961
3962 // For image detection
3963 static int deth=0,detd=0; // detected header/data size in bytes
3964 static Filetype dett; // detected block type
3965 if (deth) return fseek(in, start+deth, SEEK_SET),deth=0,dett;
3966 else if (detd) return fseek(in, start+detd, SEEK_SET),detd=0,DEFAULT;
3967
3968 for (int i=0; i<n; ++i) {
3969 int c=getc(in);
3970 if (c==EOF) return (Filetype)(-1);
3971 buf3=buf3<<8|buf2>>24;
3972 buf2=buf2<<8|buf1>>24;
3973 buf1=buf1<<8|buf0>>24;
3974 buf0=buf0<<8|c;
3975
3976 // detect PNG images
3977 if (!png && buf3==0x89504E47 /*%PNG*/ && buf2==0x0D0A1A0A && buf1==0x0000000D && buf0==0x49484452) png=i, lastchunk=buf3;
3978 if (png){
3979 const int p=i-png;
3980 if (p==12){
3981 pngw = buf2;
3982 pngh = buf1;
3983 pngbps = buf0>>24;
3984 pngtype = (U8)(buf0>>16);
3985 png*=((buf0&0xFFFF)==0 && pngw && pngh && pngbps==8 && (!pngtype || pngtype==2 || pngtype==3 || pngtype==4 || pngtype==6));
3986 }
3987 else if (p==17){
3988 png*=((buf1&0xFF)==0);
3989 nextchunk =(png)?i+8:0;
3990 }
3991 else if (p>17 && i==nextchunk){
3992 nextchunk+=buf1+4/*CRC*/+8/*Chunk length+Id*/;
3993 lastchunk = buf0;
3994 png*=(lastchunk!=0x49454E44/*IEND*/);
3995 }
3996 }
3997
3998 // ZLIB stream detection
3999 zbuf[zbufpos]=c;
4000 zbufpos=(zbufpos+1)%32;
4001 int zh=parse_zlib_header(((int)zbuf[zbufpos])*256+(int)zbuf[(zbufpos+1)%32]);
4002 if ((i>=31 && zh!=-1) || zzippos==i) {
4003 int streamLength=0, ret=0;
4004
4005 // Quick check possible stream by decompressing first 32 bytes
4006 z_stream strm;
4007 strm.zalloc=Z_NULL; strm.zfree=Z_NULL; strm.opaque=Z_NULL;
4008 strm.next_in=Z_NULL; strm.avail_in=0;
4009 if (zlib_inflateInit(&strm,zh)==Z_OK) {
4010 unsigned char tmp[32];
4011 for (int j=0; j<32; j++) tmp[j]=zbuf[(zbufpos+j)%32];
4012 strm.next_in=tmp; strm.avail_in=32;
4013 strm.next_out=zout; strm.avail_out=1<<16;
4014 ret=inflate(&strm, Z_FINISH);
4015 ret=(inflateEnd(&strm)==Z_OK && (ret==Z_STREAM_END || ret==Z_BUF_ERROR) && strm.total_in>=16);
4016 }
4017 if (ret) {
4018 // Verify valid stream and determine stream length
4019 long savedpos=ftell(in);
4020 strm.zalloc=Z_NULL; strm.zfree=Z_NULL; strm.opaque=Z_NULL;
4021 strm.next_in=Z_NULL; strm.avail_in=0; strm.total_in=strm.total_out=0;
4022 if (zlib_inflateInit(&strm,zh)==Z_OK) {
4023 for (int j=i-31; j<n; j+=1<<16) {
4024 unsigned int blsize=min(n-j,1<<16);
4025 fseek(in, start+j, SEEK_SET);
4026 if (fread(zin, 1, blsize, in)!=blsize) break;
4027 strm.next_in=zin; strm.avail_in=blsize;
4028 do {
4029 strm.next_out=zout; strm.avail_out=1<<16;
4030 ret=inflate(&strm, Z_FINISH);
4031 } while (strm.avail_out==0 && ret==Z_BUF_ERROR);
4032 if (ret==Z_STREAM_END) streamLength=strm.total_in;
4033 if (ret!=Z_BUF_ERROR) break;
4034 }
4035 if (inflateEnd(&strm)!=Z_OK) streamLength=0;
4036 }
4037 fseek(in, savedpos, SEEK_SET);
4038 }
4039 if (streamLength>0) {
4040 info=0;
4041 if (pdfimw>0 && pdfimw<0x1000000 && pdfimh>0) {
4042 if (pdfimb==8 && (int)strm.total_out==pdfimw*pdfimh) info=(((pdfgray<<6)|8)<<24)+pdfimw;
4043 if (pdfimb==8 && (int)strm.total_out==pdfimw*pdfimh*3) info=(24<<24)+pdfimw*3;
4044 if (pdfimb==4 && (int)strm.total_out==((pdfimw+1)/2)*pdfimh) info=(8<<24)+((pdfimw+1)/2);
4045 if (pdfimb==1 && (int)strm.total_out==((pdfimw+7)/8)*pdfimh) info=(1<<24)+((pdfimw+7)/8);
4046 pdfgray=0;
4047 }
4048 else if (png && pngw<0x1000000 && lastchunk==0x49444154/*IDAT*/){
4049 if (pngbps==8 && pngtype==2 && (int)strm.total_out==(pngw*3+1)*pngh) info=PNGFlag|(24<<24)|(pngw*3), png=0;
4050 else if (pngbps==8 && pngtype==6 && (int)strm.total_out==(pngw*4+1)*pngh) info=PNGFlag|(32<<24)|(pngw*4), png=0;
4051 }
4052 return fseek(in, start+i-31, SEEK_SET),detd=streamLength,ZLIB;
4053 }
4054 }
4055 if (zh==-1 && zbuf[zbufpos]=='P' && zbuf[(zbufpos+1)%32]=='K' && zbuf[(zbufpos+2)%32]=='\x3'
4056 && zbuf[(zbufpos+3)%32]=='\x4' && zbuf[(zbufpos+8)%32]=='\x8' && zbuf[(zbufpos+9)%32]=='\0') {
4057 int nlen=(int)zbuf[(zbufpos+26)%32]+((int)zbuf[(zbufpos+27)%32])*256
4058 +(int)zbuf[(zbufpos+28)%32]+((int)zbuf[(zbufpos+29)%32])*256;
4059 if (nlen<256 && i+30+nlen<n) zzippos=i+30+nlen;
4060 }
4061 if (i-pdfimp>1024) pdfim=pdfimw=pdfimh=pdfimb=pdfgray=0;
4062 if (pdfim>1 && !(isspace(c) || isdigit(c))) pdfim=1;
4063 if (pdfim==2 && isdigit(c)) pdfimw=pdfimw*10+(c-'0');
4064 if (pdfim==3 && isdigit(c)) pdfimh=pdfimh*10+(c-'0');
4065 if (pdfim==4 && isdigit(c)) pdfimb=pdfimb*10+(c-'0');
4066 if ((buf0&0xffff)==0x3c3c) pdfimp=i,pdfim=1; // <<
4067 if (pdfim && (buf1&0xffff)==0x2f57 && buf0==0x69647468) pdfim=2,pdfimw=0; // /Width
4068 if (pdfim && (buf1&0xffffff)==0x2f4865 && buf0==0x69676874) pdfim=3,pdfimh=0; // /Height
4069 if (pdfim && buf3==0x42697473 && buf2==0x50657243 && buf1==0x6f6d706f
4070 && buf0==0x6e656e74 && zbuf[(zbufpos+15)%32]=='/') pdfim=4,pdfimb=0; // /BitsPerComponent
4071 if (pdfim && (buf2&0xFFFFFF)==0x2F4465 && buf1==0x76696365 && buf0==0x47726179) pdfgray=1; // /DeviceGray
4072
4073 // CD sectors detection (mode 1 and mode 2 form 1+2 - 2352 bytes)
4074 if (buf1==0x00ffffff && buf0==0xffffffff && !cdi) cdi=i,cda=-1,cdm=0;
4075 if (cdi && i>cdi) {
4076 const int p=(i-cdi)%2352;
4077 if (p==8 && (buf1!=0xffffff00 || ((buf0&0xff)!=1 && (buf0&0xff)!=2))) cdi=0;
4078 else if (p==16 && i+2336<n) {
4079 U8 data[2352];
4080 long savedpos=ftell(in);
4081 fseek(in, start+i-23, SEEK_SET);
4082 fread(data, 1, 2352, in);
4083 fseek(in, savedpos, SEEK_SET);
4084 int t=expand_cd_sector(data, cda, 1);
4085 if (t!=cdm) cdm=t*(i-cdi<2352);
4086 if (cdm && cda!=10 && (cdm==1 || buf0==buf1)) {
4087 if (type!=CD) return info=cdm,fseek(in, start+cdi-7, SEEK_SET), CD;
4088 cda=(data[12]<<16)+(data[13]<<8)+data[14];
4089 if (cdm!=1 && i-cdi>2352 && buf0!=cdf) cda=10;
4090 if (cdm!=1) cdf=buf0;
4091 } else cdi=0;
4092 }
4093 if (!cdi && type==CD) return fseek(in, start+i-p-7, SEEK_SET), DEFAULT;
4094 }
4095 if (type==CD) continue;
4096
4097 // Detect JPEG by code SOI APPx (FF D8 FF Ex) followed by
4098 // SOF0 (FF C0 xx xx 08) and SOS (FF DA) within a reasonable distance.
4099 // Detect end by any code other than RST0-RST7 (FF D9-D7) or
4100 // a byte stuff (FF 00).
4101
4102 if (!soi && i>=3 && (buf0&0xffffff00)==0xffd8ff00 && ((buf0&0xFE)==0xC0 || (U8)buf0==0xC4 || ((U8)buf0>=0xDB && (U8)buf0<=0xFE) )) soi=i, app=i+2, sos=sof=0;
4103 if (soi) {
4104 if (app==i && (buf0>>24)==0xff &&
4105 ((buf0>>16)&0xff)>0xc1 && ((buf0>>16)&0xff)<0xff) app=i+(buf0&0xffff)+2;
4106 if (app<i && (buf1&0xff)==0xff && (buf0&0xfe0000ff)==0xc0000008) sof=i;
4107 if (sof && sof>soi && i-sof<0x1000 && (buf0&0xffff)==0xffda) {
4108 sos=i;
4109 if (type!=JPEG) return fseek(in, start+soi-3, SEEK_SET), JPEG;
4110 }
4111 if (i-soi>0x40000 && !sos) soi=0;
4112 }
4113 if (type==JPEG && sos && i>sos && (buf0&0xff00)==0xff00
4114 && (buf0&0xff)!=0 && (buf0&0xf8)!=0xd0) return DEFAULT;
4115
4116 // Detect .wav file header
4117 if (buf0==0x52494646) wavi=i,wavm=0;
4118 if (wavi) {
4119 int p=i-wavi;
4120 if (p==4) wavsize=bswap(buf0);
4121 else if (p==8){
4122 wavtype=(buf0==0x57415645)?1:(buf0==0x7366626B)?2:0;
4123 if (!wavtype) wavi=0;
4124 }
4125 else if (wavtype){
4126 if (wavtype==1) {
4127 if (p==16 && (buf1!=0x666d7420 || bswap(buf0)!=16)) wavi=0;
4128 else if (p==22) wavch=bswap(buf0)&0xffff;
4129 else if (p==34) wavbps=bswap(buf0)&0xffff;
4130 else if (p==40+wavm && buf1!=0x64617461) wavm+=bswap(buf0)+8,wavi=(wavm>0xfffff?0:wavi);
4131 else if (p==40+wavm) {
4132 int wavd=bswap(buf0);
4133 if ((wavch==1 || wavch==2) && (wavbps==8 || wavbps==16) && wavd>0 && wavsize>=wavd+36
4134 && wavd%((wavbps/8)*wavch)==0) AUD_DET(AUDIO,wavi-3,44+wavm,wavd,wavch+wavbps/4-3);
4135 wavi=0;
4136 }
4137 }
4138 else{
4139 if ((p==16 && buf1!=0x4C495354) || (p==20 && buf0!=0x494E464F))
4140 wavi=0;
4141 else if (p>20 && buf1==0x4C495354 && (wavi*=(buf0!=0))){
4142 wavlen = bswap(buf0);
4143 wavlist = i;
4144 }
4145 else if (wavlist){
4146 p=i-wavlist;
4147 if (p==8 && (buf1!=0x73647461 || buf0!=0x736D706C))
4148 wavi=0;
4149 else if (p==12){
4150 int wavd = bswap(buf0);
4151 if (wavd && (wavd+12)==wavlen)
4152 AUD_DET(AUDIO,wavi-3,(12+wavlist-(wavi-3)+1)&~1,wavd,1+16/4-3);
4153 wavi=0;
4154 }
4155 }
4156 }
4157 }
4158 }
4159
4160 // Detect .aiff file header
4161 if (buf0==0x464f524d) aiff=i,aiffs=0; // FORM
4162 if (aiff) {
4163 const int p=i-aiff;
4164 if (p==12 && (buf1!=0x41494646 || buf0!=0x434f4d4d)) aiff=0; // AIFF COMM
4165 else if (p==24) {
4166 const int bits=buf0&0xffff, chn=buf1>>16;
4167 if ((bits==8 || bits==16) && (chn==1 || chn==2)) aiffm=chn+bits/4+1; else aiff=0;
4168 } else if (p==42+aiffs && buf1!=0x53534e44) aiffs+=(buf0+8)+(buf0&1),aiff=(aiffs>0x400?0:aiff);
4169 else if (p==42+aiffs) AUD_DET(AUDIO,aiff-3,54+aiffs,buf0-8,aiffm);
4170 }
4171
4172 // Detect .mod file header
4173 if ((buf0==0x4d2e4b2e || buf0==0x3643484e || buf0==0x3843484e // M.K. 6CHN 8CHN
4174 || buf0==0x464c5434 || buf0==0x464c5438) && (buf1&0xc0c0c0c0)==0 && i>=1083) {
4175 long savedpos=ftell(in);
4176 const int chn=((buf0>>24)==0x36?6:(((buf0>>24)==0x38 || (buf0&0xff)==0x38)?8:4));
4177 int len=0; // total length of samples
4178 int numpat=1; // number of patterns
4179 for (int j=0; j<31; j++) {
4180 fseek(in, start+i-1083+42+j*30, SEEK_SET);
4181 const int i1=getc(in);
4182 const int i2=getc(in);
4183 len+=i1*512+i2*2;
4184 }
4185 fseek(in, start+i-131, SEEK_SET);
4186 for (int j=0; j<128; j++) {
4187 int x=getc(in);
4188 if (x+1>numpat) numpat=x+1;
4189 }
4190 if (numpat<65) AUD_DET(AUDIO,i-1083,1084+numpat*256*chn,len,4);
4191 fseek(in, savedpos, SEEK_SET);
4192 }
4193
4194 // Detect .s3m file header
4195 if (buf0==0x1a100000) s3mi=i,s3mno=s3mni=0;
4196 if (s3mi) {
4197 const int p=i-s3mi;
4198 if (p==4) s3mno=bswap(buf0)&0xffff,s3mni=(bswap(buf0)>>16);
4199 else if (p==16 && (((buf1>>16)&0xff)!=0x13 || buf0!=0x5343524d)) s3mi=0;
4200 else if (p==16) {
4201 long savedpos=ftell(in);
4202 int b[31],sam_start=(1<<16),sam_end=0,ok=1;
4203 for (int j=0;j<s3mni;j++) {
4204 fseek(in, start+s3mi-31+0x60+s3mno+j*2, SEEK_SET);
4205 int i1=getc(in);
4206 i1+=getc(in)*256;
4207 fseek(in, start+s3mi-31+i1*16, SEEK_SET);
4208 i1=getc(in);
4209 if (i1==1) { // type: sample
4210 for (int k=0;k<31;k++) b[k]=fgetc(in);
4211 int len=b[15]+(b[16]<<8);
4212 int ofs=b[13]+(b[14]<<8);
4213 if (b[30]>1) ok=0;
4214 if (ofs*16<sam_start) sam_start=ofs*16;
4215 if (ofs*16+len>sam_end) sam_end=ofs*16+len;
4216 }
4217 }
4218 if (ok && sam_start<(1<<16)) AUD_DET(AUDIO,s3mi-31,sam_start,sam_end-sam_start,0);
4219 s3mi=0;
4220 fseek(in, savedpos, SEEK_SET);
4221 }
4222 }
4223
4224 // Detect .bmp image
4225 if ( !(bmp || hdrless) && (((buf0&0xffff)==16973) || (!(buf0&0xFFFFFF) && ((buf0>>24)==0x28))) ) //possible 'BM' or headerless DIB
4226 imgbpp=bmpx=bmpy=0,hdrless=!(U8)buf0,bmpof=hdrless*54,bmp=i-hdrless*16;
4227 if (bmp || hdrless) {
4228 const int p=i-bmp;
4229 if (p==12) bmpof=bswap(buf0);
4230 else if (p==16 && buf0!=0x28000000) bmp=hdrless=0; //BITMAPINFOHEADER (0x28)
4231 else if (p==20) bmpx=bswap(buf0),bmp=((bmpx==0||bmpx>0x30000)?(hdrless=0):bmp); //width
4232 else if (p==24) bmpy=abs((int)bswap(buf0)),bmp=((bmpy==0||bmpy>0x10000)?(hdrless=0):bmp); //height
4233 else if (p==27) imgbpp=c,bmp=((imgbpp!=1 && imgbpp!=4 && imgbpp!=8 && imgbpp!=24 && imgbpp!=32)?(hdrless=0):bmp);
4234 else if ((p==31) && buf0) bmp=hdrless=0;
4235 else if (p==36) bmps=bswap(buf0);
4236 // check number of colors in palette (4 bytes), must be 0 (default) or <= 1<<bpp.
4237 // also check if image is too small, since it might not be worth it to use the image models
4238 else if (p==48){
4239 if ( (!buf0 || ((bswap(buf0)<=(U32)(1<<imgbpp)) && (imgbpp<=8))) && (((bmpx*bmpy*imgbpp)>>3)>64) ) {
4240 // possible icon/cursor?
4241 if (hdrless && (bmpx*2==bmpy) && imgbpp>1 &&
4242 (
4243 (bmps>0 && bmps==( (bmpx*bmpy*(imgbpp+1))>>4 )) ||
4244 ((!bmps || bmps<((bmpx*bmpy*imgbpp)>>3)) && (
4245 (bmpx==8) || (bmpx==10) || (bmpx==14) || (bmpx==16) || (bmpx==20) ||
4246 (bmpx==22) || (bmpx==24) || (bmpx==32) || (bmpx==40) || (bmpx==48) ||
4247 (bmpx==60) || (bmpx==64) || (bmpx==72) || (bmpx==80) || (bmpx==96) ||
4248 (bmpx==128) || (bmpx==256)
4249 ))
4250 )
4251 )
4252 bmpy=bmpx;
4253
4254 // if DIB and not 24bpp, we must calculate the data offset based on BPP or num. of entries in color palette
4255 if (hdrless && (imgbpp<=24))
4256 bmpof+=((buf0)?bswap(buf0)*4:4<<imgbpp);
4257 bmpof+=(bmp-1)*(bmp<1);
4258
4259 if (hdrless && bmps && bmps<((bmpx*bmpy*imgbpp)>>3)) { /*Guard against erroneous DIB detections*/ }
4260 else if (imgbpp==1) IMG_DET(IMAGE1,max(0,bmp-1),bmpof,(((bmpx-1)>>5)+1)*4,bmpy);
4261 else if (imgbpp==4) IMG_DET(IMAGE4,max(0,bmp-1),bmpof,((bmpx*4+31)>>5)*4,bmpy);
4262 else if (imgbpp==8){
4263 fseek(in, start+bmp+53, SEEK_SET);
4264 IMG_DET( (IsGrayscalePalette(in, (buf0)?bswap(buf0):1<<imgbpp, 1))?IMAGE8GRAY:IMAGE8,max(0,bmp-1),bmpof,(bmpx+3)&-4,bmpy);
4265 }
4266 else if (imgbpp==24) IMG_DET(IMAGE24,max(0,bmp-1),bmpof,((bmpx*3)+3)&-4,bmpy);
4267 else if (imgbpp==32) IMG_DET(IMAGE32,max(0,bmp-1),bmpof,bmpx*4,bmpy);
4268 }
4269 bmp=hdrless=0;
4270 }
4271 }
4272
4273 // Detect .pbm .pgm .ppm .pam image
4274 if ((buf0&0xfff0ff)==0x50300a) {
4275 pgmn=(buf0&0xf00)>>8;
4276 if ((pgmn>=4 && pgmn<=6) || pgmn==7) pgm=i,pgm_ptr=pgmw=pgmh=pgmc=pgmcomment=pamatr=pamd=0;
4277 }
4278 if (pgm) {
4279 if (i-pgm==1 && c==0x23) pgmcomment=1; //pgm comment
4280 if (!pgmcomment && pgm_ptr) {
4281 int s=0;
4282 if (pgmn==7) {
4283 if ((buf1&0xdfdf)==0x5749 && (buf0&0xdfdfdfff)==0x44544820) pgm_ptr=0, pamatr=1; // WIDTH
4284 if ((buf1&0xdfdfdf)==0x484549 && (buf0&0xdfdfdfff)==0x47485420) pgm_ptr=0, pamatr=2; // HEIGHT
4285 if ((buf1&0xdfdfdf)==0x4d4158 && (buf0&0xdfdfdfff)==0x56414c20) pgm_ptr=0, pamatr=3; // MAXVAL
4286 if ((buf1&0xdfdf)==0x4445 && (buf0&0xdfdfdfff)==0x50544820) pgm_ptr=0, pamatr=4; // DEPTH
4287 if ((buf2&0xdf)==0x54 && (buf1&0xdfdfdfdf)==0x55504c54 && (buf0&0xdfdfdfff)==0x59504520) pgm_ptr=0, pamatr=5; // TUPLTYPE
4288 if ((buf1&0xdfdfdf)==0x454e44 && (buf0&0xdfdfdfff)==0x4844520a) pgm_ptr=0, pamatr=6; // ENDHDR
4289 if (c==0x0a) {
4290 if (pamatr==0) pgm=0;
4291 else if (pamatr<5) s=pamatr;
4292 if (pamatr!=6) pamatr=0;
4293 }
4294 } else if (c==0x20 && !pgmw) s=1;
4295 else if (c==0x0a && !pgmh) s=2;
4296 else if (c==0x0a && !pgmc && pgmn!=4) s=3;
4297 if (s) {
4298 pgm_buf[pgm_ptr++]=0;
4299 int v=atoi(pgm_buf);
4300 if (s==1) pgmw=v; else if (s==2) pgmh=v; else if (s==3) pgmc=v; else if (s==4) pamd=v;
4301 if (v==0 || (s==3 && v>255)) pgm=0; else pgm_ptr=0;
4302 }
4303 }
4304 if (!pgmcomment) pgm_buf[pgm_ptr++]=c;
4305 if (pgm_ptr>=32) pgm=0;
4306 if (pgmcomment && c==0x0a) pgmcomment=0;
4307 if (pgmw && pgmh && !pgmc && pgmn==4) IMG_DET(IMAGE1,pgm-2,i-pgm+3,(pgmw+7)/8,pgmh);
4308 if (pgmw && pgmh && pgmc && (pgmn==5 || (pgmn==7 && pamd==1 && pamatr==6))) IMG_DET(IMAGE8GRAY,pgm-2,i-pgm+3,pgmw,pgmh);
4309 if (pgmw && pgmh && pgmc && (pgmn==6 || (pgmn==7 && pamd==3 && pamatr==6))) IMG_DET(IMAGE24,pgm-2,i-pgm+3,pgmw*3,pgmh);
4310 if (pgmw && pgmh && pgmc && (pgmn==7 && pamd==4 && pamatr==6)) IMG_DET(IMAGE32,pgm-2,i-pgm+3,pgmw*4,pgmh);
4311 }
4312
4313 // Detect .rgb image
4314 if ((buf0&0xffff)==0x01da) rgbi=i,rgbx=rgby=0;
4315 if (rgbi) {
4316 const int p=i-rgbi;
4317 if (p==1 && c!=0) rgbi=0;
4318 else if (p==2 && c!=1) rgbi=0;
4319 else if (p==4 && (buf0&0xffff)!=1 && (buf0&0xffff)!=2 && (buf0&0xffff)!=3) rgbi=0;
4320 else if (p==6) rgbx=buf0&0xffff,rgbi=(rgbx==0?0:rgbi);
4321 else if (p==8) rgby=buf0&0xffff,rgbi=(rgby==0?0:rgbi);
4322 else if (p==10) {
4323 int z=buf0&0xffff;
4324 if (rgbx && rgby && (z==1 || z==3 || z==4)) IMG_DET(IMAGE8,rgbi-1,512,rgbx,rgby*z);
4325 rgbi=0;
4326 }
4327 }
4328
4329 // Detect .tiff file header (2/8/24 bit color, not compressed).
4330 if (buf1==0x49492a00 && n>i+(int)bswap(buf0)) {
4331 long savedpos=ftell(in);
4332 fseek(in, start+i+bswap(buf0)-7, SEEK_SET);
4333
4334 // read directory
4335 int dirsize=getc(in);
4336 int tifx=0,tify=0,tifz=0,tifzb=0,tifc=0,tifofs=0,tifofval=0,b[12];
4337 if (getc(in)==0) {
4338 for (int i=0; i<dirsize; i++) {
4339 for (int j=0; j<12; j++) b[j]=getc(in);
4340 if (b[11]==EOF) break;
4341 int tag=b[0]+(b[1]<<8);
4342 int tagfmt=b[2]+(b[3]<<8);
4343 int taglen=b[4]+(b[5]<<8)+(b[6]<<16)+(b[7]<<24);
4344 int tagval=b[8]+(b[9]<<8)+(b[10]<<16)+(b[11]<<24);
4345 if (tagfmt==3||tagfmt==4) {
4346 if (tag==256) tifx=tagval;
4347 else if (tag==257) tify=tagval;
4348 else if (tag==258) tifzb=taglen==1?tagval:8; // bits per component
4349 else if (tag==259) tifc=tagval; // 1 = no compression
4350 else if (tag==273 && tagfmt==4) tifofs=tagval,tifofval=(taglen<=1);
4351 else if (tag==277) tifz=tagval; // components per pixel
4352 }
4353 }
4354 }
4355 if (tifx && tify && tifzb && (tifz==1 || tifz==3) && (tifc==1) && (tifofs && tifofs+i<n)) {
4356 if (!tifofval) {
4357 fseek(in, start+i+tifofs-7, SEEK_SET);
4358 for (int j=0; j<4; j++) b[j]=getc(in);
4359 tifofs=b[0]+(b[1]<<8)+(b[2]<<16)+(b[3]<<24);
4360 }
4361 if (tifofs && tifofs<(1<<18) && tifofs+i<n) {
4362 if (tifz==1 && tifzb==1) IMG_DET(IMAGE1,i-7,tifofs,((tifx-1)>>3)+1,tify);
4363 else if (tifz==1 && tifzb==8) IMG_DET(IMAGE8,i-7,tifofs,tifx,tify);
4364 else if (tifz==3 && tifzb==8) IMG_DET(IMAGE24,i-7,tifofs,tifx*3,tify);
4365 }
4366 }
4367 fseek(in, savedpos, SEEK_SET);
4368 }
4369
4370 // Detect .tga image (8-bit 256 colors or 24-bit uncompressed)
4371 if (buf1==0x00010100 && buf0==0x00000118) tga=i,tgax=tgay,tgaz=8,tgat=1;
4372 else if (buf1==0x00000200 && buf0==0x00000000) tga=i,tgax=tgay,tgaz=24,tgat=2;
4373 else if (buf1==0x00000300 && buf0==0x00000000) tga=i,tgax=tgay,tgaz=8,tgat=3;
4374 if (tga) {
4375 if (i-tga==8) tga=(buf1==0?tga:0),tgax=(bswap(buf0)&0xffff),tgay=(bswap(buf0)>>16);
4376 else if (i-tga==10) {
4377 if (tgaz==(int)((buf0&0xffff)>>8) && tgax && tgay) {
4378 if (tgat==1){
4379 fseek(in, start+tga+11, SEEK_SET);
4380 IMG_DET( (IsGrayscalePalette(in))?IMAGE8GRAY:IMAGE8,tga-7,18+256*3,tgax,tgay);
4381 }
4382 else if (tgat==2) IMG_DET(IMAGE24,tga-7,18,tgax*3,tgay);
4383 else if (tgat==3) IMG_DET(IMAGE8,tga-7,18,tgax,tgay);
4384 }
4385 tga=0;
4386 }
4387 }
4388
4389 // Detect .gif
4390 if (type==DEFAULT && dett==GIF && i==0) {
4391 dett=DEFAULT;
4392 if (c==0x2c || c==0x21) gif=2,gifi=2;
4393 }
4394 if (!gif && (buf1&0xffff)==0x4749 && (buf0==0x46383961 || buf0==0x46383761)) gif=1,gifi=i+5;
4395 if (gif) {
4396 if (gif==1 && i==gifi) gif=2,gifi=i+5+((c&128)?(3*(2<<(c&7))):0);
4397 if (gif==2 && i==gifi) {
4398 if ((buf0&0xff0000)==0x210000) gif=5,gifi=i;
4399 else if ((buf0&0xff0000)==0x2c0000) gif=3,gifi=i;
4400 else gif=0;
4401 }
4402 if (gif==3 && i==gifi+6) gifw=(bswap(buf0)&0xffff);
4403 if (gif==3 && i==gifi+7) gif=4,gifc=gifb=0,gifa=gifi=i+2+((c&128)?(3*(2<<(c&7))):0);
4404 if (gif==4 && i==gifi) {
4405 if (c>0 && gifb && gifc!=gifb) gifw=0;
4406 if (c>0) gifb=gifc,gifc=c,gifi+=c+1;
4407 else if (!gifw) gif=2,gifi=i+3;
4408 else return fseek(in, start+gifa-1, SEEK_SET),detd=i-gifa+2,info=gifw,dett=GIF;
4409 }
4410 if (gif==5 && i==gifi) {
4411 if (c>0) gifi+=c+1; else gif=2,gifi=i+3;
4412 }
4413 }
4414
4415 // Detect EXE if the low order byte (little-endian) XX is more
4416 // recently seen (and within 4K) if a relative to absolute address
4417 // conversion is done in the context CALL/JMP (E8/E9) XX xx xx 00/FF
4418 // 4 times in a row. Detect end of EXE at the last
4419 // place this happens when it does not happen for 64KB.
4420
4421 if (((buf1&0xfe)==0xe8 || (buf1&0xfff0)==0x0f80) && ((buf0+1)&0xfe)==0) {
4422 int r=buf0>>24; // relative address low 8 bits
4423 int a=((buf0>>24)+i)&0xff; // absolute address low 8 bits
4424 int rdist=i-relpos[r];
4425 int adist=i-abspos[a];
4426 if (adist<rdist && adist<0x800 && abspos[a]>5) {
4427 e8e9last=i;
4428 ++e8e9count;
4429 if (e8e9pos==0 || e8e9pos>abspos[a]) e8e9pos=abspos[a];
4430 }
4431 else e8e9count=0;
4432 if (type==DEFAULT && e8e9count>=4 && e8e9pos>5)
4433 return fseek(in, start+e8e9pos-5, SEEK_SET), EXE;
4434 abspos[a]=i;
4435 relpos[r]=i;
4436 }
4437 if (i-e8e9last>0x4000) {
4438 if (type==EXE) return fseek(in, start+e8e9last, SEEK_SET), DEFAULT;
4439 e8e9count=e8e9pos=0;
4440 }
4441
4442 // Detect base64 encoded data
4443 if (b64s==0 && buf0==0x73653634 && ((buf1&0xffffff)==0x206261 || (buf1&0xffffff)==0x204261)) b64s=1,b64i=i-6; //' base64' ' Base64'
4444 if (b64s==0 && ((buf1==0x3b626173 && buf0==0x6536342c) || (buf1==0x215b4344 && buf0==0x4154415b))) b64s=3,b64i=i+1; // ';base64,' '![CDATA['
4445 if (b64s>0) {
4446 if (b64s==1 && buf0==0x0d0a0d0a) b64s=((i-b64i>=128)?0:2),b64i=i+1,b64line=0;
4447 else if (b64s==2 && (buf0&0xffff)==0x0d0a && b64line==0) b64line=i+1-b64i,b64nl=i;
4448 else if (b64s==2 && (buf0&0xffff)==0x0d0a && b64line>0 && (buf0&0xffffff)!=0x3d0d0a) {
4449 if (i-b64nl<b64line && buf0!=0x0d0a0d0a) i-=1,b64s=5;
4450 else if (buf0==0x0d0a0d0a) i-=3,b64s=5;
4451 else if (i-b64nl==b64line) b64nl=i;
4452 else b64s=0;
4453 }
4454 else if (b64s==2 && (buf0&0xffffff)==0x3d0d0a) i-=1,b64s=5; // '=' or '=='
4455 else if (b64s==2 && !(isalnum(c) || c=='+' || c=='/' || c==10 || c==13 || c=='=')) b64s=0;
4456 if (b64line>0 && (b64line<=4 || b64line>255)) b64s=0;
4457 if (b64s==3 && i>=b64i && !(isalnum(c) || c=='+' || c=='/' || c=='=')) b64s=4;
4458 if ((b64s==4 && i-b64i>128) || (b64s==5 && i-b64i>512 && i-b64i<(1<<27))) return fseek(in, start+b64i, SEEK_SET),detd=i-b64i,BASE64;
4459 if (b64s>3) b64s=0;
4460 }
4461 }
4462 return type;
4463}
4464
4465typedef enum {FDECOMPRESS, FCOMPARE, FDISCARD} FMode;
4466
4467void encode_cd(FILE* in, FILE* out, int len, int info) {
4468 const int BLOCK=2352;
4469 U8 blk[BLOCK];
4470 fputc((len%BLOCK)>>8,out);
4471 fputc(len%BLOCK,out);
4472 for (int offset=0; offset<len; offset+=BLOCK) {
4473 if (offset+BLOCK > len) {
4474 fread(&blk[0], 1, len-offset, in);
4475 fwrite(&blk[0], 1, len-offset, out);
4476 } else {
4477 fread(&blk[0], 1, BLOCK, in);
4478 if (info==3) blk[15]=3;
4479 if (offset==0) fwrite(&blk[12], 1, 4+4*(blk[15]!=1), out);
4480 fwrite(&blk[16+8*(blk[15]!=1)], 1, 2048+276*(info==3), out);
4481 if (offset+BLOCK*2 > len && blk[15]!=1) fwrite(&blk[16], 1, 4, out);
4482 }
4483 }
4484}
4485
4486int decode_cd(FILE *in, int size, FILE *out, FMode mode, int &diffFound) {
4487 const int BLOCK=2352;
4488 U8 blk[BLOCK];
4489 long i=0, i2=0;
4490 int a=-1, bsize=0, q=fgetc(in);
4491 q=(q<<8)+fgetc(in);
4492 size-=2;
4493 while (i<size) {
4494 if (size-i==q) {
4495 fread(blk, q, 1, in);
4496 fwrite(blk, q, 1, out);
4497 i+=q;
4498 i2+=q;
4499 } else if (i==0) {
4500 fread(blk+12, 4, 1, in);
4501 if (blk[15]!=1) fread(blk+16, 4, 1, in);
4502 bsize=2048+(blk[15]==3)*276;
4503 i+=4*(blk[15]!=1)+4;
4504 } else {
4505 a=(blk[12]<<16)+(blk[13]<<8)+blk[14];
4506 }
4507 fread(blk+16+(blk[15]!=1)*8, bsize, 1, in);
4508 i+=bsize;
4509 if (bsize>2048) blk[15]=3;
4510 if (blk[15]!=1 && size-q-i==4) {
4511 fread(blk+16, 4, 1, in);
4512 i+=4;
4513 }
4514 expand_cd_sector(blk, a, 0);
4515 if (mode==FDECOMPRESS) fwrite(blk, BLOCK, 1, out);
4516 else if (mode==FCOMPARE) for (int j=0; j<BLOCK; ++j) if (blk[j]!=getc(out) && !diffFound) diffFound=i2+j+1;
4517 i2+=BLOCK;
4518 }
4519 return i2;
4520}
4521
4522
4523// 24-bit image data transform:
4524// simple color transform (b, g, r) -> (g, g-r, g-b)
4525
4526void encode_bmp(FILE* in, FILE* out, int len, int width) {
4527 int r,g,b;
4528 for (int i=0; i<len/width; i++) {
4529 for (int j=0; j<width/3; j++) {
4530 b=fgetc(in), g=fgetc(in), r=fgetc(in);
4531 fputc(g, out);
4532 fputc(g-r, out);
4533 fputc(g-b, out);
4534 }
4535 for (int j=0; j<width%3; j++) fputc(fgetc(in), out);
4536 }
4537}
4538
4539int decode_bmp(Encoder& en, int size, int width, FILE *out, FMode mode, int &diffFound) {
4540 int r,g,b,p;
4541 for (int i=0; i<size/width; i++) {
4542 p=i*width;
4543 for (int j=0; j<width/3; j++) {
4544 b=en.decompress(), g=en.decompress(), r=en.decompress();
4545 if (mode==FDECOMPRESS) {
4546 fputc(b-r, out);
4547 fputc(b, out);
4548 fputc(b-g, out);
4549 if (!j && !(i&0xf)) en.print_status();
4550 }
4551 else if (mode==FCOMPARE) {
4552 if (((b-r)&255)!=getc(out) && !diffFound) diffFound=p+1;
4553 if (b!=getc(out) && !diffFound) diffFound=p+2;
4554 if (((b-g)&255)!=getc(out) && !diffFound) diffFound=p+3;
4555 p+=3;
4556 }
4557 }
4558 for (int j=0; j<width%3; j++) {
4559 if (mode==FDECOMPRESS) {
4560 fputc(en.decompress(), out);
4561 }
4562 else if (mode==FCOMPARE) {
4563 if (en.decompress()!=getc(out) && !diffFound) diffFound=p+j+1;
4564 }
4565 }
4566 }
4567 return size;
4568}
4569
4570// 32-bit image
4571void encode_im32(FILE* in, FILE* out, int len, int width) {
4572 int r,g,b,a;
4573 for (int i=0; i<len/width; i++) {
4574 for (int j=0; j<width/4; j++) {
4575 b=fgetc(in), g=fgetc(in), r=fgetc(in); a=fgetc(in);
4576 fputc(g, out);
4577 fputc(g-r, out);
4578 fputc(g-b, out);
4579 fputc(a, out);
4580 }
4581 for (int j=0; j<width%4; j++) fputc(fgetc(in), out);
4582 }
4583}
4584
4585int decode_im32(Encoder& en, int size, int width, FILE *out, FMode mode, int &diffFound) {
4586 int r,g,b,a,p;
4587 bool rgb = (width&(1<<31))>0;
4588 if (rgb) width^=(1<<31);
4589 for (int i=0; i<size/width; i++) {
4590 p=i*width;
4591 for (int j=0; j<width/4; j++) {
4592 b=en.decompress(), g=en.decompress(), r=en.decompress(), a=en.decompress();
4593 if (mode==FDECOMPRESS) {
4594 fputc(b-r, out); fputc(b, out); fputc(b-g, out); fputc(a, out);
4595 if (!j && !(i&0xf)) en.print_status();
4596 }
4597 else if (mode==FCOMPARE) {
4598 if (((b-r)&255)!=getc(out) && !diffFound) diffFound=p+1;
4599 if (b!=getc(out) && !diffFound) diffFound=p+2;
4600 if (((b-g)&255)!=getc(out) && !diffFound) diffFound=p+3;
4601 if (((a)&255)!=getc(out) && !diffFound) diffFound=p+4;
4602 p+=4;
4603 }
4604 }
4605 for (int j=0; j<width%4; j++) {
4606 if (mode==FDECOMPRESS) {
4607 fputc(en.decompress(), out);
4608 }
4609 else if (mode==FCOMPARE) {
4610 if (en.decompress()!=getc(out) && !diffFound) diffFound=p+j+1;
4611 }
4612 }
4613 }
4614 return size;
4615}
4616
4617// EXE transform: <encoded-size> <begin> <block>...
4618// Encoded-size is 4 bytes, MSB first.
4619// begin is the offset of the start of the input file, 4 bytes, MSB first.
4620// Each block applies the e8e9 transform to strings falling entirely
4621// within the block starting from the end and working backwards.
4622// The 5 byte pattern is E8/E9 xx xx xx 00/FF (x86 CALL/JMP xxxxxxxx)
4623// where xxxxxxxx is a relative address LSB first. The address is
4624// converted to an absolute address by adding the offset mod 2^25
4625// (in range +-2^24).
4626
4627void encode_exe(FILE* in, FILE* out, int len, int begin) {
4628 const int BLOCK=0x10000;
4629 Array<U8> blk(BLOCK);
4630 fprintf(out, "%c%c%c%c", begin>>24, begin>>16, begin>>8, begin);
4631
4632 // Transform
4633 for (int offset=0; offset<len; offset+=BLOCK) {
4634 int size=min(len-offset, BLOCK);
4635 int bytesRead=fread(&blk[0], 1, size, in);
4636 if (bytesRead!=size) quit("encode_exe read error");
4637 for (int i=bytesRead-1; i>=5; --i) {
4638 if ((blk[i-4]==0xe8 || blk[i-4]==0xe9 || (blk[i-5]==0x0f && (blk[i-4]&0xf0)==0x80))
4639 && (blk[i]==0||blk[i]==0xff)) {
4640 int a=(blk[i-3]|blk[i-2]<<8|blk[i-1]<<16|blk[i]<<24)+offset+begin+i+1;
4641 a<<=7;
4642 a>>=7;
4643 blk[i]=a>>24;
4644 blk[i-1]=a^176;
4645 blk[i-2]=(a>>8)^176;
4646 blk[i-3]=(a>>16)^176;
4647 }
4648 }
4649 fwrite(&blk[0], 1, bytesRead, out);
4650 }
4651}
4652
4653int decode_exe(Encoder& en, int size, FILE *out, FMode mode, int &diffFound) {
4654 const int BLOCK=0x10000; // block size
4655 int begin, offset=6, a;
4656 U8 c[6];
4657 begin=en.decompress()<<24;
4658 begin|=en.decompress()<<16;
4659 begin|=en.decompress()<<8;
4660 begin|=en.decompress();
4661 size-=4;
4662 for (int i=4; i>=0; i--) c[i]=en.decompress(); // Fill queue
4663
4664 while (offset<size+6) {
4665 memmove(c+1, c, 5);
4666 if (offset<=size) c[0]=en.decompress();
4667 // E8E9 transform: E8/E9 xx xx xx 00/FF -> subtract location from x
4668 if ((c[0]==0x00 || c[0]==0xFF) && (c[4]==0xE8 || c[4]==0xE9 || (c[5]==0x0F && (c[4]&0xF0)==0x80))
4669 && (((offset-1)^(offset-6))&-BLOCK)==0 && offset<=size) { // not crossing block boundary
4670 a=((c[1]^176)|(c[2]^176)<<8|(c[3]^176)<<16|c[0]<<24)-offset-begin;
4671 a<<=7;
4672 a>>=7;
4673 c[3]=a;
4674 c[2]=a>>8;
4675 c[1]=a>>16;
4676 c[0]=a>>24;
4677 }
4678 if (mode==FDECOMPRESS) putc(c[5], out);
4679 else if (mode==FCOMPARE && c[5]!=getc(out) && !diffFound) diffFound=offset-6+1;
4680 if (mode==FDECOMPRESS && !(offset&0xfff)) en.print_status();
4681 offset++;
4682 }
4683 return size;
4684}
4685
4686int encode_zlib(FILE* in, FILE* out, int len) {
4687 const int BLOCK=1<<16, LIMIT=128;
4688 U8 zin[BLOCK*2],zout[BLOCK],zrec[BLOCK*2], diffByte[81*LIMIT];
4689 int diffPos[81*LIMIT];
4690
4691 // Step 1 - parse offset type form zlib stream header
4692 long pos=ftell(in);
4693 unsigned int h1=fgetc(in), h2=fgetc(in);
4694 fseek(in, pos, SEEK_SET);
4695 int zh=parse_zlib_header(h1*256+h2);
4696 int memlevel,clevel,window=zh==-1?0:MAX_WBITS+10+zh/4,ctype=zh%4;
4697 int minclevel=window==0?1:ctype==3?7:ctype==2?6:ctype==1?2:1;
4698 int maxclevel=window==0?9:ctype==3?9:ctype==2?6:ctype==1?5:1;
4699
4700 // Step 2 - check recompressiblitiy, determine parameters and save differences
4701 z_stream main_strm, rec_strm[81];
4702 int diffCount[81], recpos[81], main_ret=Z_STREAM_END;
4703 main_strm.zalloc=Z_NULL; main_strm.zfree=Z_NULL; main_strm.opaque=Z_NULL;
4704 main_strm.next_in=Z_NULL; main_strm.avail_in=0;
4705 if (zlib_inflateInit(&main_strm,zh)!=Z_OK) return false;
4706 for (int i=0; i<81; i++) {
4707 memlevel=(i%9)+1;
4708 clevel=(i/9)+1;
4709 rec_strm[i].zalloc=Z_NULL; rec_strm[i].zfree=Z_NULL; rec_strm[i].opaque=Z_NULL;
4710 rec_strm[i].next_in=Z_NULL; rec_strm[i].avail_in=0;
4711 int ret=deflateInit2(&rec_strm[i], clevel, Z_DEFLATED, window-MAX_WBITS, memlevel, Z_DEFAULT_STRATEGY);
4712 diffCount[i]=(clevel>=minclevel && clevel<=maxclevel && ret==Z_OK)?0:LIMIT;
4713 recpos[i]=BLOCK*2;
4714 diffPos[i*LIMIT]=-1;
4715 diffByte[i*LIMIT]=0;
4716 }
4717 for (int i=0; i<len; i+=BLOCK) {
4718 unsigned int blsize=min(len-i,BLOCK);
4719 for (int j=0; j<81; j++) {
4720 if (diffCount[j]>=LIMIT) continue;
4721 memmove(&zrec[0], &zrec[BLOCK], BLOCK);
4722 recpos[j]-=BLOCK;
4723 }
4724 memmove(&zin[0], &zin[BLOCK], BLOCK);
4725 fread(&zin[BLOCK], 1, blsize, in); // Read block from input file
4726
4727 // Decompress/inflate block
4728 main_strm.next_in=&zin[BLOCK]; main_strm.avail_in=blsize;
4729 do {
4730 main_strm.next_out=&zout[0]; main_strm.avail_out=BLOCK;
4731 main_ret=inflate(&main_strm, Z_FINISH);
4732
4733 // Recompress/deflate block with all possible parameters
4734 for (int j=0; j<81; j++) {
4735 if (diffCount[j]>=LIMIT) continue;
4736 rec_strm[j].next_in=&zout[0]; rec_strm[j].avail_in=BLOCK-main_strm.avail_out;
4737 rec_strm[j].next_out=&zrec[recpos[j]]; rec_strm[j].avail_out=BLOCK*2-recpos[j];
4738 int ret=deflate(&rec_strm[j], (int)main_strm.total_in == len ? Z_FINISH : Z_NO_FLUSH);
4739 if (ret!=Z_BUF_ERROR && ret!=Z_STREAM_END && ret!=Z_OK) { diffCount[j]=LIMIT; continue; }
4740
4741 // Compare
4742 int end=2*BLOCK-(int)rec_strm[j].avail_out;
4743 int tail=max(main_ret==Z_STREAM_END ? len-(int)rec_strm[j].total_out : 0,0);
4744 for (int k=recpos[j]; k<end+tail; k++) {
4745 if ((k<end && i+k-BLOCK<len && zrec[k]!=zin[k]) || k>=end) {
4746 if (++diffCount[j]<LIMIT) {
4747 const int p=j*LIMIT+diffCount[j];
4748 diffPos[p]=i+k-BLOCK;
4749 diffByte[p]=zin[k];
4750 }
4751 }
4752 }
4753 recpos[j]=2*BLOCK-rec_strm[j].avail_out;
4754 }
4755 } while (main_strm.avail_out==0 && main_ret==Z_BUF_ERROR);
4756 if (main_ret!=Z_BUF_ERROR && main_ret!=Z_STREAM_END) break;
4757 }
4758 int minCount=LIMIT, index;
4759 for (int i=80; i>=0; i--) {
4760 deflateEnd(&rec_strm[i]);
4761 if (diffCount[i]<minCount) {
4762 minCount=diffCount[i];
4763 memlevel=(i%9)+1;
4764 clevel=(i/9)+1;
4765 index=i;
4766 }
4767 }
4768 inflateEnd(&main_strm);
4769 if (minCount==LIMIT) return false;
4770
4771 // Step 3 - write parameters, differences and precompressed (inflated) data
4772 fputc(diffCount[index], out);
4773 fputc(window, out);
4774 fputc(index, out);
4775 for (int i=0; i<=diffCount[index]; i++) {
4776 const int v=i==diffCount[index] ? len-diffPos[index*LIMIT+i]
4777 : diffPos[index*LIMIT+i+1]-diffPos[index*LIMIT+i]-1;
4778 fputc(v>>24, out); fputc(v>>16, out); fputc(v>>8, out); fputc(v, out);
4779 }
4780 for (int i=0; i<diffCount[index]; i++) fputc(diffByte[index*LIMIT+i+1], out);
4781
4782 fseek(in, pos, SEEK_SET);
4783 main_strm.zalloc=Z_NULL; main_strm.zfree=Z_NULL; main_strm.opaque=Z_NULL;
4784 main_strm.next_in=Z_NULL; main_strm.avail_in=0;
4785 if (zlib_inflateInit(&main_strm,zh)!=Z_OK) return false;
4786 for (int i=0; i<len; i+=BLOCK) {
4787 unsigned int blsize=min(len-i,BLOCK);
4788 fread(&zin[0], 1, blsize, in);
4789 main_strm.next_in=&zin[0]; main_strm.avail_in=blsize;
4790 do {
4791 main_strm.next_out=&zout[0]; main_strm.avail_out=BLOCK;
4792 main_ret=inflate(&main_strm, Z_FINISH);
4793 fwrite(&zout[0], 1, BLOCK-main_strm.avail_out, out);
4794 } while (main_strm.avail_out==0 && main_ret==Z_BUF_ERROR);
4795 if (main_ret!=Z_BUF_ERROR && main_ret!=Z_STREAM_END) break;
4796 }
4797 return main_ret==Z_STREAM_END;
4798}
4799
4800int decode_zlib(FILE* in, int size, FILE *out, FMode mode, int &diffFound) {
4801 const int BLOCK=1<<16, LIMIT=128;
4802 U8 zin[BLOCK],zout[BLOCK];
4803 int diffCount=min(fgetc(in),LIMIT-1);
4804 int window=fgetc(in)-MAX_WBITS;
4805 int index=fgetc(in);
4806 int memlevel=(index%9)+1;
4807 int clevel=(index/9)+1;
4808 int len=0;
4809 int diffPos[LIMIT];
4810 diffPos[0]=-1;
4811 for (int i=0; i<=diffCount; i++) {
4812 int v=fgetc(in)<<24; v|=fgetc(in)<<16; v|=fgetc(in)<<8; v|=fgetc(in);
4813 if (i==diffCount) len=v+diffPos[i]; else diffPos[i+1]=v+diffPos[i]+1;
4814 }
4815 U8 diffByte[LIMIT];
4816 diffByte[0]=0;
4817 for (int i=0; i<diffCount; i++) diffByte[i+1]=fgetc(in);
4818 size-=7+5*diffCount;
4819
4820 z_stream rec_strm;
4821 int diffIndex=1,recpos=0;
4822 rec_strm.zalloc=Z_NULL; rec_strm.zfree=Z_NULL; rec_strm.opaque=Z_NULL;
4823 rec_strm.next_in=Z_NULL; rec_strm.avail_in=0;
4824 int ret=deflateInit2(&rec_strm, clevel, Z_DEFLATED, window, memlevel, Z_DEFAULT_STRATEGY);
4825 if (ret!=Z_OK) return 0;
4826 for (int i=0; i<size; i+=BLOCK) {
4827 int blsize=min(size-i,BLOCK);
4828 fread(&zin[0], 1, blsize, in);
4829 rec_strm.next_in=&zin[0]; rec_strm.avail_in=blsize;
4830 do {
4831 rec_strm.next_out=&zout[0]; rec_strm.avail_out=BLOCK;
4832 ret=deflate(&rec_strm, i+blsize==size ? Z_FINISH : Z_NO_FLUSH);
4833 if (ret!=Z_BUF_ERROR && ret!=Z_STREAM_END && ret!=Z_OK) break;
4834 const int have=min(BLOCK-rec_strm.avail_out,len-recpos);
4835 while (diffIndex<=diffCount && diffPos[diffIndex]>=recpos && diffPos[diffIndex]<recpos+have) {
4836 zout[diffPos[diffIndex]-recpos]=diffByte[diffIndex];
4837 diffIndex++;
4838 }
4839 if (mode==FDECOMPRESS) fwrite(&zout[0], 1, have, out);
4840 else if (mode==FCOMPARE) for (int j=0; j<have; j++) if (zout[j]!=getc(out) && !diffFound) diffFound=recpos+j+1;
4841 recpos+=have;
4842
4843 } while (rec_strm.avail_out==0);
4844 }
4845 while (diffIndex<=diffCount) {
4846 if (mode==FDECOMPRESS) fputc(diffByte[diffIndex], out);
4847 else if (mode==FCOMPARE) if (diffByte[diffIndex]!=getc(out) && !diffFound) diffFound=recpos+1;
4848 diffIndex++;
4849 recpos++;
4850 }
4851 deflateEnd(&rec_strm);
4852 return recpos==len ? len : 0;
4853}
4854
4855//
4856// decode/encode base64
4857//
4858static const char table1[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4859bool isbase64(unsigned char c) {
4860 return (isalnum(c) || (c == '+') || (c == '/')|| (c == 10) || (c == 13));
4861}
4862
4863int decode_base64(FILE *in, FILE *out, FMode mode, int &diffFound){
4864 U8 inn[3];
4865 int i,len1=0, len=0, blocksout = 0;
4866 int fle=0;
4867 int linesize=0;
4868 int outlen=0;
4869 int tlf=0;
4870 linesize=getc(in);
4871 outlen=getc(in);
4872 outlen+=(getc(in)<<8);
4873 outlen+=(getc(in)<<16);
4874 tlf=(getc(in));
4875 outlen+=((tlf&63)<<24);
4876 U8 *ptr,*fptr;
4877 ptr = (U8*)calloc((outlen>>2)*4+10, 1);
4878 if (!ptr) quit("Out of memory (d_B64)");
4879 fptr=&ptr[0];
4880 tlf=(tlf&192);
4881 if (tlf==128)
4882 tlf=10; // LF: 10
4883 else if (tlf==64)
4884 tlf=13; // LF: 13
4885 else
4886 tlf=0;
4887
4888 while(fle<outlen){
4889 len=0;
4890 for(i=0;i<3;i++){
4891 inn[i] = getc( in );
4892 if(!feof(in)){
4893 len++;
4894 len1++;
4895 }
4896 else {
4897 inn[i] = 0;
4898 }
4899 }
4900 if(len){
4901 U8 in0,in1,in2;
4902 in0=inn[0],in1=inn[1],in2=inn[2];
4903 fptr[fle++]=(table1[in0>>2]);
4904 fptr[fle++]=(table1[((in0&0x03)<<4)|((in1&0xf0)>>4)]);
4905 fptr[fle++]=((len>1?table1[((in1&0x0f)<<2)|((in2&0xc0)>>6)]:'='));
4906 fptr[fle++]=((len>2?table1[in2&0x3f]:'='));
4907 blocksout++;
4908 }
4909 if(blocksout>=(linesize/4) && linesize!=0){ //no lf if linesize==0
4910 if( blocksout && !feof(in) && fle<=outlen) { //no lf if eof
4911 if (tlf) fptr[fle++]=(tlf);
4912 else fptr[fle++]=13,fptr[fle++]=10;
4913 }
4914 blocksout = 0;
4915 }
4916 }
4917 //Write out or compare
4918 if (mode==FDECOMPRESS){
4919 fwrite(&ptr[0], 1, outlen, out);
4920
4921 }
4922 else if (mode==FCOMPARE){
4923 for(i=0;i<outlen;i++){
4924 U8 b=fptr[i];
4925
4926
4927 if (b!=fgetc(out) && !diffFound) diffFound=ftell(out);
4928 }
4929 }
4930 free(ptr);
4931 return outlen;
4932}
4933
4934inline char valueb(char c){
4935 const char *p = strchr(table1, c);
4936 if(p) {
4937 return p-table1;
4938 } else {
4939 return 0;
4940 }
4941}
4942
4943void encode_base64(FILE* in, FILE* out, int len) {
4944 int in_len = 0;
4945 int i = 0;
4946 int j = 0;
4947 int b=0;
4948 int lfp=0;
4949 int tlf=0;
4950 char src[4];
4951 U8 *ptr,*fptr;
4952 int b64mem=(len>>2)*3+10;
4953 ptr = (U8*)calloc(b64mem, 1);
4954 if (!ptr) quit("Out of memory (e_B64)");
4955 fptr=&ptr[0];
4956 int olen=5;
4957
4958 while (b=fgetc(in),in_len++ , ( b != '=') && isbase64(b) && in_len<=len) {
4959 if (b==13 || b==10) {
4960 if (lfp==0) lfp=in_len ,tlf=b;
4961 if (tlf!=b) tlf=0;
4962 continue;
4963 }
4964 src[i++] = b;
4965 if (i ==4){
4966 for (j = 0; j <4; j++) src[j] = valueb(src[j]);
4967 src[0] = (src[0] << 2) + ((src[1] & 0x30) >> 4);
4968 src[1] = ((src[1] & 0xf) << 4) + ((src[2] & 0x3c) >> 2);
4969 src[2] = ((src[2] & 0x3) << 6) + src[3];
4970
4971 fptr[olen++]=src[0];
4972 fptr[olen++]=src[1];
4973 fptr[olen++]=src[2];
4974 i = 0;
4975 }
4976 }
4977
4978 if (i){
4979 for (j=i;j<4;j++)
4980 src[j] = 0;
4981
4982 for (j=0;j<4;j++)
4983 src[j] = valueb(src[j]);
4984
4985 src[0] = (src[0] << 2) + ((src[1] & 0x30) >> 4);
4986 src[1] = ((src[1] & 0xf) << 4) + ((src[2] & 0x3c) >> 2);
4987 src[2] = ((src[2] & 0x3) << 6) + src[3];
4988
4989 for (j=0;(j<i-1);j++) {
4990 fptr[olen++]=src[j];
4991 }
4992 }
4993 fptr[0]=lfp&255; //nl lenght
4994 fptr[1]=len&255;
4995 fptr[2]=len>>8&255;
4996 fptr[3]=len>>16&255;
4997 if (tlf!=0) {
4998 if (tlf==10) fptr[4]=128;
4999 else fptr[4]=64;
5000 }
5001 else
5002 fptr[4]=len>>24&63; //1100 0000
5003 fwrite(&ptr[0], 1, olen, out);
5004 free(ptr);
5005}
5006
5007int encode_gif(FILE* in, FILE* out, int len) {
5008 int codesize=fgetc(in),diffpos=0,hdrsize=6,clearpos=0,bsize=0;
5009 int beginin=ftell(in),beginout=ftell(out);
5010 U8 output[4096];
5011 fputc(hdrsize>>8, out);
5012 fputc(hdrsize&255, out);
5013 fputc(bsize, out);
5014 fputc(clearpos>>8, out);
5015 fputc(clearpos&255, out);
5016 fputc(codesize, out);
5017 for (int phase=0; phase<2; phase++) {
5018 fseek(in, beginin, SEEK_SET);
5019 int bits=codesize+1,shift=0,buf=0;
5020 int blocksize=0,maxcode=(1<<codesize)+1,last=-1,dict[4096];
5021 bool end=false;
5022 while ((blocksize=fgetc(in))>0 && ftell(in)-beginin<len && !end) {
5023 for (int i=0; i<blocksize; i++) {
5024 buf|=fgetc(in)<<shift;
5025 shift+=8;
5026 while (shift>=bits && !end) {
5027 int code=buf&((1<<bits)-1);
5028 buf>>=bits;
5029 shift-=bits;
5030 if (!bsize && code!=(1<<codesize)) {
5031 hdrsize+=4; fputc(0, out); fputc(0, out); fputc(0, out); fputc(0, out);
5032 }
5033 if (!bsize) bsize=blocksize;
5034 if (code==(1<<codesize)) {
5035 if (maxcode>(1<<codesize)+1) {
5036 if (clearpos && clearpos!=69631-maxcode) return 0;
5037 clearpos=69631-maxcode;
5038 }
5039 bits=codesize+1, maxcode=(1<<codesize)+1, last=-1;
5040 }
5041 else if (code==(1<<codesize)+1) end=true;
5042 else if (code>maxcode+1) return 0;
5043 else {
5044 int j=(code<=maxcode?code:last),size=1;
5045 while (j>=(1<<codesize)) {
5046 output[4096-(size++)]=dict[j]&255;
5047 j=dict[j]>>8;
5048 }
5049 output[4096-size]=j;
5050 if (phase==1) fwrite(&output[4096-size], 1, size, out); else diffpos+=size;
5051 if (code==maxcode+1) { if (phase==1) fputc(j, out); else diffpos++; }
5052 if (last!=-1) {
5053 if (++maxcode>=8191) return 0;
5054 if (maxcode<=4095)
5055 {
5056 dict[maxcode]=(last<<8)+j;
5057 if (phase==0) {
5058 bool diff=false;
5059 for (int m=(1<<codesize)+2;m<min(maxcode,4095);m++) if (dict[maxcode]==dict[m]) { diff=true; break; }
5060 if (diff) {
5061 hdrsize+=4;
5062 j=diffpos-size-(code==maxcode);
5063 fputc((j>>24)&255, out); fputc((j>>16)&255, out); fputc((j>>8)&255, out); fputc(j&255, out);
5064 diffpos=size+(code==maxcode);
5065 }
5066 }
5067 }
5068 if (maxcode>=((1<<bits)-1) && bits<12) bits++;
5069 }
5070 last=code;
5071 }
5072 }
5073 }
5074 }
5075 }
5076 diffpos=ftell(out);
5077 fseek(out, beginout, SEEK_SET);
5078 fputc(hdrsize>>8, out);
5079 fputc(hdrsize&255, out);
5080 fputc(255-bsize, out);
5081 fputc((clearpos>>8)&255, out);
5082 fputc(clearpos&255, out);
5083 fseek(out, diffpos, SEEK_SET);
5084 return ftell(in)-beginin==len-1;
5085}
5086
5087#define gif_write_block(count) { output[0]=(count);\
5088if (mode==FDECOMPRESS) fwrite(&output[0], 1, (count)+1, out);\
5089else if (mode==FCOMPARE) for (int j=0; j<(count)+1; j++) if (output[j]!=getc(out) && !diffFound) diffFound=outsize+j+1;\
5090outsize+=(count)+1; blocksize=0; }
5091
5092#define gif_write_code(c) { buf+=(c)<<shift; shift+=bits;\
5093while (shift>=8) { output[++blocksize]=buf&255; buf>>=8;shift-=8;\
5094if (blocksize==bsize) gif_write_block(bsize); }}
5095
5096int decode_gif(FILE* in, int size, FILE *out, FMode mode, int &diffFound) {
5097 int diffcount=fgetc(in), curdiff=0, diffpos[4096];
5098 diffcount=((diffcount<<8)+fgetc(in)-6)/4;
5099 int bsize=255-fgetc(in);
5100 int clearpos=fgetc(in); clearpos=(clearpos<<8)+fgetc(in);
5101 clearpos=(69631-clearpos)&0xffff;
5102 int codesize=fgetc(in),bits=codesize+1,shift=0,buf=0,blocksize=0;
5103 if (diffcount>4096 || clearpos<=(1<<codesize)+2) return 1;
5104 int maxcode=(1<<codesize)+1,dict[4096],input;
5105 for (int i=0; i<diffcount; i++) {
5106 diffpos[i]=fgetc(in);
5107 diffpos[i]=(diffpos[i]<<8)+fgetc(in);
5108 diffpos[i]=(diffpos[i]<<8)+fgetc(in);
5109 diffpos[i]=(diffpos[i]<<8)+fgetc(in);
5110 if (i>0) diffpos[i]+=diffpos[i-1];
5111 }
5112 U8 output[256];
5113 size-=6+diffcount*4;
5114 int last=fgetc(in),total=size+1,outsize=1;
5115 if (mode==FDECOMPRESS) fputc(codesize, out);
5116 else if (mode==FCOMPARE) if (codesize!=getc(out) && !diffFound) diffFound=1;
5117 if (diffcount==0 || diffpos[0]!=0) gif_write_code(1<<codesize) else curdiff++;
5118 while (size-->=0 && (input=fgetc(in))>=0) {
5119 int code=-1, key=(last<<8)+input;
5120 for (int i=(1<<codesize)+2; i<=min(maxcode,4095); i++) if (dict[i]==key) code=i;
5121 if (curdiff<diffcount && total-size>diffpos[curdiff]) curdiff++,code=-1;
5122 if (code==-1) {
5123 gif_write_code(last);
5124 if (maxcode==clearpos) { gif_write_code(1<<codesize); bits=codesize+1, maxcode=(1<<codesize)+1; }
5125 else
5126 {
5127 ++maxcode;
5128 if (maxcode<=4095) dict[maxcode]=key;
5129 if (maxcode>=(1<<bits) && bits<12) bits++;
5130 }
5131 code=input;
5132 }
5133 last=code;
5134 }
5135 gif_write_code(last);
5136 gif_write_code((1<<codesize)+1);
5137 if (shift>0) {
5138 output[++blocksize]=buf&255;
5139 if (blocksize==bsize) gif_write_block(bsize);
5140 }
5141 if (blocksize>0) gif_write_block(blocksize);
5142 if (mode==FDECOMPRESS) fputc(0, out);
5143 else if (mode==FCOMPARE) if (0!=getc(out) && !diffFound) diffFound=outsize+1;
5144 return outsize+1;
5145}
5146
5147
5148//////////////////// Compress, Decompress ////////////////////////////
5149
5150void direct_encode_block(Filetype type, FILE *in, int len, Encoder &en, int info=-1) {
5151 en.compress(type);
5152 en.compress(len>>24);
5153 en.compress(len>>16);
5154 en.compress(len>>8);
5155 en.compress(len);
5156 if (info!=-1) {
5157 en.compress(info>>24);
5158 en.compress(info>>16);
5159 en.compress(info>>8);
5160 en.compress(info);
5161 }
5162 printf("Compressing... ");
5163 for (int j=0; j<len; ++j) {
5164 if (!(j&0xfff)) en.print_status(j, len);
5165 en.compress(getc(in));
5166 }
5167 printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b");
5168}
5169
5170void compressRecursive(FILE *in, long n, Encoder &en, char *blstr, int it=0, float p1=0.0, float p2=1.0);
5171
5172void transform_encode_block(Filetype type, FILE *in, int len, Encoder &en, int info, char *blstr, int it, float p1, float p2, long begin) {
5173 if (type==EXE || type==CD || type==IMAGE24 || type==IMAGE32 || type==ZLIB || type==BASE64 || type==GIF) {
5174 FILE* tmp=tmpfile(); // temporary encoded file
5175 if (!tmp) perror("tmpfile"), quit();
5176 int diffFound=0;
5177 if (type==IMAGE24) encode_bmp(in, tmp, len, info);
5178 else if (type==IMAGE32) encode_im32(in, tmp, len, info);
5179 else if (type==EXE) encode_exe(in, tmp, len, begin);
5180 else if (type==CD) encode_cd(in, tmp, len, info);
5181 else if (type==ZLIB) diffFound=encode_zlib(in, tmp, len)?0:1;
5182 else if (type==BASE64) encode_base64(in, tmp, len);
5183 else if (type==GIF) diffFound=encode_gif(in, tmp, len)?0:1;
5184 const long tmpsize=ftell(tmp);
5185 fseek(tmp, tmpsize, SEEK_SET);
5186 if (!diffFound) {
5187 rewind(tmp);
5188 en.setFile(tmp);
5189 fseek(in, begin, SEEK_SET);
5190 if (type==IMAGE24) decode_bmp(en, tmpsize, info, in, FCOMPARE, diffFound);
5191 else if (type==IMAGE32) decode_im32(en, tmpsize, info, in, FCOMPARE, diffFound);
5192 else if (type==EXE) decode_exe(en, tmpsize, in, FCOMPARE, diffFound);
5193 else if (type==CD) decode_cd(tmp, tmpsize, in, FCOMPARE, diffFound);
5194 else if (type==ZLIB) decode_zlib(tmp, tmpsize, in, FCOMPARE, diffFound);
5195 else if (type==BASE64) decode_base64(tmp, in, FCOMPARE, diffFound);
5196 else if (type==GIF) decode_gif(tmp, tmpsize, in, FCOMPARE, diffFound);
5197 }
5198 // Test fails, compress without transform
5199 if (diffFound || fgetc(tmp)!=EOF) {
5200 printf("Transform fails at %d, skipping...\n", diffFound-1);
5201 fseek(in, begin, SEEK_SET);
5202 direct_encode_block(DEFAULT, in, len, en);
5203 } else {
5204 rewind(tmp);
5205 if (type==CD || type==ZLIB || type==BASE64 || type==GIF) {
5206 en.compress(type), en.compress(tmpsize>>24), en.compress(tmpsize>>16);
5207 en.compress(tmpsize>>8), en.compress(tmpsize);
5208 if (type==ZLIB && info) {// PDF or PNG image
5209 Filetype type2 = DEFAULT;
5210 switch ((info>>24)&0x3F){
5211 case 32 : type2 = IMAGE32; break;
5212 case 24 : type2 = IMAGE24; break;
5213 case 8 : type2 = ((info&GrayFlag)>0)?IMAGE8GRAY:IMAGE8; break;
5214 case 1 : type2 = IMAGE1;
5215 }
5216 int hdrsize=7+5*fgetc(tmp);
5217 rewind(tmp);
5218 direct_encode_block(HDR, tmp, hdrsize, en);
5219 if (!(info&PNGFlag))
5220 transform_encode_block(type2, tmp, tmpsize-hdrsize, en, info&0xffffff, blstr, it, p1, p2, hdrsize);
5221 else
5222 direct_encode_block(type2, tmp, tmpsize-hdrsize, en, info&(0xFFFFFF|PNGFlag|GrayFlag));
5223 } else if (type==GIF) {
5224 int hdrsize=fgetc(tmp);
5225 hdrsize=(hdrsize<<8)+fgetc(tmp);
5226 rewind(tmp);
5227 direct_encode_block(HDR, tmp, hdrsize, en);
5228 direct_encode_block(IMAGE8, tmp, tmpsize-hdrsize, en, info);
5229 } else {
5230 compressRecursive(tmp, tmpsize, en, blstr, it+1, p1, p2);
5231 }
5232 } else if (type==EXE) {
5233 direct_encode_block(type, tmp, tmpsize, en);
5234 } else if (type==IMAGE24 || type==IMAGE32) {
5235 direct_encode_block(type, tmp, tmpsize, en, info);
5236 }
5237 }
5238 fclose(tmp); // deletes
5239 } else {
5240 const int i1=(type==IMAGE1 || type==IMAGE4 || type==IMAGE8 || type==IMAGE8GRAY || type==AUDIO)?info:-1;
5241 direct_encode_block(type, in, len, en, i1);
5242 }
5243}
5244
5245void compressRecursive(FILE *in, long n, Encoder &en, char *blstr, int it, float p1, float p2) {
5246 static const char* typenames[15]={"default", "jpeg", "hdr",
5247 "1b-image", "4b-image", "8b-image", "8b-img-grayscale", "24b-image", "32b-image", "audio", "exe", "cd", "zlib", "base64", "gif"};
5248 static const char* audiotypes[4]={"8b mono", "8b stereo", "16b mono",
5249 "16b stereo"};
5250 Filetype type=DEFAULT;
5251 int blnum=0, info; // image width or audio type
5252 long begin=ftell(in), end0=begin+n;
5253 char b2[32];
5254 strcpy(b2, blstr);
5255 if (b2[0]) strcat(b2, "-");
5256 if (it==5) {
5257 direct_encode_block(DEFAULT, in, n, en);
5258 return;
5259 }
5260 float pscale=n>0?(p2-p1)/n:0;
5261
5262 // Transform and test in blocks
5263 while (n>0) {
5264 Filetype nextType=detect(in, n, type, info);
5265 long end=ftell(in);
5266 fseek(in, begin, SEEK_SET);
5267 if (end>end0) { // if some detection reports longer then actual size file is
5268 end=begin+1;
5269 type=DEFAULT;
5270 }
5271 int len=int(end-begin);
5272 if (len>0) {
5273 en.set_status_range(p1,p2=p1+pscale*len);
5274 sprintf(blstr,"%s%d",b2,blnum++);
5275 printf(" %-11s | %-16s |%10d bytes [%ld - %ld]",blstr,typenames[type],len,begin,end-1);
5276 if (type==AUDIO) printf(" (%s)", audiotypes[info%4]);
5277 else if (type==IMAGE1 || type==IMAGE4 || type==IMAGE8 || type==IMAGE8GRAY || type==IMAGE24 || type==IMAGE32) printf(" (width: %d)", info);
5278 else if (type==CD) printf(" (m%d/f%d)", info==1?1:2, info!=3?1:2);
5279 else if (type==ZLIB && info) printf(" (image %dbpp%s)",(info>>24)&0x3F,((info&GrayFlag)>0)?" grayscale":"");
5280 printf("\n");
5281 transform_encode_block(type, in, len, en, info, blstr, it, p1, p2, begin);
5282 p1=p2;
5283 n-=len;
5284 }
5285 type=nextType;
5286 begin=end;
5287 }
5288}
5289
5290// Compress a file. Split filesize bytes into blocks by type.
5291// For each block, output
5292// <type> <size> and call encode_X to convert to type X.
5293// Test transform and compress.
5294void compress(const char* filename, long filesize, Encoder& en) {
5295 assert(en.getMode()==COMPRESS);
5296 assert(filename && filename[0]);
5297 FILE *in=fopen(filename, "rb");
5298 if (!in) perror(filename), quit();
5299 long start=en.size();
5300 printf("Block segmentation:\n");
5301 char blstr[32]="";
5302 compressRecursive(in, filesize, en, blstr);
5303 if (in) fclose(in);
5304 printf("Compressed from %ld to %ld bytes.\n",filesize,en.size()-start);
5305}
5306
5307#ifdef WINDOWS
5308#include <windows.h>
5309#endif
5310
5311// Try to make a directory, return true if successful
5312bool makedir(const char* dir) {
5313#ifdef WINDOWS
5314 return CreateDirectory(dir, 0)==TRUE;
5315#else
5316#ifdef UNIX
5317 return mkdir(dir, 0777)==0;
5318#else
5319 return false;
5320#endif
5321#endif
5322}
5323
5324int decompressRecursive(FILE *out, long n, Encoder& en, FMode mode, int it=0) {
5325 Filetype type;
5326 long len, i=0;
5327 int diffFound=0, info;
5328 FILE *tmp;
5329 while (i<n) {
5330 type=(Filetype)en.decompress();
5331 len=en.decompress()<<24;
5332 len|=en.decompress()<<16;
5333 len|=en.decompress()<<8;
5334 len|=en.decompress();
5335
5336 if (type==IMAGE1 || type==IMAGE4 || type==IMAGE8 || type==IMAGE8GRAY || type==IMAGE24 || type==IMAGE32 || type==AUDIO) {
5337 info=0; for (int i=0; i<4; ++i) { info<<=8; info+=en.decompress(); }
5338 }
5339 if (type==IMAGE24 && !(info&PNGFlag) /*To skip the color transform on PNG filtered data*/) len=decode_bmp(en, len, info, out, mode, diffFound);
5340 else if (type==IMAGE32 && !(info&PNGFlag) /*To skip the color transform on PNG filtered data*/) decode_im32(en, len, info, out, mode, diffFound);
5341 else if (type==EXE) len=decode_exe(en, len, out, mode, diffFound);
5342 else if (type==CD || type==ZLIB || type==GIF || type==BASE64) {
5343 tmp=tmpfile();
5344 if (!tmp) perror("tmpfile"), quit();
5345 decompressRecursive(tmp, len, en, FDECOMPRESS, it+1);
5346 if (mode!=FDISCARD) {
5347 rewind(tmp);
5348 if (type==CD) len=decode_cd(tmp, len, out, mode, diffFound);
5349 if (type==ZLIB) len=decode_zlib(tmp, len, out, mode, diffFound);
5350 if (type==BASE64) len=decode_base64(tmp, out, mode, diffFound);
5351 if (type==GIF) len=decode_gif(tmp, len, out, mode, diffFound);
5352 }
5353 fclose(tmp);
5354 } else {
5355 for (int j=0; j<len; ++j) {
5356 if (!(j&0xfff)) en.print_status();
5357 if (mode==FDECOMPRESS) putc(en.decompress(), out);
5358 else if (mode==FCOMPARE) {
5359 if (en.decompress()!=fgetc(out) && !diffFound) {
5360 mode=FDISCARD;
5361 diffFound=i+j+1;
5362 }
5363 } else en.decompress();
5364 }
5365 }
5366 i+=len;
5367 }
5368 return diffFound;
5369}
5370
5371// Decompress a file
5372void decompress(const char* filename, long filesize, Encoder& en) {
5373 FMode mode=FDECOMPRESS;
5374 assert(en.getMode()==DECOMPRESS);
5375 assert(filename && filename[0]);
5376
5377 // Test if output file exists. If so, then compare.
5378 FILE* f=fopen(filename, "rb");
5379 if (f) mode=FCOMPARE,printf("Comparing");
5380 else {
5381 // Create file
5382 f=fopen(filename, "wb");
5383 if (!f) { // Try creating directories in path and try again
5384 String path(filename);
5385 for (int i=0; path[i]; ++i) {
5386 if (path[i]=='/' || path[i]=='\\') {
5387 char savechar=path[i];
5388 path[i]=0;
5389 if (makedir(path.c_str()))
5390 printf("Created directory %s\n", path.c_str());
5391 path[i]=savechar;
5392 }
5393 }
5394 f=fopen(filename, "wb");
5395 }
5396 if (!f) mode=FDISCARD,printf("Skipping"); else printf("Extracting");
5397 }
5398 printf(" %s %ld -> ", filename, filesize);
5399
5400 // Decompress/Compare
5401 int r=decompressRecursive(f, filesize, en, mode);
5402 if (mode==FCOMPARE && !r && getc(f)!=EOF) printf("file is longer\n");
5403 else if (mode==FCOMPARE && r) printf("differ at %d\n",r-1);
5404 else if (mode==FCOMPARE) printf("identical\n");
5405 else printf("done \n");
5406 if (f) fclose(f);
5407}
5408
5409//////////////////////////// User Interface ////////////////////////////
5410
5411
5412// int expand(String& archive, String& s, const char* fname, int base) {
5413// Given file name fname, print its length and base name (beginning
5414// at fname+base) to archive in format "%ld\t%s\r\n" and append the
5415// full name (including path) to String s in format "%s\n". If fname
5416// is a directory then substitute all of its regular files and recursively
5417// expand any subdirectories. Base initially points to the first
5418// character after the last / in fname, but in subdirectories includes
5419// the path from the topmost directory. Return the number of files
5420// whose names are appended to s and archive.
5421
5422// Same as expand() except fname is an ordinary file
5423int putsize(String& archive, String& s, const char* fname, int base) {
5424 int result=0;
5425 FILE *f=fopen(fname, "rb");
5426 if (f) {
5427 fseek(f, 0, SEEK_END);
5428 long len=ftell(f);
5429 if (len>=0) {
5430 static char blk[24];
5431 sprintf(blk, "%ld\t", len);
5432 archive+=blk;
5433 archive+=(fname+base);
5434 archive+="\n";
5435 s+=fname;
5436 s+="\n";
5437 ++result;
5438 }
5439 fclose(f);
5440 }
5441 return result;
5442}
5443
5444#ifdef WINDOWS
5445
5446int expand(String& archive, String& s, const char* fname, int base) {
5447 int result=0;
5448 DWORD attr=GetFileAttributes(fname);
5449 if ((attr != 0xFFFFFFFF) && (attr & FILE_ATTRIBUTE_DIRECTORY)) {
5450 WIN32_FIND_DATA ffd;
5451 String fdir(fname);
5452 fdir+="/*";
5453 HANDLE h=FindFirstFile(fdir.c_str(), &ffd);
5454 while (h!=INVALID_HANDLE_VALUE) {
5455 if (!equals(ffd.cFileName, ".") && !equals(ffd.cFileName, "..")) {
5456 String d(fname);
5457 d+="/";
5458 d+=ffd.cFileName;
5459 result+=expand(archive, s, d.c_str(), base);
5460 }
5461 if (FindNextFile(h, &ffd)!=TRUE) break;
5462 }
5463 FindClose(h);
5464 }
5465 else // ordinary file
5466 result=putsize(archive, s, fname, base);
5467 return result;
5468}
5469
5470#else
5471#ifdef UNIX
5472
5473int expand(String& archive, String& s, const char* fname, int base) {
5474 int result=0;
5475 struct stat sb;
5476 if (stat(fname, &sb)<0) return 0;
5477
5478 // If a regular file and readable, get file size
5479 if (sb.st_mode & S_IFREG && sb.st_mode & 0400)
5480 result+=putsize(archive, s, fname, base);
5481
5482 // If a directory with read and execute permission, traverse it
5483 else if (sb.st_mode & S_IFDIR && sb.st_mode & 0400 && sb.st_mode & 0100) {
5484 DIR *dirp=opendir(fname);
5485 if (!dirp) {
5486 perror("opendir");
5487 return result;
5488 }
5489 dirent *dp;
5490 while(errno=0, (dp=readdir(dirp))!=0) {
5491 if (!equals(dp->d_name, ".") && !equals(dp->d_name, "..")) {
5492 String d(fname);
5493 d+="/";
5494 d+=dp->d_name;
5495 result+=expand(archive, s, d.c_str(), base);
5496 }
5497 }
5498 if (errno) perror("readdir");
5499 closedir(dirp);
5500 }
5501 else printf("%s is not a readable file or directory\n", fname);
5502 return result;
5503}
5504
5505#else // Not WINDOWS or UNIX, ignore directories
5506
5507int expand(String& archive, String& s, const char* fname, int base) {
5508 return putsize(archive, s, fname, base);
5509}
5510
5511#endif
5512#endif
5513
5514
5515// To compress to file1.fp8: fp8 [-n] file1 [file2...]
5516// To decompress: fp8 file1.fp8 [output_dir]
5517int main(int argc, char** argv) {
5518 bool pause=argc<=2; // Pause when done?
5519 try {
5520
5521 // Get option
5522 bool doExtract=false; // -d option
5523 bool doList=false; // -l option
5524 if (argc>1 && argv[1][0]=='-' && argv[1][1] && !argv[1][2]) {
5525 if (argv[1][1]>='0' && argv[1][1]<='8')
5526 level=argv[1][1]-'0';
5527 else if (argv[1][1]=='d')
5528 doExtract=true;
5529 else if (argv[1][1]=='l')
5530 doList=true;
5531 else
5532 quit("Valid options are -0 through -8, -d, -l\n");
5533 --argc;
5534 ++argv;
5535 pause=false;
5536 }
5537
5538 // Print help message
5539 if (argc<2) {
5540 printf(PROGNAME " archiver (C) 2016, Matt Mahoney et al.\n"
5541 "Free under GPL, http://www.gnu.org/licenses/gpl.txt\n\n"
5542#ifdef WINDOWS
5543 "To compress or extract, drop a file or folder on the "
5544 PROGNAME " icon.\n"
5545 "The output will be put in the same folder as the input.\n"
5546 "\n"
5547 "Or from a command window: "
5548#endif
5549 "To compress:\n"
5550 " " PROGNAME " -level file (compresses to file." PROGNAME ")\n"
5551 " " PROGNAME " -level archive files... (creates archive." PROGNAME ")\n"
5552 " " PROGNAME " file (level -%d, pause when done)\n"
5553 "level: -0 = store, -1 -2 -3 = faster (uses 35, 48, 59 MB)\n"
5554 "-4 -5 -6 -7 -8 = smaller (uses 133, 233, 435, 837, 1643 MB)\n"
5555#if defined(WINDOWS) || defined (UNIX)
5556 "You may also compress directories.\n"
5557#endif
5558 "\n"
5559 "To extract or compare:\n"
5560 " " PROGNAME " -d dir1/archive." PROGNAME " (extract to dir1)\n"
5561 " " PROGNAME " -d dir1/archive." PROGNAME " dir2 (extract to dir2)\n"
5562 " " PROGNAME " archive." PROGNAME " (extract, pause when done)\n"
5563 "\n"
5564 "To view contents: " PROGNAME " -l archive." PROGNAME "\n"
5565 "\n",
5566 DEFAULT_OPTION);
5567 quit();
5568 }
5569
5570 FILE* archive=0; // compressed file
5571 int files=0; // number of files to compress/decompress
5572 Array<const char*> fname(1); // file names (resized to files)
5573 Array<long> fsize(1); // file lengths (resized to files)
5574
5575 // Compress or decompress? Get archive name
5576 Mode mode=COMPRESS;
5577 String archiveName(argv[1]);
5578 {
5579 const int prognamesize=strlen(PROGNAME);
5580 const int arg1size=strlen(argv[1]);
5581 if (arg1size>prognamesize+1 && argv[1][arg1size-prognamesize-1]=='.'
5582 && equals(PROGNAME, argv[1]+arg1size-prognamesize)) {
5583 mode=DECOMPRESS;
5584 }
5585 else if (doExtract || doList)
5586 mode=DECOMPRESS;
5587 else {
5588 archiveName+=".";
5589 archiveName+=PROGNAME;
5590 }
5591 }
5592
5593 // Compress: write archive header, get file names and sizes
5594 String header_string;
5595 String filenames;
5596 if (mode==COMPRESS) {
5597
5598 // Expand filenames to read later. Write their base names and sizes
5599 // to archive.
5600 int i;
5601 for (i=1; i<argc; ++i) {
5602 String name(argv[i]);
5603 int len=name.size()-1;
5604 for (int j=0; j<=len; ++j) // change \ to /
5605 if (name[j]=='\\') name[j]='/';
5606 while (len>0 && name[len-1]=='/') // remove trailing /
5607 name[--len]=0;
5608 int base=len-1;
5609 while (base>=0 && name[base]!='/') --base; // find last /
5610 ++base;
5611 if (base==0 && len>=2 && name[1]==':') base=2; // chop "C:"
5612 int expanded=expand(header_string, filenames, name.c_str(), base);
5613 if (!expanded && (i>1||argc==2))
5614 printf("%s: not found, skipping...\n", name.c_str());
5615 files+=expanded;
5616 }
5617
5618 // If there is at least one file to compress
5619 // then create the archive header.
5620 if (files<1) quit("Nothing to compress\n");
5621 archive=fopen(archiveName.c_str(), "wb+");
5622 if (!archive) perror(archiveName.c_str()), quit();
5623 fprintf(archive, PROGNAME "%c%d", 0, level);
5624 printf("Creating archive %s with %d file(s)...\n",
5625 archiveName.c_str(), files);
5626 }
5627
5628 // Decompress: open archive for reading and store file names and sizes
5629 if (mode==DECOMPRESS) {
5630 archive=fopen(archiveName.c_str(), "rb+");
5631 if (!archive) perror(archiveName.c_str()), quit();
5632
5633 // Check for proper format and get option
5634 String header;
5635 int len=strlen(PROGNAME)+2, c, i=0;
5636 header.resize(len+1);
5637 while (i<len && (c=getc(archive))!=EOF) {
5638 header[i]=c;
5639 i++;
5640 }
5641 header[i]=0;
5642 if (strncmp(header.c_str(), PROGNAME "\0", strlen(PROGNAME)+1))
5643 printf("%s: not a %s file\n", archiveName.c_str(), PROGNAME), quit();
5644 level=header[strlen(PROGNAME)+1]-'0';
5645 if (level<0||level>8) level=DEFAULT_OPTION;
5646 }
5647
5648 // Set globals according to option
5649 assert(level>=0 && level<=8);
5650 buf.setsize(MEM*8);
5651 Encoder en(mode, archive);
5652
5653 // Compress header
5654 if (mode==COMPRESS) {
5655 int len=header_string.size();
5656 printf("\nFile list (%d bytes)\n", len);
5657 assert(en.getMode()==COMPRESS);
5658 long start=en.size();
5659 en.compress(0); // block type 0
5660 en.compress(len>>24); en.compress(len>>16); en.compress(len>>8); en.compress(len); // block length
5661 for (int i=0; i<len; i++) en.compress(header_string[i]);
5662 printf("Compressed from %d to %ld bytes.\n",len,en.size()-start);
5663 }
5664
5665 // Deompress header
5666 if (mode==DECOMPRESS) {
5667 if (en.decompress()!=0) printf("%s: header corrupted\n", archiveName.c_str()), quit();
5668 int len=0;
5669 len+=en.decompress()<<24;
5670 len+=en.decompress()<<16;
5671 len+=en.decompress()<<8;
5672 len+=en.decompress();
5673 header_string.resize(len);
5674 for (int i=0; i<len; i++) {
5675 header_string[i]=en.decompress();
5676 if (header_string[i]=='\n') files++;
5677 }
5678 if (doList) printf("File list of %s archive:\n%s", archiveName.c_str(), header_string.c_str());
5679 }
5680
5681 // Fill fname[files], fsize[files] with input filenames and sizes
5682 fname.resize(files);
5683 fsize.resize(files);
5684 char *p=&header_string[0];
5685 char* q=&filenames[0];
5686 for (int i=0; i<files; ++i) {
5687 assert(p);
5688 fsize[i]=atol(p);
5689 assert(fsize[i]>=0);
5690 while (*p!='\t') ++p;
5691 *(p++)='\0';
5692 fname[i]=mode==COMPRESS?q:p;
5693 while (*p!='\n') ++p;
5694 *(p++)='\0';
5695 if (mode==COMPRESS) { while (*q!='\n') ++q; *(q++)='\0'; }
5696 }
5697
5698 // Compress or decompress files
5699 assert(fname.size()==files);
5700 assert(fsize.size()==files);
5701 long total_size=0; // sum of file sizes
5702 for (int i=0; i<files; ++i) total_size+=fsize[i];
5703 if (mode==COMPRESS) {
5704 for (int i=0; i<files; ++i) {
5705 printf("\n%d/%d Filename: %s (%ld bytes)\n", i+1, files, fname[i], fsize[i]);
5706 compress(fname[i], fsize[i], en);
5707 }
5708 en.flush();
5709 printf("\nTotal %ld bytes compressed to %ld bytes.\n", total_size, en.size());
5710 }
5711
5712 // Decompress files to dir2: fp8 -d dir1/archive.fp8 dir2
5713 // If there is no dir2, then extract to dir1
5714 // If there is no dir1, then extract to .
5715 else if (!doList) {
5716 assert(argc>=2);
5717 String dir(argc>2?argv[2]:argv[1]);
5718 if (argc==2) { // chop "/archive.fp8"
5719 int i;
5720 for (i=dir.size()-2; i>=0; --i) {
5721 if (dir[i]=='/' || dir[i]=='\\') {
5722 dir[i]=0;
5723 break;
5724 }
5725 if (i==1 && dir[i]==':') { // leave "C:"
5726 dir[i+1]=0;
5727 break;
5728 }
5729 }
5730 if (i==-1) dir="."; // "/" not found
5731 }
5732 dir=dir.c_str();
5733 if (dir[0] && (dir.size()!=3 || dir[1]!=':')) dir+="/";
5734 for (int i=0; i<files; ++i) {
5735 String out(dir.c_str());
5736 out+=fname[i];
5737 decompress(out.c_str(), fsize[i], en);
5738 }
5739 }
5740 fclose(archive);
5741 if (!doList) programChecker.print();
5742 }
5743 catch(const char* s) {
5744 if (s) printf("%s\n", s);
5745 }
5746 if (pause) {
5747 printf("\nClose this window or press ENTER to continue...\n");
5748 getchar();
5749 }
5750 return 0;
5751}