· 6 years ago · Sep 10, 2019, 03:22 PM
1// Dump Qt embedded resources (QRC) from a manually extracted region (in PE .rdata section).
2// https://stackoverflow.com/questions/35772103/qt-applications-replacing-embedded-resources/51281796#51281796
3
4// Format:
5// <4 byte:big-endian:resource size><resource data>
6
7/*/
8 * Configuration: Release
9 * Platform: x64
10 *
11 * Additional compiler options:
12 * /MP /Os
13 * https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-by-category
14 *
15 * Additional linker options:
16 * "kernel32.lib" /MANIFEST:NO
17 * https://docs.microsoft.com/en-us/cpp/build/reference/linker-options
18/*/
19
20
21//==================[targetver.h]==================//
22#pragma once
23
24#include <SDKDDKVer.h>
25//===============[end: targetver.h]================//
26
27
28//===================[stdafx.h]====================//
29#pragma once
30
31#include "targetver.h"
32
33//_byteswap_*
34#include <intrin.h>
35
36#include <stdio.h>
37#include <wchar.h>
38
39#include <windows.h>
40
41//Little-endian Test (and IntelliSense adequacy Test)
42//https://docs.microsoft.com/en-us/cpp/cpp/string-and-character-literals-cpp
43//https://en.wikipedia.org/wiki/Escape_sequences_in_C#Table_of_escape_sequences
44#if '\x00\x01'==0x0100u
45#define LITTLE_ENDIAN
46//ntohl() - call to ws2_32.dll
47//_byteswap_ulong() needs /Oi compiler flag (properties: C/C++ > Optimization: Enable Intristic Functions)
48#define be32_to_cpu(be32) (_byteswap_ulong(be32))
49#else
50#error "Little-endian Test: fail - this wasn't supposed to happen on Windows https://stackoverflow.com/questions/2100331/c-macro-definition-to-determine-big-endian-or-little-endian-machine#comment94541545_2100363"
51//IntelliSense BUG
52#define be32_to_cpu(be32) (be32)
53#endif
54
55#define _0P nullptr
56//================[end: stdafx.h]==================//
57
58
59//===================[stdafx.cpp]==================//
60#include "stdafx.h"
61//================[end: stdafx.cpp]================//
62
63
64//=========[Qt-QRC-resources-extractor.cpp]========//
65#include "stdafx.h"
66
67#define MAIN_EXTRACTOR
68#ifdef MAIN_EXTRACTOR
69//_getch(): conio.h + VC/crt/src/getch.c
70//_wassert(): assert.h + VC/crt/src/assert.c
71//For Single-threaded mode (unsafe in Multi-threaded mode)
72void err_pause()
73{
74 //ReadConsoleInput https://github.com/MicrosoftDocs/Console-Docs/blob/master/docs/readconsoleinput.md#user-content-readconsoleinput-function
75 //example https://github.com/MicrosoftDocs/Console-Docs/blob/master/docs/reading-input-buffer-events.md
76
77 //Standard Handle can be redirected to a file -> some functions fail
78 //HANDLE hIn = GetStdHandle(STD_INPUT_HANDLE);
79 //HANDLE hErr = GetStdHandle(STD_ERROR_HANDLE);
80 //better not do that:
81 static HANDLE hIn = CreateFile(L"CONIN$", GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, _0P, OPEN_EXISTING, 0, _0P);
82 static HANDLE hErr = CreateFile(L"CONOUT$", GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, _0P, OPEN_EXISTING, 0, _0P);
83
84 //ENABLE_PROCESSED_INPUT:
85 //https://github.com/MicrosoftDocs/Console-Docs/blob/master/docs/setconsolemode.md#user-content-parameters
86 //https://github.com/MicrosoftDocs/Console-Docs/blob/master/docs/setconsolemode.md#user-content-remarks
87 //HandlerRoutine for [Ctrl+C] https://github.com/MicrosoftDocs/Console-Docs/blob/master/docs/handlerroutine.md#user-content-handlerroutine-callback-function
88 DWORD const newMode = ENABLE_PROCESSED_INPUT;
89 DWORD oldMode;
90
91 GetConsoleMode(hIn, &oldMode);
92 SetConsoleMode(hIn, newMode);
93
94 FlushConsoleInputBuffer(hIn);
95
96 INPUT_RECORD event;
97 DWORD eventsNum;
98 do{
99 if(!ReadConsoleInput(hIn, &event, 1, &eventsNum) || eventsNum!=1){
100 //https://github.com/MicrosoftDocs/Console-Docs/blob/master/docs/writeconsole.md#user-content-remarks
101 #define c_str__len(str) str, sizeof(str)/sizeof(*str)-1
102 WriteConsoleA(hErr, c_str__len("All bad :-(\n"),_0P,_0P); //may fail...
103 #undef c_str__len
104 //fputs("All bad :-(\n", stderr); //too complex and unsafe (see _assert() in assert.c) for panic state
105 SetConsoleMode(hIn, oldMode);
106 SuspendThread(GetCurrentThread()); //resume in Process Explorer | or __debugbreak() or DebugBreak()
107 SetConsoleMode(hIn, newMode);
108 }else{
109 if(event.EventType==KEY_EVENT && !event.Event.KeyEvent.bKeyDown) break; //"!" to pass [Ctrl] for [Ctrl+C]
110 }
111 }while(1);
112
113 SetConsoleMode(hIn, oldMode);
114}
115
116BOOL _pause_before_exit_ = 0; //pause to see errors
117
118typedef BOOL (WINAPI &rw_file)(HANDLE,LPVOID,DWORD,LPDWORD,LPOVERLAPPED);
119typedef BOOL (WINAPI &w_console)(HANDLE,LPCSTR,DWORD,LPDWORD,LPVOID);
120
121// Try* functions - pause the program if an error is occurred - to Fix the Error and Continue
122
123void TryRWFile_(rw_file RW, void *__restrict const hFile, void *__restrict const dataBuf, const DWORD dataSize, DWORD &bytesDone, const UINT caller_line)
124{
125 /*//for: DWORD *__restrict _bytesDone
126 DWORD rec;
127 *bytesDone = _bytesDone==nullptr ? &rec : _bytesDone;
128 //but breaks down optimization*/
129
130 if(!RW(hFile, dataBuf, dataSize, &bytesDone,_0P)){
131 ///Start of func///////////////////////////////////////////////////////////////
132 ///==========================================================================//
133 ///compiller optimize this: extracts (inline) `if` above into caller,
134 ///if compile (Release) with /O2 (Maximize Speed) /Os (Favor small code) flags
135 ///(properties: C/C++ > Optimization)
136 ///(or https://docs.microsoft.com/en-us/cpp/preprocessor/optimize)
137 ///>>>see in Disassembly tab: Debug [F5],
138 ///>>>Debug menu > Windows > Disassembly [Alt+8]
139 ///==========================================================================//
140 DWORD err = GetLastError();
141 _pause_before_exit_ = 1;
142
143 //https://stackoverflow.com/questions/21374548/setstdhandle-has-no-effect-on-cout-printf
144 HANDLE hErr; do{
145 hErr = GetStdHandle(STD_ERROR_HANDLE);
146 }while((hErr==INVALID_HANDLE_VALUE || hErr==_0P) && (__debugbreak(), 1));
147
148 w_console WConsole = GetFileType(hErr)==FILE_TYPE_CHAR ? (w_console)WriteConsoleA:(w_console)WriteFile;
149
150 #define c_str__len(str) str, sizeof(str)/sizeof(*str)-1
151 WConsole(hErr, c_str__len("Error Codes: docs.microsoft.com/en-us/windows/win32/debug/system-error-codes\n"), _0P,_0P);
152 //https://docs.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-getlasterror
153 //https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-formatmessage
154 //https://docs.microsoft.com/en-us/windows/win32/debug/retrieving-the-last-error-code
155 //https://docs.microsoft.com/en-us/windows/win32/debug/error-handling-functions
156
157 auto &msg_0 = "TryRWFile(): line %u, error #%u";
158 const size_t msg_0_size = sizeof(msg_0) + 2*(-2+10);
159
160 auto &msg_0_0 = ", bytes written %u of %u\n";
161 const size_t msg_0_0_size = sizeof(msg_0_0) + 2*(-2+10);
162
163 auto &msg_0_1 = "\nFix and press any key to continue...";
164
165 static_assert((sizeof(char)|sizeof(*msg_0)|sizeof(*msg_0_0)|sizeof(*msg_0_1))==1, "char must be 1 byte");
166
167 size_t bufOffset; //buffer owerflow mitigation [compiler depended - safe to put in struct]: before buf in stack ...really, no
168 char buf[msg_0_size-1 + max(msg_0_0_size, sizeof(msg_0_1))]; //`-1` is '\0'
169
170 do{
171 #pragma warning(suppress:4996)
172 bufOffset = _snprintf(buf, sizeof(buf), msg_0, caller_line, err); //if return `-1` ...
173
174 if(bytesDone > 0){
175 #pragma warning(suppress:4996)
176 WConsole(hErr, buf, (DWORD)(bufOffset+_snprintf(buf+bufOffset, sizeof(buf)-bufOffset, msg_0_0, bytesDone, dataSize)), _0P,_0P);
177 return;
178 }else{
179 memcpy(buf+bufOffset, msg_0_1, sizeof(msg_0_1));
180 WConsole(hErr, buf, (DWORD)(bufOffset+sizeof(msg_0_1)-1), _0P,_0P); //`-1` is '\0'
181 err_pause();
182 WConsole(hErr, c_str__len("\n"), _0P,_0P);
183 }
184 }while(!RW(hFile, dataBuf, dataSize, &bytesDone,_0P) && (err=GetLastError(), 1));
185 #undef c_str__len
186 ///End of func/////////////////////////////////////////////////////////////////
187 }
188}
189#define TryRFile(fIn, dataBuf, bufSize, bytesRead) TryRWFile_((rw_file)ReadFile, fIn, dataBuf, bufSize, bytesRead, __LINE__)
190#define TryWFile(fOut, dataBuf, dataSize, bytesWritten) TryRWFile_((rw_file)WriteFile, fOut, dataBuf, dataSize, bytesWritten, __LINE__)
191
192void TryCreateDir_(LPCWSTR path, const UINT caller_line)
193{
194 if(!CreateDirectoryW(path,_0P)){
195 ///Start of func///////////////////////////////////////////////////////////////
196 ///==========================================================================//
197 ///compiller optimize this: extracts (inline) `if` above into caller,
198 ///if compile (Release) with /O2 (Maximize Speed) /Os (Favor small code) flags
199 ///(properties: C/C++ > Optimization)
200 ///(or https://docs.microsoft.com/en-us/cpp/preprocessor/optimize)
201 ///>>>see in Disassembly tab: Debug [F5],
202 ///>>>Debug menu > Windows > Disassembly [Alt+8]
203 ///==========================================================================//
204 DWORD err = GetLastError();
205 if(err == ERROR_ALREADY_EXISTS) return;
206 _pause_before_exit_ = 1;
207
208 //https://stackoverflow.com/questions/21374548/setstdhandle-has-no-effect-on-cout-printf
209 HANDLE hErr; do{
210 hErr = GetStdHandle(STD_ERROR_HANDLE);
211 }while((hErr==INVALID_HANDLE_VALUE || hErr==_0P) && (__debugbreak(), 1));
212
213 w_console WConsole = GetFileType(hErr)==FILE_TYPE_CHAR ? (w_console)WriteConsoleA:(w_console)WriteFile;
214
215 #define c_str__len(str) str, sizeof(str)/sizeof(*str)-1
216 WConsole(hErr, c_str__len("Error Codes: docs.microsoft.com/en-us/windows/win32/debug/system-error-codes\n"), _0P,_0P);
217 //https://docs.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-getlasterror
218 //https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-formatmessage
219 //https://docs.microsoft.com/en-us/windows/win32/debug/retrieving-the-last-error-code
220 //https://docs.microsoft.com/en-us/windows/win32/debug/error-handling-functions
221
222 auto &msg_0 = "TryCreateDir(): line %u, error #%u";
223 const size_t msg_0_size = sizeof(msg_0) + 2*(-2+10);
224
225 auto &msg_0_0 = "\nFix and press any key to continue...";
226
227 static_assert((sizeof(char)|sizeof(*msg_0)|sizeof(*msg_0_0))==1, "char must be 1 byte");
228
229 size_t bufOffset;
230 char buf[msg_0_size-1 + sizeof(msg_0_0)]; //`-1` is '\0'
231
232 do{
233 #pragma warning(suppress:4996)
234 bufOffset = _snprintf(buf, sizeof(buf), msg_0, caller_line, err); //if return `-1` ...
235 memcpy(buf+bufOffset, msg_0_0, sizeof(msg_0_0));
236
237 WConsole(hErr, buf, (DWORD)(bufOffset+sizeof(msg_0_0)-1), _0P,_0P); //`-1` is '\0'
238 err_pause();
239 WConsole(hErr, c_str__len("\n"), _0P,_0P);
240 }while(!CreateDirectoryW(path,_0P) && (err=GetLastError(), err!=ERROR_ALREADY_EXISTS));
241 #undef c_str__len
242 ///End of func/////////////////////////////////////////////////////////////////
243 }
244}
245#define TryCreateDir(path) TryCreateDir_(path, __LINE__)
246
247HANDLE TryCreateFile_(LPCWSTR path, const DWORD desired_access, const DWORD share_mode, const DWORD creation_disposition, const DWORD flags_and_attributes, const UINT caller_line)
248{
249 HANDLE ret_hFile = CreateFileW(path, desired_access, share_mode,_0P, creation_disposition, flags_and_attributes,_0P);
250 if(ret_hFile==INVALID_HANDLE_VALUE){
251 DWORD err = GetLastError();
252 _pause_before_exit_ = 1;
253
254 //https://stackoverflow.com/questions/21374548/setstdhandle-has-no-effect-on-cout-printf
255 HANDLE hErr; do{
256 hErr = GetStdHandle(STD_ERROR_HANDLE);
257 }while((hErr==INVALID_HANDLE_VALUE || hErr==_0P) && (__debugbreak(), 1));
258
259 w_console WConsole = GetFileType(hErr)==FILE_TYPE_CHAR ? (w_console)WriteConsoleA:(w_console)WriteFile;
260
261 #define c_str__len(str) str, sizeof(str)/sizeof(*str)-1
262 WConsole(hErr, c_str__len("Error Codes: docs.microsoft.com/en-us/windows/win32/debug/system-error-codes\n"), _0P,_0P);
263 //https://docs.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-getlasterror
264 //https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-formatmessage
265 //https://docs.microsoft.com/en-us/windows/win32/debug/retrieving-the-last-error-code
266 //https://docs.microsoft.com/en-us/windows/win32/debug/error-handling-functions
267
268 auto &msg_0 = "TryCreateFile(): line %u, error #%u";
269 const size_t msg_0_size = sizeof(msg_0) + 2*(-2+10);
270
271 auto &msg_0_0 = "\nFix and press any key to continue...";
272
273 static_assert((sizeof(char)|sizeof(*msg_0)|sizeof(*msg_0_0))==1, "char must be 1 byte");
274
275 size_t bufOffset;
276 char buf[msg_0_size-1 + sizeof(msg_0_0)]; //`-1` is '\0'
277
278 do{
279 #pragma warning(suppress:4996)
280 bufOffset = _snprintf(buf, sizeof(buf), msg_0, caller_line, err); //if return `-1` ...
281 memcpy(buf+bufOffset, msg_0_0, sizeof(msg_0_0));
282
283 WConsole(hErr, buf, (DWORD)(bufOffset+sizeof(msg_0_0)-1), _0P,_0P); //`-1` is '\0'
284 err_pause();
285 WConsole(hErr, c_str__len("\n"), _0P,_0P);
286
287 ret_hFile = CreateFileW(path, desired_access, share_mode,_0P, creation_disposition, flags_and_attributes,_0P);
288 }while(ret_hFile==INVALID_HANDLE_VALUE && (err=GetLastError(), 1));
289 #undef c_str__len
290 }
291
292 return ret_hFile;
293}
294#define TryCreateFile(path, desired_access, share_mode, creation_disposition, flags_and_attributes) TryCreateFile_(path, desired_access, share_mode, creation_disposition, flags_and_attributes, __LINE__)
295
296LPVOID TryVirtualAllocBuf_(const size_t bytes, const UINT caller_line)
297{
298 LPVOID ret_lpAddress = VirtualAlloc(_0P, bytes, MEM_COMMIT, PAGE_READWRITE);
299 if(ret_lpAddress==_0P){
300 DWORD err = GetLastError();
301 _pause_before_exit_ = 1;
302
303 //https://stackoverflow.com/questions/21374548/setstdhandle-has-no-effect-on-cout-printf
304 HANDLE hErr; do{
305 hErr = GetStdHandle(STD_ERROR_HANDLE);
306 }while((hErr==INVALID_HANDLE_VALUE || hErr==_0P) && (__debugbreak(), 1));
307
308 w_console WConsole = GetFileType(hErr)==FILE_TYPE_CHAR ? (w_console)WriteConsoleA:(w_console)WriteFile;
309
310 #define c_str__len(str) str, sizeof(str)/sizeof(*str)-1
311 WConsole(hErr, c_str__len("Error Codes: docs.microsoft.com/en-us/windows/win32/debug/system-error-codes\n"), _0P,_0P);
312 //https://docs.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-getlasterror
313 //https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-formatmessage
314 //https://docs.microsoft.com/en-us/windows/win32/debug/retrieving-the-last-error-code
315 //https://docs.microsoft.com/en-us/windows/win32/debug/error-handling-functions
316
317 auto &msg_0 = "TryVirtualAllocBuf(): line %u, error #%u";
318 const size_t msg_0_size = sizeof(msg_0) + 2*(-2+10);
319
320 auto &msg_0_0 = "\nFix and press any key to continue...";
321
322 static_assert((sizeof(char)|sizeof(*msg_0)|sizeof(*msg_0_0))==1, "char must be 1 byte");
323
324 size_t bufOffset;
325 char buf[msg_0_size-1 + sizeof(msg_0_0)]; //`-1` is '\0'
326
327 do{
328 #pragma warning(suppress:4996)
329 bufOffset = _snprintf(buf, sizeof(buf), msg_0, caller_line, err); //if return `-1` ...
330 memcpy(buf+bufOffset, msg_0_0, sizeof(msg_0_0));
331
332 WConsole(hErr, buf, (DWORD)(bufOffset+sizeof(msg_0_0)-1), _0P,_0P); //`-1` is '\0'
333 err_pause();
334 WConsole(hErr, c_str__len("\n"), _0P,_0P);
335
336 ret_lpAddress = VirtualAlloc(_0P, bytes, MEM_COMMIT, PAGE_READWRITE);
337 }while(ret_lpAddress==_0P && (err=GetLastError(), 1));
338 #undef c_str__len
339 }
340 return ret_lpAddress;
341}
342#define TryVirtualAllocBuf(bytes) TryVirtualAllocBuf_(bytes, __LINE__)
343
344const wchar_t* GetFileExt(const BYTE *const data, const DWORD dataSize)
345{
346 if(dataSize>=6){
347 //GCK'S FILE SIGNATURES TABLE https://www.garykessler.net/Library/file_sigs.html
348 switch(*(unsigned __int16 *)data){
349 case '\x2F\x2A': return L"xpm"; //XPM3 https://en.wikipedia.org/wiki/X_PixMap
350 case '\x3C\x3F': return L"svg"; //SVG (xml)
351 case '\x42\x4D': return L"bmp"; //BMP
352 case '\x89\x50': return L"png"; //PNG
353 default:switch(*(unsigned __int32 *)data){
354 case '\x00\x01\x00\x00': return L"ttf"; //TTF
355 default:switch(((unsigned __int16 *)data)[2]){
356 case '\x78\x9C': return L"offzip"; //zlib: `offzip -a .rdata unzip 0` http://aluigi.altervista.org/mytoolz/offzip.zip
357 }
358 }
359 }
360 }
361
362 return L""; //unknown
363}
364
365int main()
366{
367 LARGE_INTEGER perfFreq, perfCountStart;
368 QueryPerformanceFrequency(&perfFreq);
369
370#define wchar_2(name,Name0,Name1,c0,c1) \
371 wchar_t name##Name0[sizeof(L##c0##L##c1)/sizeof(wchar_t)] = L##c0; \
372 wchar_t *const &name##Name1 = name##Name0 + sizeof(L##c0)/sizeof(wchar_t)-1; \
373 const size_t name##Name1##Cap = sizeof(L##c1)/sizeof(wchar_t);
374
375 wchar_2(fOut,Path,Name,"qrc-res/","\0#########.extext");
376
377#undef wchar_2
378
379 HANDLE fIn = TryCreateFile(L".rdata", GENERIC_READ, 0, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN);
380
381 TryCreateDir(fOutPath);
382
383 const DWORD bufSize = 64*1024;
384 LPBYTE buf = (LPBYTE)TryVirtualAllocBuf(bufSize);
385
386 QueryPerformanceCounter(&perfCountStart);
387
388 DWORD rec;
389 unsigned __int32 resSize;
390 unsigned int resCounter = 0;
391 while(TryRFile(fIn, &resSize, sizeof(resSize), rec), rec==sizeof(resSize)){ //`rec==sizeof(resSize)` -- weak point (if rec<sizeof(resSize))
392 resCounter++;
393 resSize = be32_to_cpu(resSize);
394
395 if(resSize && (TryRFile(fIn, buf, min(resSize,bufSize), rec), rec>0)){
396 swprintf(fOutName, fOutNameCap, L"%u.%s", resCounter, GetFileExt(buf, rec));
397
398 HANDLE fOut = TryCreateFile(fOutPath, GENERIC_WRITE, 0, CREATE_NEW, FILE_ATTRIBUTE_NORMAL);
399 do{
400 resSize -= rec;
401 LPBYTE data = buf;
402 for(DWORD size=rec; size>0; data+=rec,size-=rec) TryWFile(fOut, data, size, rec);
403 }while(resSize && (TryRFile(fIn, buf, min(resSize,bufSize), rec), rec>0));
404
405 CloseHandle(fOut);
406 }else break; //>>> comment if remain ??? bytes
407 }
408
409 {LARGE_INTEGER perfCount;
410 QueryPerformanceCounter(&perfCount);
411 perfCount.QuadPart -= perfCountStart.QuadPart;
412 printf("%f [s] (%I64d)\n", perfCount.QuadPart/(double)perfFreq.QuadPart, perfCount.QuadPart);
413 }
414 printf("res count: %u\n", resCounter);
415
416 {LARGE_INTEGER currPos={}, fileSize={};
417 SetFilePointerEx(fIn, currPos, &currPos, FILE_CURRENT) &&
418 GetFileSizeEx(fIn, &fileSize) &&
419 (fileSize.QuadPart -= currPos.QuadPart) && (
420 _pause_before_exit_ = ~0,
421 fprintf(stderr, "remain: %I64u bytes\n", fileSize.QuadPart) //try to comment `break` above
422 );
423 }
424
425 CloseHandle(fIn);
426 VirtualFree(buf, 0, MEM_RELEASE);
427
428 if(_pause_before_exit_) err_pause();
429
430 return _pause_before_exit_;
431}
432
433//Try: CreateFileMapping + MapViewOfFile+UnmapViewOfFile
434//caution:
435//CreateFileMapping: SEC_WRITECOMBINE SEC_NOCACHE is http://ntsecurity.nu/onmymind/2006/2006-06-01.html
436//EXCEPTION_IN_PAGE_ERROR https://docs.microsoft.com/en-us/windows/win32/memory/reading-and-writing-from-a-file-view
437
438//FILE_FLAG_SEQUENTIAL_SCAN vs FILE_FLAG_NO_BUFFERING https://stackoverflow.com/questions/7136184/readfile-file-flag-no-buffering-how-to-read-data-between-two-sectors#comment8556514_7136230
439//https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering
440//Windows via C/C++ (Jeffrey Richter) https://www.microsoftpressstore.com/articles/article.aspx?p=2224047 ("CreateFile Cache Flags" section)
441//https://devblogs.microsoft.com/oldnewthing/?p=8493 (cache hints have no effect on memory-mapped file I/O)
442//https://social.technet.microsoft.com/Forums/en-US/09dd046e-8127-4550-8e26-5fba7a5a0743/performance-flaw-in-file-system-decreases-writefile-throughput-towards-0?forum=winserverfiles
443//write last (partial) sector: WriteFile() [write full sector size] + SetFileInformationByHandle() [decrease file size] https://stackoverflow.com/questions/25703256/how-to-do-an-unbuffered-file-transfer-using-api-readfile-writefile/25703631#25703631
444
445#elif defined(MAIN_TEST__FILE_IO_API)
446int main()
447{
448 LARGE_INTEGER perfFreq, perfCountStart;
449 QueryPerformanceFrequency(&perfFreq);
450
451 HANDLE fIn = CreateFile(L"0.rdata", GENERIC_READ, 0,_0P, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN,_0P); //5 850 624 bytes
452 HANDLE fOut = CreateFile(L"out", GENERIC_WRITE, 0,_0P, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL,_0P);
453
454 const DWORD bufSize = 64*1024;
455 LPVOID buf = VirtualAlloc(_0P, bufSize, MEM_COMMIT, PAGE_READWRITE);
456
457 QueryPerformanceCounter(&perfCountStart);
458 DWORD recR=0, recW=0;
459 while(ReadFile(fIn, buf, bufSize, &recR,_0P), recR>0){
460 WriteFile(fOut, buf, bufSize, &recW,_0P);
461 //if(recR!=bufSize || recW!=bufSize) printf("R:%d\tW:%d\n", recR, recW);
462 }
463
464 CloseHandle(fOut);
465
466 {LARGE_INTEGER perfCount;
467 QueryPerformanceCounter(&perfCount);
468 perfCount.QuadPart -= perfCountStart.QuadPart;
469 printf("%f [s] (%I64d)\n", perfCount.QuadPart/(double)perfFreq.QuadPart, perfCount.QuadPart);
470 }SuspendThread(GetCurrentThread());//Wait:Suspend (can resume by Process Explorer), Sleep(INFINITE) - Wait:DelayExecution
471
472 CloseHandle(fIn);
473 VirtualFree(buf, 0, MEM_RELEASE);
474
475 //HAB v1.3a (Statistic [>>]): testmem.tz.ru
476 //0. bufR 56KB; bufW 64KB: 0.038084 [s] (89261), 0.008331 [s] (19527), 0.007772 [s] (18217); CPU Time: 0, 0.015, 0.031
477 //1. bufR 64KB; nobW 3KB,63KB: 0.092528 [s] (216866), 0.079237 [s] (185715), 0.073985 [s] (173405); CPU Time: 0, 0.031
478 //2. nobR 1KB,54KB; bufW: 0.061560 [s] (144283), 0.052413 [s] (122845), 0.051144 [s] (119871); CPU Time: 0
479 //3. nobR 59KB; nobW 5KB,64KB: 0.114394 [s] (268115), 0.111947 [s] (262379), 0.107965 [s] (253048); CPU Time: 0
480
481 return 0;
482}
483#elif defined(MAIN_TEST__FILE_MAPPING_API)
484//Theoretically, File Mapping is able:
485// - if a data does not exist in the File Cache:
486// - load part of the file into the File Cache
487// - associate Virtual Pages of a process with Physical Pages of the File Cache
488// (i.e. w/o copying from the File Cache to the process address space
489// that occurs when using File I/O API - buffered I/O)
490// - if the data exists in the File Cache:
491// - associate Virtual Pages of a process with Physical Pages of the File Cache
492// (w/o copying)
493//Disadvantages of File Mapping compared to File I/O API at sequential scan:
494// - need to use 2 system calls (UnmapViewOfFile() и MapViewOfFile())
495// instead of one (ReadFile()) - need a new function:
496// - [or general] UnmapAndMapViewOfFile or RemapViewOfFile(lpOldBaseAddress, dwDesiredAccess, dwFileOffsetHigh, dwFileOffsetLow, dwNumberOfBytesToMap)
497// - [or specific] OffsetViewOfFile(lpOldBaseAddress, dwNumberOfBytesToMap) is analog to ReadFile()
498// or OffsetViewOfFile(lpOldBaseAddress, dwOffsetHigh, dwOffsetLow)
499//Advantages of File Mapping compared to File I/O API:
500// - File I/O buffer is backed by the system paging file (consumes space)
501// or you can VirtualLock() into physical memory (process's Working Set) - bad idea:
502// - https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-virtuallock#remarks
503// - https://devblogs.microsoft.com/oldnewthing/?p=24573
504// vs
505// File Mapping object is backed by a file in the file system
506int main()
507{
508 LARGE_INTEGER perfFreq, perfCountStart;
509 QueryPerformanceFrequency(&perfFreq);
510
511 HANDLE fIn = CreateFile(L"0.rdata", GENERIC_READ, 0,_0P, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN,_0P); //5 850 624 bytes
512 HANDLE fOut = CreateFile(L"out", GENERIC_WRITE, 0,_0P, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL,_0P);
513
514 HANDLE fmIn = CreateFileMapping(fIn, 0, PAGE_READONLY, 0, 0, 0);
515 //Although an application may close the file handle used to create a file mapping object,
516 //the system holds the corresponding file open until the last view of the file is unmapped.
517 //https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-unmapviewoffile#remarks
518 CloseHandle(fIn);
519
520 const DWORD bufSize = 64*1024;
521 LARGE_INTEGER bufOffset = {};
522 LPCVOID buf = _0P;
523
524 QueryPerformanceCounter(&perfCountStart);
525 DWORD recW=0;
526 while(buf = MapViewOfFile(fmIn, FILE_MAP_READ, bufOffset.HighPart, bufOffset.LowPart, bufSize)){
527 WriteFile(fOut, buf, bufSize, &recW,_0P);
528 //if(recW!=bufSize) printf("W:%d\n", recW);
529 UnmapViewOfFile(buf);
530 bufOffset.QuadPart += bufSize;
531 }
532
533 CloseHandle(fOut);
534
535 {LARGE_INTEGER perfCount;
536 QueryPerformanceCounter(&perfCount);
537 perfCount.QuadPart -= perfCountStart.QuadPart;
538 printf("%f [s] (%I64d)\n", perfCount.QuadPart/(double)perfFreq.QuadPart, perfCount.QuadPart);
539 }SuspendThread(GetCurrentThread());//Wait:Suspended (can resume by Process Explorer), Sleep(INFINITE) - Wait:DelayExecution
540
541 //a file mapping object does not close until all references to it are released.
542 //Therefore, to fully close a file mapping object, an application must unmap
543 //all mapped views of the file mapping object by calling UnmapViewOfFile and
544 //close the file mapping object handle by calling CloseHandle.
545 //These functions can be called in any order.
546 //https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createfilemappinga#remarks
547 CloseHandle(fmIn);
548
549 //HAB v1.3a (Statistic [>>]): testmem.tz.ru
550 //0. bufR 580KB; bufW 53KB: 0.083979 [s] (196830), 0.011111 [s] (26043), 0.008886 [s] (20828); CPU Time: 0.015
551 //1. bufR 340KB; nobW 58KB: 0.188818 [s] (442554), 0.245481 [s] (575360), 0.133576 [s] (313077); CPU Time: 0
552 //2. nobR 488KB; bufW 64KB: 0.089158 [s] (208970), 0.065768 [s] (154147), 0.065941 [s] (154553); CPU Time: 0.015
553 //3. nobR 304KB; nobW 66KB: 0.168522 [s] (394983), 0.166814 [s] (390981), 0.167571 [s] (392755); CPU Time: 0.015
554
555 return 0;
556}
557#endif
558//======[end: Qt-QRC-resources-extractor.cpp]======//
559
560// Out of Berne Convention
561// Unlicense or BOLA https://blitiri.com.ar/p/bola/ or Pastebin default (CC BY-SA 3.0)