Zstandard for Lazarus, "without" DLL, Windows only

[domasz]

Great, thanks!

[paule32]

As far as I know, grep is for text files...
But yes, you can use grep to search the file of String Matches.

But for:  grep your_regex_pattern -n file.txt
grep output the line number of the Match.

I am a little bit the Fan of minimal Executables.
But FPC is very fast for creating them.

*.a Archive Files provide you a import.a file which contains the Procedures and Functions with a import Link to the DLL.
Windows itself use DLL's to save Memory.
And for me, it makes no sense, to store 100 MB Executables on the Computer disk.
It makes me more Sense to save Memory and Disk Space by using DLL files

FPC create a import library .a File, which can be used for second using when the Project is a Library.

I find it very intressting to pack the DLL and EXE Files.
Because the normal user of a Computer Desktop tends to be install all stuff that comes from a monthly Paper and CD.
Then the Computer hard disk is overswapped with Tools, that use the same Functions and Procedures...

I think it is Time to open a FPC Store, where the trusted Application's can be download after a check of Viruses and Trojan...

But this is my point of mind... others: please make your own points of mind...
My Problems are not of others... others Problems are not mine, too ...

[domasz]

Linux guys like to store libraries in system directory. The result is- you can't just copy an app to another computer because dependencies are missing.
So I think embedding everything in EXE, or at least app's dir, is the best approach. Users don't need to care about dependencies. And even a 100 MB EXE today is pretty much nothing. For an SSD drive that's instant loading.

[msintle]

Linux guys like to store libraries in system directory. The result is- you can't just copy an app to another computer because dependencies are missing.

Or you can use InstallAware for Linux and copy the libraries you need over to the system directory in an automatically elevated setup.

[Fibonacci]

Compression ratio is really low IIRC.

yes, it is relatively low but, I'd say it's decent. That said, definitely not nearly as good as what is obtained from more sophisticated algorithms used in 7zip and Winrar.

The really nice feature is: no external dll needed because the routines are part of the O/S.  That's a nice plus.

Found a new API. Introduced in Windows 10 1511.

Code: Pascal  [Select][+][-]

1. program app;
2.  
3. uses SysUtils, Windows, Classes;
4.  
5. type
6.   PD3D_SHADER_DATA = ^D3D_SHADER_DATA;
7.   D3D_SHADER_DATA = record
8.     pBytecode: Pointer;
9.     BytecodeLength: SIZE_T;
10.   end;
11.  
12. type
13.   PID3DBlob = ^ID3DBlob;
14.   ID3DBlob = interface(IUnknown)
15.     ['{8BA5FB08-5195-40e2-AC58-0D989C3A0102}']
16.     function GetBufferPointer: Pointer; stdcall;
17.     function GetBufferSize: SIZE_T; stdcall;
18.   end;
19.  
20. const
21.   D3D_COMPRESS_SHADER_KEEP_ALL_PARTS = $00000001;
22.  
23. function D3DCompressShaders(
24.   uNumShaders: UINT;
25.   pShaderData: PD3D_SHADER_DATA;
26.   uFlags: UINT;
27.   out ppCompressedData: ID3DBlob
28. ): HRESULT; stdcall; external 'D3DCompiler_47.dll';
29.  
30. function D3DDecompressShaders(
31.   pSrcData: Pointer;
32.   SrcDataSize: SIZE_T;
33.   uNumShaders: UINT;
34.   uStartIndex: UINT;
35.   pIndices: PUINT;
36.   uFlags: UINT;
37.   ppShaders: PID3DBlob;
38.   pTotalShaders: PUINT
39. ): HRESULT; stdcall; external 'D3DCompiler_47.dll';
40.  
41. function getfile(p: string): string;
42. var
43.   m: TMemoryStream;
44. begin
45.   m := TMemoryStream.Create;
46.   m.LoadFromFile(p);
47.   setlength(result, m.Size);
48.   m.Read(result[1], m.Size);
49.   m.Free;
50. end;
51.  
52. procedure main;
53. var
54.   dsa: D3D_SHADER_DATA;
55.   blob: ID3DBlob;
56.   s, compressed: string;
57. begin
58.   s := getfile('C:\Windows\explorer.exe');
59.   writeln('uncompressed len = ', length(s));
60.  
61.   dsa.pBytecode := @s[1];
62.   dsa.BytecodeLength := length(s);
63.  
64.   if D3DCompressShaders(1, @dsa, D3D_COMPRESS_SHADER_KEEP_ALL_PARTS, blob)  <> S_OK then begin
65.     writeln('fail compress');
66.     exit;
67.   end;
68.  
69.   writeln('compressed len   = ', blob.GetBufferSize);
70.  
71.   setlength(compressed, blob.GetBufferSize);
72.   move(blob.GetBufferPointer^, compressed[1], length(compressed));
73.  
74.   if D3DDecompressShaders(@compressed[1], length(compressed), 1, 0, nil, 0, @blob, nil) <> S_OK then begin
75.     writeln('fail decompress');
76.     exit;
77.   end;
78.  
79.   writeln('decompressed len = ', blob.GetBufferSize);
80.   writeln('decompress ok? ', CompareMem(blob.GetBufferPointer, @s[1], length(s)));
81.   writeln('compressed to ', Round((length(compressed)/length(s))*100), '% of the original size');
82. end;
83.  
84. begin
85.   try
86.     main;
87.   finally
88.     readln;
89.   end;
90. end.

Code: Text  [Select][+][-]

1. uncompressed len = 5973416
2. compressed len   = 2096961
3. decompressed len = 5973416
4. decompress ok? TRUE
5. compressed to 35% of the original size

Code: Text  [Select][+][-]

1. explorer.exe        5834 KB
2. D3DCompressShaders: 2048 KB
3. ZIP Ultra:          2567 KB
4. 7zip Ultra:         1875 KB

[gues1]

Found a new API. Introduced in Windows 10 1511.

https://learn.microsoft.com/en-us/windows/win32/api/d3dcompiler/nf-d3dcompiler-d3dcompressshaders

Note  You can use this API to develop your Windows Store apps, but you can't use it in apps that you submit to the Windows Store.

[Fibonacci]

Who cares. I dont put my apps in the Windows Store, do you?

[gues1]

I am a little bit the Fan of minimal Executables.
But FPC is very fast for creating them.
*.a Archive Files provide you a import.a file which contains the Procedures and Functions with a import Link to the DLL.
Windows itself use DLL's to save Memory.
And for me, it makes no sense, to store 100 MB Executables on the Computer disk.
It makes me more Sense to save Memory and Disk Space by using DLL files

Delphi as this option using the runtime BPLs (that are DDLs), and you can link to them so you exe is very minimal size (but you must distribute all BPLs that are runtime linked).

But, once upon a time .... is not the begin of a novel, but a past nightmare.
Who doesn't know about DLLs HELL ?

One time the applications are linked to a lot of DLLs, not only system DLLs but also thirdy part DLLs. And this DLL stay in a common path, so the distribution of the applications is very compact. This also occurs 'cause internet speed that was low at that time, but not only for this.

When the DLLs started to change there were an infinite problems (remember the moltitude of msvcrt.dll ?).

Lazarus and Delphi normally create a monolithic EXE (I speak about Windows apps), without the use of external DLLs (of course excepts the Windows systems DLLs).
I love this, expecially after lived a "DLLs HELL" time. And I think that more people love this. I used BPLs some time ago, but the patch released (that not always patched the BPLs) made me give up their use.

And, when FPC or LCL will be updated, what about the old DLL the linked functions of LCL (now updated) ? You must recompile ALL your DLLs and verifiy the all functions, procedures, methods, etc ... use the same "interfaces".

Should be better that like in Delphi there were standard DLLs available from LAZARUS and FPC and the possibility to link them at compile time, and of course the possibility to recompile them every time is needed (like IDE of Lazarus).

But this is really a big big works.

[gues1]

Who cares. I dont put my apps in the Windows Store, do you?

Never put anything in Windows store, but ... normally I see that those advertisement mean that you applications will not be "accepted" very well from security apps.

I can sign my exes, so may be that overload the security check but not all can sign their apps.

Anyway thanks for looking and finding this feature.

[Fibonacci]

Another compression API. DEFLATE via PresentationNative_v0300.dll, available since Windows 7.

Code: Pascal  [Select][+][-]

1. program deflate;
2.  
3. type
4.   z_streamp = ^z_stream;
5.   z_stream = record
6.     next_in   : PByte;   // next input byte
7.     avail_in  : DWord;   // number of bytes available at next_in
8.     total_in  : DWord;   // total nb of input bytes read so far
9.     next_out  : PByte;   // next output byte should be put there
10.     avail_out : DWord;   // remaining free space at next_out
11.     total_out : DWord;   // total nb of bytes output so far
12.     msg       : PByte;   // last error message, NULL if no error
13.     state     : Pointer; // not visible by applications
14.     zalloc    : Pointer; // used to allocate the internal state
15.     zfree     : Pointer; // used to free the internal state
16.     opaque    : Pointer; // private data object passed to zalloc and zfree
17.     data_type : Integer; // best guess about the data type: binary or text
18.     adler     : DWord;   // adler32 value of the uncompressed data
19.     reserved  : DWord;   // reserved for future use
20.   end;
21.  
22. const
23.   Z_FINISH     = 4;
24.   Z_STREAM_END = 1;
25.   Z_BUF_ERROR  = -5;
26.  
27. function ums_deflate_init(stream: z_streamp; level: integer; version: pchar; stream_size: integer): integer; stdcall; external 'PresentationNative_v0300.dll';
28. function ums_deflate(stream: z_streamp; flush: integer): integer; stdcall; external 'PresentationNative_v0300.dll';
29. function ums_inflate_init(stream: z_streamp; version: pchar; stream_size: integer): integer; stdcall; external 'PresentationNative_v0300.dll';
30. function ums_inflate(stream: z_streamp; flush: integer): integer; stdcall; external 'PresentationNative_v0300.dll';
31.  
32. function _lopen(lpPathName: PChar; iReadWrite: Integer): THandle; stdcall; external 'kernel32.dll';
33. function _lread(hFile: THandle; lpBuffer: Pointer; uBytes: Cardinal): Cardinal; stdcall; external 'kernel32.dll';
34. function _lclose(hFile: THandle): Integer; stdcall; external 'kernel32.dll';
35. function GetFileSize(hFile: THandle; lpFileSizeHigh: PCardinal): Cardinal; stdcall; external 'kernel32.dll';
36.  
37. function deflate(s: string): string;
38. var
39.   z: z_stream;
40.   e: integer;
41. begin
42.   // alloc at least 64 bytes for the output
43.   if length(s) > 64 then setlength(result, length(s)) else setlength(result, 64);
44.  
45.   z.zalloc := nil;
46.   z.zfree := nil;
47.   z.opaque := nil;
48.   z.avail_in := length(s);
49.   z.next_in := @s[1];
50.   z.avail_out := length(result);
51.   z.next_out := @result[1];
52.  
53.   if ums_deflate_init(@z, 9, '1', sizeof(z)) <> 0 then exit('');
54.  
55.   e := ums_deflate(@z, Z_FINISH);
56.   if e <> Z_STREAM_END then exit('');
57.  
58.   setlength(result, z.total_out);
59. end;
60.  
61. function inflate(s: string): string;
62. var
63.   z: z_stream;
64.   e: integer;
65. begin
66.   setlength(result, length(s)*2);
67.  
68.   while true do begin
69.     z.zalloc := nil;
70.     z.zfree := nil;
71.     z.opaque := nil;
72.     z.avail_in := length(s);
73.     z.next_in := @s[1];
74.     z.avail_out := length(result);
75.     z.next_out := @result[1];
76.  
77.     if ums_inflate_init(@z, '1', sizeof(z)) <> 0 then exit('');
78.  
79.     e := ums_inflate(@z, Z_FINISH);
80.     if e = Z_BUF_ERROR then begin
81.       // double the buffer
82.       setlength(result, length(result)*2);
83.       continue;
84.     end;
85.     if e <> Z_STREAM_END then exit('');
86.     break;
87.   end;
88.  
89.   setlength(result, z.total_out);
90. end;
91.  
92. var
93.   h: thandle;
94.   original, compressed, decompressed: string;
95.  
96. begin
97.   // test file
98.   h := _lopen('C:\Windows\explorer.exe', 0);
99.   setlength(original, GetFileSize(h, nil));
100.   _lread(h, @original[1], length(original));
101.   _lclose(h);
102.   writeln('original = ':30, length(original));
103.  
104.   // compress
105.   compressed := deflate(original);
106.   writeln('compressed = ':30, length(compressed));
107.  
108.   // decompress
109.   decompressed := inflate(compressed);
110.   writeln('decompressed = ':30, length(decompressed));
111.  
112.   writeln('decompressed = original? ':30, decompressed=original);
113.   writeln('compression ratio = ':30, trunc((1-(length(compressed)/length(original)))*100), '%');
114.  
115.   readln;
116. end.

[jamie]

Let me get this straight, the current TZipper and TUnzipper does not generate correct ZIP files?

I caught this thread because I just got done using the zStream unit to compress data in memory or optionally out to file as part of a larger code base.

 As far as I know from looking at the sources it has no external LIB requirement.

Jamie

[Fibonacci]

Deflate – yes, TZipper/ZLib is just fine. Using undocumented APIs is a form of art. It allows you to create binaries that are an order of magnitude smaller than what you would get with TZipper.

Why would anyone need such tiny binaries? Well, your imagination is the only limit. Shellcodes, packers, crypters, demoscene...

As for D3D compression API - it outperforms ZIP by far.