Question about bit fields in bitpacked record

[440bx]

Hello,

I suspect that I am misunderstanding something about bitpacked records. 

Referring to the attached image, the DWORD value is shown is $72.  I expected the field "unknown28" (the first 28 bits of that DWORD) to be $7, not $72 (which is the entire 32bit DWORD.)  Also, I expected the field "DepDisabled" to be $1 because the second bit in the DWORD is $1.

The program that produces that output (which might be different on your system) is :

Code: Pascal  [Select][+][-]

1. {$APPTYPE      CONSOLE}
2.  
3. {$TYPEDADDRESS ON}
4.  
5. {$ifdef WIN64}
6.   {$FATAL DEP policy cannot be queried in 64bit - compile for 32bit}
7. {$endif}
8.  
9. program _QueryInformationProcess;
10.  
11. uses
12.   Windows,
13.   sysutils
14.   ;
15.  
16. {$ifdef VER90}
17. type
18.   DWORD = longint;
19.   PDWORD = ^DWORD;
20. {$endif}
21.  
22. type
23.   NTSTATUS = DWORD;
24.  
25.   pboolean = ^boolean;
26.  
27. const
28.    { 22   34 } ProcessExecuteFlags = 34;  { TPROCESS_DEP_POLICY               }
29.  
30.   ntdll = 'ntdll';
31.  
32.   PROCESS_ALL_ACCESS        = STANDARD_RIGHTS_REQUIRED or SYNCHRONIZE or $FFFF;
33.  
34.   STATUS_SUCCESS            = 0;
35.  
36. type
37.   { bit types for bitfields - since Delphi 2 does not support bitfields       }
38.   {                           these types are FPC only                        }
39.  
40.   T1BIT      = 0 ..        $1;
41.   T2BITS     = 0 ..        $3;
42.   T3BITS     = 0 ..        $7;
43.   T4BITS     = 0 ..        $F;
44.   T5BITS     = 0 ..       $1F;
45.   T6BITS     = 0 ..       $3F;
46.   T7BITS     = 0 ..       $7F;
47.   T8BITS     = 0 ..       $FF;            { byte                              }
48.   T9BITS     = 0 ..      $1FF;
49.  
50.   T10BITS    = 0 ..      $3FF;
51.   T11BITS    = 0 ..      $7FF;
52.   T12BITS    = 0 ..      $FFF;
53.   T13BITS    = 0 ..     $1FFF;
54.   T14BITS    = 0 ..     $3FFF;
55.   T15BITS    = 0 ..     $7FFF;
56.   T16BITS    = 0 ..     $FFFF;            { word                              }
57.   T17BITS    = 0 ..    $1FFFF;
58.   T18BITS    = 0 ..    $3FFFF;
59.   T19BITS    = 0 ..    $7FFFF;
60.  
61.   T20BITS    = 0 ..    $FFFFF;
62.   T21BITS    = 0 ..   $1FFFFF;
63.   T22BITS    = 0 ..   $3FFFFF;
64.   T23BITS    = 0 ..   $7FFFFF;
65.   T24BITS    = 0 ..   $FFFFFF;
66.   T25BITS    = 0 ..  $1FFFFFF;
67.   T26BITS    = 0 ..  $3FFFFFF;
68.   T27BITS    = 0 ..  $7FFFFFF;
69.   T28BITS    = 0 ..  $FFFFFFF;
70.   T29BITS    = 0 .. $1FFFFFFF;
71.  
72.   T30BITS    = 0 .. $3FFFFFFF;
73.   T31BITS    = 0 .. $7FFFFFFF;
74.  
75. {$ifdef FPC}
76.   type
77.     { _PROCESS_DEP_POLICY                             - 32bit only            }
78.  
79.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
80.     TPROCESS_DEP_POLICY = bitpacked record
81.       Unknown28                       : T28BITS;
82.       Permanent                       : T1BIT;
83.  
84.       DepEnabledThunk                 : T1BIT;
85.       DepDisabled                     : T1BIT;
86.       Unknown1                        : T1BIT;
87.     end;
88. {$endif}
89.  
90. {$ifdef VER90}
91.   type
92.     { _PROCESS_DEP_POLICY                             - 32bit only            }
93.  
94.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
95.     TPROCESS_DEP_POLICY = record
96.       ExecuteFlags          : DWORD;
97.     end;
98. {$endif}
99.  
100. function NtQueryInformationProcess(ProcessHandle            : THANDLE;
101.                                    ProcessInformationClass  : DWORD;
102.                                    ProcessInformation       : pointer;
103.                                    ProcessInformationLength : DWORD;
104.                                    ReturnLength             : PDWORD)
105.          : NTSTATUS; stdcall; external ntdll;
106.  
107. function IsDebuggerPresent {$ifdef FPC} () {$endif}
108.        : BOOL; stdcall external kernel32;
109.  
110. function _itoa
111.              (
112.               { _in_  } In32bitInteger  : integer;
113.               { _out_ } OutAsciizString : pchar;
114.               { _in_  } InRadix         : DWORD
115.              )
116.          : pchar; cdecl external ntdll;                            { !! CDECL }
117.  
118. function GetProcessDEPPolicy
119.                    (
120.                     { _in_  } InProcessHandle : THANDLE;
121.                     { _out_ } OutFlags        : PDWORD;
122.                     { _out_ } OutPermanent    : pboolean      { 4th bit set   }
123.                    )
124.        : BOOL; stdcall external kernel32;
125.  
126. var
127.   ProcessHandle                   : THANDLE;
128.  
129. var
130.   { variable for GetProcessDEPPolicy                                          }
131.  
132.   Flags                           : DWORD;
133.   Permanent                       : boolean;
134.  
135. var
136.   { variables for NtQueryInformationProcess                                   }
137.  
138.   ProcessExecuteFlagsData         : TPROCESS_DEP_POLICY;
139.  
140.   BufferSize    : DWORD;
141.  
142.   ReturnLength  : DWORD;
143.  
144.   Status        : NTSTATUS;
145.  
146.  
147.   itoa_buf      : packed array[0..127] of char;
148.  
149.  
150. begin
151.   writeln;
152.   writeln;
153.  
154.   // since we are dealing with our own process we can specify PROCESS_ALL_ACCESS
155.  
156.   ProcessHandle := OpenProcess(PROCESS_ALL_ACCESS,
157.                                FALSE,
158.                                GetCurrentProcessId());
159.  
160.   if ProcessHandle = 0 then Halt(1);
161.  
162.   { use the API to verify the results                                         }
163.  
164.   writeln;
165.   writeln;
166.   if not GetProcessDEPPolicy(ProcessHandle,
167.                             @Flags,
168.                             @Permanent) then
169.   begin
170.     { this shouldn't happen                                                   }
171.     writeln;
172.     writeln;
173.     writeln('  GetProcessDEPPolicy failed.');
174.   end;
175.  
176.   writeln;
177.   writeln('  Flags     = ', Flags);
178.   writeln('  Permanent = ', Permanent);    { 4th bit of DWORD                 }
179.  
180.  
181.   { ------------------------------------------------------------------------- }
182.   { get the same results using NtQueryInformationProcess                      }
183.  
184.   writeln;
185.   writeln;
186.   writeln('  ProcessExecuteFlags : ', ord(ProcessExecuteFlags), ' (d) ',
187.              IntToHex(ord(ProcessExecuteFlags), 0), ' (h)');
188.  
189.   BufferSize   := sizeof(ProcessExecuteFlagsData);
190.   ReturnLength := 0;
191.  
192.   writeln;
193.   writeln;
194.   writeln('  Buffer size : ', BufferSize);
195.  
196.   ZeroMemory(@ProcessExecuteFlagsData, sizeof(ProcessExecuteFlagsData));
197.  
198.   if IsDebuggerPresent then DebugBreak();
199.   Status := NtQueryInformationProcess(ProcessHandle,
200.                                       ProcessExecuteFlags,
201.                                      @ProcessExecuteFlagsData,
202.                                       BufferSize,
203.                                      @ReturnLength);
204.  
205.   if IsDebuggerPresent then DebugBreak();
206.   if Status <> STATUS_SUCCESS then
207.   begin
208.     writeln;
209.     writeln;
210.     writeln('  NtQueryInformationProcess failed.  NTSTATUS : ',
211.                IntToHex(Status, 0));
212.   end;
213.  
214.   writeln;
215.   writeln('  returned length : ', ReturnLength);
216.  
217.  
218.   writeln;
219.   writeln;
220.  
221.   { output the result in binary for reference                                 }
222.  
223.   writeln('  ', _itoa(DWORD(ProcessExecuteFlagsData), itoa_buf, 2));
224.   writeln;
225.  
226.  
227.   {$ifdef FPC}
228.   with ProcessExecuteFlagsData do
229.   begin
230.     writeln('  ProcessExecuteFlagsData = $', IntToHex(DWORD(ProcessExecuteFlagsData), 0));
231.     writeln;
232.  
233.     writeln('  Unknown28       = $', IntToHex(Unknown28, 0));
234.     writeln('  Permanent       = ',  Permanent);
235.     writeln('  DepEnabledThunk = ',  DepEnabledThunk);
236.     writeln('  DepDisabled     = ',  DepDisabled);
237.     writeln('  Unknown1        = ',  Unknown1);
238.   end;
239.   {$endif}
240.  
241.   {$ifdef VER90}
242.     writeln;
243.     writeln;
244.     writeln('  Process execute flags : ',
245.                IntToHex(ProcessExecuteFlagsData.ExecuteFlags, 0));
246.   {$endif}
247.  
248.  
249.  
250.   writeln;
251.   writeln('Press <enter>/<result> to end this program.');
252.   readln;
253. end.
254.  

My questions are: why isn't DepDisabled = 1 and why isn't Unknown28 = 7 ?  Any suggestion on how to get the result I want is definitely appreciated.

Thank you for your help.

[MathMan]

I may be off-track totally here, but shouldn't

Code: Pascal  [Select][+][-]

1. {$ifdef FPC}
2.   type
3.     { _PROCESS_DEP_POLICY                             - 32bit only            }
4.  
5.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
6.     TPROCESS_DEP_POLICY = bitpacked record
7.       Unknown28                       : T28BITS;
8.       Permanent                       : T1BIT;
9.  
10.       DepEnabledThunk                 : T1BIT;
11.       DepDisabled                     : T1BIT;
12.       Unknown1                        : T1BIT;
13.     end;
14. {$endif}

be defined as follows

Code: Pascal  [Select][+][-]

1. {$ifdef FPC}
2.   type
3.     { _PROCESS_DEP_POLICY                             - 32bit only            }
4.  
5.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
6.     TPROCESS_DEP_POLICY = bitpacked record
7.       Unknown1                        : T1BIT;
8.       DepDisabled                     : T1BIT;
9.       DepEnabledThunk                 : T1BIT;
10.       Permanent                       : T1BIT;
11.       Unknown28                       : T28BITS;
12.     end;
13. {$endif}

To me it looks like you have your definition upside down - that's why Unknown28 reflects the low 28 bit of the Flags value (when seen as DWORD).

Cheers,
MathMan

[Martin_fr]

You are probably on a little endian machine (like intel)?

So $00000072 writes to mem like this

Record_addr: $72 $00 $00 $00

[440bx]

@MathMan, @Martin_fr

Yes, I am on a little endian machine (intel)

I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72.

IOW, the only values that make sense to me is either "0000000" (which is 0) or "7200000". 

I don't see how "Unknown28" could be just "72".  In either endianness that value does not seem possible.

[MathMan]

@MathMan, @Martin_fr

Yes, I am on a little endian machine (intel)

I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72.

IOW, the only values that make sense to me is either "0000000" (which is 0) or "7200000". 

I don't see how "Unknown28" could be just "72".  In either endianness that value does not seem possible.

Huh - why would you expect that? I would expect "Unknown28" to be little endian too on an Intel machine ...

[Martin_fr]

@MathMan, @Martin_fr

Yes, I am on a little endian machine (intel)

I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72.

Not really, Lets name the bytes 11, 22, 33 and 44
Then lets look at the DWORD $44332211  and the packed subrange (fitting in 3 bytes) $332211

DWORD in mem: 11 22 33 44
That is so, if I access that mem as PByte I get $11 (the last byte of the DWORD $44332211)

Or that I can do

Code: Pascal  [Select][+][-]

1. Fillbyte(mem, 0, MAXLEN);
2. PByte(mem)^ := 7
3. wirteln(PDWord(mem)^); // 7

Code: [Select]

DWORD in mem: 11 22 33 44
RANGE in mem: 11 22 33
So if I read range from the DWORD address then i get the bytes 11 22 and 33, and in the variable they will have the value $332211 (the first byte from memory has the lowest significance and is the last byte of the value)

And all that is intended.

Your $00000072

Code: [Select]

DWORD in mem: 72 00 00 00
RANGE in mem: 72 00 00 // 72 is the least significant byte, it will be at the end of the written number $000072 (that are 28 bits)

The bytes in memory are always in reverse order to what they are in human readable representation.

Off topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages)
English: 23 => Twenty-three
German 23 => Drei-und-Zwanzig (three and twenty)

But both have 14 => four-teen (the four move before the ten)

[440bx]

Huh - why would you expect that? I would expect "Unknown28" to be little endian too on an Intel machine ...

Because the compiler should be using one endianness or the other, not mix the two of them.

if the compiler chooses big endian then, the value of "Unknown28" is $7200000 (28 bits), if it chooses little endian then the value of "Unknown28" is 0000000 which is 0 (zero).  There is no endianness that yields the value $72 for "Unknown28".

I wrote a little test program to better see what is happening, its output is:

Code: Text  [Select][+][-]

1. i :  0  : 0
2. i :  1  : 1
3. i :  2  : 0
4. i :  3  : 0
5. i :  4  : 1
6. i :  5  : 1
7. i :  6  : 1
8. i :  7  : 0
9. i :  8  : 0
10. i :  9  : 0
11. i : 10  : 0
12. i : 11  : 0
13. i : 12  : 0
14. i : 13  : 0
15. i : 14  : 0
16. i : 15  : 0
17. i : 16  : 0
18. i : 17  : 0
19. i : 18  : 0
20. i : 19  : 0
21. i : 20  : 0
22. i : 21  : 0
23. i : 22  : 0
24. i : 23  : 0
25. i : 24  : 0
26. i : 25  : 0
27. i : 26  : 0
28. i : 27  : 0
29. i : 28  : 0
30. i : 29  : 0
31. i : 30  : 0
32. i : 31  : 0
33.  
34.  
35. TwentyEight : 72
36. Four        : 0[/size][/font]

if the compiler is going to use the big endian memory ordering then the above shows that "Unknown28" should be $7200000 not $72 (that $72 is followed by 6 bytes of zeroes) and those cannot simply disappear because in big endian they are to the right of the value ($72 in this case).

What the compiler seems to be doing is taking the first 28 bits, adding a nibble at the end and then evaluating the resulting 32bits in little endian mode, not big endian but, when doing individual bits it uses big endian.  IOW, it's mixing endianness-es depending on the field sizes.


@Martin,

I was typing this when you made your post.  I'm going to review it after posting this.


the test program that produced that output is:

Code: Pascal  [Select][+][-]

1. {$APPTYPE      CONSOLE}
2.  
3. {$TYPEDADDRESS ON}
4.  
5.  
6. program _BitFields2;
7.  
8.  
9. uses
10.   Windows,
11.   sysutils
12.   ;
13.  
14.  
15. type
16.   { bit types for bitfields - since Delphi 2 does not support bitfields       }
17.   {                           these types are FPC only                        }
18.  
19.   T1BIT      = 0 ..        $1;
20.   T2BITS     = 0 ..        $3;
21.   T3BITS     = 0 ..        $7;
22.   T4BITS     = 0 ..        $F;
23.   T5BITS     = 0 ..       $1F;
24.   T6BITS     = 0 ..       $3F;
25.   T7BITS     = 0 ..       $7F;
26.   T8BITS     = 0 ..       $FF;            { byte                              }
27.   T9BITS     = 0 ..      $1FF;
28.  
29.   T10BITS    = 0 ..      $3FF;
30.   T11BITS    = 0 ..      $7FF;
31.   T12BITS    = 0 ..      $FFF;
32.   T13BITS    = 0 ..     $1FFF;
33.   T14BITS    = 0 ..     $3FFF;
34.   T15BITS    = 0 ..     $7FFF;
35.   T16BITS    = 0 ..     $FFFF;            { word                              }
36.   T17BITS    = 0 ..    $1FFFF;
37.   T18BITS    = 0 ..    $3FFFF;
38.   T19BITS    = 0 ..    $7FFFF;
39.  
40.   T20BITS    = 0 ..    $FFFFF;
41.   T21BITS    = 0 ..   $1FFFFF;
42.   T22BITS    = 0 ..   $3FFFFF;
43.   T23BITS    = 0 ..   $7FFFFF;
44.   T24BITS    = 0 ..   $FFFFFF;
45.   T25BITS    = 0 ..  $1FFFFFF;
46.   T26BITS    = 0 ..  $3FFFFFF;
47.   T27BITS    = 0 ..  $7FFFFFF;
48.   T28BITS    = 0 ..  $FFFFFFF;
49.   T29BITS    = 0 .. $1FFFFFFF;
50.  
51.   T30BITS    = 0 .. $3FFFFFFF;
52.   T31BITS    = 0 .. $7FFFFFFF;
53.  
54.  
55. const
56.   ntdll = 'ntdll';
57.  
58.  
59. function _itoa
60.              (
61.               { _in_  } In32bitInteger  : integer;
62.               { _out_ } OutAsciizString : pchar;
63.               { _in_  } InRadix         : DWORD
64.              )
65.          : pchar; cdecl external ntdll;                            { !! CDECL }
66.  
67. var
68.   BitTest                         : DWORD = $72;
69.  
70.   BitFields                       : bitpacked record
71.     Bits                              : bitpacked array[0..31] of T1BIT;
72.   end absolute BitTest;
73.  
74.   BitFields2                      : bitpacked record
75.     case integer of
76.       1 : (
77.            TwentyEight                 : T28BITS;
78.            Four                        : T4BITS;
79.           );
80.  
81.       2 : (
82.            Bits                        : bitpacked array[0..31] of T1BIT;
83.           );
84.   end absolute BitTest;
85.  
86.   i                               : integer;
87.  
88.  
89. begin
90.   writeln;
91.   writeln;
92.  
93.   with BitFields2 do
94.   begin
95.     for i := low(Bits) to high(Bits) do
96.     begin
97.       writeln('  i : ', i:2, '  : ', Bits[i]);
98.     end;
99.  
100.     writeln;
101.     writeln;
102.     writeln('  TwentyEight : ', IntToHex(TwentyEight, 0));
103.     writeln('  Four        : ', Four);
104.   end;
105.  
106.  
107.   writeln;
108.   writeln('Press ENTER/RETURN to end this program.');
109.   readln;
110. end.

[440bx]

And all that is intended.

Your $00000072

Code: [Select]

DWORD in mem: 72 00 00 00
RANGE in mem: 72 00 00 // 72 is the least significant byte, it will be at the end of the written number $000072 (that are 28 bits)


I understand and agree with everything you've said but, as you showed, the "RANGE in mem" is 72 00 00 0 (28 bits), once the compiler decides to look at bits in memory order, it cannot change the way of looking at them.  In memory order the value is 72 00 00 0

The bytes in memory are always in reverse order to what they are in human readable representation.

Completely agree with that.  what causes a real problem is that the compiler starts looking at bytes in memory-order and then switches to little-endian order which is what made the 72 00 00 0 show up as just 72.

I'm going to have to review every bitpacked record definition I have and test thoroughly to ensure they behave as they should (the majority of them defined just as they are in C but, now I have doubts that they'll work as they do in C.)

Thank you Martin and thank you MathMan for your help.  It was very useful.

[dseligo]

Off topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages)
English: 23 => Twenty-three
German 23 => Drei-und-Zwanzig (three and twenty)

Slovene also has this.
But this is only for the last two digits. So there is also mixed endianness, like 440bx said 

[howardpc]

Off topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages)
English: 23 => Twenty-three
German 23 => Drei-und-Zwanzig (three and twenty)

Poetic and Shakespearean English also has this, such as the nursery rhyme:
"Four and twenty blackbirds baked in a pie"