Recent

Author Topic: Question about bit fields in bitpacked record  (Read 796 times)

440bx

  • Hero Member
  • *****
  • Posts: 2441
Question about bit fields in bitpacked record
« on: June 05, 2021, 08:27:19 am »
Hello,

I suspect that I am misunderstanding something about bitpacked records. 

Referring to the attached image, the DWORD value is shown is $72.  I expected the field "unknown28" (the first 28 bits of that DWORD) to be $7, not $72 (which is the entire 32bit DWORD.)  Also, I expected the field "DepDisabled" to be $1 because the second bit in the DWORD is $1.

The program that produces that output (which might be different on your system) is :
Code: Pascal  [Select][+][-]
  1. {$APPTYPE      CONSOLE}
  2.  
  3. {$TYPEDADDRESS ON}
  4.  
  5. {$ifdef WIN64}
  6.   {$FATAL DEP policy cannot be queried in 64bit - compile for 32bit}
  7. {$endif}
  8.  
  9. program _QueryInformationProcess;
  10.  
  11. uses
  12.   Windows,
  13.   sysutils
  14.   ;
  15.  
  16. {$ifdef VER90}
  17. type
  18.   DWORD = longint;
  19.   PDWORD = ^DWORD;
  20. {$endif}
  21.  
  22. type
  23.   NTSTATUS = DWORD;
  24.  
  25.   pboolean = ^boolean;
  26.  
  27. const
  28.    { 22   34 } ProcessExecuteFlags = 34;  { TPROCESS_DEP_POLICY               }
  29.  
  30.   ntdll = 'ntdll';
  31.  
  32.   PROCESS_ALL_ACCESS        = STANDARD_RIGHTS_REQUIRED or SYNCHRONIZE or $FFFF;
  33.  
  34.   STATUS_SUCCESS            = 0;
  35.  
  36. type
  37.   { bit types for bitfields - since Delphi 2 does not support bitfields       }
  38.   {                           these types are FPC only                        }
  39.  
  40.   T1BIT      = 0 ..        $1;
  41.   T2BITS     = 0 ..        $3;
  42.   T3BITS     = 0 ..        $7;
  43.   T4BITS     = 0 ..        $F;
  44.   T5BITS     = 0 ..       $1F;
  45.   T6BITS     = 0 ..       $3F;
  46.   T7BITS     = 0 ..       $7F;
  47.   T8BITS     = 0 ..       $FF;            { byte                              }
  48.   T9BITS     = 0 ..      $1FF;
  49.  
  50.   T10BITS    = 0 ..      $3FF;
  51.   T11BITS    = 0 ..      $7FF;
  52.   T12BITS    = 0 ..      $FFF;
  53.   T13BITS    = 0 ..     $1FFF;
  54.   T14BITS    = 0 ..     $3FFF;
  55.   T15BITS    = 0 ..     $7FFF;
  56.   T16BITS    = 0 ..     $FFFF;            { word                              }
  57.   T17BITS    = 0 ..    $1FFFF;
  58.   T18BITS    = 0 ..    $3FFFF;
  59.   T19BITS    = 0 ..    $7FFFF;
  60.  
  61.   T20BITS    = 0 ..    $FFFFF;
  62.   T21BITS    = 0 ..   $1FFFFF;
  63.   T22BITS    = 0 ..   $3FFFFF;
  64.   T23BITS    = 0 ..   $7FFFFF;
  65.   T24BITS    = 0 ..   $FFFFFF;
  66.   T25BITS    = 0 ..  $1FFFFFF;
  67.   T26BITS    = 0 ..  $3FFFFFF;
  68.   T27BITS    = 0 ..  $7FFFFFF;
  69.   T28BITS    = 0 ..  $FFFFFFF;
  70.   T29BITS    = 0 .. $1FFFFFFF;
  71.  
  72.   T30BITS    = 0 .. $3FFFFFFF;
  73.   T31BITS    = 0 .. $7FFFFFFF;
  74.  
  75. {$ifdef FPC}
  76.   type
  77.     { _PROCESS_DEP_POLICY                             - 32bit only            }
  78.  
  79.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
  80.     TPROCESS_DEP_POLICY = bitpacked record
  81.       Unknown28                       : T28BITS;
  82.       Permanent                       : T1BIT;
  83.  
  84.       DepEnabledThunk                 : T1BIT;
  85.       DepDisabled                     : T1BIT;
  86.       Unknown1                        : T1BIT;
  87.     end;
  88. {$endif}
  89.  
  90. {$ifdef VER90}
  91.   type
  92.     { _PROCESS_DEP_POLICY                             - 32bit only            }
  93.  
  94.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
  95.     TPROCESS_DEP_POLICY = record
  96.       ExecuteFlags          : DWORD;
  97.     end;
  98. {$endif}
  99.  
  100. function NtQueryInformationProcess(ProcessHandle            : THANDLE;
  101.                                    ProcessInformationClass  : DWORD;
  102.                                    ProcessInformation       : pointer;
  103.                                    ProcessInformationLength : DWORD;
  104.                                    ReturnLength             : PDWORD)
  105.          : NTSTATUS; stdcall; external ntdll;
  106.  
  107. function IsDebuggerPresent {$ifdef FPC} () {$endif}
  108.        : BOOL; stdcall external kernel32;
  109.  
  110. function _itoa
  111.              (
  112.               { _in_  } In32bitInteger  : integer;
  113.               { _out_ } OutAsciizString : pchar;
  114.               { _in_  } InRadix         : DWORD
  115.              )
  116.          : pchar; cdecl external ntdll;                            { !! CDECL }
  117.  
  118. function GetProcessDEPPolicy
  119.                    (
  120.                     { _in_  } InProcessHandle : THANDLE;
  121.                     { _out_ } OutFlags        : PDWORD;
  122.                     { _out_ } OutPermanent    : pboolean      { 4th bit set   }
  123.                    )
  124.        : BOOL; stdcall external kernel32;
  125.  
  126. var
  127.   ProcessHandle                   : THANDLE;
  128.  
  129. var
  130.   { variable for GetProcessDEPPolicy                                          }
  131.  
  132.   Flags                           : DWORD;
  133.   Permanent                       : boolean;
  134.  
  135. var
  136.   { variables for NtQueryInformationProcess                                   }
  137.  
  138.   ProcessExecuteFlagsData         : TPROCESS_DEP_POLICY;
  139.  
  140.   BufferSize    : DWORD;
  141.  
  142.   ReturnLength  : DWORD;
  143.  
  144.   Status        : NTSTATUS;
  145.  
  146.  
  147.   itoa_buf      : packed array[0..127] of char;
  148.  
  149.  
  150. begin
  151.   writeln;
  152.   writeln;
  153.  
  154.   // since we are dealing with our own process we can specify PROCESS_ALL_ACCESS
  155.  
  156.   ProcessHandle := OpenProcess(PROCESS_ALL_ACCESS,
  157.                                FALSE,
  158.                                GetCurrentProcessId());
  159.  
  160.   if ProcessHandle = 0 then Halt(1);
  161.  
  162.   { use the API to verify the results                                         }
  163.  
  164.   writeln;
  165.   writeln;
  166.   if not GetProcessDEPPolicy(ProcessHandle,
  167.                             @Flags,
  168.                             @Permanent) then
  169.   begin
  170.     { this shouldn't happen                                                   }
  171.     writeln;
  172.     writeln;
  173.     writeln('  GetProcessDEPPolicy failed.');
  174.   end;
  175.  
  176.   writeln;
  177.   writeln('  Flags     = ', Flags);
  178.   writeln('  Permanent = ', Permanent);    { 4th bit of DWORD                 }
  179.  
  180.  
  181.   { ------------------------------------------------------------------------- }
  182.   { get the same results using NtQueryInformationProcess                      }
  183.  
  184.   writeln;
  185.   writeln;
  186.   writeln('  ProcessExecuteFlags : ', ord(ProcessExecuteFlags), ' (d) ',
  187.              IntToHex(ord(ProcessExecuteFlags), 0), ' (h)');
  188.  
  189.   BufferSize   := sizeof(ProcessExecuteFlagsData);
  190.   ReturnLength := 0;
  191.  
  192.   writeln;
  193.   writeln;
  194.   writeln('  Buffer size : ', BufferSize);
  195.  
  196.   ZeroMemory(@ProcessExecuteFlagsData, sizeof(ProcessExecuteFlagsData));
  197.  
  198.   if IsDebuggerPresent then DebugBreak();
  199.   Status := NtQueryInformationProcess(ProcessHandle,
  200.                                       ProcessExecuteFlags,
  201.                                      @ProcessExecuteFlagsData,
  202.                                       BufferSize,
  203.                                      @ReturnLength);
  204.  
  205.   if IsDebuggerPresent then DebugBreak();
  206.   if Status <> STATUS_SUCCESS then
  207.   begin
  208.     writeln;
  209.     writeln;
  210.     writeln('  NtQueryInformationProcess failed.  NTSTATUS : ',
  211.                IntToHex(Status, 0));
  212.   end;
  213.  
  214.   writeln;
  215.   writeln('  returned length : ', ReturnLength);
  216.  
  217.  
  218.   writeln;
  219.   writeln;
  220.  
  221.   { output the result in binary for reference                                 }
  222.  
  223.   writeln('  ', _itoa(DWORD(ProcessExecuteFlagsData), itoa_buf, 2));
  224.   writeln;
  225.  
  226.  
  227.   {$ifdef FPC}
  228.   with ProcessExecuteFlagsData do
  229.   begin
  230.     writeln('  ProcessExecuteFlagsData = $', IntToHex(DWORD(ProcessExecuteFlagsData), 0));
  231.     writeln;
  232.  
  233.     writeln('  Unknown28       = $', IntToHex(Unknown28, 0));
  234.     writeln('  Permanent       = ',  Permanent);
  235.     writeln('  DepEnabledThunk = ',  DepEnabledThunk);
  236.     writeln('  DepDisabled     = ',  DepDisabled);
  237.     writeln('  Unknown1        = ',  Unknown1);
  238.   end;
  239.   {$endif}
  240.  
  241.   {$ifdef VER90}
  242.     writeln;
  243.     writeln;
  244.     writeln('  Process execute flags : ',
  245.                IntToHex(ProcessExecuteFlagsData.ExecuteFlags, 0));
  246.   {$endif}
  247.  
  248.  
  249.  
  250.   writeln;
  251.   writeln('Press <enter>/<result> to end this program.');
  252.   readln;
  253. end.
  254.  

My questions are: why isn't DepDisabled = 1 and why isn't Unknown28 = 7 ?  Any suggestion on how to get the result I want is definitely appreciated.

Thank you for your help.
FPC v3.0.4 and Lazarus 1.8.2 on Windows 7 64bit.

MathMan

  • Full Member
  • ***
  • Posts: 227
Re: Question about bit fields in bitpacked record
« Reply #1 on: June 05, 2021, 12:01:57 pm »
I may be off-track totally here, but shouldn't

Code: Pascal  [Select][+][-]
  1. {$ifdef FPC}
  2.   type
  3.     { _PROCESS_DEP_POLICY                             - 32bit only            }
  4.  
  5.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
  6.     TPROCESS_DEP_POLICY = bitpacked record
  7.       Unknown28                       : T28BITS;
  8.       Permanent                       : T1BIT;
  9.  
  10.       DepEnabledThunk                 : T1BIT;
  11.       DepDisabled                     : T1BIT;
  12.       Unknown1                        : T1BIT;
  13.     end;
  14. {$endif}

be defined as follows

Code: Pascal  [Select][+][-]
  1. {$ifdef FPC}
  2.   type
  3.     { _PROCESS_DEP_POLICY                             - 32bit only            }
  4.  
  5.     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;
  6.     TPROCESS_DEP_POLICY = bitpacked record
  7.       Unknown1                        : T1BIT;
  8.       DepDisabled                     : T1BIT;
  9.       DepEnabledThunk                 : T1BIT;
  10.       Permanent                       : T1BIT;
  11.       Unknown28                       : T28BITS;
  12.     end;
  13. {$endif}

To me it looks like you have your definition upside down - that's why Unknown28 reflects the low 28 bit of the Flags value (when seen as DWORD).

Cheers,
MathMan

Martin_fr

  • Administrator
  • Hero Member
  • *
  • Posts: 7301
  • Debugger - SynEdit - and more
    • wiki
Re: Question about bit fields in bitpacked record
« Reply #2 on: June 05, 2021, 12:03:19 pm »
You are probably on a little endian machine (like intel)?

So $00000072 writes to mem like this

Record_addr: $72 $00 $00 $00

440bx

  • Hero Member
  • *****
  • Posts: 2441
Re: Question about bit fields in bitpacked record
« Reply #3 on: June 05, 2021, 12:38:31 pm »
@MathMan, @Martin_fr

Yes, I am on a little endian machine (intel)

I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72.

IOW, the only values that make sense to me is either "0000000" (which is 0) or "7200000". 

I don't see how "Unknown28" could be just "72".  In either endianness that value does not seem possible.
FPC v3.0.4 and Lazarus 1.8.2 on Windows 7 64bit.

MathMan

  • Full Member
  • ***
  • Posts: 227
Re: Question about bit fields in bitpacked record
« Reply #4 on: June 05, 2021, 12:54:43 pm »
@MathMan, @Martin_fr

Yes, I am on a little endian machine (intel)

I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72.

IOW, the only values that make sense to me is either "0000000" (which is 0) or "7200000". 

I don't see how "Unknown28" could be just "72".  In either endianness that value does not seem possible.

Huh - why would you expect that? I would expect "Unknown28" to be little endian too on an Intel machine ...

Martin_fr

  • Administrator
  • Hero Member
  • *
  • Posts: 7301
  • Debugger - SynEdit - and more
    • wiki
Re: Question about bit fields in bitpacked record
« Reply #5 on: June 05, 2021, 01:05:03 pm »
@MathMan, @Martin_fr

Yes, I am on a little endian machine (intel)

I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72.

Not really, Lets name the bytes 11, 22, 33 and 44
Then lets look at the DWORD $44332211  and the packed subrange (fitting in 3 bytes) $332211

DWORD in mem: 11 22 33 44
That is so, if I access that mem as PByte I get $11 (the last byte of the DWORD $44332211)

Or that I can do
Code: Pascal  [Select][+][-]
  1. Fillbyte(mem, 0, MAXLEN);
  2. PByte(mem)^ := 7
  3. wirteln(PDWord(mem)^); // 7

Code: [Select]
DWORD in mem: 11 22 33 44
RANGE in mem: 11 22 33
So if I read range from the DWORD address then i get the bytes 11 22 and 33, and in the variable they will have the value $332211 (the first byte from memory has the lowest significance and is the last byte of the value)

And all that is intended.

Your $00000072
Code: [Select]
DWORD in mem: 72 00 00 00
RANGE in mem: 72 00 00 // 72 is the least significant byte, it will be at the end of the written number $000072 (that are 28 bits)

The bytes in memory are always in reverse order to what they are in human readable representation.



Off topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages)
English: 23 => Twenty-three
German 23 => Drei-und-Zwanzig (three and twenty)

But both have 14 => four-teen (the four move before the ten)

« Last Edit: June 05, 2021, 01:13:05 pm by Martin_fr »

440bx

  • Hero Member
  • *****
  • Posts: 2441
Re: Question about bit fields in bitpacked record
« Reply #6 on: June 05, 2021, 01:26:53 pm »
Huh - why would you expect that? I would expect "Unknown28" to be little endian too on an Intel machine ...
Because the compiler should be using one endianness or the other, not mix the two of them.

if the compiler chooses big endian then, the value of "Unknown28" is $7200000 (28 bits), if it chooses little endian then the value of "Unknown28" is 0000000 which is 0 (zero).  There is no endianness that yields the value $72 for "Unknown28".

I wrote a little test program to better see what is happening, its output is:
Code: Text  [Select][+][-]
  1. i :  0  : 0
  2. i :  1  : 1
  3. i :  2  : 0
  4. i :  3  : 0
  5. i :  4  : 1
  6. i :  5  : 1
  7. i :  6  : 1
  8. i :  7  : 0
  9. i :  8  : 0
  10. i :  9  : 0
  11. i : 10  : 0
  12. i : 11  : 0
  13. i : 12  : 0
  14. i : 13  : 0
  15. i : 14  : 0
  16. i : 15  : 0
  17. i : 16  : 0
  18. i : 17  : 0
  19. i : 18  : 0
  20. i : 19  : 0
  21. i : 20  : 0
  22. i : 21  : 0
  23. i : 22  : 0
  24. i : 23  : 0
  25. i : 24  : 0
  26. i : 25  : 0
  27. i : 26  : 0
  28. i : 27  : 0
  29. i : 28  : 0
  30. i : 29  : 0
  31. i : 30  : 0
  32. i : 31  : 0
  33.  
  34.  
  35. TwentyEight : 72
  36. Four        : 0[/size][/font]
if the compiler is going to use the big endian memory ordering then the above shows that "Unknown28" should be $7200000 not $72 (that $72 is followed by 6 bytes of zeroes) and those cannot simply disappear because in big endian they are to the right of the value ($72 in this case).

What the compiler seems to be doing is taking the first 28 bits, adding a nibble at the end and then evaluating the resulting 32bits in little endian mode, not big endian but, when doing individual bits it uses big endian.  IOW, it's mixing endianness-es depending on the field sizes.



@Martin,

I was typing this when you made your post.  I'm going to review it after posting this.



the test program that produced that output is:
Code: Pascal  [Select][+][-]
  1. {$APPTYPE      CONSOLE}
  2.  
  3. {$TYPEDADDRESS ON}
  4.  
  5.  
  6. program _BitFields2;
  7.  
  8.  
  9. uses
  10.   Windows,
  11.   sysutils
  12.   ;
  13.  
  14.  
  15. type
  16.   { bit types for bitfields - since Delphi 2 does not support bitfields       }
  17.   {                           these types are FPC only                        }
  18.  
  19.   T1BIT      = 0 ..        $1;
  20.   T2BITS     = 0 ..        $3;
  21.   T3BITS     = 0 ..        $7;
  22.   T4BITS     = 0 ..        $F;
  23.   T5BITS     = 0 ..       $1F;
  24.   T6BITS     = 0 ..       $3F;
  25.   T7BITS     = 0 ..       $7F;
  26.   T8BITS     = 0 ..       $FF;            { byte                              }
  27.   T9BITS     = 0 ..      $1FF;
  28.  
  29.   T10BITS    = 0 ..      $3FF;
  30.   T11BITS    = 0 ..      $7FF;
  31.   T12BITS    = 0 ..      $FFF;
  32.   T13BITS    = 0 ..     $1FFF;
  33.   T14BITS    = 0 ..     $3FFF;
  34.   T15BITS    = 0 ..     $7FFF;
  35.   T16BITS    = 0 ..     $FFFF;            { word                              }
  36.   T17BITS    = 0 ..    $1FFFF;
  37.   T18BITS    = 0 ..    $3FFFF;
  38.   T19BITS    = 0 ..    $7FFFF;
  39.  
  40.   T20BITS    = 0 ..    $FFFFF;
  41.   T21BITS    = 0 ..   $1FFFFF;
  42.   T22BITS    = 0 ..   $3FFFFF;
  43.   T23BITS    = 0 ..   $7FFFFF;
  44.   T24BITS    = 0 ..   $FFFFFF;
  45.   T25BITS    = 0 ..  $1FFFFFF;
  46.   T26BITS    = 0 ..  $3FFFFFF;
  47.   T27BITS    = 0 ..  $7FFFFFF;
  48.   T28BITS    = 0 ..  $FFFFFFF;
  49.   T29BITS    = 0 .. $1FFFFFFF;
  50.  
  51.   T30BITS    = 0 .. $3FFFFFFF;
  52.   T31BITS    = 0 .. $7FFFFFFF;
  53.  
  54.  
  55. const
  56.   ntdll = 'ntdll';
  57.  
  58.  
  59. function _itoa
  60.              (
  61.               { _in_  } In32bitInteger  : integer;
  62.               { _out_ } OutAsciizString : pchar;
  63.               { _in_  } InRadix         : DWORD
  64.              )
  65.          : pchar; cdecl external ntdll;                            { !! CDECL }
  66.  
  67. var
  68.   BitTest                         : DWORD = $72;
  69.  
  70.   BitFields                       : bitpacked record
  71.     Bits                              : bitpacked array[0..31] of T1BIT;
  72.   end absolute BitTest;
  73.  
  74.   BitFields2                      : bitpacked record
  75.     case integer of
  76.       1 : (
  77.            TwentyEight                 : T28BITS;
  78.            Four                        : T4BITS;
  79.           );
  80.  
  81.       2 : (
  82.            Bits                        : bitpacked array[0..31] of T1BIT;
  83.           );
  84.   end absolute BitTest;
  85.  
  86.   i                               : integer;
  87.  
  88.  
  89. begin
  90.   writeln;
  91.   writeln;
  92.  
  93.   with BitFields2 do
  94.   begin
  95.     for i := low(Bits) to high(Bits) do
  96.     begin
  97.       writeln('  i : ', i:2, '  : ', Bits[i]);
  98.     end;
  99.  
  100.     writeln;
  101.     writeln;
  102.     writeln('  TwentyEight : ', IntToHex(TwentyEight, 0));
  103.     writeln('  Four        : ', Four);
  104.   end;
  105.  
  106.  
  107.   writeln;
  108.   writeln('Press ENTER/RETURN to end this program.');
  109.   readln;
  110. end.

FPC v3.0.4 and Lazarus 1.8.2 on Windows 7 64bit.

440bx

  • Hero Member
  • *****
  • Posts: 2441
Re: Question about bit fields in bitpacked record
« Reply #7 on: June 05, 2021, 01:46:09 pm »
And all that is intended.

Your $00000072
Code: [Select]
DWORD in mem: 72 00 00 00
RANGE in mem: 72 00 00 // 72 is the least significant byte, it will be at the end of the written number $000072 (that are 28 bits)
I understand and agree with everything you've said but, as you showed, the "RANGE in mem" is 72 00 00 0 (28 bits), once the compiler decides to look at bits in memory order, it cannot change the way of looking at them.  In memory order the value is 72 00 00 0

The bytes in memory are always in reverse order to what they are in human readable representation.
Completely agree with that.  what causes a real problem is that the compiler starts looking at bytes in memory-order and then switches to little-endian order which is what made the 72 00 00 0 show up as just 72.

I'm going to have to review every bitpacked record definition I have and test thoroughly to ensure they behave as they should (the majority of them defined just as they are in C but, now I have doubts that they'll work as they do in C.)

Thank you Martin and thank you MathMan for your help.  It was very useful.
« Last Edit: June 05, 2021, 01:49:50 pm by 440bx »
FPC v3.0.4 and Lazarus 1.8.2 on Windows 7 64bit.

dseligo

  • Sr. Member
  • ****
  • Posts: 315
Re: Question about bit fields in bitpacked record
« Reply #8 on: June 05, 2021, 01:47:48 pm »
Off topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages)
English: 23 => Twenty-three
German 23 => Drei-und-Zwanzig (three and twenty)

Slovene also has this.
But this is only for the last two digits. So there is also mixed endianness, like 440bx said  :)

howardpc

  • Hero Member
  • *****
  • Posts: 3728
Re: Question about bit fields in bitpacked record
« Reply #9 on: June 05, 2021, 01:51:16 pm »
Off topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages)
English: 23 => Twenty-three
German 23 => Drei-und-Zwanzig (three and twenty)
Poetic and Shakespearean English also has this, such as the nursery rhyme:
"Four and twenty blackbirds baked in a pie"

 

TinyPortal © 2005-2018