Recent

Author Topic: Unexpected structure size  (Read 2704 times)

440bx

  • Hero Member
  • *****
  • Posts: 3921
Unexpected structure size
« on: September 18, 2021, 01:13:28 am »
Hello,

A bitpacked sub record is causing the size of the complete record to be different than expected.  Refer to the test code below (compilable as-is):
Code: Pascal  [Select][+][-]
  1. {$APPTYPE       CONSOLE}
  2.  
  3. {$TYPEDADDRESS  ON}
  4.  
  5. {$LONGSTRINGS   OFF}
  6.  
  7. { --------------------------------------------------------------------------- }
  8.  
  9. {$DEFINE ENABLE_BIT_FIELDS}
  10. {$DEFINE PROBLEMATIC_DEFINITION}
  11.  
  12. program _TRTL_BALANCED_NODE;
  13.   { program to verify the size of the TRTL_BALANCED_NODE structure           }
  14.  
  15.  
  16. type
  17.   { bit types for bitfields                                                  }
  18.  
  19.    _1bit     = 0 ..                $1;
  20.    _2bits    = 0 ..                $3;
  21.    _3bits    = 0 ..                $7;
  22.    _4bits    = 0 ..                $F;
  23.    _5bits    = 0 ..               $1F;
  24.    _6bits    = 0 ..               $3F;
  25.    _7bits    = 0 ..               $7F;
  26.    _8bits    = 0 ..               $FF;    { byte                              }
  27.    _9bits    = 0 ..              $1FF;
  28.  
  29.   _10bits    = 0 ..              $3FF;
  30.   _11bits    = 0 ..              $7FF;
  31.   _12bits    = 0 ..              $FFF;
  32.   _13bits    = 0 ..             $1FFF;
  33.   _14bits    = 0 ..             $3FFF;
  34.   _15bits    = 0 ..             $7FFF;
  35.   _16bits    = 0 ..             $FFFF;    { word                              }
  36.   _17bits    = 0 ..            $1FFFF;
  37.   _18bits    = 0 ..            $3FFFF;
  38.   _19bits    = 0 ..            $7FFFF;
  39.  
  40.   _20bits    = 0 ..            $FFFFF;
  41.   _21bits    = 0 ..           $1FFFFF;
  42.   _22bits    = 0 ..           $3FFFFF;
  43.   _23bits    = 0 ..           $7FFFFF;
  44.   _24bits    = 0 ..           $FFFFFF;
  45.   _25bits    = 0 ..          $1FFFFFF;
  46.   _26bits    = 0 ..          $3FFFFFF;
  47.   _27bits    = 0 ..          $7FFFFFF;
  48.   _28bits    = 0 ..          $FFFFFFF;
  49.   _29bits    = 0 ..         $1FFFFFFF;
  50.  
  51.   _30bits    = 0 ..         $3FFFFFFF;
  52.   _31bits    = 0 ..         $7FFFFFFF;
  53.  
  54.   _32bits    = 0 ..         $FFFFFFFF;    { DWORD                             }
  55.  
  56.   _33bits    = 0 ..        $1FFFFFFFF;
  57.   _34bits    = 0 ..        $3FFFFFFFF;
  58.   _35bits    = 0 ..        $7FFFFFFFF;
  59.   _36bits    = 0 ..        $FFFFFFFFF;
  60.   _37bits    = 0 ..       $1FFFFFFFFF;
  61.   _38bits    = 0 ..       $3FFFFFFFFF;
  62.   _39bits    = 0 ..       $7FFFFFFFFF;
  63.  
  64.   _40bits    = 0 ..       $FFFFFFFFFF;
  65.   _41bits    = 0 ..      $1FFFFFFFFFF;
  66.   _42bits    = 0 ..      $3FFFFFFFFFF;
  67.   _43bits    = 0 ..      $7FFFFFFFFFF;
  68.   _44bits    = 0 ..      $FFFFFFFFFFF;
  69.   _45bits    = 0 ..     $1FFFFFFFFFFF;
  70.   _46bits    = 0 ..     $3FFFFFFFFFFF;
  71.   _47bits    = 0 ..     $7FFFFFFFFFFF;
  72.   _48bits    = 0 ..     $FFFFFFFFFFFF;
  73.   _49bits    = 0 ..    $1FFFFFFFFFFFF;
  74.  
  75.   _50bits    = 0 ..    $3FFFFFFFFFFFF;
  76.   _51bits    = 0 ..    $7FFFFFFFFFFFF;
  77.   _52bits    = 0 ..    $FFFFFFFFFFFFF;
  78.   _53bits    = 0 ..   $1FFFFFFFFFFFFF;
  79.   _54bits    = 0 ..   $3FFFFFFFFFFFFF;
  80.   _55bits    = 0 ..   $7FFFFFFFFFFFFF;
  81.   _56bits    = 0 ..   $FFFFFFFFFFFFFF;
  82.   _57bits    = 0 ..  $1FFFFFFFFFFFFFF;
  83.   _58bits    = 0 ..  $3FFFFFFFFFFFFFF;
  84.   _59bits    = 0 ..  $7FFFFFFFFFFFFFF;
  85.  
  86.   _60bits    = 0 ..  $FFFFFFFFFFFFFFF;
  87.   _61bits    = 0 .. $1FFFFFFFFFFFFFFF;
  88.   _62bits    = 0 .. $3FFFFFFFFFFFFFFF;
  89.   _63bits    = 0 .. $7FFFFFFFFFFFFFFF;
  90.  
  91. type
  92.   PRTL_BALANCED_NODE = ^TRTL_BALANCED_NODE;
  93.   TRTL_BALANCED_NODE = packed record
  94.     Node                 : record
  95.       case integer of
  96.         1 : (
  97.              Children        : array[0..1] of PRTL_BALANCED_NODE;
  98.             );
  99.  
  100.         2 : (
  101.              Left            : PRTL_BALANCED_NODE;
  102.              Right           : PRTL_BALANCED_NODE;
  103.             );
  104.     end;
  105.  
  106.     Value                : {$ifdef FPC} bitpacked {$endif} record
  107.       case integer of
  108.         1 : (
  109.              ParentValue     : ptruint;
  110.             );
  111.  
  112.  
  113.         { ------------------------------------------------------------------- }
  114.         { something in these bit fields is not right                          }
  115.  
  116.         {$ifdef ENABLE_BIT_FIELDS}
  117.         {$ifdef PROBLEMATIC_DEFINITION}
  118.         {$ifdef FPC}
  119.         2 : (
  120.              Red             :  _1bit;
  121.  
  122.              {$ifdef WIN32}                  { no problem in 32bits           }
  123.                FillerRed     : _31bits;
  124.              {$endif}
  125.  
  126.              {$ifdef WIN64}                  { PROBLEM: causes one extra byte }
  127.                FillerRed     : _63bits;
  128.              {$endif}
  129.             );
  130.         {$endif}
  131.         {$endif}
  132.  
  133.  
  134.         {$ifdef FPC}
  135.         3 : (
  136.              Balance         :  _2bits;
  137.  
  138.              {$ifdef WIN32}                  { no problem here                }
  139.                FillerBalance : _30bits;
  140.              {$endif}
  141.  
  142.              {$ifdef WIN64}                  { no problem here either         }
  143.                FillerBalance : _62bits;
  144.              {$endif}
  145.             );
  146.         {$endif}
  147.         {$endif}
  148.     end;
  149.   end;
  150.  
  151.  
  152. var
  153.   V1 : record
  154.     F1 : TRTL_BALANCED_NODE;
  155.     F2 : TRTL_BALANCED_NODE;
  156.   end;
  157.  
  158.  
  159. begin
  160.   writeln;
  161.   writeln;
  162.  
  163.   writeln('  sizeof(TRTL_BALANCED_NODE) : ', sizeof(TRTL_BALANCED_NODE));
  164.   writeln('  sizeof(V1)                 : ', sizeof(V1));
  165.  
  166.  
  167.   writeln;
  168.   writeln;
  169.   writeln('press ENTER/RETURN to end this program');
  170.   readln;
  171. end.
  172.  

The problem only occurs in 64bit.  if PROBLEMATIC_DEFINITION is defined, the size is reported as 25 instead of 24.  IOW, somehow the structure "acquired" an extra byte. 

if PROBLEMATIC_DEFINITION is not enabled then the size is 24 as expected.

Is there a reason I am not seeing for that extra byte to be present when PROBLEMATIC_DEFINITION is defined or is the extra byte caused by a bug in the compiler ? (note: using FPC v3.0.4)

Thank you for your help.
(FPC v3.0.4 and Lazarus 1.8.2) or (FPC v3.2.2 and Lazarus v3.2) on Windows 7 SP1 64bit.

Kays

  • Hero Member
  • *****
  • Posts: 569
  • Whasup!?
    • KaiBurghardt.de
Re: Unexpected structure size
« Reply #1 on: September 18, 2021, 11:19:04 pm »
[…] Is there a reason I am not seeing for that extra byte to be present when PROBLEMATIC_DEFINITION is defined […]
The host data type of a range is the common data type of its limits. Even though you don’t explicitly specify it hexadecimal literals are signed.

[…] is the extra byte caused by a bug in the compiler ? […]
I’d say so. FPC apparently doesn’t eliminate the unneeded sign-bit in this border case. If I compile your program with GPC it’ll output 24 as expected (presuming appropriate $defines and using packed instead of bitpacked).

As a workaround you can deliberately denote the necessity of a sign:
Code: Pascal  [Select][+][-]
  1.   _63bits    = -$3FFFFFFFFFFFFFFF .. $3FFFFFFFFFFFFFFE;
For some inexplicable reason the absolute value of the upper limit needs to be smaller than the absolute value of the lower limit (otherwise I get an internal error 200706094).

Just a question though: Do you need fillerRed/fillerBalance? I mean, will you access their values? Because, you know, the largest variant of a variant record determines the total size. Reading your trtl_balanced_node.value definition ptrUInt will always determine or coincide with the intended size.
Yours Sincerely
Kai Burghardt

440bx

  • Hero Member
  • *****
  • Posts: 3921
Re: Unexpected structure size
« Reply #2 on: September 18, 2021, 11:34:42 pm »
Kays, thank you for looking into it.

Just a question though: Do you need fillerRed/fillerBalance? I mean, will you access their values?
The "Filler" fields are there for documentation purposes.  I can get rid of them but, I like making it explicit that they are unused/filler.

After playing a bit with the definition, I found that if instead of using _63bits in a single field, I have one field of _62bits and another of _1bit then the compiler computes the correct size.  It's hokey but, at least it serves the documentation purpose.

I have a question for you, did you use FPC v3.0.4 (the version I use) or a later version to do your testing ?
(FPC v3.0.4 and Lazarus 1.8.2) or (FPC v3.2.2 and Lazarus v3.2) on Windows 7 SP1 64bit.

ASerge

  • Hero Member
  • *****
  • Posts: 2212
Re: Unexpected structure size
« Reply #3 on: September 19, 2021, 06:20:22 am »
I think that for large digits, signed arithmetic is always used.
Code: Pascal  [Select][+][-]
  1. {$MODE OBJFPC}
  2. {$LONGSTRINGS ON}
  3. {$APPTYPE CONSOLE}
  4.  
  5. type
  6.   T2Bit = 0..UInt64((1 shl 2) - 1);
  7.   T31Bit = 0..UInt64((1 shl 31) - 1);
  8.   T32Bit = 0..UInt64((1 shl 32) - 1);
  9.   T33Bit = 0..UInt64((1 shl 33) - 1);
  10.   T62Bit = 0..UInt64((1 shl 62) - 1);
  11.  
  12.   TSome32_32 = bitpacked record
  13.     case Byte of
  14.       0: (A: T32Bit; B: T32Bit);
  15.   end;
  16.  
  17.   TSome31_32 = bitpacked record
  18.     case Byte of
  19.       0: (A: T31Bit; B: T33Bit);
  20.   end;
  21.  
  22.   TSome2_62 = bitpacked record
  23.     case Byte of
  24.       0: (A: T2Bit; B: T62Bit);
  25.   end;
  26.  
  27. begin
  28.   Writeln(BitSizeOf(T2Bit), '  ', HexStr(High(T2Bit), 16));
  29.   Writeln(BitSizeOf(T31Bit), ' ', HexStr(High(T31Bit), 16));
  30.   Writeln(BitSizeOf(T32Bit), ' ', HexStr(High(T32Bit), 16));
  31.   Writeln(BitSizeOf(T33Bit), ' ', HexStr(High(T33Bit), 16));
  32.   Writeln(BitSizeOf(T62Bit), ' ', HexStr(High(T62Bit), 16));
  33.   Writeln(BitSizeOf(TSome32_32.A), ' + ', BitSizeOf(TSome32_32.B), ' = ', BitSizeOf(TSome32_32));
  34.   Writeln(BitSizeOf(TSome31_32.A), ' + ', BitSizeOf(TSome31_32.B), ' = ', BitSizeOf(TSome31_32));
  35.   Writeln(BitSizeOf(TSome2_62.A), ' + ', BitSizeOf(TSome2_62.B), ' = ', BitSizeOf(TSome2_62));
  36.   Readln;
  37. end.

FPC x32 3.2.0
Quote
8  0000000000000003
32 000000007FFFFFFF
32 00000000FFFFFFFF
64 00000001FFFFFFFF
64 3FFFFFFFFFFFFFFF
32 + 32 = 64
31 + 64 = 96
2 + 64 = 72

FPC x64 3.2.2
Quote
8  0000000000000003
32 000000007FFFFFFF
32 00000000FFFFFFFF
64 00000001FFFFFFFF
64 3FFFFFFFFFFFFFFF
32 + 32 = 64
31 + 33 = 64
2 + -1 = 8

440bx

  • Hero Member
  • *****
  • Posts: 3921
Re: Unexpected structure size
« Reply #4 on: September 19, 2021, 07:12:32 am »
Hello Serge,

In this particular case, I'm not concerned about the BitSizeof.  My concern is the size in bytes of the packed record and, what FPC (v3.0.4) is doing doesn't make any sense.

Using your sample code:
Code: Pascal  [Select][+][-]
  1. {$MODE OBJFPC}
  2. {$LONGSTRINGS ON}
  3. {$APPTYPE CONSOLE}
  4.  
  5. type
  6.   T2Bit  = 0..UInt64((1 shl  2) - 1);
  7.   T31Bit = 0..UInt64((1 shl 31) - 1);
  8.   T32Bit = 0..UInt64((1 shl 32) - 1);
  9.   T33Bit = 0..UInt64((1 shl 33) - 1);
  10.   T62Bit = 0..UInt64((1 shl 62) - 1);
  11.  
  12.   TSome32_32 = bitpacked record
  13.     case Byte of
  14.       0: (A: T32Bit; B: T32Bit);
  15.   end;
  16.  
  17.   TSome31_32 = bitpacked record
  18.     case Byte of
  19.       0: (A: T31Bit; B: T33Bit);
  20.   end;
  21.  
  22.   TSome2_62 = bitpacked record
  23.     case Byte of
  24.       0: (A: T2Bit; B: T62Bit);
  25.   end;
  26.  
  27.  
  28. begin
  29.   case sizeof(pointer) of
  30.     4 : writeln('  32 bit results');
  31.     8 : writeln('  64 bit results');
  32.   end;
  33.  
  34.   writeln;
  35.  
  36.   writeln('  ', sizeof(TSome32_32));
  37.   writeln('  ', sizeof(TSome31_32));
  38.   writeln('  ', sizeof(TSome2_62));
  39.  
  40.   Readln;
  41. end.
and taking sizeof (instead of bitsizeof) the bitpacked records here are the results:

32bits:
Code: Text  [Select][+][-]
  1.   32 bit results
  2.  
  3.   8
  4.   12
  5.   9
  6.  

64bits:
Code: Text  [Select][+][-]
  1.   64 bit results
  2.  
  3.   8
  4.   8
  5.   1
  6.  

In 32bit, the size of the first record is as expected.  The size of the second record is off by 4 bytes 31 bits + 33bits don't occupy 12 bytes (though, I think I know what FPC is doing - which is not right).  The size of the third record is off by 1 byte, 64bits don't occupy 9 bytes. 

For the second record, it looks like what FPC is doing is allocating a DWORD (for T31bit) followed by a QWORD (for T33Bit), that would explain how it's getting 12 bytes.

For the third record, it looks like it's allocating a byte for T2BIT and a qword for T62Bit.  I wouldn't call that bitpacking.

In 64bit, the first two sizes reported are ok but, the size of the third record being 1 is very "peculiar". Packing 64bits into 1 byte is quite a feat :)
(FPC v3.0.4 and Lazarus 1.8.2) or (FPC v3.2.2 and Lazarus v3.2) on Windows 7 SP1 64bit.

ASerge

  • Hero Member
  • *****
  • Posts: 2212
Re: Unexpected structure size
« Reply #5 on: September 19, 2021, 09:11:21 am »
And the conclusion is this - for bitpacked, types that are larger than 32 bits can cause ambiguities.

440bx

  • Hero Member
  • *****
  • Posts: 3921
Re: Unexpected structure size
« Reply #6 on: September 19, 2021, 10:05:25 am »
And the conclusion is this - for bitpacked, types that are larger than 32 bits can cause ambiguities.
The word that comes to mind isn't "ambiguities"... it's more like "ambuguitis"  :D
« Last Edit: September 19, 2021, 10:07:00 am by 440bx »
(FPC v3.0.4 and Lazarus 1.8.2) or (FPC v3.2.2 and Lazarus v3.2) on Windows 7 SP1 64bit.

Kays

  • Hero Member
  • *****
  • Posts: 569
  • Whasup!?
    • KaiBurghardt.de
Re: Unexpected structure size
« Reply #7 on: September 21, 2021, 12:35:45 am »
[…] The "Filler" fields are there for documentation purposes. […]
Huh, alright, although I’d say just as it is a bad idea to keep unreachable code it’s a bad idea to include unnecessary declarations. It simply clutters your (generated) documentation.

I would insert some safeguard though:
Code: Pascal  [Select][+][-]
  1.         {$if sizeOf(trtl_balanced_node) <> 3 * sizeOf(ptrUInt)}
  2.                 {$fatal size mismatch}
  3.         {$endIf}

[…] In 64bit, the first two sizes reported are ok but, the size of the third record being 1 is very "peculiar". […]
Have you tried using this data type? Like:
Code: Pascal  [Select][+][-]
  1. var
  2.         x:  TSome2_62;
  3. begin
  4.         x.B := 0;
I can’t even compile it without getting the internal error 2006081613. Interestingly, bitSizeOf(tSome2_62.B) is −1. I didn’t know this was even possible. I think this warrants a bug report.
Yours Sincerely
Kai Burghardt

440bx

  • Hero Member
  • *****
  • Posts: 3921
Re: Unexpected structure size
« Reply #8 on: September 21, 2021, 12:47:34 am »
Huh, alright, although I’d say just as it is a bad idea to keep unreachable code it’s a bad idea to include unnecessary declarations. It simply clutters your (generated) documentation.
It's a good idea to make it explicitly known the bits are unused.  That way the person who reads the code knows that, some bit definitions aren't missing, they are simply unused.  Knowing that also aids in future maintenance.

Have you tried using this data type? Like:
Code: Pascal  [Select][+][-]
  1. var
  2.         x:  TSome2_62;
  3. begin
  4.         x.B := 0;
I can’t even compile it without getting the internal error 2006081613. Interestingly, bitSizeOf(tSome2_62.B) is −1. I didn’t know this was even possible. I think this warrants a bug report.
I haven't tried your suggestion.  I noticed that if instead of having one field of one (1) bit and another of 63 bits, I have two fields of 1 bit each followed by a field of 62 bits then the size is correct. 

I would have reported it as a bug if I were using the latest version of FPC but, since I am not, I don't know if the problem exists in the current version.  Additionally, none of the FPC developers have said anything about this problem.

I believe it's a bug and, it's not the only "peculiar" behavior I've seen associated with bit fields.  If a developer confirms it is a bug then, I'll gladly create a bug report for it.
(FPC v3.0.4 and Lazarus 1.8.2) or (FPC v3.2.2 and Lazarus v3.2) on Windows 7 SP1 64bit.

 

TinyPortal © 2005-2018