Recent

Author Topic: fcl-json Slow  (Read 1477 times)

gii

  • Jr. Member
  • **
  • Posts: 53
fcl-json Slow
« on: November 21, 2020, 03:22:55 am »
I have the same code in PHP and Pascal.

PHP:
Code: PHP  [Select][+][-]
  1. for ( $k = 0; $k < 1000; $k++ ) {
  2.  
  3.         for ( $i = 1; $i <= 15; $i++ ) {
  4.  
  5.                 $file = file_get_contents($i . '.txt');
  6.  
  7.                 $arr = json_decode( $file, false );
  8.  
  9.                 for ( $j = 0; $j < count($arr->Data); $j++ ) {
  10.  
  11.                         $a = $arr->Data[$j]->Data;
  12.                         $b = $arr->Data[$j]->Open;
  13.                         $c = $arr->Data[$j]->Close;
  14.                         $d = $arr->Data[$j]->High;
  15.                         $e = $arr->Data[$j]->Low;
  16.                         $f = $arr->Data[$j]->Volume;                   
  17.  
  18.                         if ( strlen($a) > 50 ) {
  19.                                
  20.                                 $a = 'sdsdsd';
  21.                                 echo $a;
  22.                         }
  23.  
  24.                         if ( $b > 10000000 ) {
  25.                                
  26.                                 $b = 100;
  27.                                 echo $b;
  28.                         }
  29.  
  30.                         if ( $c > 10000000 ) {
  31.                                
  32.                                 $c = 100;
  33.                                 echo $c;
  34.                         }
  35.  
  36.                         if ( $d > 10000000 ) {
  37.                                
  38.                                 $d = 100;
  39.                                 echo $d;
  40.                         }
  41.  
  42.                         if ( $e > 10000000 ) {
  43.                                
  44.                                 $e = 100;
  45.                                 echo $e;
  46.                         }
  47.  
  48.                         if ( $f > 10000000 ) {
  49.                                
  50.                                 $f = 100;
  51.                                 echo $f;
  52.                         }                      
  53.                 }
  54.         }
  55.  
  56.         $k++;
  57. }
  58.  

Pascal:
Code: Pascal  [Select][+][-]
  1. procedure TMyApplication.DoRun;
  2. var
  3.   AItens : TJSONArray;
  4.   AData : TJSONData;
  5.   AFile : TFileStream;
  6.   i, j, k : Integer;
  7.   a : String;
  8.   b, c, d, e : Double;
  9.   f : Integer;
  10. begin
  11.   for i := 0 to 999 do
  12.   begin
  13.     for j := 1 to 15 do
  14.     begin
  15.         AFile := TFileStream.Create( IntToStr( j ) + '.txt', fmOpenRead or fmShareDenyNone);
  16.         try
  17.           AData := GetJSON( AFile );
  18.           try
  19.             AItens := ( AData as TJSONObject ).Arrays['Data'];
  20.  
  21.             for k := 0 to ( AItens.Count - 1 ) do
  22.             begin
  23.                 with AItens[k] as TJSONObject do
  24.                 begin
  25.                     a := Strings['Data'];
  26.                     b := Floats['Open'];
  27.                     c := Floats['Close'];
  28.                     d := Floats['High'];
  29.                     e := Floats['Low'];
  30.                     f := Integers['Volume'];
  31.                 end;
  32.  
  33.                 if ( Length(a) > 50 ) then
  34.                 begin
  35.                                         a := 'sdsdsd';
  36.                     WriteLn( a );
  37.                             end;
  38.  
  39.                                 if ( b > 10000000 ) then
  40.                 begin
  41.                                         b := 100;
  42.                     WriteLn( FloatToStr( b ) );
  43.                             end;
  44.  
  45.                                 if ( c > 10000000 ) then
  46.                 begin
  47.                                         c := 100;
  48.                     WriteLn( FloatToStr( c ) );
  49.                             end;
  50.  
  51.                 if ( d > 10000000 ) then
  52.                 begin
  53.                                         d := 100;
  54.                     WriteLn( FloatToStr( d ) );
  55.                             end;
  56.  
  57.                 if ( e > 10000000 ) then
  58.                 begin
  59.                                         e := 100;
  60.                     WriteLn( FloatToStr( e ) );
  61.                             end;
  62.  
  63.                 if ( f > 10000000 ) then
  64.                 begin
  65.                                         f := 100;
  66.                     WriteLn( FloatToStr( f ) );
  67.                             end;
  68.             end;
  69.           finally
  70.             AData.Free;
  71.           end;
  72.         finally
  73.           AFile.Free;
  74.         end;
  75.     end;
  76.   end;
  77.   // stop program loop
  78.   Terminate;
  79. end;
  80.  

PHP takes 30 seconds to execute it.

Pascal takes 141 seconds to execute it.

The files were generated using the following PHP code:

Code: PHP  [Select][+][-]
  1. $dtInicial = date_create_from_format('Y-m-d H:i:s', '1970-01-01 00:00:00', new DateTimeZone('America/Sao_Paulo'));
  2. $dtFinal = new DateTime("now", new DateTimeZone('America/Sao_Paulo') );
  3.  
  4. $i = 1;
  5. $arr = array( "Data" => array() );
  6.  
  7. while ( $dtInicial <= $dtFinal ) {
  8.  
  9.         array_push( $arr["Data"], array(
  10.                 "Data" => $dtInicial->format('Y-m-d H:i:s'),
  11.                 "Open" => mt_rand(300, 1500) / mt_rand(1, 3),
  12.                 "Close" => mt_rand(300, 1500) / mt_rand(1, 3),
  13.                 "High" => mt_rand(300, 1500) / mt_rand(1, 3),
  14.                 "Low" => mt_rand(300, 1500) / mt_rand(1, 3),
  15.                 "Volume" => mt_rand(100000, 5000000)
  16.         ) );
  17.  
  18.         if ( count( $arr["Data"] ) == 1200 ) {
  19.  
  20.                 @file_put_contents( $i . '.txt', json_encode( $arr, JSON_UNESCAPED_SLASHES ) );
  21.                
  22.                 $arr = array( "Data" => array() );
  23.                 $i++;
  24.         }
  25.  
  26.         $dtInicial->add(new DateInterval('P1D'));
  27. }
  28.  

What is the reason for the slowness?

Is there any better pascal JSON implementation?

Is there anything I can improve on the current code?

lainz

  • Hero Member
  • *****
  • Posts: 4463
    • https://lainz.github.io/
Re: fcl-json Slow
« Reply #1 on: November 21, 2020, 03:56:14 am »
Search for jsontools is a lot faster.

Okoba

  • Hero Member
  • *****
  • Posts: 528
Re: fcl-json Slow
« Reply #2 on: November 21, 2020, 10:13:22 am »
It is not clear what is the code you are benchmarking.
For a better benchmark, please use GetTickCount and check only GetJSON and getting values, as reading files and doing writeln is not your focus, and writeln is particularly slow.
You should probably check this portion of your code in PHP and Pascal:
Code: Pascal  [Select][+][-]
  1.      
  2.  AData := GetJSON( AFile );
  3.             AItens := ( AData as TJSONObject ).Arrays['Data'];
  4.  
  5.             for k := 0 to ( AItens.Count - 1 ) do
  6.             begin
  7.                 with AItens[k] as TJSONObject do
  8.                 begin
  9.                     a := Strings['Data'];
  10.                     b := Floats['Open'];
  11.                     c := Floats['Close'];
  12.                     d := Floats['High'];
  13.                     e := Floats['Low'];
  14.                     f := Integers['Volume'];
  15.                 end;
  16. AData.Free;

If the speed was still not satisfying, you check other JSON libraries like what Lainz mentioned: https://github.com/sysrpl/JsonTools/

gii

  • Jr. Member
  • **
  • Posts: 53
Re: fcl-json Slow
« Reply #3 on: November 21, 2020, 05:17:51 pm »
Reading files is an important part of my project.

I didn't share the complete code, but I'm using GetTickCount64 () to calculate the time.

The code has WriteLn(), but they are never called. Just to display the benchmark.

I updated the code to work with JSONTools.

With JSONTools the code is 20 seconds faster. However, 120 seconds is still slow.

Code: Pascal  [Select][+][-]
  1. procedure TMyApplication.DoRun;
  2. var
  3.   ACmd : String;
  4.   ATime : QWord;
  5.   AItens : TJsonNode;
  6.   N : TJsonNode;
  7.   i, j, k : Integer;
  8.   a : String;
  9.   b, c, d, e : Double;
  10.   f : Integer;
  11. begin
  12.   ATime := GetTickCount64();
  13.  
  14.   for i := 0 to 999 do
  15.   begin
  16.     for j := 1 to 15 do
  17.     begin
  18.         N := TJsonNode.Create;
  19.         try
  20.           N.LoadFromFile( 'C:\xampp\htdocs\teste_json\' + IntToStr( j ) + '.txt' );
  21.  
  22.           AItens := N.Child('Data').AsArray;
  23.  
  24.           for k := 0 to ( AItens.Count - 1 ) do
  25.           begin
  26.               with AItens.Child( k ) do
  27.               begin
  28.                   a := Find('Data').AsString;
  29.                   b := Find('Open').AsNumber;
  30.                   c := Find('Close').AsNumber;
  31.                   d := Find('High').AsNumber;
  32.                   e := Find('Low').AsNumber;
  33.                   f := Trunc( Find('Volume').AsNumber );
  34.               end;
  35.  
  36.               if ( Length(a) > 50 ) then
  37.               begin
  38.                                   a := 'sdsdsd';
  39.                   //WriteLn( a );
  40.                           end;
  41.  
  42.                           if ( b > 10000000 ) then
  43.               begin
  44.                                   b := 100;
  45.                   //WriteLn( FloatToStr( b ) );
  46.                           end;
  47.  
  48.                           if ( c > 10000000 ) then
  49.               begin
  50.                                   c := 100;
  51.                   //WriteLn( FloatToStr( c ) );
  52.                           end;
  53.  
  54.               if ( d > 10000000 ) then
  55.               begin
  56.                                   d := 100;
  57.                   //WriteLn( FloatToStr( d ) );
  58.                           end;
  59.  
  60.               if ( e > 10000000 ) then
  61.               begin
  62.                                   e := 100;
  63.                   //WriteLn( FloatToStr( e ) );
  64.                           end;
  65.  
  66.               if ( f > 10000000 ) then
  67.               begin
  68.                                   f := 100;
  69.                   //WriteLn( FloatToStr( f ) );
  70.                           end;
  71.           end;
  72.         finally
  73.           N.Free;
  74.         end;
  75.     end;
  76.   end;
  77.  
  78.   WriteLn( IntToStr( GetTickCount64() - ATime ) );
  79.   ReadLn( ACmd );
  80.  
  81.   // stop program loop
  82.   Terminate;
  83. end;
  84.  

lainz

  • Hero Member
  • *****
  • Posts: 4463
    • https://lainz.github.io/
Re: fcl-json Slow
« Reply #4 on: November 21, 2020, 06:00:26 pm »
You have used some optimization level in the project when compiling for release?

What if you do "Run without debugging" so the program runs faster?

gii

  • Jr. Member
  • **
  • Posts: 53
Re: fcl-json Slow
« Reply #5 on: November 21, 2020, 06:18:46 pm »
The tests were performed in the Release compilation mode (generated by the IDE).

I'm testing without debugging.

lainz

  • Hero Member
  • *****
  • Posts: 4463
    • https://lainz.github.io/
Re: fcl-json Slow
« Reply #6 on: November 21, 2020, 07:01:33 pm »
Ok. I think you should follow Okoba suggestion and measure what is the slow spot, if loading from file or the JSON parsing.

gii

  • Jr. Member
  • **
  • Posts: 53
Re: fcl-json Slow
« Reply #7 on: November 21, 2020, 07:30:28 pm »
Following suggestions I modified the code (PHP and Pascal) to ignore the file loading.

Result:

PHP: 28842 ms x Pascal: 43813 ms

Code: Pascal  [Select][+][-]
  1. procedure TMyApplication.DoRun;
  2. var
  3.   ACmd : String;
  4.   ATime : QWord;
  5.   AItens : TJsonNode;
  6.   N : array [ 0 .. 14 ] of TJsonNode;
  7.   i, j, k : Integer;
  8.   a : String;
  9.   b, c, d, e : Double;
  10.   f : Integer;
  11. begin
  12.   for j := 0 to 14 do
  13.   begin
  14.     N[j] := TJsonNode.Create;
  15.     N[j].LoadFromFile( 'C:\xampp\htdocs\teste_json\' + IntToStr( j + 1 ) + '.txt' );
  16.   end;
  17.  
  18.   ATime := GetTickCount64();
  19.  
  20.   for i := 0 to 999 do
  21.   begin
  22.     for j := 0 to 14 do
  23.     begin
  24.         AItens := N[j].Child('Data').AsArray;
  25.  
  26.         for k := 0 to ( AItens.Count - 1 ) do
  27.         begin
  28.             with AItens.Child( k ) do
  29.             begin
  30.                 a := Find('Data').AsString;
  31.                 b := Find('Open').AsNumber;
  32.                 c := Find('Close').AsNumber;
  33.                 d := Find('High').AsNumber;
  34.                 e := Find('Low').AsNumber;
  35.                 f := Trunc( Find('Volume').AsNumber );
  36.             end;
  37.         end;
  38.     end;
  39.   end;
  40.  
  41.   WriteLn( IntToStr( GetTickCount64() - ATime ) );
  42.  
  43.   for j := 0 to 14 do
  44.     N[j].Free;
  45.  
  46.   ReadLn( ACmd );
  47.  
  48.   // stop program loop
  49.   Terminate;
  50. end;
  51.  

I decided to test using Delphi Community Edition, and to my surprise, it is slower.

** The test also ignored reading the file.

Delphi: 60359 ms.

Okoba

  • Hero Member
  • *****
  • Posts: 528
Re: fcl-json Slow
« Reply #8 on: November 21, 2020, 07:36:30 pm »
Now this is more accurate. Can you please check JSONTools and report the result and the code?

gii

  • Jr. Member
  • **
  • Posts: 53
Re: fcl-json Slow
« Reply #9 on: November 21, 2020, 07:42:52 pm »
Now this is more accurate. Can you please check JSONTools and report the result and the code?

Excuse me. Tests have already been done using JSONTools.

I found out what caused the slowness.

The Find () method is very slow.

Accessing the nodes through the index number, I got better results than in PHP.

Code: Pascal  [Select][+][-]
  1. procedure TMyApplication.DoRun;
  2. var
  3.   ACmd : String;
  4.   ATime : QWord;
  5.   AItens : TJsonNode;
  6.   N : array [ 0 .. 14 ] of TJsonNode;
  7.   i, j, k : Integer;
  8.   a : String;
  9.   b, c, d, e : Double;
  10.   f : Integer;
  11. begin
  12.   for j := 0 to 14 do
  13.   begin
  14.     N[j] := TJsonNode.Create;
  15.     N[j].LoadFromFile( 'C:\xampp\htdocs\teste_json\' + IntToStr( j + 1 ) + '.txt' );
  16.   end;
  17.  
  18.   ATime := GetTickCount64();
  19.  
  20.   for i := 0 to 999 do
  21.   begin
  22.     for j := 0 to 14 do
  23.     begin
  24.         AItens := N[j].Child(0).AsArray;
  25.  
  26.         for k := 0 to ( AItens.Count - 1 ) do
  27.         begin
  28.             with AItens.Child( k ) do
  29.             begin
  30.                 a := Child(0).AsString;
  31.                 b := Child(1).AsNumber;
  32.                 c := Child(2).AsNumber;
  33.                 d := Child(3).AsNumber;
  34.                 e := Child(4).AsNumber;
  35.                 f := Trunc( Child(5).AsNumber );
  36.             end;
  37.         end;
  38.     end;
  39.   end;
  40.  
  41.   WriteLn( IntToStr( GetTickCount64() - ATime ) );
  42.  
  43.   for j := 0 to 14 do
  44.     N[j].Free;
  45.  
  46.   ReadLn( ACmd );
  47.  
  48.   // stop program loop
  49.   Terminate;
  50. end;
  51.  

Pascal: 28047 ms

Thanks a lot for the help.
« Last Edit: November 21, 2020, 07:44:58 pm by gii »

Okoba

  • Hero Member
  • *****
  • Posts: 528
Re: fcl-json Slow
« Reply #10 on: November 21, 2020, 07:45:25 pm »
Welcome. If I remember correctly both Pascal list do not use dictionary so finding nodes by name will be slightly slow if you do it repeatedly.

PascalDragon

  • Hero Member
  • *****
  • Posts: 5448
  • Compiler Developer
Re: fcl-json Slow
« Reply #11 on: November 21, 2020, 09:13:19 pm »
Welcome. If I remember correctly both Pascal list do not use dictionary so finding nodes by name will be slightly slow if you do it repeatedly.

It totally depends on how it's implemented inside the JSON classes and e.g. fpJson uses a hash table for that.

jamie

  • Hero Member
  • *****
  • Posts: 6091
Re: fcl-json Slow
« Reply #12 on: November 21, 2020, 09:30:48 pm »
I didn't see it mentioned or maybe I missed it but did you try that run without debugging?

Debugging is very slow due to the overhead.
The only true wisdom is knowing you know nothing

bytebites

  • Hero Member
  • *****
  • Posts: 633
Re: fcl-json Slow
« Reply #13 on: November 21, 2020, 09:51:29 pm »
The tests were performed in the Release compilation mode (generated by the IDE).

I'm testing without debugging.

jamie

  • Hero Member
  • *****
  • Posts: 6091
Re: fcl-json Slow
« Reply #14 on: November 21, 2020, 10:54:03 pm »
must be the memory frag taking place maybe..

 In the past what I've done with search strings in a large list is to place the most recent viewed at the top of the list in a cache area of recent searched items..

 That usually speeds things up if you are in a loop for example, looking at the same one..

 some search engines just remember where they were the last time they searched and start from there the next time.


 But the better approach to this would be a quick pass down through the code and convert it to a token stream where as all the identifiers get converted to a index number looking into the list of identifiers

The only true wisdom is knowing you know nothing

 

TinyPortal © 2005-2018