Recent

Author Topic: 7,970,312 As it turns out.  (Read 3468 times)

Handoko

  • Hero Member
  • *****
  • Posts: 3085
  • My goal: build my own game engine using Lazarus
Re: 7,970,312 As it turns out.
« Reply #30 on: January 06, 2019, 05:34:08 am »
I don't entirely understand.

Loading all data into memory is not wise.

Do you play computer games?
You may notice games usually pause for loading the game data each time you finish a round/stage/campaign or entering a new town/building/area.

You have a high-end computer. It maybe okay for it to loads all the 7 millions items and still runs smoothly. But you need to think about the other not-so-high-end computers.

Below is a demo I wrote to show how I split the food items into several categories and present it to users.

Code: Pascal  [Select]
  1. unit Unit1;
  2.  
  3. {$mode objfpc}{$H+}
  4.  
  5. interface
  6.  
  7. uses
  8.   Classes, SysUtils, Forms, Controls, Dialogs, StdCtrls;
  9.  
  10. type
  11.  
  12.   { TForm1 }
  13.  
  14.   TForm1 = class(TForm)
  15.     btnStart: TButton;
  16.     btnOK: TButton;
  17.     Label1: TLabel;
  18.     Label2: TLabel;
  19.     lblCategory1: TLabel;
  20.     lblCategory2: TLabel;
  21.     lblItem1: TLabel;
  22.     lblItem2: TLabel;
  23.     lbxCategory1: TListBox;
  24.     lbxCategory2: TListBox;
  25.     lbxItem1: TListBox;
  26.     lbxItem2: TListBox;
  27.     procedure btnOKClick(Sender: TObject);
  28.     procedure btnStartClick(Sender: TObject);
  29.     procedure FormCreate(Sender: TObject);
  30.     procedure FormDestroy(Sender: TObject);
  31.     procedure lbxCategory1SelectionChange(Sender: TObject; User: boolean);
  32.     procedure lbxCategory2SelectionChange(Sender: TObject; User: boolean);
  33.     procedure lbxItem1SelectionChange(Sender: TObject; User: boolean);
  34.   private
  35.     procedure LoadCategory(AListBox: TListBox);
  36.   end;
  37.  
  38. var
  39.   Form1: TForm1;
  40.   ListOfFileNames: TStringList;
  41.  
  42. implementation
  43.  
  44. {$R *.lfm}
  45.  
  46. { TForm1 }
  47.  
  48. procedure TForm1.btnOKClick(Sender: TObject);
  49. begin
  50.  
  51.   if lbxItem1.ItemIndex < 0 then
  52.   begin
  53.     ShowMessage('Please tell me what you want to eat tonight.');
  54.     Exit;
  55.   end;
  56.   if lbxItem2.ItemIndex < 0 then
  57.   begin
  58.     ShowMessage('You haven''t told me what you want to eat tomorrow morning.');
  59.     Exit;
  60.   end;
  61.  
  62.   ShowMessage('I''m now prepraring the food you''ve ordered.' + LineEnding +
  63.     'See you later!');
  64.   Close;
  65.  
  66. end;
  67.  
  68. procedure TForm1.btnStartClick(Sender: TObject);
  69. begin
  70.   btnStart.Visible     := False;
  71.   lblCategory1.Visible := True;
  72.   lbxCategory1.Visible := True;
  73.   lblItem1.Visible     := True;
  74.   lbxItem1.Visible     := True;
  75.   LoadCategory(lbxCategory1);
  76. end;
  77.  
  78. procedure TForm1.FormCreate(Sender: TObject);
  79. begin
  80.   ListOfFileNames := TStringList.Create;
  81. end;
  82.  
  83. procedure TForm1.FormDestroy(Sender: TObject);
  84. begin
  85.   ListOfFileNames.Free;
  86. end;
  87.  
  88. procedure TForm1.lbxCategory1SelectionChange(Sender: TObject; User: boolean);
  89. var
  90.   FileName  : string;
  91.   InputFile : TextFile;
  92.   S         : string;
  93. begin
  94.  
  95.   if lbxCategory1.ItemIndex < 0 then
  96.     Exit;
  97.  
  98.   FileName := ListOfFileNames[lbxCategory1.ItemIndex];
  99.   if not(FileExists(FileName)) then
  100.   begin
  101.     ShowMessage('Missing ' + '!');
  102.     Exit;
  103.   end;
  104.  
  105.   // Preparing
  106.   lbxItem1.Clear;
  107.   AssignFile(InputFile, FileName);
  108.   Reset(InputFile);
  109.  
  110.   // Loading
  111.   while not EOF(InputFile) do
  112.   begin
  113.     ReadLn(InputFile, S);
  114.     lbxItem1.AddItem(S, nil);
  115.   end;
  116.  
  117.   // Finishing
  118.   CloseFile(InputFile);
  119.  
  120. end;
  121.  
  122. procedure TForm1.lbxCategory2SelectionChange(Sender: TObject; User: boolean);
  123. var
  124.   FileName  : string;
  125.   InputFile : TextFile;
  126.   S         : string;
  127. begin
  128.  
  129.   if lbxCategory2.ItemIndex < 0 then
  130.     Exit;
  131.  
  132.   FileName := ListOfFileNames[lbxCategory2.ItemIndex];
  133.   if not(FileExists(FileName)) then
  134.   begin
  135.     ShowMessage('Missing ' + '!');
  136.     Exit;
  137.   end;
  138.  
  139.   // Preparing
  140.   lbxItem2.Clear;
  141.   AssignFile(InputFile, FileName);
  142.   Reset(InputFile);
  143.  
  144.   // Loading
  145.   while not EOF(InputFile) do
  146.   begin
  147.     ReadLn(InputFile, S);
  148.     lbxItem2.AddItem(S, nil);
  149.   end;
  150.  
  151.   // Finishing
  152.   CloseFile(InputFile);
  153.  
  154. end;
  155.  
  156. procedure TForm1.lbxItem1SelectionChange(Sender: TObject; User: boolean);
  157. begin
  158.  
  159.   if (Label2.Visible) then
  160.     Exit;
  161.  
  162.   Label2.Visible       := True;
  163.   lblCategory2.Visible := True;
  164.   lbxCategory2.Visible := True;
  165.   lblItem2.Visible     := True;
  166.   lbxItem2.Visible     := True;
  167.  
  168.   LoadCategory(lbxCategory2);
  169.  
  170. end;
  171.  
  172. procedure TForm1.LoadCategory(AListBox: TListBox);
  173. const
  174.   CategoryFileName = 'category.txt';
  175.   Separator        = '|';
  176. var
  177.   InputFile : TextFile;
  178.   SplitPos  : Integer;
  179.   S         : string;
  180. begin
  181.  
  182.   if not(FileExists(CategoryFileName)) then
  183.   begin
  184.     ShowMessage('Missing category.txt!');
  185.     Exit;
  186.   end;
  187.  
  188.   // Preparing
  189.   ListOfFileNames.Clear;
  190.   AListBox.Clear;
  191.   AssignFile(InputFile, CategoryFileName);
  192.   Reset(InputFile);
  193.  
  194.   // Loading
  195.   while not EOF(InputFile) do
  196.   begin
  197.     ReadLn(InputFile, S);
  198.     SplitPos := S.IndexOf(Separator);
  199.     ListOfFileNames.Add(Trim(S.Substring(0, SplitPos)));
  200.     AListBox.AddItem(S.Remove(0, SplitPos+2), nil);
  201.   end;
  202.  
  203.   // Finishing
  204.   CloseFile(InputFile);
  205.  
  206. end;
  207.  
  208. end.

You may do the similar on your program. Split the items based on its countries for example. It will be more user friendly. If you let user to pick an item from a listbox that contains thousand of items, it is not good user interface design.

So my suggestion is try to split the items into several categories. This design approach has a benefit, you don't load all the data at once. And it can solve your issue. But, of course you have to split the data first.

JLWest

  • Hero Member
  • *****
  • Posts: 545
Re: 7,970,312 As it turns out.
« Reply #31 on: January 06, 2019, 06:48:16 am »
@Josh No Unfortunately that's a different set of data.

My data is specific to X-Plane 11 and it is all the airports [H] Heliport and Seaplane bases in X-Plane 11. I'm sure that a lot, if not all the data is in those files. However, X-Plane is only going to look at it's Apt.Dat file for airport information, runways, ramp starts taxiways, ILS or GGPS landing info. If you have a corrupted file you will fly right into the ground on an ILS landing (KSFO) runway 25 right I think in a B747-800. No one believed me until I gave them the step by step. Navigraph had to correct the data and re-download the data to their subscribers. It was a mess. Navigraph is a paid subscription. A costly paid subscription. On the order of 150 euros. 
JLWEST
Lazuras ver 2.0.2 
 FPC 3.0.4, Lazarus IDE v1.8.2 Windows 10 Pro
 Intel i7 770K CPU 4.2GHz 32702MB Ram
GeForce GTX 1080 Graphics - 8 Gig
3952 GB (1.5 SSD)

JLWest

  • Hero Member
  • *****
  • Posts: 545
Re: 7,970,312 As it turns out.
« Reply #32 on: January 06, 2019, 07:05:10 am »
@ lucmar

No hurry. I just got the reads done in the quick and dirty for a sample data set. Should have it done sometime tomorrow.
 
Anyone know how to do a dropbox and what that's all about.

When I buy a new Boeing I sometimes pick it up from a dropbox. They will take large data set. The download of the B737-800 Max took several min. in a zip format.

Nice plane. A small team took an existing 737-800 from X-Plane 11 and started modifying it to be whats called study level.

After 1,331 versions it pretty solid. I won't upgrade from here for a while. And it Free.

Payware aircraft cost 35 to 90 dollars. His is by far the best and it's free.

I gave him a $100 donation felt so bad with the best Boeing on the line for nothing.
 
JLWEST
Lazuras ver 2.0.2 
 FPC 3.0.4, Lazarus IDE v1.8.2 Windows 10 Pro
 Intel i7 770K CPU 4.2GHz 32702MB Ram
GeForce GTX 1080 Graphics - 8 Gig
3952 GB (1.5 SSD)