Recent

Author Topic: Static site generator in 200 LOC  (Read 1188 times)

tennis

  • New Member
  • *
  • Posts: 11
Static site generator in 200 LOC
« on: November 30, 2020, 08:51:05 pm »
I was looking into static site generators in order to replace a Wordpress site with outdated
plugins. I found the typical recommended ones like Hugo to be overly complex. The time
i used to actually read through the documentation rougly correspond to the time it took
to just "roll your own" in 200 lines:

Code: Pascal  [Select][+][-]
  1. program sitegen;
  2. {$mode objfpc}{$H+}
  3.  
  4. uses
  5.   Classes, SysUtils, RegExpr, fpTemplate, httpdefs,
  6.   MarkdownProcessor in 'external/MarkdownProcessor.pas',
  7.   MarkdownDaringFireball in 'external/MarkdownDaringFireball.pas',
  8.   MarkdownCommonMark in 'external/MarkdownCommonMark.pas',
  9.   MarkdownUnicodeUtils in 'external/MarkdownUnicodeUtils.pas',
  10.   MarkdownHTMLEntities in 'external/MarkdownHTMLEntities.pas';
  11.  
  12. const
  13.   SiteMapHeader = '<?xml version="1.0" encoding="UTF-8"?>' + LineEnding +
  14. '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'  + LineEnding;
  15.  
  16. var
  17.   DefsRe : TRegExpr;
  18.  
  19. function ExtractHeader(ARawContent: String; Defs : TStringList) : String;
  20. var
  21.   SList: TStringList;
  22. begin
  23.   SList := TStringList.Create;
  24.   try
  25.     SList.Text := ARawContent;
  26.     while SList.Count > 0 do
  27.     begin
  28.       if not DefsRe.Exec(SList.Strings[0]) then
  29.         Break;
  30.       if Defs.IndexOfName(DefsRe.Match[1]) <> -1 then
  31.           WriteLn('Warning : skipping duplicate header ' + DefsRe.Match[1]);
  32.       Defs.AddPair(DefsRe.Match[1], DefsRe.Match[2]);
  33.       SList.Delete(0);
  34.     end;
  35.     Result := SList.Text;
  36.   finally
  37.     FreeAndNil(SList);
  38.   end;
  39. end;
  40.  
  41. function ListFiles(APath: String): TStringArray;
  42. var
  43.   Info : TSearchRec;
  44. begin
  45.   Result := [];
  46.   if FindFirst(APath, faAnyFile - faDirectory, Info) = 0 then
  47.   repeat
  48.     Result := Concat(Result, [String(Info.Name)]);
  49.   until FindNext(info) <> 0;
  50.   FindClose(Info);
  51. end;
  52.  
  53. function LoadFromFile(AFileName: String) : String;
  54. var
  55.   FStream: TFileStream;
  56.   SData: RawByteString = '';
  57. begin
  58.   FStream := TFileStream.Create(AFileName, fmOpenRead, fmShareDenyWrite);
  59.   SetLength(SData, FStream.Size);
  60.   if FStream.Size > 0 then
  61.     FStream.Read(SData[1], FStream.Size);
  62.   Result := UTF8String(SData);
  63.   FreeAndNil(FStream);
  64. end;
  65.  
  66. procedure Main;
  67. var
  68.   T : TTemplateParser;
  69.   Processor : TMarkdownProcessor;
  70.   FStream: TFileStream;
  71.   Global, Content, Defs : TStringList;
  72.   S, Ext, Name, FileName, Tpl : String;
  73.   I, J : Integer;
  74. begin
  75.   try
  76.     DefsRe := TRegExpr.Create('^:(\S+): \s*(.*)\s*$');
  77.     Processor := TMarkdownProcessor.createDialect(mdDaringFireball);
  78.     Processor.Unsafe := True;
  79.     Global := Nil;
  80.     Content := TStringList.Create;
  81.     Content.Sorted := True;
  82.     Content.Duplicates := dupIgnore;
  83.  
  84.     for FileName in ListFiles('content' + DirectorySeparator + '*') do
  85.     begin
  86.         Ext := LowerCase(ExtractFileExt(FileName));
  87.         case Ext of
  88.           '.md', '.html' :
  89.           begin
  90.             S := LoadFromFile('content' + DirectorySeparator + FileName);
  91.             if Length(S) = 0 then
  92.             begin
  93.               WriteLn('Warning : file empty ' + FileName);
  94.               Continue;
  95.             end;
  96.             Name := ChangeFileExt(FileName, '.html');
  97.             if Content.IndexOfName(Name) <> -1 then
  98.             begin
  99.               WriteLn('Warning : skipping duplicate ' + Name);
  100.               Continue;
  101.             end;
  102.             Defs := TStringList.Create;
  103.             Defs.Sorted := True;
  104.             Defs.Duplicates := dupIgnore;
  105.             S := ExtractHeader(S, Defs);
  106.             if LowerCase(Name) = 'global.html' then
  107.             begin
  108.               if Assigned(Global) then
  109.                 WriteLn('Warning : global redefined');
  110.               Global := Defs;
  111.               Continue;
  112.             end;
  113.             if Ext = '.md' then
  114.               S := Processor.Process(S);
  115.             Content.AddPair(Name, S , Defs);
  116.           end;
  117.         else
  118.           WriteLn('Warning : skipping unknown file ' + FileName);
  119.         end;
  120.     end;
  121.  
  122.     if Content.Count = 0 then
  123.     begin
  124.       WriteLn('Error : content not found');
  125.       Halt(1);
  126.     end;
  127.  
  128.     for FileName in ListFiles('public' + DirectorySeparator + '*') do
  129.     begin
  130.       Ext := LowerCase(ExtractFileExt(FileName));
  131.       if (Ext = '.html') or (Ext = '.xml') or (Ext = '.txt') then
  132.         if not DeleteFile('public' + DirectorySeparator + FileName) then
  133.           WriteLn('Warning : could not delte file ' + FileName);
  134.     end;
  135.  
  136.     if not DirectoryExists('public') then
  137.       CreateDir('public');
  138.  
  139.     FStream := TFileStream.Create('public' + DirectorySeparator + 'sitemap.xml', fmCreate);
  140.     S := SiteMapHeader;
  141.     FStream.Write(S[1], Length(S));
  142.  
  143.     T := TTemplateParser.Create;
  144.     T.StartDelimiter := '{%';
  145.     T.EndDelimiter := '%}';
  146.     T.AllowTagParams := False;
  147.     T.Recursive := True;
  148.     for I := 0 to Content.Count - 1 do
  149.     begin
  150.       T.Clear;
  151.       T.Values['content'] := Content.ValueFromIndex[I];
  152.       Defs := TStringList(Content.Objects[I]);
  153.       if Assigned(Global) then
  154.       begin
  155.         for J := 0 to Global.Count - 1 do
  156.           T.Values[Global.Names[J]] := Global.ValueFromIndex[J];
  157.       end;
  158.       for J := 0 to Defs.Count - 1 do
  159.         T.Values[Defs.Names[J]] := Defs.ValueFromIndex[J];
  160.  
  161.       Tpl := Defs.Values['template'];
  162.       if Length(Tpl) = 0 then
  163.         Tpl := 'default.html';
  164.  
  165.       Name := Content.Names[I];
  166.       T.ParseFiles('template' + DirectorySeparator + Tpl, 'public' + DirectorySeparator + Name);
  167.       WriteLn('Processed ' + Name);
  168.       FreeAndNil(Defs);
  169.  
  170.       S := HTTPEncode(T.ParseString('{%root%}' + Name));
  171.       S := '<url><loc>' + S + '</loc></url>' + LineEnding;
  172.       FStream.Write(S[1], Length(S));
  173.     end;
  174.     S := '</urlset>'  + LineEnding;
  175.     FStream.Write(S[1], Length(S));
  176.     WriteLn('Processed sitemap.xml');
  177.  
  178.     if FileExists('template' + DirectorySeparator + 'robots.txt') then
  179.     begin
  180.       T.Clear;
  181.       if Assigned(Global) then
  182.       begin
  183.         for J := 0 to Global.Count - 1 do
  184.           T.Values[Global.Names[J]] := Global.ValueFromIndex[J];
  185.       end;
  186.       T.ParseFiles('template' + DirectorySeparator + 'robots.txt', 'public' + DirectorySeparator + 'robots.txt');
  187.       WriteLn('Processed robots.txt');
  188.     end;
  189.   finally
  190.     FreeAndNil(FStream);
  191.     FreeAndNil(Global);
  192.     FreeAndNil(Content);
  193.     FreeAndNil(DefsRe);
  194.     FreeAndNil(T);
  195.   end;
  196. end;
  197.  
  198. begin
  199.   Main;
  200. end.
  201.  

The Markdown processer is taken from https://github.com/grahamegrieve/delphi-markdown/tree/master/source.

Pages are places in the content folder with header
variables extracted in the form of :name: content.

Pages are transformed with fpTemplate and placed
in the public folder.

A file named global.html/global.md in the
content folder is used for global site config.

sitemap.xml is automatic generated.

There may be some low hanging improvments in:
  • Multiline front matter
  • Include other templates in current template
  • Preserve file hierarchy in content folder

However, for my simple needs this was sufficient
and I did not pursue down the lane of constant
adding new "features" and handling of corner
cases ending with complexity.

And yes, there are probably several bugs.

I leve this for inspiration to others, but it may not fit
your particular setup directly.

 
« Last Edit: November 30, 2020, 09:22:56 pm by tennis »

 

TinyPortal © 2005-2018