Rest API chunk file download

[daniel_sap]

Hi,
I'm writing a rest api endpoint which should return a file chunk by chunk.
I wrote some code which tries to do it

Code: Pascal  [Select][+][-]

1. program ChunkDownload;
2.  
3. {$mode objfpc}{$H+}
4.  
5. uses
6.   {$IFDEF UNIX}
7.   cthreads,
8.   {$ENDIF}
9.   SysUtils, fphttpapp, httproute, HTTPDefs, Classes;
10.  
11.  
12. procedure downloadFileWithChunks(aRequest: TRequest; aResponse: TResponse);
13. const
14.   CHUNK_SIZE = 8192; // 8 KB chunks
15. var
16.   FilePath: string;
17.   FileStream: TFileStream;
18.   Buffer: array[0..CHUNK_SIZE-1] of Byte;
19.   BytesRead: Integer;
20.   CustomFileName: string;
21.   TempStream: TMemoryStream;
22. begin
23.   FilePath := 'D:\SomeBigFile.exe';
24.  
25.   if FilePath = '' then
26.   begin
27.     aResponse.Content := '{"result": "Missing file path"}';
28.     aResponse.Code := 400;
29.     aResponse.ContentLength := Length(aResponse.Content);
30.     aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
31.     aResponse.SendContent;
32.     WriteLn('Sent 400 response');
33.     Exit;
34.   end;
35.  
36.   if not FileExists(FilePath) then
37.   begin
38.     aResponse.Content := '{"result": "File not found: ' + FilePath + '"}';
39.     aResponse.Code := 404;
40.     aResponse.ContentLength := Length(aResponse.Content);
41.     aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
42.     aResponse.SendContent;
43.     WriteLn('Sent 404 response for path: ', FilePath);
44.     Exit;
45.   end;
46.  
47.   try
48.     FileStream := TFileStream.Create(FilePath, fmOpenRead or fmShareDenyWrite);
49.     try
50.       CustomFileName := ExtractFileName(FilePath);
51.  
52.       // Set headers
53.       aResponse.ContentType := 'application/x-msdownload';
54.       aResponse.ContentLength := FileStream.Size;
55.       aResponse.SetCustomHeader('Content-Disposition', 'attachment; filename="' + CustomFileName + '"');
56.       aResponse.SetCustomHeader('Accept-Ranges', 'bytes');
57.       aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
58.  
59.       WriteLn('Headers set:');
60.       WriteLn('Content-Type: ', aResponse.ContentType);
61.       WriteLn('Content-Length: ', IntToStr(aResponse.ContentLength));
62.       WriteLn('Content-Disposition: attachment; filename="', CustomFileName, '"');
63.       WriteLn('Access-Control-Allow-Origin: *');
64.  
65.       // Send headers
66.       aResponse.SendResponse;
67.       WriteLn('Headers sent via SendResponse');
68.  
69.       // Prepare and send content
70.  
71.       TempStream := TMemoryStream.Create;
72.       aResponse.ContentStream := TempStream;
73.       try
74.         while FileStream.Position < FileStream.Size do
75.         begin
76. //          BytesRead := FileStream.Read(Buffer, CHUNK_SIZE);
77.           TempStream.Position := 0;
78.           BytesRead := TempStream.CopyFrom(FileStream, CHUNK_SIZE);
79.  
80.           if BytesRead <= 0 then
81.             break;
82.  
83.           TempStream.Position := 0;
84.           aResponse.SendContent;
85.         end;
86.       finally
87.         TempStream.Free; // Framework takes ownership after assignment
88.       end;
89.     finally
90.       FileStream.Free;
91.     end;
92.   except
93.     on E: Exception do
94.     begin
95.       aResponse.Content := '{"result": "Error: ' + E.ClassName + ' - ' + E.Message + '"}';
96.       aResponse.Code := 500;
97.       aResponse.ContentLength := Length(aResponse.Content);
98.       aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
99.       aResponse.SendContent;
100.       WriteLn('Error: ', E.ClassName, ' - ', E.Message);
101.     end;
102.   end;
103.  
104. end;
105.  
106. begin
107.   Application.Port := 9090;
108.  
109.   HTTPRouter.RegisterRoute('/file', TRouteMethod.rmGet, @downloadFileWithChunks);
110.  
111.   Application.Threaded := true;
112.   Application.Initialize();
113.  
114.   WriteLn('Server starter at port: ' + IntToStr(Application.Port));
115.   Application.Run;
116. end.
117.  

It's not working cause after the SendResponse content is allready sent and SendContent causes an error.
May be you can advice how to properly send chunk HTTP Response

Here is also the code of the HTML file which triggers the download

Code: Text  [Select][+][-]

1. <!DOCTYPE html>
2. <html lang="en">
3. <head>
4.   <meta charset="UTF-8" />
5.   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6.   <title>Download File in Chunks</title>
7. </head>
8. <body>
9.   <h1>Download File from REST API</h1>
10.   <label for="filePath">Enter File Path:</label>
11.   <input type="text" id="filePath" placeholder="/path/to/file.txt" />
12.   <button onclick="downloadFile()">Download File</button>
13.  
14.   <script>
15.     async function downloadFile() {
16.       const filePath = document.getElementById('filePath').value;
17.       if (!filePath) {
18.         alert('Please enter a valid file path');
19.         return;
20.       }
21.  
22.       const downloadUrl = `http://localhost:9090/file?path=${encodeURIComponent(filePath)}`;
23.  
24.       try {
25.         const response = await fetch(downloadUrl);
26.         if (!response.ok) {
27.           throw new Error(`Server returned ${response.status}: ${await response.text()}`);
28.         }
29.  
30.         // Log all response headers
31.         console.log('Response headers:');
32.         for (const [key, value] of response.headers) {
33.           console.log(`${key}: ${value}`);
34.         }
35.  
36.         const totalSize = parseInt(response.headers.get('Content-Length'), 10);
37.         let receivedSize = 0;
38.  
39.         // Extract filename from Content-Disposition
40.         let fileName = 'downloaded_file';
41.         const disposition = response.headers.get('Content-Disposition');
42.         if (disposition && disposition.includes('filename=')) {
43.           const matches = disposition.match(/filename="([^"]+)"/);
44.           if (matches && matches[1]) {
45.             fileName = matches[1];
46.           }
47.         }
48.         console.log('Using filename:', fileName);
49.  
50.         const reader = response.body.getReader();
51.         const chunks = [];
52.        
53.         while (true) {
54.           const { done, value } = await reader.read();
55.           if (done) {
56.                     console.log('done');
57.                         break;
58.                   }
59.  
60.           chunks.push(value);
61.           receivedSize += value.length;
62.           console.log(`Received ${receivedSize} of ${totalSize} bytes`);
63.         }
64.  
65.         const blob = new Blob(chunks);
66.         const url = window.URL.createObjectURL(blob);
67.         const link = document.createElement('a');
68.         link.href = url;
69.         link.download = fileName;
70.         document.body.appendChild(link);
71.         link.click();
72.         document.body.removeChild(link);
73.         window.URL.revokeObjectURL(url);
74.  
75.         console.log('Download completed successfully');
76.       } catch (error) {
77.         console.error('Download failed:', error);
78.         alert('Download failed: ' + error.message);
79.       }
80.     }
81.   </script>
82. </body>
83. </html>
84.  
85.  

[Leledumbo]

Chunked transfer encoding is not yet implemented, every TResponse instance is a one time use by calling SendContent.

There is a way, but it breaks TResponse abstraction. Since you're using fphttpapp, the backend server is fphttpserver. You can downcast TResponse to TFPHTTPConnectionResponse then instead of calling AResponse.SendContent, call AResponse.Connection.Socket.WriteBuffer() directly in a loop, where each iteration is a chunk. Don't forget to send the correct headers beforehand, as you must also follow the chunked transfer encoding protocol. At the most basic, you forgot to set:

Code: [Select]

Transfer-Encoding: chunkedYour code also doesn't follow the chunk format, which should be:

Code: [Select]

<chunk size>CRLF<chunk data>CRLFfinally terminated by:

Code: [Select]

0CRLFCRLF

[daniel_sap]

Thank you, Leledumbo, for providing everything needed for understanding and implementing

I made the changes and here is the working code

Code: Pascal  [Select][+][-]

1. program ChunkDownload;
2.  
3. {$mode objfpc}{$H+}
4.  
5. uses
6.   {$IFDEF UNIX}
7.   cthreads,
8.   {$ENDIF}
9.   SysUtils, fphttpapp, httproute, HTTPDefs, Classes, fphttpserver, fpWeb;
10.  
11.  
12. type
13.  
14.   TFPHTTPConnectionResponseHack = class(TFPHTTPConnectionResponse);
15.  
16. procedure downloadFileWithChunks(aRequest: TRequest; aResponse: TResponse);
17. const
18.   CHUNK_SIZE = 8192; // 8 KB chunks
19. var
20.   FilePath: string;
21.   FileStream: TFileStream;
22.   Buffer: array of Byte;
23.   BytesRead: Integer;
24.   CustomFileName: string;
25.   connection: TFPHTTPConnection;
26.   ChunkHeader: string;
27. begin
28.   if not (aResponse is TFPHTTPConnectionResponse) then
29.     raise Exception.Create('Invalid response type, not TFPHTTPConnectionResponse');
30.  
31.   if TFPHTTPConnectionResponseHack(aResponse).Connection = nil then
32.     raise Exception.Create('Connection is nil. Ensure the request is handled via fphttpserver.');
33.  
34.   FilePath := 'D:\Project\Lazarus\Applications\Silhouette\SilSetup\windows\x86_64-win64\Output\Silhouette_6_x86_64-win64.exe';
35.  
36.   if FilePath = '' then
37.   begin
38.     aResponse.Content := '{"result": "Missing file path"}';
39.     aResponse.Code := 400;
40.     aResponse.ContentLength := Length(aResponse.Content);
41.     aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
42.     aResponse.SendContent;
43.     WriteLn('Sent 400 response');
44.     Exit;
45.   end;
46.  
47.   if not FileExists(FilePath) then
48.   begin
49.     aResponse.Content := '{"result": "File not found: ' + FilePath + '"}';
50.     aResponse.Code := 404;
51.     aResponse.ContentLength := Length(aResponse.Content);
52.     aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
53.     aResponse.SendContent;
54.     WriteLn('Sent 404 response for path: ', FilePath);
55.     Exit;
56.   end;
57.  
58.   try
59.     FileStream := TFileStream.Create(FilePath, fmOpenRead or fmShareDenyWrite);
60.     try
61.       connection := TFPHTTPConnectionResponseHack(aResponse).Connection;
62.       CustomFileName := ExtractFileName(FilePath);
63.  
64.       // Set headers
65.       aResponse.ContentType := 'application/x-msdownload';
66.       aResponse.ContentLength := FileStream.Size;
67.       aResponse.SetCustomHeader('Content-Disposition', 'attachment; filename="' + CustomFileName + '"');
68.       aResponse.SetCustomHeader('Accept-Ranges', 'bytes');
69.       aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
70.       AResponse.SetCustomHeader('Access-Control-Expose-Headers', 'Content-Disposition, Content-Length, Transfer-Encoding');
71.       aResponse.SetCustomHeader('Transfer-Encoding', 'chunked');
72.       TFPHTTPConnectionResponse(aResponse).SendHeaders;
73.  
74.       WriteLn('Headers set:');
75.       WriteLn('Content-Type: ', aResponse.ContentType);
76.       WriteLn('Content-Length: ', IntToStr(aResponse.ContentLength));
77.       WriteLn('Content-Disposition: attachment; filename="', CustomFileName, '"');
78.       WriteLn('Access-Control-Allow-Origin: *');
79.  
80.      // Allocate buffer
81.      SetLength(Buffer, CHUNK_SIZE);
82.  
83.      while FileStream.Position < FileStream.Size do
84.       begin
85.         BytesRead := FileStream.Read(Buffer[0], CHUNK_SIZE);
86.         if BytesRead <= 0 then
87.           break;
88.  
89.         // Write chunk size in hexadecimal followed by CRLF
90.         ChunkHeader := IntToHex(BytesRead, 1) + #13#10;
91.         Connection.Socket.WriteBuffer(ChunkHeader[1], Length(ChunkHeader));
92.  
93.         // Write the actual data followed by CRLF
94.         Connection.Socket.WriteBuffer(Buffer[0], BytesRead);
95.         Connection.Socket.WriteBuffer(#13#10, 2);
96.       end;
97.  
98.       // Send final chunk (0 length) to indicate end of transfer
99.       ChunkHeader := '0'#13#10#13#10;
100.       Connection.Socket.WriteBuffer(ChunkHeader[1], Length(ChunkHeader));
101.     finally
102.       FileStream.Free;
103.     end;
104.  
105.   except
106.     on E: Exception do
107.     begin
108.       aResponse.Content := '{"result": "Error: ' + E.ClassName + ' - ' + E.Message + '"}';
109.       aResponse.Code := 500;
110.       aResponse.ContentLength := Length(aResponse.Content);
111.       aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');
112.       aResponse.SendContent;
113.       WriteLn('Error: ', E.ClassName, ' - ', E.Message);
114.     end;
115.   end;
116.  
117. end;
118.  
119. begin
120.   Application.Port := 9090;
121.  
122.   HTTPRouter.RegisterRoute('/file', TRouteMethod.rmGet, @downloadFileWithChunks);
123.  
124.   Application.Threaded := true;
125.   Application.Initialize();
126.  
127.   WriteLn('Server starter at port: ' + IntToStr(Application.Port));
128.   Application.Run;
129. end.
130.  

I had to introduce TFPHTTPConnectionResponseHack class
cause Connection property is not accessible in TFPHTTPConnectionResponse
and it's parent introduces property Connection: String

[Leledumbo]

I had to introduce TFPHTTPConnectionResponseHack class
cause Connection property is not accessible in TFPHTTPConnectionResponse
and it's parent introduces property Connection: String

Ah yes, I forgot its visibility is protected. Despite working, I guess the protected visibility is broken here since you didn't reintroduce the property with higher visibility. Compiler maintainers should be able to ensure, hope they see this.