Increase receiving capacity size (http methods, wsarecv)

[tudi_x]

hi All,
as per the below code i am using a socket to listen for POST or GET messages containing JSON information.
the issue that happens is that every 6 - 8 times the connection breaks and the application that sends the information (written in go) receives something like:

Code: Pascal  [Select][+][-]

1. error: Post http://127.0.0.1:55552/: net/http: HTTP/1.x transport connection broken: read tcp 127.0.0.1:5441->127.0.0.1:55552: wsarecv: An existing connection was forcibly closed by the remote host.

it looks like the truncation occurs after 4096 characters.

Code: Pascal  [Select][+][-]

1. unit sock_ls;
2.  
3. {$mode objfpc}{$H+}
4.  
5. interface
6.  
7. uses
8.   Classes, SysUtils, Sockets,
9.   blcksock,
10.   fpjson, jsonparser,
11.   strutils;
12.  
13. type
14.   TPassMessage = procedure(AMsg: string) of object;
15.  
16. type
17.   TQSockListen = class(TThread)
18.   private
19.     _PassMessage: TPassMessage;
20.  
21.     _MainSocket: word;
22.     _IAddr: TINetSockAddr; //internet address, needs Sockets unit
23.     _ASocket: integer;
24.     _RequestBody: string;
25.  
26.     procedure Connect;
27.     procedure sockRead;
28.     procedure sockWrite();
29.     function Rfc822DateTime(t: TDateTime): string;
30.  
31.   protected
32.     procedure Execute; override;
33.     procedure TriggerMessage(AMsg: string);
34.  
35.   public
36.     constructor Create(APort: cardinal; AIP: string); reintroduce;
37.     destructor Destroy(); override;
38.  
39.     property OnPassMessage: TPassMessage read _PassMessage write _PassMessage;
40.   end;
41.  
42. implementation
43.  
44. const
45.   packet_size = 10485760;
46.  
47. constructor TQSockListen.Create(APort: cardinal; AIP: string);
48. begin
49.   inherited Create(False);
50.   self.FreeOnTerminate := True;
51.  
52.   _MainSocket := fpsocket(AF_INET, SOCK_STREAM, 0);
53.   _IAddr.sin_family := AF_INET;
54.   _IAddr.sin_port := htons(APort);
55.   _IAddr.sin_addr.s_addr := longword(StrToNetAddr(AIP));
56.  
57.   fpbind(_MainSocket, @_IAddr, SizeOf(_IAddr));
58.   fplisten(_MainSocket, 1);
59. end;
60.  
61. procedure TQSockListen.Execute;
62. begin
63.   while not Terminated do
64.   begin
65.     Connect;
66.     sockRead;
67.  
68.     sockWrite;
69.  
70.     {closing socket}
71.     if _ASocket > 0 then
72.     begin
73.       CloseSocket(_ASocket);
74.     end;
75.   end;
76. end;
77.  
78. procedure TQSockListen.Connect;
79. var
80.   sAddrSize: longint;
81.  
82. begin
83.   try
84.     sAddrSize := SizeOf(_IAddr);
85.     _ASocket := fpaccept(_MainSocket, @_IAddr, @sAddrSize);
86.   except
87.     on E: Exception do
88.       TriggerMessage('TQSocket.Connect: Error ' + E.Message);
89.   end;
90. end;
91.  
92. procedure TQSockListen.sockRead;
93. var
94.   buf: string;
95.   c: integer;
96.   j: TJSONData;
97.   json: string;
98.  
99. begin
100.   try
101.     setLength(buf, packet_size);
102.     c := fprecv(_ASocket, PChar(buf), packet_size, 0);
103.     setLength(buf, c);
104.  
105.     _RequestBody := buf;
106.  
107.     writeln(_RequestBody);
108.     writeln('--------- end: ' + FormatDateTime('hh:nn:ss.zzz', Now()) + '----------');
109.  
110.     TriggerMessage('TQSocket.sRead: request body length: ' + IntToStr(Length(_RequestBody)) + sLineBreak + 'start|' + sLineBreak + _RequestBody + sLineBreak + '|end');
111.  
112.     try
113.       {verify body has left bracket}
114.       if AnsiContainsStr(_RequestBody, '{') then
115.       begin
116.         json := Copy(_RequestBody, Pos('{', _RequestBody), Length(_RequestBody));
117.  
118.         j := GetJSON(json);
119.         TriggerMessage('JSON parsed : ' + json);
120.       end;
121.  
122.     except
123.       on E: Exception do
124.       begin
125.         writeln('JSON conversion failed');
126.         TriggerMessage('position of {: ' + IntToStr(Pos('{', _RequestBody)));
127.         TriggerMessage('JSON conversion failed with error: ' + E.Message);
128.       end;
129.     end;
130.     FreeAndNil(j);
131.  
132.   except
133.     on E: Exception do
134.       TriggerMessage('TQSocket.sRead: Error ' + E.Message);
135.   end;
136. end;
137.  
138. procedure TQSockListen.sockWrite();
139. var
140.   resp: string;
141.  
142. begin
143.   try
144.     resp := 'HTTP/1.0 200' + CRLF;
145.     fpsend(_ASocket, PChar(resp), Length(resp), 0);
146.  
147.     resp := 'Content-type: Text/Html' + CRLF;
148.     fpsend(_ASocket, PChar(resp), Length(resp), 0);
149.  
150.     resp := 'Content-length: 0' + CRLF;
151.     fpsend(_ASocket, PChar(resp), Length(resp), 0);
152.  
153.     resp := 'Connection: close' + CRLF;
154.     fpsend(_ASocket, PChar(resp), Length(resp), 0);
155.  
156.     resp := 'Date: ' + Rfc822DateTime(now) + CRLF;
157.     fpsend(_ASocket, PChar(resp), Length(resp), 0);
158.  
159.     resp := 'Server: Lazarus Synapse' + CRLF;
160.     fpsend(_ASocket, PChar(resp), Length(resp), 0);
161.  
162.     resp := '' + CRLF;
163.     fpsend(_ASocket, PChar(resp), Length(resp), 0);
164.   except
165.     on E: Exception do
166.       TriggerMessage('TQSocket.sWrite: Error ' + E.Message);
167.   end;
168. end;
169.  
170. procedure TQSockListen.TriggerMessage(AMsg: string);
171. begin
172.   if Assigned(_PassMessage) then
173.     _PassMessage(AMsg);
174. end;
175.  
176. function TQSockListen.Rfc822DateTime(t: TDateTime): string;
177. var
178.   wYear, wMonth, wDay: word;
179.  
180. const
181.   MyDayNames: array[1..7] of ansistring = ('Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat');
182.   MyMonthNames: array[1..12] of ansistring = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec');
183.  
184. begin
185.   DecodeDate(t, wYear, wMonth, wDay);
186.   Result := Format('%s, %d %s %s %s', [MyDayNames[DayOfWeek(t)], wDay, MyMonthNames[wMonth], FormatDateTime('yyyy hh":"nn":"ss', t), '+0200']);
187. end;
188.  
189. destructor TQSockListen.Destroy();
190. begin
191.   inherited Destroy;
192. end;
193.  
194. end.

please advise how i could increase the capacity so this truncation does not occur ever.
thank you!

[Remy Lebeau]

the issue that happens is that every 6 - 8 times the connection breaks and the application that sends the information (written in go) receives something like:

Code: Pascal  [Select][+][-]

1. error: Post http://127.0.0.1:55552/: net/http: HTTP/1.x transport connection broken: read tcp 127.0.0.1:5441->127.0.0.1:55552: wsarecv: An existing connection was forcibly closed by the remote host.

it looks like the truncation occurs after 4096 characters.

TCP is a byte stream, there is no 1-to-1 correlation between sends and reads.

You are using a 10MB memory buffer for reading, but fprecv() will never be able to return that much data in a single read.  Its return value is -1 on error, 0 on graceful disconnect, and > 0 to specify the actual number of bytes received.  Receiving an HTTP request requires many calls to fprecv(), so you need to call it in a loop, appending received data to _RequestBody, until you reach the end of the HTTP request.  Only then can you process _RequestBody correctly.

To detect the end of the HTTP request, you need to actually parse the HTTP request as you receive it so you know when to stop reading.  Refer to RFC 2616 Section 4.4 for the formal rules to determine the transmission size of an HTTP message.  In a nutshell, you have to read a CRLF-delimited line for the HTTP request line, then read CRLF-delimited lines for the HTTP headers until you reach a CRLF CRLF sequence denoting the end of the headers, then you have to process the request line and headers to determine the type of message being sent and the format of the message's body, if there is any, before you can then read the message body according to that format.

Similarly, sending an HTTP response requires many calls to fpsend().  Even just sending a single string may require more than 1 call to  fpsend(), as it can send fewer bytes than requested.  So you need to call fpsend() in a loop as well.  Its return value is -1 on error and > 0 to specify the actual umber of bytes sent (or more accurately, the number of bytes accepted into the kernel buffer for eventual transmission).  Continue calling it until there are more more bytes to send.

You are best off not handling this manually at all.  There are plenty of HTTP server implementations available if you look around.

[tudi_x]

thank you for the clarifications. i do not intend to reinvent the wheel.
how much data can fprecv() return? because i sent 10000 characters in a POST request with the current code and did not have any issues fully reading it even without a for loop.

if content length is below 200 i do not have an issue and receive it full:

Code: Pascal  [Select][+][-]

1. POST / HTTP/1.1
2. Host: 127.0.0.1:55552
3. User-Agent: Go-http-client/1.1
4. Content-Length: 51
5. Content-Type: application/json
6. Accept-Encoding: gzip
7.  
8. {"request":"http://127.0.0.1:55556/v1/sube/status"}

it looks like fprecv is not the one causing the issue but maybe something underneath.


in regards to looking around, i looked around and with my beginner skills found:
- nYume server uses about the same code as the below on receive
- Synapse web server has in wiki a light server example with request body read incomplete and opened http://forum.lazarus.freepascal.org/index.php/topic,39088.0.html .
- standard library as per TFPHTTPServer has an issue when shutting down. is reported in
https://bugs.freepascal.org/view.php?id=32727
- Indy is quite big but will consider as last option

are you referring to other HTTP implementations then the above? if yes, could you please point me with a URL?
if not i see only option is to adapt the part that works from standard library for reading a body which is a pity because i think in 2017 HTTP issues related to the standard library should not arise

[Remy Lebeau]

i do not intend to reinvent the wheel.

Then don't.  Like I said, there are plenty of existing HTTP server implementations available.

how much data can fprecv() return?

That entirely depends on:

- the size of the kernel receive buffer that backs the socket

- the size of the memory buffer that you receive into.

You can customize the size of the kernel buffer using setsockopt() with the SO_RCVBUF option, and use getsockopt() to retrieve the actual buffer size (setsockopt() might not assign the exact size you specify).

But in any case, it doesn't really matter what the buffer size is.  You still need to be prepared to call fprecv() in a loop, and you MUST parse the HTTP message data to detect end-of-message correctly.

because i sent 10000 characters in a POST request with the current code and did not have any issues fully reading it even without a for loop.

You still need to be prepared to call fprecv() multiple times, if for no other reason than you have to stop reading when you reach the end of the HTTP headers, because they tell you how to read the rest of the data, so you would have to adjust your subsequent reading logic accordingly until you reach the end of the message.

if content length is below 200 i do not have an issue and receive it full:

You can't rely on that behavior.  Again, TCP is a streaming transport, and fprecv() can return fewer bytes than requested.  The contract between you and fprecv() is that it can return as few as 1 byte, and at most the number of bytes you request.  Which means it can return anything in between.  You MUST account for that possibility.  fprecv() returns whatever is *currently* in the kernel buffer, or if the buffer is empty then it waits for at least 1 byte to arrive (if using a blocking socket) and then returns whatever it can.

it looks like fprecv is not the one causing the issue but maybe something underneath.

Yeah - YOU.  You are not reading and processing the HTTP data correctly.  HTTP is a structured protocol, but you are not following the structure at all.

are you referring to other HTTP implementations then the above?

I'm biased towards Indy (but only because I develop it!).  It has a working TIdHTTPServer component.

I'm sure other HTTP server implementations also work.

[tudi_x]

thank you!
based on your feedback i managed the below which serves my purpose of receiving JSON for now.

Code: Pascal  [Select][+][-]

1. {https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html}
2.       {message type}
3.       message.Append(ASocket.RecvString(Timeout));
4.       {$IF DEFINED(Debug)}
5.       WriteLn(message.Strings[message.Count - 1]);
6.       {$EndIF}
7.  
8.       {headers}
9.       repeat
10.         message.Append(ASocket.RecvString(Timeout));
11.         WriteLn(message.Strings[message.Count - 1]);
12.  
13.         if AnsiContainsText(message.Strings[message.Count - 1], 'Content-Length') then
14.         begin
15.           content := StrToInt(copy(message.Strings[message.Count - 1], pos(':', message.Strings[message.Count - 1]) + 1, Length(message.Strings[message.Count - 1])));
16.         end;
17.       until message.Strings[message.Count - 1] = '';
18.  
19.       {body}
20.       if content > 0 then
21.       begin
22.         without_body := Length(message.Text);
23.         writeln('content: ' + IntToStr(content) + '|' + 'headers: ' + IntToStr(without_body));
24.         TriggerMessage('content: ' + IntToStr(content) + '|' + 'headers: ' + IntToStr(without_body));
25.  
26.         repeat
27.           message.Append(ASocket.RecvString(Timeout));
28.  
29.           {$IF DEFINED(Debug)}
30.           writeln('all: ' + IntToStr(length(message.Text)));
31.           WriteLn(message.Strings[message.Count - 1]);
32.           writeln(IntToStr(length(message.Text) - without_body - content));
33.           {$EndIF}
34.  
35.         until length(message.Text) - without_body >= content;
36.       end;
37.  
38.       TriggerMessage(message.Text);
39.       message.Clear;    

[Remy Lebeau]

based on your feedback i managed the below which serves my purpose of receiving JSON for now.

That code is very wrong, and will likely break eventually.

When reading the request line and headers, you MUST read CRLF-delimited strings, which a function like RecvString() is likely NOT doing.

You are not accounting for the possibility that the 'Content-Length' header may not always be present (for instance 'Transfer-Encoding: chunked' may be used instead), or the fact that not all message types even have a message body to begin with, even if there is a 'Content-Length' present.

And even if there is a message body to read, you are not taking the 'Content-Length' value into account correctly when reading the body.  You can't just blindly read arbitrary strings until you reach the desired size, you might read too much data (such as if the client is using HTTP pipelining, for instance).  You have to read EXACTLY how much 'Content-Length' says to read, no more, no less.  And in the case that something other than 'Content-Length' is used to denote the message size, you are not reading the message body at all.

In short, you are not following the rules of RFC 2616 Section 4.4 (and its successor, RFC 7230 Section 3.3.3).  Until you do, your code is risking corrupting the HTTP communications.