[SOLVED] Sending "User-agent" using HTTPClient library

[maurobio]

Dear ALL,

When testing a simple program for downloading image files from Wikimedia (reproduced below), I keep getting an error message Unexpected response status code: 403, which shows that a client is forbidden from accessing a valid URL because of client side issues.

I suspect (but of course I am not sure) that the problem is that this simple test program is blocked by Wikimedia because the program does not provide a "User-Agent header" to fake a browser visit. If this is really the case, how could I provide this information using the HTTPClient library?

Here is the code:

Code: Pascal  [Select][+][-]

1. program wikimedia;
2.  
3. {$MODE OBJFPC}{$H+}
4.  
5. uses
6.   Classes, SysUtils, fphttpclient, openssl, opensslsockets;
7.  
8. procedure DumpExceptionCallStack(E: Exception);
9. var
10.   I: Integer;
11.   Frames: PPointer;
12.   Report: string;
13. begin
14.   Report := 'Program exception! ' + LineEnding + 'Stacktrace:' + LineEnding + LineEnding;
15.   if E <> nil then
16.   begin
17.     Report := Report + 'Exception class: ' + E.ClassName + LineEnding + 'Message: ' + E.Message + LineEnding;
18.   end;
19.   Report := Report + BackTraceStrFunc(ExceptAddr);
20.   Frames := ExceptFrames;
21.   for I := 0 to ExceptFrameCount - 1
22.     do Report := Report + LineEnding + BackTraceStrFunc(Frames[I]);
23.   WriteLn(Report);
24.   Halt;
25. end;
26.  
27. function GetUrlAs(Url: String; AsName: String): Boolean;
28. begin
29.   Result := False;
30.   with TFPHttpClient.Create(nil) do
31.   try
32.         AllowRedirect := True;
33.     if (ExtractFilePath(AsName) <> '') then
34.       if not DirectoryExists(ExtractFilePath(AsName)) then
35.         if not ForceDirectories(ExtractFilePath(AsName)) then Exit;
36.     try
37.       WriteLn(trim(FormatDateTime('h:nn:ss AM/PM MM/DD/YYYY', now)) + ' GET: ' + Url);
38.       Get(Url, AsName);
39.       Result := True;
40.     finally
41.       Free;
42.     end;
43.   except
44.     on E: Exception do DumpExceptionCallStack(E);
45.   end;
46. end;
47.  
48. procedure ReportStatus(Success: Boolean); inline;
49. begin
50.   if Success then WriteLn('Succes') else WriteLn('Failure');
51. end;
52.  
53. const
54.   URL1 = 'http://commons.wikimedia.org/wiki/Special:Filepath/Illustration_Vicia_faba1.jpg';
55.   URL2 = 'http://commons.wikimedia.org/wiki/Special:Filepath/Broadbean_Yield.png';
56.   URL3 = 'http://commons.wikimedia.org/wiki/Special:Filepath/Fava_beans_1.jpg';
57. var
58.   TargetDir : String;
59.   SomeDirectory : String;
60.   SomeFolderName : String;
61.   RetVal : boolean;
62. begin
63.   InitSSLInterface;
64.   SomeDirectory := IncludeTrailingPathDelimiter(GetCurrentDir);
65.   SomeFolderName := 'pictures';
66.   TargetDir := IncludeTrailingPathDelimiter(SomeDirectory + SomeFolderName);
67.  
68.   WriteLn('Attempting to download URL''s into directory ', TargetDir);
69.  
70.   //RetVal := GetUrlAs(URL1, TargetDir + 'pic1.jpg');
71.   RetVal := GetUrlAs(URL1, TargetDir + ExtractFileName(URL1));
72.   ReportStatus(RetVal);
73.  
74.   Sleep(1000);
75.  
76.   RetVal := GetUrlAs(URL2, TargetDir + ExtractFileName(URL2));
77.   ReportStatus(RetVal);
78.  
79.   Sleep(1000);
80.  
81.   RetVal := GetUrlAs(URL3, TargetDir + ExtractFileName(URL3));
82.   ReportStatus(RetVal);
83.  
84.   WriteLn('Done!');
85. end.
86.  

Could some of you wizards give me some hints on how to get this working?

Thanks in advance!

With warmest regards,

[dseligo]

Add this after line 31:

Code: Pascal  [Select][+][-]

1. AddHeader('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0');

[maurobio]

Hi, @dseligo!

Great, it now works fine!

Thank you very much!

With warmest regards,