How to Convert ANSI format file into UTF8 format

[RaviMore9]

Hello All,

I would like to convert Same ANSI format file into UTF8 file without generating another file. (I want to convert same ANSI file into UTF8 format).
Is there any function available to convert ANSI file format to UTF8 format?

Eg,
Test_file.txt(ANSI format) ---> Test_file.txt (UTF8 format) 



Thanks in advance.

[KodeZwerg]

Code: Pascal  [Select][+][-]

1. //https://forum.lazarus.freepascal.org/index.php/topic,60635.msg454159.html#msg454159
2. program Project1;
3.  
4. {$APPTYPE CONSOLE}
5.  
6. Uses
7.   SysUtils, Classes;
8.  
9. function FileANSItoUTF8(const AFilename: string; const WriteBOM: Boolean = True): Boolean;
10. var
11.   Input, Output: TStringList;
12.   i: Integer;
13. begin
14.   Result := False;
15.   Input  := nil;
16.   Output := nil;
17.   if (not FileExists(AFilename)) then
18.     Exit;
19.   try
20.     Input  := TStringList.Create;
21.     try
22.       Input.LoadFromFile(AFilename, TEncoding.ANSI);
23.       Output := TStringList.Create;
24.       try
25.         Output.WriteBOM := WriteBOM;
26.         for i := 0 to Pred(Input.Count) do
27.           Output.Add(AnsiToUTF8(Input.Strings[i]));
28.         Output.SaveToFile(AFilename, TEncoding.UTF8);
29.       finally
30.         Output.Free;
31.         Output := nil;
32.       end;
33.     finally
34.       Input.Free;
35.       Input := nil;
36.     end;
37.     Result := i > 0;
38.   except
39.     if (Output <> nil) then
40.       Output.Free;
41.     if (Input <> nil) then
42.       Input.Free;
43.   end;
44. end;
45.  
46. var
47.   s: string;
48. begin
49.   WriteLn('Convert ANSI to UTF8 - by KodeZwerg 2022 for forum.lazarus.freepascal.org');
50.   if (ParamCount > 0) then
51.     begin
52.       s := ParamStr(1);
53.       WriteLn('Converting: ' + s);
54.       if FileANSItoUTF8(s) then
55.         WriteLn('Success!')
56.         else
57.         WriteLn('Failed!');
58.     end
59.     else
60.       WriteLn('Usage: ' + ParamStr(0) + ' "ANSI encoded file.txt"');
61.   WriteLn('Press RETURN to continue.');
62.   ReadLn;
63. end.

Please test this. You might play a little with second Parameter (WriteBOM) to get what you want.

[Thaddy]

Why do you use a loop? You can convert in one go using

Code: Pascal  [Select][+][-]

1. Output.Text :=AnsiToUTF8(Output.Text);
2. Output.WriteBom := true;  // will be called before save and savetofile goes through a savetostream
3. Output.SaveToFile(<your file>);

This will result in an UTF8 file with BOM. BOM is optional.
And nilling evreywhere is also strange since the stringlists are local. I would rewrite your code something like this:

Code: Text  [Select][+][-]

1. {$mode objfpc}{$H+}
2. uses classes, sysutils;
3. function FileANSItoUTF8(const AFilename: string; const WriteBOM: Boolean = True): Boolean;
4. var
5.   Input, Output: TStringList;
6. begin
7.   Result := False;
8.   if FileExists(AFilename) then
9.   try
10.     Input  := TStringList.Create;
11.     try
12.       Input.LoadFromFile(AFilename, TEncoding.ANSI);
13.       Output := TStringList.Create;
14.       try
15.         Output.WriteBOM := WriteBOM;
16.         Output.Text := AnsiToUTF8(Input.Text);
17.       finally
18.         Output.Free;
19.       end;
20.     finally  
21.       Input.Free;
22.     end;
23.     Result := true;
24.   except
25.     On E:Exception do
26.         writeln (E.Message);
27.   end;
28. end;
29.  
30. var
31.   s: string;
32. begin
33.   WriteLn('Convert ANSI to UTF8 - by KodeZwerg 2022 for forum.lazarus.freepascal.org');
34.   if ParamCount > 0 then
35.     begin
36.       s := ParamStr(1);
37.       WriteLn('Converting: ' + s);
38.       if FileANSItoUTF8(s) then
39.         WriteLn('Success!')
40.         else
41.         WriteLn('Failed!');
42.     end
43.     else
44.       WriteLn('Usage: ' + ParamStr(0) + ' "ANSI encoded file.txt"');
45.   WriteLn('Press RETURN to continue.');
46.   ReadLn;
47. end.

[KodeZwerg]

I do nil to free if something bad happen and not produce memory leaks.
I do loop to set Result.
I do set BOM optional with a switch and it does not matter where you set it since it is just a boolean.

[KodeZwerg]

Why do you use a loop? You can convert in one go using

Code: Pascal  [Select][+][-]

1. Output.Text :=AnsiToUTF8(Output.Text);
2. Output.WriteBom := true;  // will be called before save and savetofile goes through a savetostream
3. Output.SaveToFile(<your file>);

This will result in an UTF8 file with BOM. BOM is optional.
And nilling evreywhere is also strange since the stringlists are local. I would rewrite your code something like this:

Code: Text  [Select][+][-]

1. {$mode objfpc}{$H+}
2. uses classes, sysutils;
3. function FileANSItoUTF8(const AFilename: string; const WriteBOM: Boolean = True): Boolean;
4. var
5.   Input, Output: TStringList;
6. begin
7.   Result := False;
8.   if FileExists(AFilename) then
9.   try
10.     Input  := TStringList.Create;
11.     try
12.       Input.LoadFromFile(AFilename, TEncoding.ANSI);
13.       Output := TStringList.Create;
14.       try
15.         Output.WriteBOM := WriteBOM;
16.         Output.Text := AnsiToUTF8(Input.Text);
17.       finally
18.         Output.Free;
19.       end;
20.     finally  
21.       Input.Free;
22.     end;
23.     Result := true;
24.   except
25.     On E:Exception do
26.         writeln (E.Message);
27.   end;
28. end;
29.  
30. var
31.   s: string;
32. begin
33.   WriteLn('Convert ANSI to UTF8 - by KodeZwerg 2022 for forum.lazarus.freepascal.org');
34.   if ParamCount > 0 then
35.     begin
36.       s := ParamStr(1);
37.       WriteLn('Converting: ' + s);
38.       if FileANSItoUTF8(s) then
39.         WriteLn('Success!')
40.         else
41.         WriteLn('Failed!');
42.     end
43.     else
44.       WriteLn('Usage: ' + ParamStr(0) + ' "ANSI encoded file.txt"');
45.   WriteLn('Press RETURN to continue.');
46.   ReadLn;
47. end.

Your code does save nothing.

[Thaddy]

1)Then you do not understand that the stringlists are local to the function. In that case there is really no need for nil, since after function exit they are out of scope.
2)If the rest of the code executes, the result is always true. On any kind of exception the result remains false.
3)True, but I did not in my small example.
4) add as needed. just before finally of output.
TIP
5) optionally add code to check read/write protection.

[KodeZwerg]

1)Then you do not understand that the stringlists are local to the function. In that case there is really no need for nil, since after function exit they are out of scope.
2)If the rest of the code executes, the result is always true. On any kind of exception the result remains false.
3)True, but I did not in my small example.
4) add as needed. just before finally of output.
TIP
5) optionally add code to check read/write protection.

1. I will try that after i found out how to detect memory leaks within Lazarus, on my old IDE it is a must to free created objects. Thank you for that information. Sorry that i am sceptical on that.
2. I agree you, i am used to do it my way but you would, if you would also write, get True for empty files, no? 
3. I do not know to what you refer 
4. I do not know to what you refer 
5. You are absolut correct, my way is just a working horse, it say Yes (True) or No (False) as Result, not much checking internal on that matter. 

[paweld]

Or:

Code: Pascal  [Select][+][-]

1. program Project1;
2.  
3. {$APPTYPE CONSOLE}
4.  
5. Uses
6.   SysUtils, Classes, LConvEncoding;
7.  
8. function FiletoUTF8(const AFilename: string): String;
9. var
10.   sl: TStringList;
11. begin
12.   Result := '';
13.   if (not FileExists(AFilename)) then
14.     Exit;
15.   sl := TStringList.Create;
16.   try
17.     sl.LoadFromFile(AFilename);
18.     Result := GuessEncoding(sl.Text);
19.     sl.Text := ConvertEncoding(sl.Text, Result, 'UTF-8');
20.     sl.SaveToFile(AFilename, TEncoding.UTF8);
21.   except
22.     Result := '';
23.   end;
24.   sl.Free;
25. end;
26.  
27. var
28.   s: string;
29. begin
30.   WriteLn('Convert to UTF8');
31.   if (ParamCount > 0) then
32.     begin
33.       s := ParamStr(1);
34.       WriteLn('Converting: ' + s);
35.       s := FiletoUTF8(s);
36.       if s <> '' then
37.         WriteLn('Success: from ' + s + ' to UTF-8!')
38.         else
39.         WriteLn('Failed!');
40.     end
41.     else
42.       WriteLn('Usage: ' + ParamStr(0) + ' "ANSI encoded file.txt"');
43.   WriteLn('Press RETURN to continue.');
44.   ReadLn;
45. end.

  Add LCLBase to requirments

[Thaddy]

Hello All,

I would like to convert Same ANSI format file into UTF8 file without generating another file. (I want to convert same ANSI file into UTF8 format).
Is there any function available to convert ANSI file format to UTF8 format?

Eg,
Test_file.txt(ANSI format) ---> Test_file.txt (UTF8 format) 



Thanks in advance.

My above code can do that.

[Thaddy]

OR:

The other code does not rely on any Lazarus specifics. Imho better.