unit UnitUTF8Conversion;
//This unit contains encoding/codepage converting routines for CSV (i.e. textual) files
{$mode objfpc}{$H+}
interface
uses
Classes, SysUtils, Dialogs,
{ FileUtil,} lazfileutils, lazutf8, LConvEncoding,
chsd_dll_intf, dynlibs;
procedure ConvertCSVFileEncodingToUTF8(pFilePathName: string);
function DetermineCSVFileEncoding(pFilePathName: string): string;
function MapEncoding(pCharsetInfo: string): string;
implementation
type
MapCharsetInfo = record
Name: PChar;
CodePage: integer;
Language: PChar;
LazEncoding: string;
end;
const
MAP_CHARSETS: array [1..32] of MapCharsetInfo = (
// UNKNOWN_CHARSET
(
Name: 'Unknown';
CodePage: -1;
Language: 'Unknown';
LazEncoding: '';
),
// PURE_ASCII_CHARSET
(
Name: 'ASCII';
CodePage: 0;
Language: 'ASCII';
LazEncoding: '';
),
// UTF8_CHARSET
(
Name: 'UTF-8';
CodePage: 65001;
Language: 'Unicode';
LazEncoding: 'utf8';
),
// UCS4_BE_CHARSET
(
Name: 'X-ISO-10646-UCS-4-3412';
CodePage: 12001;
Language: 'Unicode';
LazEncoding: '';
),
// UTF16_BE_CHARSET
(
Name: 'UTF-16BE';
CodePage: 1201;
Language: 'Unicode';
LazEncoding: '';
),
// UTF32_BE_CHARSET
(
Name: 'UTF-32BE';
CodePage: 12001;
Language: 'Unicode';
LazEncoding: '';
),
// UCS4_LE_CHARSET
(
Name: 'X-ISO-10646-UCS-4-2143';
CodePage: 12000;
Language: 'Unicode';
LazEncoding: '';
),
// UTF32_LE_CHARSET
(
Name: 'UTF-32LE';
CodePage: 12000;
Language: 'Unicode';
LazEncoding: '';
),
// UTF16_LE_CHARSET
(
Name: 'UTF-16LE';
CodePage: 1200;
Language: 'Unicode';
LazEncoding: '';
),
// LATIN5_BULGARIAN_CHARSET
(
Name: 'ISO-8859-5';
CodePage: 28595;
Language: 'Bulgarian';
LazEncoding: '';
),
// WINDOWS_BULGARIAN_CHARSET
(
Name: 'windows-1251';
CodePage: 1251;
Language: 'Bulgarian';
LazEncoding: 'cp1251';
),
// KOI8_R_CHARSET
(
Name: 'KOI8-R';
CodePage: 20866;
Language: 'russian';
LazEncoding: 'koi8';
),
// WINDOWS_1251_CHARSET
(
Name: 'windows-1251';
CodePage: 1251;
Language: 'russian';
LazEncoding: 'cp1251';
),
// ISO_8859_5_CHARSET
(
Name: 'ISO-8859-5';
CodePage: 28595;
Language: 'russian';
LazEncoding: '';
),
// X_MAC_CYRILLIC_CHARSET
(
Name: 'x-mac-cyrillic';
CodePage: 10007;
Language: 'russian';
LazEncoding: '';
),
// IBM866_CHARSET
(
Name: 'IBM866';
CodePage: 866;
Language: 'russian';
LazEncoding: '';
),
// IBM855_CHARSET
(
Name: 'IBM855';
CodePage: 855;
Language: 'russian';
LazEncoding: '';
),
// ISO_8859_7_CHARSET
(
Name: 'ISO-8859-7';
CodePage: 28597;
Language: 'greek';
LazEncoding: '';
),
// WINDOWS_1253_CHARSET
(
Name: 'windows-1253';
CodePage: 1253;
Language: 'greek';
LazEncoding: 'cp1253';
),
// ISO_8859_8_CHARSET
(
Name: 'ISO-8859-8';
CodePage: 28598;
Language: 'hebrew';
LazEncoding: '';
),
// WINDOWS_1255_CHARSET
(
Name: 'windows-1255';
CodePage: 1255;
Language: 'hebrew';
LazEncoding: 'cp1255';
),
// BIG5_CHARSET
(
Name: 'Big5';
CodePage: 950;
Language: 'ch';
LazEncoding: '';
),
// ISO_2022_CN_CHARSET
(
Name: 'ISO-2022-CN';
CodePage: 50227;
Language: 'ch';
LazEncoding: '';
),
// ISO_2022_JP_CHARSET
(
Name: 'ISO-2022-JP';
CodePage: 50222;
Language: 'japanese';
LazEncoding: '';
),
// ISO_2022_KR_CHARSET
(
Name: 'ISO-2022-KR';
CodePage: 50225;
Language: 'kr';
LazEncoding: '';
),
// EUC_JP_CHARSET
(
Name: 'EUC-JP';
CodePage: 51932;
Language: 'japanese';
LazEncoding: '';
),
// EUC_KR_CHARSET
(
Name: 'EUC-KR';
CodePage: 51949;
Language: 'kr';
LazEncoding: '';
),
// X_EUC_TW_CHARSET
(
Name: 'x-euc-tw';
CodePage: 51936;
Language: 'ch';
LazEncoding: '';
),
// SHIFT_JIS_CHARSET
(
Name: 'Shift_JIS';
CodePage: 932;
Language: 'japanese';
LazEncoding: '';
),
// GB18030_CHARSET
(
Name: 'GB18030';
CodePage: 54936;
Language: 'ch';
LazEncoding: '';
),
// HZ_GB_2312_CHARSET
(
Name: 'HZ-GB-2312';
CodePage: 52936;
Language: 'ch';
LazEncoding: '';
),
// WINDOWS_1252_CHARSET
(
Name: 'windows-1252';
CodePage: 1252;
Language: 'eu';
LazEncoding: 'cp1252';
)
);
procedure ConvertCSVFileEncodingToUTF8(pFilePathName: string);
var
About: rAboutHolder;
Info: rCharsetInfo;
Fs: TFilestream;
P: PChar;
vOriginalEncoding: string;
sl: TStringList;
begin
if FileExistsUTF8(pFilePathName) then begin
try
if LibHandle = NilHandle then begin
ShowMessage ('Error! I could not load Charset Detector library! Alternative method for charset detection will be used.');
Exit;
end;
csd_GetAbout(About);
{
ShowMessage(About.About);
}
// Backup original CSV file
sl:=TStringList.Create;
sl.LoadFromFile(pFilePathName);
sl.SaveToFile(pFilePathName+'.bcp');
// Read CSV file
Fs := TFilestream.Create(pFilePathName, fmOpenReadWrite);
P := AllocMem(Fs.Size);
Fs.ReadBuffer(P^, Fs.Size);
csd_Reset;
csd_HandleData(PChar(P), Length(P));
if not csd_Done() then
csd_DataEnd;
Info := csd_GetDetectedCharset();
{
ShowMessage(Info.Name + sLineBreak +
InttoStr(Info.CodePage) + sLineBreak +
Info.Language);
}
//Convert File Charset Encoding
if (Info.Name<>'') then begin
vOriginalEncoding:=MapEncoding(Info.Name);
P:=PChar(ConvertEncoding(P, vOriginalEncoding, EncodingUTF8));
end
else begin
vOriginalEncoding:=GuessEncoding(P);
P:=PChar(ConvertEncoding(P, vOriginalEncoding, EncodingUTF8));
end;
// Save CSV File
Fs.Seek(0, soFromBeginning);
Fs.WriteBuffer(Pointer(P)^, UTF8Length(P));
ShowMessage('The CSV file "' + pFilePathName + '" is analyzed. '
+ 'Charset detected by Charset Detection Library: ' + Info.Name + ','
+ 'Charset detected by GuessEncoding function: ' + GuessEncoding(P) + '.' + LineEnding
+ 'The CSV file "' + pFilePathName + '" is converted to UTF8, while original CSV file is preserved as backup file: "' + pFilePathName + '.bcp".');
finally
{
FreeMem(P); // I had to comment this, because it crashes application...Why?
}
Fs.Free;
sl.Free;
end;
end
else begin
ShowMessage('The file: ' + pFilePathName + ' does not exist!');
end;
end;
function DetermineCSVFileEncoding(pFilePathName: string): string;
var
About: rAboutHolder;
Info: rCharsetInfo;
Fs: TFilestream;
P: PChar;
sl: TStringList;
begin
if FileExistsUTF8(pFilePathName) then begin
try
Result:='';
if LibHandle = NilHandle then begin
ShowMessage ('Error! I could not load Charset Detector library! Alternative method for charset detection will be used.');
Result:='';
Exit;
end;
csd_GetAbout(About);
{
ShowMessage(About.About);
}
// Read CSV file
Fs := TFilestream.Create(pFilePathName, fmOpenReadWrite);
P := AllocMem(Fs.Size);
Fs.ReadBuffer(P^, Fs.Size);
csd_Reset;
csd_HandleData(PChar(P), Length(P));
if not csd_Done() then
csd_DataEnd;
Info := csd_GetDetectedCharset();
{
ShowMessage(Info.Name + sLineBreak +
InttoStr(Info.CodePage) + sLineBreak +
Info.Language);
}
Result:=Info.Name;
ShowMessage('The CSV file "' + pFilePathName + '" is analyzed. '
+ 'Charset detected by Charset Detection Library: ' + Info.Name + ','
+ 'Charset detected by GuessEncoding function: ' + GuessEncoding(P) + '.');
finally
FreeMem(P);
Fs.Free;
sl.Free;
end;
end
else begin
ShowMessage('The file: ' + pFilePathName + ' does not exist!');
end;
end;
function MapEncoding(pCharsetInfo: string): string;
var i:Integer;
begin
Result:='';
for i:=Low(MAP_CHARSETS) to High(MAP_CHARSETS) do begin
if (MAP_CHARSETS[i].Name=pCharsetInfo) then begin
Result:=MAP_CHARSETS[i].LazEncoding;
end;
end;
end;
end.
Recent
