Set in Style cbCustomColors to true
And use the event OnGetColors:
procedure TForm1.ColorListBox1GetColors(Sender: TCustomColorListBox;
Items: TStrings);
begin
Items.Clear;
Items.AddObject('red', TObject(PtrInt(clRed)));
Items.AddObject('yellow', TObject(PtrInt(clYellow)));
end;