Embedding translations in binary (with country flag)

[Virchanza]

This is all still a little half-baked but I've put some stuff together for embedding translations in the binary file for your application (for every platform -- not just for MS-Windows).

I have all of this working for my network analysis application called 'Dynamo'. So far I have it in English, German and Irish.

First I wrote a small separate program that processes all the ".mo" and ".png" files in the current directoy, making a char array for each file. Here's a typical output from this small separate program:

Code: Select all

char unsigned const g_binary_de_mo[8869u] = { 0xde, 0x12, 0x04, 0x95,  /* and so on */ };
char unsigned const g_binary_ga_mo[8208u] = { 0xde, 0x12, 0x04, 0x95,  /* and so on */ };

char unsigned const g_binary_de_png[1188u] = { 0x89, 0x50, 0x4e, 0x47  /* and so on */ };
char unsigned const g_binary_ga_png[1189u] = { 0x89, 0x50, 0x4e, 0x47  /* and so on */ };
char unsigned const g_binary_en_png[3653u] = { 0x89, 0x50, 0x4e, 0x47  /* and so on */ };

typedef std::tuple<std::string,char unsigned const *,std::size_t> TupleType;
typedef std::vector<TupleType> VecType;

std::map<std::string,VecType> g_binaries = {
    { ".mo", {
               {"de_mo", g_binary_de_mo, 8869u },
               {"ga_mo", g_binary_ga_mo, 8208u },
             }
    },
    { ".png", {
               {"de_png", g_binary_de_png, 1188u },
               {"ga_png", g_binary_ga_png, 1189u },
               {"en_png", g_binary_en_png, 3653u },
             }
    },
};

And here's the source code for the small separate program:

Code: Select all

#include <iostream>
#include <iomanip>
#include <fstream>
#include <string>
#include <vector>
#include <algorithm>  /* replace */
#include <map>  /* map */
#include <iterator>
#include <tuple>  /* tuple */
#include <boost/filesystem.hpp>

namespace fs = boost::filesystem;

using std::string;
using std::ostream;
using std::ofstream;
using std::ifstream;
using std::cout;
using std::endl;

using std::vector;
using std::map;
using std::tuple;

char const *const g_extensions[] = {
	".mo",
	".png"
};

//Each extension has a list of filenames
typedef tuple<string,string,size_t> TupleType;
typedef vector<TupleType> VecType;
map<string,VecType> g_archive;

void Initalise_Global_Archive(void)
{
	for ( auto &e : g_extensions )
	{
		g_archive[e];
	}
}

void Normalise_Filename(string &str)
{
	std::replace(str.begin(), str.end(), '.', '_');
}

void Append_Binary_Catalogue(ostream &os_hpp, ostream &os_cpp, string str_mofile, VecType &arg_vec)
{
	ifstream mofile(str_mofile, std::ios::binary);
	
	if ( !mofile.is_open() )
		return;
		
	vector<char unsigned> tmp_vec;
	tmp_vec.reserve(10000ull);
	
	char ctmp;
	
	while ( mofile.get(ctmp) )  //std::copy doesn't work here
	{
		tmp_vec.push_back( *reinterpret_cast<char unsigned*>(&ctmp) );
	}
	
	Normalise_Filename(str_mofile);
	
	arg_vec.push_back(TupleType(str_mofile, "g_binary_" + str_mofile, tmp_vec.size()));
	
	os_hpp << "extern char unsigned const g_binary_" << str_mofile
	       << "["
	       << std::dec << std::setw(0)
	       << tmp_vec.size()
	       << "u];\n";
	
	os_cpp << "char unsigned const g_binary_" << str_mofile
	       << "["
	       << std::dec << std::setw(0)
	       << tmp_vec.size()
	       << "u] = {\n ";
	
	unsigned constexpr bytes_per_row = 12u;
	unsigned counter = bytes_per_row + 1u;
	
	for (auto const &c : tmp_vec)
	{
		if ( 0 == --counter )
		{
			os_cpp << "\n ";
			
			counter = bytes_per_row;
		}
		
		os_cpp << " 0x" << std::hex << std::setfill('0')
		       << std::setw(2) << std::nouppercase
		       << static_cast<unsigned>(c);
		   
		if ( &c != &tmp_vec.back() )
			os_cpp << ",";
	}
	
	os_cpp << "\n};\n";
}

auto main(void) -> int
{
	cout << "Searching current directory for translation catalogue files" << endl;
	
	Initalise_Global_Archive();
	
	ofstream os_hpp("./binaries_char_arrays.hpp", std::ios::trunc),
	         os_cpp("./binaries_char_arrays.cpp", std::ios::trunc); // Truncate and overwrite

	static char const str_stuff_at_top[] =
		"#include <cstddef>\n"
		"#include <vector>\n"
		"#include <map>\n"
		"#include <string>\n"
		"#include <tuple>\n\n";
		
	os_hpp << str_stuff_at_top;
	os_cpp << str_stuff_at_top;
	
	for ( auto const &entry : fs::directory_iterator(fs::current_path()) )
	{
		auto const filenameStr = entry.path().filename().string();
		
		if ( fs::is_regular_file(entry.path()) )
		{
			for ( auto &e : g_archive )
			{
				if ( e.first == entry.path().extension() )
				{
					cout << "Found " << entry.path().filename().string() << endl;
					
					Append_Binary_Catalogue(os_hpp, os_cpp, entry.path().filename().string(), e.second);
					break;
				}
			}
		}
	}
	
	static char const str_stuff_at_end[] =
	    "\ntypedef std::tuple<std::string,char unsigned const *,std::size_t> TupleType;\n"
		"typedef std::vector<TupleType> VecType;\n";
		
	os_hpp << str_stuff_at_end;
	os_cpp << str_stuff_at_end;
	
	os_hpp << "extern ";
	
	static char const str_stuff_at_end2[] =
		"std::map<std::string,VecType> g_binaries";
		
	os_hpp << str_stuff_at_end2;
	os_hpp << ";\n";
	
	os_cpp << str_stuff_at_end2;
	
	os_cpp << " = {\n";
	
	for ( auto &e : g_archive )
	{
		os_cpp << "    { \"" << e.first << "\", {\n";
		       
		for ( auto &f : e.second )
		{
			os_cpp << "               {";
			os_cpp << "\"" << std::get<0>(f) << "\""
			       << ", " << std::get<1>(f)
			       << ", " << std::dec << std::setw(0) << std::get<2>(f)
			       << "u },\n";
		}
		
		os_cpp << "             }\n";
		os_cpp << "    },\n";
	}
	
	os_cpp << "};";
}

My plan is to compile this small separate program as one of the compilation steps of compiling my bigger application. I want this to be streamlined so that if you change a ".mo" file or ".png" file in the current directory, then the main Makefile will automatically regenerate all the char arrays.

Next you'll want a special translations loader that can read from an input iterator. Again this is half-baked and I will do more work on it, but here's the header file:

Code: Select all

#ifndef HPP__BOMBASTIC_TRANSLATOR

#define HPP__BOMBASTIC_TRANSLATOR



#include <wx/app.h>     /* wxApp */

#include <wx/arrstr.h>  /* wxArrayString, wxString */



class BombasticTranslator {

public:



	char current_two_letter_code[2 + 1];

	

	wxApp &m_app;



	bool m_UseNativeConfig;



	BombasticTranslator(wxApp &app);

	

	void GetInstalledLanguages(wxArrayString &names, wxArrayString &codes);

	bool AskUserForLanguage(wxArrayString &names, wxArrayString &codes);

};



#endif

And here's the source file:

Code: Select all

#include "multilingual.hpp"
#include "bombastic_translator.hpp"
#include <wx/translation.h>
#include <wx/choicdlg.h>  /* wxGetSingleChoiceIndex */
#include <wx/defs.h>  /* wxOVERRIDE */
#include <vector>
#include <tuple>
#include <iterator>  /* std::begin, std::end */

#include <iostream>  /* REVISIT FIX */

#include "binaries_char_arrays.hpp"

template<typename InputIterator>
class wxInputIteratorTranslationsLoader : public wxTranslationsLoader {
protected:

    typedef std::tuple<wxString,InputIterator,InputIterator> CatalogueTuple;

    std::vector<CatalogueTuple> m_catalogues;
      
public:

    void MakeCatalogueVisible(wxString const &arg_lang, InputIterator arg_begin, InputIterator arg_end)
    {
		std::cout << "===== ===== ===== ===== Thomas : MakeCatalogueVisible : " << arg_lang << std::endl;
        m_catalogues.push_back(
            CatalogueTuple(arg_lang, arg_begin, arg_end)
        );
    }
    
    virtual wxArrayString GetAvailableTranslations(wxString const &domain) const;

    virtual wxMsgCatalog *LoadCatalog(wxString const &domain, wxString const &lang);
};

template<typename InputIterator>
wxArrayString wxInputIteratorTranslationsLoader<InputIterator>::
              GetAvailableTranslations(wxString const &domain) const
{
	std::cout << "===== ===== ===== ===== Thomas : GetAvailableTranslations\n";
	wxArrayString arrstr;
	
    for (auto &catuple : m_catalogues)
    {
        arrstr.Add(std::get<0>(catuple));
    }
	
	return arrstr;
}

template<typename InputIterator>
wxMsgCatalog *wxInputIteratorTranslationsLoader<InputIterator>::
              LoadCatalog(wxString const &domain, wxString const &lang)
{
	std::cout << "===== ===== ===== ===== Thomas : LoadCatalog : " << lang << "\n";
    CatalogueTuple *pcatuple = nullptr;

    for (auto &catuple : m_catalogues)
    {
        if ( lang == std::get<0>(catuple) )
        {
	        pcatuple = &catuple;
            break;
        }
    }

    if ( !pcatuple )
    {
		std::cout << "===== ===== ===== ===== Thomas : Requested catalogue has not been made visible\n";
		return nullptr;
	}
    
    InputIterator &it_begin = std::get<1>(*pcatuple);

    InputIterator &it_end = std::get<2>(*pcatuple);

    wxMsgCatalog *const pcat = wxMsgCatalog::CreateFromData(
        wxCharBuffer::CreateNonOwned(
            reinterpret_cast<char const*>(it_begin),
            it_end - it_begin),
        domain);

    if ( !pcat )
    {
		std::cout << "===== ===== ===== ===== Thomas : Catalogue is visible but failed to load\n";
        wxLogWarning(_("Not a valid message catalog."));
    }

    return pcat;
}

static void Create_Custom_Translation_Loader(
                wxInputIteratorTranslationsLoader<char unsigned const *> *&p_iitl
            )
{
	p_iitl = new std::remove_pointer<
	                 std::remove_reference<decltype(p_iitl)>::type
									>::type;

	VecType &vec = g_binaries[".mo"];
	
	for (auto const &e : vec)
	{
		p_iitl->MakeCatalogueVisible(
			std::get<0>(e).substr(0,2),
			std::get<1>(e),
			std::get<1>(e) + std::get<2>(e)
		);
	}	
}

BombasticTranslator::BombasticTranslator(wxApp &app)
: m_app(app)
{	
	strcpy(this->current_two_letter_code, "en");

	wxLog::AddTraceMask("i18n");

    wxInputIteratorTranslationsLoader<char unsigned const *> *p_iitl;
    
	Create_Custom_Translation_Loader(p_iitl);
	
	wxTranslations *const p = new wxTranslations;
	p->SetLoader(p_iitl);
	
	wxTranslations::Set(p);
}

void BombasticTranslator::GetInstalledLanguages(wxArrayString &names, wxArrayString &codes)
{
	names.Clear();
	codes.Clear();
	
	names.Add("English");
	codes.Add("en");
	
	wxArrayString const &arrstr = wxTranslations::Get()->GetAvailableTranslations(wxEmptyString);
	
	for (auto &str : arrstr)
	{
		names.Add(str);
		codes.Add(str);
	}
}

bool BombasticTranslator::AskUserForLanguage(wxArrayString &names, wxArrayString &codes)
{
	long const index = wxGetSingleChoiceIndex(_("Select the language"), _("Language"), names);
	
	if ( -1 == index )
		return false;
		
	//m_p_wxtrans->Init(identifiers[i]);  REVISIT FIX

	memcpy(this->current_two_letter_code, codes[index].GetData(), 2);
	this->current_two_letter_code[2] = '\0';

    wxInputIteratorTranslationsLoader<char unsigned const *> *p_iitl;
    
	Create_Custom_Translation_Loader(p_iitl);
			
	wxTranslations *const p = new wxTranslations;
	p->SetLoader(p_iitl);
	wxTranslations::Set(p);
		
	//wxTranslations::Get()->SetLanguage((wxLanguage)(wxLANGUAGE_USER_DEFINED + 1));
	wxTranslations::Get()->SetLanguage(codes[index]);
	wxTranslations::Get()->AddCatalog(codes[index]);

	return true;
}

Then to use this, you have a "BombasticTranslator" inside your "wxApp", like this:

Code: Select all

class App_Dynamo : public wxApp {
public:
	
	BombasticTranslator *m_bombtrans;
	
	bool SelectLanguage(void)
	{
		wxArrayString names;
		wxArrayString codes;		
		m_bombtrans->GetInstalledLanguages(names, codes);
		return m_bombtrans->AskUserForLanguage(names, codes);
	}
};

I have all of this working for my Dynamo application, and it changes language in a split second at runtime (and you can keep switching languages as much as you want while the program's running).

If any of these ideas are good enough to make it into wxWidgets 3.1.x, I'm enthusiastic to contribute.

[ONEEYEMAN]

Hi,
Please post to wx-dev ML.
This forum is by users for users of the library. No core developers coming in here and check the postings, unless the ML references a thread.

However, I seriously doubt that. wxWidgets philosophy is - follow a native platform guidelines. And I don't think any of the 3 major platform guidelines says - translations can be embedded.

But I maybe wrong nevertheless.

Thank you.

[Virchanza]

However, I seriously doubt that. wxWidgets philosophy is - follow a native platform guidelines. And I don't think any of the 3 major platform guidelines says - translations can be embedded.

wxMSW has the class "wxResourceTranslationsLoader" for loading ".mo" files embedded in the binary.