//! Helper tool to convert translations from gettext messages to Android string resources. //! //! The procedure for converting the translations is relatively simple. The base Android string //! resources file is first loaded, and then each gettext translation file is loaded and compared to //! the Android base strings. For every translation string that matches exactly the Android base //! string value (after a normalization pass described below), the translated string is used in the //! new Android strings file for the respective locale. //! //! To make the comparison work on most strings, the Android and gettext messages are normalized //! first. This means that new lines in the XML files are removed and collapsed into a single space, //! the message parameters are changed so that they are in a common format, and there is also a //! small workaround for having different apostrophe characters in the GUI in some messages. //! //! Android's plural resources are also translated using the same principle. It's important to note //! that the singular quantity item (i.e., the item where `quantity="one"`) for each Android plural //! resource will be used as the `msgid` to be search for in the gettext translations file. //! //! Missing translations are appended to the gettext messages template file (`messages.pot`). These //! are the entries for which no translation in any locale was found. When missing plurals are //! appended to the template file, the new message entries are created using the singular quantity //! item as the `msgid` and the other quantity item as the `msgid_plural`. Because of this, it is //! important to note that the former can't have parameters, while the latter can. Otherwise, the //! entries will have to be manually added. //! //! Note that this conversion procedure is very raw and likely very brittle, so while it works for //! most cases, it is important to keep in mind that this is just a helper tool and manual steps are //! likely to be needed from time to time. mod android; mod gettext; mod normalize; use crate::android::{ PluralResource, PluralResources, StringResource, StringResources, StringValue, }; use crate::gettext::MsgValue; use crate::normalize::Normalize; use itertools::Itertools; use std::path::PathBuf; use std::{ collections::HashMap, fs::{self}, path::Path, }; fn main() -> Result<(), Box> { let resources_dir = Path::new("../lib/resource/src/main/res"); let string_resources = StringResources::try_from(resources_dir.join("values/strings.xml").as_ref())?; let plural_resources = PluralResources::try_from(resources_dir.join("values/plurals.xml").as_ref())?; check_duplicates(&string_resources); let known_strings: HashMap = string_resources .into_iter() .map(|resource| (resource.value.normalize(), resource)) .collect(); let known_plurals: HashMap = plural_resources .iter() .map(|plural| { let singular = plural .items .iter() .find(|variant| variant.quantity == android::PluralQuantity::One) .map(|variant| variant.string.to_string()) .expect("Missing singular plural variant"); (singular, plural) }) .collect(); let locale_dir = Path::new("../../desktop/packages/mullvad-vpn/locales"); generate_translated_strings_xml_files( locale_dir, resources_dir, &known_strings, &known_plurals, ); let template_path = locale_dir.join("messages.pot"); let template = gettext::Messages::from_file(&template_path)?; let mut missing_translations = known_strings; let mut missing_plurals: HashMap<_, _> = known_plurals; for message in template { match message.value { MsgValue::Invariant(_, _) => { missing_translations.remove(&message.id.normalize()); } MsgValue::Plural { .. } => { missing_plurals.remove(&message.id.normalize()); } }; } add_missing_translations(&template_path, missing_translations); add_missing_plurals(&template_path, &plural_resources, missing_plurals); generate_relay_locale_files(locale_dir); Ok(()) } fn generate_translated_strings_xml_files( locale_dir: &Path, resources_dir: &Path, known_strings: &HashMap, known_plurals: &HashMap, ) { let locale_files = fs::read_dir(locale_dir) .expect("Failed to open root locale directory") .filter_map(|dir_entry_result| dir_entry_result.ok().map(|dir_entry| dir_entry.path())) .filter(|dir_entry_path| dir_entry_path.is_dir()) .map(|dir_path| dir_path.join("messages.po")) .filter(|file_path| file_path.exists()); for locale_file in locale_files { let locale = locale_file .parent() .unwrap() .file_name() .unwrap() .to_str() .unwrap(); let destination_dir = resources_dir.join(android_locale_directory(locale)); if !destination_dir.exists() { fs::create_dir(&destination_dir).expect("Failed to create Android locale directory"); } let translations = gettext::Messages::from_file(&locale_file) .expect("Failed to load translations for a locale"); generate_translations( known_strings.clone(), known_plurals.clone(), translations, destination_dir.join("strings.xml"), destination_dir.join("plurals.xml"), ); } } fn check_duplicates(string_resources: &StringResources) { // The current format is not built to handle multiple strings with the same values // so we check for duplicates and panic if they are present let duplicates: HashMap<&StringValue, Vec<&StringResource>> = string_resources .iter() .into_group_map_by(|res| &res.value) .into_iter() .filter(|(_, string_resources)| string_resources.len() > 1) .collect(); if !duplicates.is_empty() { duplicates .iter() .for_each(|(string_value, string_resources)| { eprintln!( "String value: '{}', exists in following resource IDs: {}", string_value, string_resources .iter() .map(|x| x.name.clone()) .collect::>() .join(", ") ) }); panic!("Duplicate string values!!"); } } fn add_missing_translations( template_path: &PathBuf, missing_translations: HashMap, ) { if !missing_translations.is_empty() { println!("Appending missing translations to template file:"); gettext::append_to_template( template_path, missing_translations .into_iter() .inspect(|(_, res)| { println!( " {}: {}", res.name, res.value.normalize_keep_parameter_indices() ) }) .map(|(_, res)| gettext::MsgEntry { id: gettext::MsgString::from_unescaped( &res.value.normalize_keep_parameter_indices(), ), value: gettext::MsgString::empty().into(), }), ) .expect("Failed to append missing translations to message template file"); } } fn add_missing_plurals( template_path: &PathBuf, plural_resources: &PluralResources, missing_plurals: HashMap, ) { if !missing_plurals.is_empty() { println!("Appending missing plural translations to template file:"); gettext::append_to_template( template_path, missing_plurals .into_iter() .filter_map(|(_, p)| plural_resources.iter().find(|plural| plural.name == p.name)) .cloned() .inspect(|plural| { let other_item = &plural .items .iter() .find(|plural| plural.quantity == android::PluralQuantity::Other) .expect("Plural items are empty") .string; println!(" {}: {}", plural.name, other_item); }) .map(|mut plural| { let singular_position = plural .items .iter() .position(|plural| plural.quantity == android::PluralQuantity::One) .expect("Missing singular variant to use as msgid"); let id = gettext::MsgString::from_escaped( plural.items.remove(singular_position).string.to_string(), ); let other_position = plural .items .iter() .position(|plural| plural.quantity == android::PluralQuantity::Other) .expect("Missing other variant to use as msgid_plural"); let plural_id = gettext::MsgString::from_escaped( plural.items.remove(other_position).string.to_string(), ); gettext::MsgEntry { id, value: MsgValue::Plural { plural_id, values: vec![gettext::MsgString::empty(), gettext::MsgString::empty()], }, } }), ) .expect("Failed to append missing plural translations to message template file"); } } fn generate_relay_locale_files(locale_dir: &Path) { let relay_template_path = locale_dir.join("relay-locations.pot"); let default_translations = gettext::Messages::from_file(&relay_template_path) .expect("Failed to load translations for a locale"); let resources_dir = Path::new("../lib/resource/src/main/res"); let relay_locations_path = resources_dir.join("xml/relay_locations.xml"); let mut localized_strings = android::StringResources::new(); for translation in default_translations { match translation.value { MsgValue::Invariant(_, arg_ordering) => { if !translation.id.is_empty() { localized_strings.push(android::StringResource::new( translation.id.normalize(), &translation.id.normalize(), arg_ordering.as_ref(), )); } } MsgValue::Plural { .. } => {} } } localized_strings.sort(); fs::write(relay_locations_path, localized_strings.to_string()) .expect("Failed to create Android locale file"); let relay_locale_files = fs::read_dir(locale_dir) .expect("Failed to open root locale directory") .filter_map(|dir_entry_result| dir_entry_result.ok()) .map(|dir_entry| dir_entry.path()) .filter(|dir_entry_path| dir_entry_path.is_dir()) .map(|dir_path| dir_path.join("relay-locations.po")) .filter(|file_path| file_path.exists()); for relay_file in relay_locale_files { let locale = relay_file .parent() .unwrap() .file_name() .unwrap() .to_str() .unwrap(); let destination_dir = resources_dir.join(android_xml_directory(locale)); if !destination_dir.exists() { fs::create_dir(&destination_dir).expect("Failed to create Android locale directory"); } let translations = gettext::Messages::from_file(&relay_file) .expect("Failed to load translations for a locale"); generate_relay_translations(translations, destination_dir.join("relay_locations.xml")); } } /// Determines the localized value resources directory name based on a locale specification. /// /// This just makes sure a locale such as `en-US' gets correctly mapped to the directory name /// `values-en-rUS`. fn android_locale_directory(locale: &str) -> String { let mut directory = String::from("values-"); let mut parts = locale.split('-'); directory.push_str(parts.next().unwrap()); if let Some(region) = parts.next() { directory.push_str("-r"); directory.push_str(region); } directory } /// Determines the localized value resources directory name based on a locale specification. /// /// This just makes sure a locale such as `en-US' gets correctly mapped to the directory name /// `xml-en-rUS`. fn android_xml_directory(locale: &str) -> String { let mut directory = String::from("xml-"); let mut parts = locale.split('-'); directory.push_str(parts.next().unwrap()); if let Some(region) = parts.next() { directory.push_str("-r"); directory.push_str(region); } directory } /// Generate translated Android relay resource strings for a locale. fn generate_relay_translations( translations: gettext::Messages, strings_output_path: impl AsRef, ) { let mut localized_strings = android::StringResources::new(); for translation in translations { match translation.value { MsgValue::Invariant(translation_value, arg_ordering) => { localized_strings.push(android::StringResource::new( translation.id.normalize(), &translation_value.normalize(), arg_ordering.as_ref(), )); } MsgValue::Plural { .. } => {} } } localized_strings.sort(); fs::write(strings_output_path, localized_strings.to_string()) .expect("Failed to create Android locale file"); } /// Generate translated Android resource strings for a locale. /// /// Based on the gettext translated message entries, it finds the messages with message IDs that /// match known Android string resource values, and obtains the string resource ID for the /// translation. An Android string resource XML file is created with the translated strings. /// /// The missing translations map is updated to only contain the strings that aren't present in the /// current locale, which means that in the end the map contains only the translations that aren't /// present in any locale. fn generate_translations( mut known_strings: HashMap, mut known_plurals: HashMap, translations: gettext::Messages, strings_output_path: impl AsRef, plurals_output_path: impl AsRef, ) { let mut localized_strings = StringResources::new(); let mut localized_plurals = PluralResources::new(); let plural_quantities = android_plural_quantities_from_gettext_plural_form( translations .plural_form .expect("Missing plural form for translation"), ); for translation in translations { match translation.value { MsgValue::Invariant(translation_value, arg_ordering) => { if let Some(android_key) = known_strings.remove(&translation.id.normalize()) { localized_strings.push(StringResource::new( android_key.name, &translation_value.normalize(), arg_ordering.as_ref(), )); } } MsgValue::Plural { values, .. } => { if let Some(android_key) = known_plurals.remove(&translation.id.normalize()) { let values = values.into_iter().map(|message| message.normalize()); localized_plurals.push(PluralResource::new( android_key.name.clone(), plural_quantities.clone().zip(values), )); } } } } localized_strings.sort(); fs::write(strings_output_path, localized_strings.to_string()) .expect("Failed to create Android locale file"); fs::write(plurals_output_path, localized_plurals.to_string()) .expect("Failed to create Android plurals file"); } /// Converts a gettext plural form into the plural quantities used by Android. /// /// Returns an iterator that can be zipped with the gettext plural variants to produce the Android /// plural string items. fn android_plural_quantities_from_gettext_plural_form( plural_form: gettext::PluralForm, ) -> impl Iterator + Clone { use android::PluralQuantity::*; use gettext::PluralForm; match plural_form { PluralForm::Single => vec![Other], PluralForm::SingularForOne | PluralForm::SingularForZeroAndOne => vec![One, Other], PluralForm::Polish | PluralForm::Russian => vec![One, Few, Many, Other], } .into_iter() }