Merge branch 'import-plural-translations-into-android'

3 files changed, 360 insertions, 52 deletions

diff --git a/android/translations-converter/src/android.rs b/android/translations-converter/src/android.rs
index ca759ffb1e..5329d95ea4 100644
--- a/android/translations-converter/src/android.rs
+++ b/android/translations-converter/src/android.rs
@@ -193,11 +193,13 @@ pub struct PluralVariant {
 }
 
 /// A valid quantity for a plural variant.
-#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
+#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum PluralQuantity {
     Zero,
     One,
+    Few,
+    Many,
     Other,
 }
 
@@ -223,3 +225,75 @@ impl IntoIterator for PluralResources {
         self.entries.into_iter()
     }
 }
+
+impl PluralResources {
+    /// Create an empty list of plural resources.
+    pub fn new() -> Self {
+        PluralResources {
+            entries: Vec::new(),
+        }
+    }
+}
+
+impl PluralResource {
+    /// Create a plural resource representation.
+    ///
+    /// The resource has a name, used as the identifier, and a list of items. Each item contains
+    /// the message and the quantity it should be used for.
+    pub fn new(name: String, values: impl Iterator<Item = (PluralQuantity, String)>) -> Self {
+        let items = values
+            .map(|(quantity, string)| PluralVariant { quantity, string })
+            .collect();
+
+        PluralResource { name, items }
+    }
+}
+
+impl Display for PluralResources {
+    fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {
+        writeln!(formatter, r#"<?xml version="1.0" encoding="utf-8"?>"#)?;
+        writeln!(formatter, "<resources>")?;
+
+        for entry in &self.entries {
+            write!(formatter, "{}", entry)?;
+        }
+
+        writeln!(formatter, "</resources>")
+    }
+}
+
+impl Display for PluralResource {
+    fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {
+        writeln!(formatter, r#"    <plurals name="{}">"#, self.name)?;
+
+        for item in &self.items {
+            writeln!(formatter, "        {}", item)?;
+        }
+
+        writeln!(formatter, "    </plurals>")
+    }
+}
+
+impl Display for PluralVariant {
+    fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {
+        write!(
+            formatter,
+            r#"<item quantity="{}">{}</item>"#,
+            self.quantity, self.string
+        )
+    }
+}
+
+impl Display for PluralQuantity {
+    fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {
+        let quantity = match self {
+            PluralQuantity::Zero => "zero",
+            PluralQuantity::One => "one",
+            PluralQuantity::Few => "few",
+            PluralQuantity::Many => "many",
+            PluralQuantity::Other => "other",
+        };
+
+        write!(formatter, "{}", quantity)
+    }
+}
diff --git a/android/translations-converter/src/gettext.rs b/android/translations-converter/src/gettext.rs
index 26abdbd577..b4962ab834 100644
--- a/android/translations-converter/src/gettext.rs
+++ b/android/translations-converter/src/gettext.rs
@@ -1,8 +1,10 @@
 use lazy_static::lazy_static;
 use regex::Regex;
 use std::{
+    collections::BTreeMap,
     fs::{File, OpenOptions},
     io::{self, BufRead, BufReader, BufWriter, Write},
+    mem,
     path::Path,
 };
 
@@ -11,6 +13,22 @@ lazy_static! {
     static ref PARAMETERS: Regex = Regex::new(r"%\([^)]*\)").unwrap();
 }
 
+/// A parsed gettext translation file.
+pub struct Translation {
+    pub plural_form: Option<PluralForm>,
+    entries: Vec<MsgEntry>,
+}
+
+/// Known plural forms.
+#[derive(Clone, Copy, Debug)]
+pub enum PluralForm {
+    Single,
+    SingularForOne,
+    SingularForZeroAndOne,
+    Polish,
+    Russian,
+}
+
 /// A message entry in a gettext translation file.
 #[derive(Clone, Debug)]
 pub struct MsgEntry {
@@ -28,41 +46,175 @@ pub enum MsgValue {
     },
 }
 
-impl From<String> for MsgValue {
-    fn from(string: String) -> Self {
-        MsgValue::Invariant(string)
-    }
+/// A helper macro to match a string to various prefix and suffix combinations.
+macro_rules! match_str {
+    (
+        ( $string:expr )
+        $( [$start:expr, $middle:ident, $end:expr] => $body:tt )*
+        _ => $else:expr $(,)*
+    ) => {
+        $(
+            if let Some($middle) = parse_line($string, $start, $end) {
+                $body
+            } else
+        )* {
+            $else
+        }
+    };
 }
 
-/// Load message entries from a gettext translation file.
-///
-/// The messages are normalized into a common format so that they can be compared to Android string
-/// resource entries.
-pub fn load_file(file_path: impl AsRef<Path>) -> Vec<MsgEntry> {
-    let mut entries = Vec::new();
-    let mut current_id = None;
-    let file = BufReader::new(File::open(file_path).expect("Failed to open gettext file"));
+impl Translation {
+    /// Load message entries from a gettext translation file.
+    ///
+    /// The messages are normalized into a common format so that they can be compared to Android
+    /// string resource entries.
+    ///
+    /// The only metadata that is parsed from the file is the "Plural-Form" header. It is assumed
+    /// that the header value is one of some hard-coded values, so if new languages that have new
+    /// plurals are added, the code will have to be updated.
+    ///
+    /// An gettext translation file has the format in the example below:
+    ///
+    /// ```
+    /// # The start of the file can contain empty entries to include some header with meta
+    /// # information. Below is the header indicating the plural format.
+    /// msgid ""
+    /// msgstr ""
+    /// "Plural-Forms: nplurals=2; plural=(n != 1);"
+    ///
+    /// # Simple translated messages
+    /// msgid "Message in original language"
+    /// msgstr "Mesaĝo en tradukita lingvo"
+    ///
+    /// # Plural translated messages (with two forms)
+    /// msgid "One translated message"
+    /// msgid_plural "%d translated messages"
+    /// msgstr[0] "Unu tradukita mesaĝo"
+    /// msgstr[1] "%d tradukitaj mesaĝoj"
+    /// ```
+    pub fn from_file(file_path: impl AsRef<Path>) -> Self {
+        let mut parsing_header = false;
+        let mut entries = Vec::new();
+        let mut current_id = None;
+        let mut current_plural_id = None;
+        let mut plural_form = None;
+        let mut variants = BTreeMap::new();
+        let file = BufReader::new(File::open(file_path).expect("Failed to open gettext file"));
+
+        for line in file.lines() {
+            let line = line.expect("Failed to read from gettext file");
+            let line = line.trim();
+
+            match_str! { (line)
+                ["msgid \"", msg_id, "\""] => {
+                    current_id = Some(normalize(msg_id));
+                }
+                ["msgstr \"", translation, "\""] => {
+                    if let Some(id) = current_id.take() {
+                        let value = MsgValue::from(normalize(translation));
+
+                        parsing_header = id.is_empty() && translation.is_empty();
+
+                        entries.push(MsgEntry { id, value });
+                    }
+
+                    current_id = None;
+                    current_plural_id = None;
+                }
+                ["msgid_plural \"", plural_id, "\""] => {
+                    current_plural_id = Some(normalize(plural_id));
+                    parsing_header = false;
+                }
+                ["msgstr[", plural_translation, "\""] => {
+                    let variant_id_end = plural_translation
+                        .chars()
+                        .position(|character| character == ']')
+                        .expect("Invalid plural msgstr");
+                    let variant_id: usize = plural_translation[..variant_id_end]
+                        .parse()
+                        .expect("Invalid variant index");
+                    let variant_msg = parse_line(&plural_translation[variant_id_end..], "] \"", "")
+                        .expect("Invalid plural msgstr");
 
-    for line in file.lines() {
-        let line = line.expect("Failed to read from gettext file");
-        let line = line.trim();
+                    variants.insert(variant_id, normalize(variant_msg));
+                    parsing_header = false;
+                }
+                ["\"", header, "\\n\""] => {
+                    if parsing_header {
+                        if let Some(plural_formula) = parse_line(header, "Plural-Forms: ", ";") {
+                            plural_form = Some(PluralForm::from_formula(plural_formula));
+                        }
+                    }
+                }
+                _ => {
+                    if let Some(plural_id) = current_plural_id.take() {
+                        let id = current_id.take().expect("Missing msgid for plural message");
+                        let values = mem::replace(&mut variants, BTreeMap::new())
+                            .into_iter()
+                            .enumerate()
+                            .inspect(|(index, (variant_id, _))| {
+                                assert_eq!(
+                                    index, variant_id,
+                                    "Unexpected variant ID for plural msgstr"
+                                )
+                            })
+                            .map(|(_, (_, value))| value)
+                            .collect();
+                        let value = MsgValue::Plural { plural_id, values };
 
-        if let Some(msg_id) = parse_line(line, "msgid \"", "\"") {
-            current_id = Some(normalize(msg_id));
-        } else {
-            if let Some(translation) = parse_line(line, "msgstr \"", "\"") {
-                if let Some(id) = current_id.take() {
-                    let value = MsgValue::from(normalize(translation));
+                        entries.push(MsgEntry { id, value });
+                    }
 
-                    entries.push(MsgEntry { id, value });
+                    current_id = None;
+                    current_plural_id = None;
+                    variants.clear();
+                    parsing_header = false;
                 }
             }
+        }
+
+        Self {
+            entries,
+            plural_form,
+        }
+    }
+}
+
+impl IntoIterator for Translation {
+    type Item = MsgEntry;
+    type IntoIter = std::vec::IntoIter<Self::Item>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.entries.into_iter()
+    }
+}
 
-            current_id = None;
+impl PluralForm {
+    /// Obtain an instance based on a known plural formula.
+    ///
+    /// Plural variants need to be obtained using a formula. However, some locales have known
+    /// formulas, so they can be represented as a known plural form. This constructor can return a
+    /// plural form based on the formulas that are known to be used in the project.
+    pub fn from_formula(formula: &str) -> Self {
+        match formula {
+            "nplurals=1; plural=0" => PluralForm::Single,
+            "nplurals=2; plural=(n != 1)" => PluralForm::SingularForOne,
+            "nplurals=2; plural=(n > 1)" => PluralForm::SingularForZeroAndOne,
+            "nplurals=4; plural=(n==1 ? 0 : (n%10>=2 && n%10<=4) && (n%100<12 || n%100>14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && n%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3)" => {
+                PluralForm::Polish
+            }
+            "nplurals=4; plural=((n%10==1 && n%100!=11) ? 0 : ((n%10 >= 2 && n%10 <=4 && (n%100 < 12 || n%100 > 14)) ? 1 : ((n%10 == 0 || (n%10 >= 5 && n%10 <=9)) || (n%100 >= 11 && n%100 <= 14)) ? 2 : 3))" => {
+                PluralForm::Russian
+            }
+            other => panic!("Unknown plural formula: {}", other),
         }
     }
+}
 
-    entries
+impl From<String> for MsgValue {
+    fn from(string: String) -> Self {
+        MsgValue::Invariant(string)
+    }
 }
 
 /// Append message entries to a translation file.
diff --git a/android/translations-converter/src/main.rs b/android/translations-converter/src/main.rs
index 36a1970c2d..3f4499dd39 100644
--- a/android/translations-converter/src/main.rs
+++ b/android/translations-converter/src/main.rs
@@ -16,12 +16,16 @@
 //! order when only named parameters are used, and Android strings only supported numbered
 //! parameters.
 //!
-//! Android's plural resources aren't currently translated, but this tool will convert them to
-//! gettext message templates and append them to the message template file. It's important to note
-//! that the first quantity item for a plural will be used as the `msgid`, so it shouldn't
-//! have any parameters. The last quantity item for a plural will be used as the `msgid_plural`,
-//! and it can contain parameters. This assumes a plural resource will have at least two items.
-//! While it would still work with a single item, this is an unlikely case for a plural resource.
+//! Android's plural resources are also translated using the same principle. It's important to note
+//! that the singular quantity item (i.e., the item where `quantity="one"`) for each Android plural
+//! resource will be used as the `msgid` to be search for in the gettext translations file.
+//!
+//! Missing translations are appended to the gettext messages template file (`messages.pot`). These
+//! are the entries for which no translation in any locale was found. When missing plurals are
+//! appended to the template file, the new message entries are created using the singular quantity
+//! item as the `msgid` and the other quantity item as the `msgid_plural`. Because of this, it is
+//! important to note that the former can't have parameters, while the latter can. Otherwise, the
+//! entries will have to be manually added.
 //!
 //! Note that this conversion procedure is very raw and likely very brittle, so while it works for
 //! most cases, it is important to keep in mind that this is just a helper tool and manual steps are
@@ -68,13 +72,29 @@ fn main() {
         }
     }
 
-    let mut missing_translations = known_strings.clone();
-
     let plurals_file = File::open(resources_dir.join("values/plurals.xml"))
         .expect("Failed to open plurals resources file");
     let plural_resources: android::PluralResources =
         serde_xml_rs::from_reader(plurals_file).expect("Failed to read plural resources file");
 
+    let known_plurals: HashMap<_, _> = plural_resources
+        .iter()
+        .map(|plural| {
+            let name = plural.name.clone();
+            let singular = plural
+                .items
+                .iter()
+                .find(|variant| variant.quantity == android::PluralQuantity::One)
+                .map(|variant| variant.string.clone())
+                .expect("Missing singular plural variant");
+
+            (singular, name)
+        })
+        .collect();
+
+    let mut missing_translations = known_strings.clone();
+    let mut missing_plurals = known_plurals.clone();
+
     let locale_dir = Path::new("../../gui/locales");
     let locale_files = fs::read_dir(&locale_dir)
         .expect("Failed to open root locale directory")
@@ -101,9 +121,12 @@ fn main() {
             locale,
             known_urls.clone(),
             known_strings.clone(),
-            gettext::load_file(&locale_file),
+            known_plurals.clone(),
+            gettext::Translation::from_file(&locale_file),
             destination_dir.join("strings.xml"),
+            destination_dir.join("plurals.xml"),
             &mut missing_translations,
+            &mut missing_plurals,
         );
     }
 
@@ -125,20 +148,37 @@ fn main() {
         .expect("Failed to append missing translations to message template file");
     }
 
-    if !plural_resources.is_empty() {
+    if !missing_plurals.is_empty() {
+        println!("Appending missing plural translations to template file:");
+
         gettext::append_to_template(
             &template_path,
             plural_resources
                 .into_iter()
                 .inspect(|plural| {
-                    let last_item = &plural.items.last().expect("Plural items are empty").string;
+                    let other_item = &plural
+                        .items
+                        .iter()
+                        .find(|plural| plural.quantity == android::PluralQuantity::Other)
+                        .expect("Plural items are empty")
+                        .string;
 
-                    println!("  {}: {}", plural.name, last_item);
+                    println!("  {}: {}", plural.name, other_item);
                 })
                 .map(|mut plural| {
-                    let plural_id = plural.items.pop().expect("Plural items are empty").string;
-                    plural.items.truncate(1);
-                    let id = plural.items.remove(0).string;
+                    let singular_position = plural
+                        .items
+                        .iter()
+                        .position(|plural| plural.quantity == android::PluralQuantity::One)
+                        .expect("Missing singular variant to use as msgid");
+                    let id = plural.items.remove(singular_position).string;
+
+                    let other_position = plural
+                        .items
+                        .iter()
+                        .position(|plural| plural.quantity == android::PluralQuantity::Other)
+                        .expect("Missing other variant to use as msgid_plural");
+                    let plural_id = plural.items.remove(other_position).string;
 
                     gettext::MsgEntry {
                         id,
@@ -188,19 +228,39 @@ fn generate_translations(
     locale: &str,
     known_urls: HashMap<String, String>,
     mut known_strings: HashMap<String, String>,
-    translations: Vec<gettext::MsgEntry>,
-    output_path: impl AsRef<Path>,
+    mut known_plurals: HashMap<String, String>,
+    translations: gettext::Translation,
+    strings_output_path: impl AsRef<Path>,
+    plurals_output_path: impl AsRef<Path>,
     missing_translations: &mut HashMap<String, String>,
+    missing_plurals: &mut HashMap<String, String>,
 ) {
-    let mut localized_resource = android::StringResources::new();
+    let mut localized_strings = android::StringResources::new();
+    let mut localized_plurals = android::PluralResources::new();
+
+    let plural_quantities = android_plural_quantities_from_gettext_plural_form(
+        translations
+            .plural_form
+            .expect("Missing plural form for translation"),
+    );
 
     for translation in translations {
-        if let gettext::MsgValue::Invariant(translation_value) = translation.value {
-            if let Some(android_key) = known_strings.remove(&translation.id) {
-                localized_resource.push(android::StringResource::new(
-                    android_key,
-                    &translation_value,
-                ));
+        match translation.value {
+            gettext::MsgValue::Invariant(translation_value) => {
+                if let Some(android_key) = known_strings.remove(&translation.id) {
+                    localized_strings.push(android::StringResource::new(
+                        android_key,
+                        &translation_value,
+                    ));
+                }
+            }
+            gettext::MsgValue::Plural { values, .. } => {
+                if let Some(android_key) = known_plurals.remove(&translation.id) {
+                    localized_plurals.push(android::PluralResource::new(
+                        android_key,
+                        plural_quantities.clone().zip(values),
+                    ));
+                }
             }
         }
     }
@@ -209,19 +269,41 @@ fn generate_translations(
         let locale_path = format!("/{}/", web_locale);
 
         for (url, android_key) in known_urls {
-            localized_resource.push(android::StringResource::new(
+            localized_strings.push(android::StringResource::new(
                 android_key,
                 &url.replacen("/en/", &locale_path, 1),
             ));
         }
     }
 
-    localized_resource.sort();
+    localized_strings.sort();
 
-    fs::write(output_path, localized_resource.to_string())
+    fs::write(strings_output_path, localized_strings.to_string())
         .expect("Failed to create Android locale file");
 
+    fs::write(plurals_output_path, localized_plurals.to_string())
+        .expect("Failed to create Android plurals file");
+
     missing_translations.retain(|translation, _| known_strings.contains_key(translation));
+    missing_plurals.retain(|translation, _| known_plurals.contains_key(translation));
+}
+
+/// Converts a gettext plural form into the plural quantities used by Android.
+///
+/// Returns an iterator that can be zipped with the gettext plural variants to produce the Android
+/// plural string items.
+fn android_plural_quantities_from_gettext_plural_form(
+    plural_form: gettext::PluralForm,
+) -> impl Iterator<Item = android::PluralQuantity> + Clone {
+    use android::PluralQuantity::*;
+    use gettext::PluralForm;
+
+    match plural_form {
+        PluralForm::Single => vec![Other],
+        PluralForm::SingularForOne | PluralForm::SingularForZeroAndOne => vec![One, Other],
+        PluralForm::Polish | PluralForm::Russian => vec![One, Few, Many, Other],
+    }
+    .into_iter()
 }
 
 /// Tries to map a translation locale to a locale used on the Mullvad website.