Merge branch 'fix-gettext-integration'

3 files changed, 106 insertions, 55 deletions

diff --git a/gui/scripts/extract-geo-data.py b/gui/scripts/extract-geo-data.py
index 82f98bf951..0719dfc92e 100644
--- a/gui/scripts/extract-geo-data.py
+++ b/gui/scripts/extract-geo-data.py
@@ -153,7 +153,7 @@ def extract_countries_po():
 
     if os.path.isdir(locale_dir):
       with fiona.open(input_path) as source:
-        po = POFile(encoding='UTF-8')
+        po = POFile(encoding='utf-8', check_for_duplicates=True)
         po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
         output_path = path.join(locale_out_dir, "countries.po")
 
@@ -212,6 +212,7 @@ def extract_countries_po():
             )
             po.append(entry)
 
+        sort_pofile_entries(po)
         po.save(output_path)
         print c.green("Extracted {} countries for {} to {}".format(len(po), locale, output_path))
 
@@ -225,7 +226,7 @@ def extract_cities_po():
     locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
 
     if os.path.isdir(locale_dir):
-      po = POFile(encoding='UTF-8')
+      po = POFile(encoding='utf-8', check_for_duplicates=True)
       po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
       output_path = path.join(locale_out_dir, "cities.po")
       hits = 0
@@ -258,11 +259,16 @@ def extract_cities_po():
                 )
 
             entry = POEntry(
-              msgid=props["name"],
+              msgid=props.get("name"),
               msgstr=translated_name
             )
-            po.append(entry)
 
+            try:
+              po.append(entry)
+            except ValueError as err:
+              print c.orange(u"Cannot add an entry: {}".format(err))
+
+      sort_pofile_entries(po)
       po.save(output_path)
       print c.green("Extracted {} cities to {}".format(len(po), output_path))
 
@@ -271,6 +277,10 @@ def extract_cities_po():
   print_stats_table("Cities translations", stats)
 
 
+def sort_pofile_entries(pofile):
+  pofile.sort(key=lambda o: o.msgid_with_context.encode('utf-8'))
+
+
 def extract_relay_translations():
   try:
     response = request_relays()
@@ -291,7 +301,7 @@ def extract_relay_translations():
 
 
 def extract_relay_locations_pot(countries):
-  pot = POFile(encoding='UTF-8')
+  pot = POFile(encoding='utf-8', check_for_duplicates=True)
   pot.metadata = {"Content-Type": "text/plain; charset=utf-8"}
   output_path = path.join(LOCALE_OUT_DIR, "relay-locations.pot")
 
@@ -356,7 +366,7 @@ def translate_relay_locations_pot(countries):
 
 
 def translate_relay_locations(place_translator, countries, locale):
-  po = POFile(encoding='UTF-8')
+  po = POFile(encoding='utf-8', check_for_duplicates=True)
   po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
   locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
   output_path = path.join(locale_out_dir, "relay-locations.po")
diff --git a/gui/scripts/integrate-into-app.py b/gui/scripts/integrate-into-app.py
index 18725fac80..81c6426cc0 100644
--- a/gui/scripts/integrate-into-app.py
+++ b/gui/scripts/integrate-into-app.py
@@ -9,11 +9,23 @@ import shutil
 import colorful as c
 
 SCRIPT_DIR = path.dirname(path.realpath(__file__))
-SOURCE_DIR = path.join(SCRIPT_DIR, "out")
-GEO_ASSETS_DEST_DIR = path.realpath(path.join(SCRIPT_DIR, "../assets/geo"))
-TRANSLATIONS_SOURCE_DIR = path.join(SOURCE_DIR, "locales")
-TRANSLATIONS_DEST_DIR = path.realpath(path.join(SCRIPT_DIR, "../locales"))
 
+# the name of the relay locations gettext catalogue template file
+RELAY_LOCATIONS_POT_FILENAME = "relay-locations.pot"
+
+# the directory with the generated content
+GENERATED_CONTENT_OUTPUT_PATH = path.join(SCRIPT_DIR, "out")
+
+# the directory with the generated localizations content
+GENERATED_TRANSLATIONS_PATH = path.join(GENERATED_CONTENT_OUTPUT_PATH, "locales")
+
+# the directory with the app's geo assets
+APP_GEO_ASSETS_PATH = path.realpath(path.join(SCRIPT_DIR, "../assets/geo"))
+
+# the directory with the existing app localizations
+APP_TRANSLATIONS_PATH = path.realpath(path.join(SCRIPT_DIR, "../locales"))
+
+# Geo assets for copying from generated content folder into the app folder
 GEO_ASSETS_TO_COPY = [
   "cities.rbush.json",
   "countries.rbush.json",
@@ -23,46 +35,43 @@ GEO_ASSETS_TO_COPY = [
   "states-provinces-lines.rbush.json",
 ]
 
+# The filenames of gettext catalogues that should be copied as is
 TRANSLATIONS_TO_COPY = [
   "cities.po",
   "countries.po"
 ]
 
+# the filenames of gettext catalogues that should be merged using msgcat
 TRANSLATIONS_TO_MERGE = [
   "relay-locations.po"
 ]
 
-def remove_common_prefix(source, destination):
-  prefix_len = len(path.commonprefix((source, destination)))
-  return (source[prefix_len:], destination[prefix_len:])
-
-def run_program(args):
-  p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-
-  print "Run: {}".format(' '.join(args))
-
-  errors = p.communicate()[1]
-  return (p.returncode, errors)
 
 def copy_geo_assets():
   for f in GEO_ASSETS_TO_COPY:
-    src = path.join(SOURCE_DIR, f)
-    dst = path.join(GEO_ASSETS_DEST_DIR, f)
+    src = path.join(GENERATED_CONTENT_OUTPUT_PATH, f)
+    dst = path.join(APP_GEO_ASSETS_PATH, f)
 
-    print "Copying {} to {}".format(*remove_common_prefix(src, dst))
+    print u"Copying {} to {}".format(*remove_common_prefix(src, dst))
 
     shutil.copyfile(src, dst)
 
+
+def merge_relay_locations_catalogue_template():
+  existing_pot_file = path.join(APP_TRANSLATIONS_PATH, RELAY_LOCATIONS_POT_FILENAME)
+  generated_pot_file = path.join(GENERATED_TRANSLATIONS_PATH, RELAY_LOCATIONS_POT_FILENAME)
+
+  merge_gettext_catalogues(existing_pot_file, generated_pot_file)
+
+
 def copy_and_merge_translations():
-  for f in os.listdir(TRANSLATIONS_SOURCE_DIR):
-    src = path.join(TRANSLATIONS_SOURCE_DIR, f)
-    dst = path.join(TRANSLATIONS_DEST_DIR, f)
+  for f in os.listdir(GENERATED_TRANSLATIONS_PATH):
+    src = path.join(GENERATED_TRANSLATIONS_PATH, f)
+    dst = path.join(APP_TRANSLATIONS_PATH, f)
 
     if path.isdir(src):
       merge_single_locale_folder(src, dst)
-    else:
-      print "Copying {} to {}".format(*remove_common_prefix(src, dst))
-      shutil.copyfile(src, dst)
+
 
 def merge_single_locale_folder(src, dst):
   for f in os.listdir(src):
@@ -70,36 +79,68 @@ def merge_single_locale_folder(src, dst):
     dst_po = path.join(dst, f)
 
     if f in TRANSLATIONS_TO_COPY:
-      print "Copying {} to {}".format(*remove_common_prefix(src_po, dst_po))
+      print u"Copying {} to {}".format(*remove_common_prefix(src_po, dst_po))
       shutil.copyfile(src_po, dst_po)
     elif f in TRANSLATIONS_TO_MERGE:
-      if path.exists(dst_po):
-        pot_basename = path.basename(path.splitext(dst_po)[0])
-        pot_path = path.join(TRANSLATIONS_DEST_DIR, pot_basename + ".pot")
+      # merge ../locales/*/file.po with ./out/locales/*/file.po
+      # use existing translation to resolve conflicts
+      merge_gettext_catalogues(dst_po, src_po)
+    else:
+      print c.orange(u"Unexpected file: {}".format(src_po))
+
+
+def merge_gettext_catalogues(existing_catalogue_file, generated_catalogue_file):
+  if path.exists(existing_catalogue_file):
+    args = (
+      existing_catalogue_file, generated_catalogue_file,
+
+      "--output-file", existing_catalogue_file,
+
+      # ensure that the first occurence takes precedence in merge conflict
+      "--use-first",
 
-        (msgmerge_code, msgmerge_errors) = run_program([
-          "msgmerge", "--update", "--no-fuzzy-matching", dst_po, pot_path])
+      # sort by msgid
+      "--sort-output",
 
-        if msgmerge_code == 0:
-          (msgcat_code, msgcat_errors) = run_program([
-            "msgcat", src_po, dst_po, "--output-file", dst_po])
+      # disable wrapping long strings because crowdin does not do that
+      "--no-wrap"
+    )
 
-          if msgcat_code == 0:
-            print c.green("Merged and concatenated the catalogues.")
-          else:
-            print c.red("msgcat exited with {}: {}".format(
-              msgcat_code, msgcat_errors.decode('utf-8').strip()))
-        else:
-          print c.red("msgmerge exited with {}: {}".format(
-            msgmerge_code, msgmerge_errors.decode('utf-8').strip()))
-      else:
-        shutil.copy(src_po, dst_po)
+    (exit_code, errors) = run_program("msgcat", *args)
+
+    if exit_code == 0:
+      print c.green(u"Merged {} into {}."
+        .format(*remove_common_prefix(generated_catalogue_file, existing_catalogue_file)))
     else:
-      print c.orange("Unexpected file: {}".format(src_po))
+      print c.red(u"msgcat exited with {}: {}".format(exit_code, errors.decode('utf-8').strip()))
+  else:
+    print c.orange(u"The existing catalogue does not exist. Copying {} to {}"
+      .format(*remove_common_prefix(generated_catalogue_file, existing_catalogue_file)))
+    shutil.copyfile(generated_catalogue_file, existing_catalogue_file)
+
+
+def remove_common_prefix(*args):
+  prefix_len = len(path.commonprefix(args))
+  return map(lambda str: str[prefix_len:], args)
+
+
+def run_program(*args):
+  p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+
+  print u"Run: {}".format(' '.join(args))
+
+  errors = p.communicate()[1]
+  return (p.returncode, errors)
+
+
+# Program main()
 
+def main():
+  if not path.exists(APP_GEO_ASSETS_PATH):
+    os.makedirs(APP_GEO_ASSETS_PATH)
 
-if not path.exists(GEO_ASSETS_DEST_DIR):
-  os.makedirs(GEO_ASSETS_DEST_DIR)
+  copy_geo_assets()
+  merge_relay_locations_catalogue_template()
+  copy_and_merge_translations()
 
-copy_geo_assets()
-copy_and_merge_translations()
+main()
diff --git a/gui/scripts/prepare-rtree.ts b/gui/scripts/prepare-rtree.ts
index 3f2519d0c0..7ec108cc75 100644
--- a/gui/scripts/prepare-rtree.ts
+++ b/gui/scripts/prepare-rtree.ts
@@ -1,6 +1,6 @@
 //
 // Script that generates r-trees for geo data.
-// run with `npx babel-node geo-data/prepare-rtree.js`
+// run with `npx ts-node geo-data/prepare-rtree.ts`
 //
 
 import * as fs from 'fs';