diff options
| -rw-r--r-- | gui/scripts/extract-geo-data.py | 22 | ||||
| -rw-r--r-- | gui/scripts/integrate-into-app.py | 137 | ||||
| -rw-r--r-- | gui/scripts/prepare-rtree.ts | 2 |
3 files changed, 106 insertions, 55 deletions
diff --git a/gui/scripts/extract-geo-data.py b/gui/scripts/extract-geo-data.py index 82f98bf951..0719dfc92e 100644 --- a/gui/scripts/extract-geo-data.py +++ b/gui/scripts/extract-geo-data.py @@ -153,7 +153,7 @@ def extract_countries_po(): if os.path.isdir(locale_dir): with fiona.open(input_path) as source: - po = POFile(encoding='UTF-8') + po = POFile(encoding='utf-8', check_for_duplicates=True) po.metadata = {"Content-Type": "text/plain; charset=utf-8"} output_path = path.join(locale_out_dir, "countries.po") @@ -212,6 +212,7 @@ def extract_countries_po(): ) po.append(entry) + sort_pofile_entries(po) po.save(output_path) print c.green("Extracted {} countries for {} to {}".format(len(po), locale, output_path)) @@ -225,7 +226,7 @@ def extract_cities_po(): locale_out_dir = path.join(LOCALE_OUT_DIR, locale) if os.path.isdir(locale_dir): - po = POFile(encoding='UTF-8') + po = POFile(encoding='utf-8', check_for_duplicates=True) po.metadata = {"Content-Type": "text/plain; charset=utf-8"} output_path = path.join(locale_out_dir, "cities.po") hits = 0 @@ -258,11 +259,16 @@ def extract_cities_po(): ) entry = POEntry( - msgid=props["name"], + msgid=props.get("name"), msgstr=translated_name ) - po.append(entry) + try: + po.append(entry) + except ValueError as err: + print c.orange(u"Cannot add an entry: {}".format(err)) + + sort_pofile_entries(po) po.save(output_path) print c.green("Extracted {} cities to {}".format(len(po), output_path)) @@ -271,6 +277,10 @@ def extract_cities_po(): print_stats_table("Cities translations", stats) +def sort_pofile_entries(pofile): + pofile.sort(key=lambda o: o.msgid_with_context.encode('utf-8')) + + def extract_relay_translations(): try: response = request_relays() @@ -291,7 +301,7 @@ def extract_relay_translations(): def extract_relay_locations_pot(countries): - pot = POFile(encoding='UTF-8') + pot = POFile(encoding='utf-8', check_for_duplicates=True) pot.metadata = {"Content-Type": "text/plain; charset=utf-8"} output_path = path.join(LOCALE_OUT_DIR, "relay-locations.pot") @@ -356,7 +366,7 @@ def translate_relay_locations_pot(countries): def translate_relay_locations(place_translator, countries, locale): - po = POFile(encoding='UTF-8') + po = POFile(encoding='utf-8', check_for_duplicates=True) po.metadata = {"Content-Type": "text/plain; charset=utf-8"} locale_out_dir = path.join(LOCALE_OUT_DIR, locale) output_path = path.join(locale_out_dir, "relay-locations.po") diff --git a/gui/scripts/integrate-into-app.py b/gui/scripts/integrate-into-app.py index 18725fac80..81c6426cc0 100644 --- a/gui/scripts/integrate-into-app.py +++ b/gui/scripts/integrate-into-app.py @@ -9,11 +9,23 @@ import shutil import colorful as c SCRIPT_DIR = path.dirname(path.realpath(__file__)) -SOURCE_DIR = path.join(SCRIPT_DIR, "out") -GEO_ASSETS_DEST_DIR = path.realpath(path.join(SCRIPT_DIR, "../assets/geo")) -TRANSLATIONS_SOURCE_DIR = path.join(SOURCE_DIR, "locales") -TRANSLATIONS_DEST_DIR = path.realpath(path.join(SCRIPT_DIR, "../locales")) +# the name of the relay locations gettext catalogue template file +RELAY_LOCATIONS_POT_FILENAME = "relay-locations.pot" + +# the directory with the generated content +GENERATED_CONTENT_OUTPUT_PATH = path.join(SCRIPT_DIR, "out") + +# the directory with the generated localizations content +GENERATED_TRANSLATIONS_PATH = path.join(GENERATED_CONTENT_OUTPUT_PATH, "locales") + +# the directory with the app's geo assets +APP_GEO_ASSETS_PATH = path.realpath(path.join(SCRIPT_DIR, "../assets/geo")) + +# the directory with the existing app localizations +APP_TRANSLATIONS_PATH = path.realpath(path.join(SCRIPT_DIR, "../locales")) + +# Geo assets for copying from generated content folder into the app folder GEO_ASSETS_TO_COPY = [ "cities.rbush.json", "countries.rbush.json", @@ -23,46 +35,43 @@ GEO_ASSETS_TO_COPY = [ "states-provinces-lines.rbush.json", ] +# The filenames of gettext catalogues that should be copied as is TRANSLATIONS_TO_COPY = [ "cities.po", "countries.po" ] +# the filenames of gettext catalogues that should be merged using msgcat TRANSLATIONS_TO_MERGE = [ "relay-locations.po" ] -def remove_common_prefix(source, destination): - prefix_len = len(path.commonprefix((source, destination))) - return (source[prefix_len:], destination[prefix_len:]) - -def run_program(args): - p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) - - print "Run: {}".format(' '.join(args)) - - errors = p.communicate()[1] - return (p.returncode, errors) def copy_geo_assets(): for f in GEO_ASSETS_TO_COPY: - src = path.join(SOURCE_DIR, f) - dst = path.join(GEO_ASSETS_DEST_DIR, f) + src = path.join(GENERATED_CONTENT_OUTPUT_PATH, f) + dst = path.join(APP_GEO_ASSETS_PATH, f) - print "Copying {} to {}".format(*remove_common_prefix(src, dst)) + print u"Copying {} to {}".format(*remove_common_prefix(src, dst)) shutil.copyfile(src, dst) + +def merge_relay_locations_catalogue_template(): + existing_pot_file = path.join(APP_TRANSLATIONS_PATH, RELAY_LOCATIONS_POT_FILENAME) + generated_pot_file = path.join(GENERATED_TRANSLATIONS_PATH, RELAY_LOCATIONS_POT_FILENAME) + + merge_gettext_catalogues(existing_pot_file, generated_pot_file) + + def copy_and_merge_translations(): - for f in os.listdir(TRANSLATIONS_SOURCE_DIR): - src = path.join(TRANSLATIONS_SOURCE_DIR, f) - dst = path.join(TRANSLATIONS_DEST_DIR, f) + for f in os.listdir(GENERATED_TRANSLATIONS_PATH): + src = path.join(GENERATED_TRANSLATIONS_PATH, f) + dst = path.join(APP_TRANSLATIONS_PATH, f) if path.isdir(src): merge_single_locale_folder(src, dst) - else: - print "Copying {} to {}".format(*remove_common_prefix(src, dst)) - shutil.copyfile(src, dst) + def merge_single_locale_folder(src, dst): for f in os.listdir(src): @@ -70,36 +79,68 @@ def merge_single_locale_folder(src, dst): dst_po = path.join(dst, f) if f in TRANSLATIONS_TO_COPY: - print "Copying {} to {}".format(*remove_common_prefix(src_po, dst_po)) + print u"Copying {} to {}".format(*remove_common_prefix(src_po, dst_po)) shutil.copyfile(src_po, dst_po) elif f in TRANSLATIONS_TO_MERGE: - if path.exists(dst_po): - pot_basename = path.basename(path.splitext(dst_po)[0]) - pot_path = path.join(TRANSLATIONS_DEST_DIR, pot_basename + ".pot") + # merge ../locales/*/file.po with ./out/locales/*/file.po + # use existing translation to resolve conflicts + merge_gettext_catalogues(dst_po, src_po) + else: + print c.orange(u"Unexpected file: {}".format(src_po)) + + +def merge_gettext_catalogues(existing_catalogue_file, generated_catalogue_file): + if path.exists(existing_catalogue_file): + args = ( + existing_catalogue_file, generated_catalogue_file, + + "--output-file", existing_catalogue_file, + + # ensure that the first occurence takes precedence in merge conflict + "--use-first", - (msgmerge_code, msgmerge_errors) = run_program([ - "msgmerge", "--update", "--no-fuzzy-matching", dst_po, pot_path]) + # sort by msgid + "--sort-output", - if msgmerge_code == 0: - (msgcat_code, msgcat_errors) = run_program([ - "msgcat", src_po, dst_po, "--output-file", dst_po]) + # disable wrapping long strings because crowdin does not do that + "--no-wrap" + ) - if msgcat_code == 0: - print c.green("Merged and concatenated the catalogues.") - else: - print c.red("msgcat exited with {}: {}".format( - msgcat_code, msgcat_errors.decode('utf-8').strip())) - else: - print c.red("msgmerge exited with {}: {}".format( - msgmerge_code, msgmerge_errors.decode('utf-8').strip())) - else: - shutil.copy(src_po, dst_po) + (exit_code, errors) = run_program("msgcat", *args) + + if exit_code == 0: + print c.green(u"Merged {} into {}." + .format(*remove_common_prefix(generated_catalogue_file, existing_catalogue_file))) else: - print c.orange("Unexpected file: {}".format(src_po)) + print c.red(u"msgcat exited with {}: {}".format(exit_code, errors.decode('utf-8').strip())) + else: + print c.orange(u"The existing catalogue does not exist. Copying {} to {}" + .format(*remove_common_prefix(generated_catalogue_file, existing_catalogue_file))) + shutil.copyfile(generated_catalogue_file, existing_catalogue_file) + + +def remove_common_prefix(*args): + prefix_len = len(path.commonprefix(args)) + return map(lambda str: str[prefix_len:], args) + + +def run_program(*args): + p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) + + print u"Run: {}".format(' '.join(args)) + + errors = p.communicate()[1] + return (p.returncode, errors) + + +# Program main() +def main(): + if not path.exists(APP_GEO_ASSETS_PATH): + os.makedirs(APP_GEO_ASSETS_PATH) -if not path.exists(GEO_ASSETS_DEST_DIR): - os.makedirs(GEO_ASSETS_DEST_DIR) + copy_geo_assets() + merge_relay_locations_catalogue_template() + copy_and_merge_translations() -copy_geo_assets() -copy_and_merge_translations() +main() diff --git a/gui/scripts/prepare-rtree.ts b/gui/scripts/prepare-rtree.ts index 3f2519d0c0..7ec108cc75 100644 --- a/gui/scripts/prepare-rtree.ts +++ b/gui/scripts/prepare-rtree.ts @@ -1,6 +1,6 @@ // // Script that generates r-trees for geo data. -// run with `npx babel-node geo-data/prepare-rtree.js` +// run with `npx ts-node geo-data/prepare-rtree.ts` // import * as fs from 'fs'; |