Translate the map and relay list

8 files changed, 834 insertions, 4 deletions

diff --git a/gui/scripts/README.md b/gui/scripts/README.md
new file mode 100644
index 0000000000..c158619925
--- /dev/null
+++ b/gui/scripts/README.md
@@ -0,0 +1,74 @@
+This is a folder with the supporting scripts written in python 2, node, bash.
+
+
+## Dependency installation notes
+
+1. Run the following command in terminal to install python dependencies:
+   `pip install -r requirements.txt`
+
+2. Run `npm install -g topojson-server` to install `geo2topo` tool which is
+   used by python scripts to convert GeoJSON to TopoJSON
+
+3. Make sure you have gettext utilities installed.
+   https://www.gnu.org/software/gettext/
+
+
+## Geo data installation notes
+
+Go to http://www.naturalearthdata.com/downloads/50m-cultural-vectors/ and
+download ZIP files with the following shapes:
+
+- Admin 0 – Countries
+- Admin 1 – States, provinces - boundary lines
+- Populated Places - simple dataset is enough
+
+or use cURL to download all ZIPs:
+
+```
+curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_countries.zip
+curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip
+curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_populated_places.zip
+curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_populated_places.zip
+```
+
+Extract the downloaded ZIP files into `scripts` folder.
+Make sure the following folders exist after extraction:
+
+- ne_50m_admin_0_countries
+- ne_50m_admin_1_states_provinces_lines
+- ne_10m_populated_places
+- ne_50m_populated_places
+
+or use the following script:
+
+```
+unzip ne_50m_admin_0_countries.zip -d ne_50m_admin_0_countries/
+unzip ne_50m_admin_1_states_provinces_lines.zip -d ne_50m_admin_1_states_provinces_lines/
+unzip ne_10m_populated_places.zip -d ne_10m_populated_places/
+unzip ne_50m_populated_places.zip -d ne_50m_populated_places/
+```
+
+## Geo data extraction notes
+
+Run the following script to produce a TopoJSON data used by the app:
+
+```
+python extract-geo-data.py
+```
+
+and finally generate the R-Tree cache:
+
+```
+npx ts-node prepare-rtree.ts
+```
+
+At this point all of the data should be saved in `gui/geo-data/out` folder.
+
+## App integration notes
+
+Once you've extracted all the geo data, run the integration script that will
+copy all files ignoring intermediate ones into the `gui/src/assets/geo` folder:
+
+```
+python integrate-into-app.py
+```
diff --git a/gui/scripts/crowdin.sh b/gui/scripts/crowdin.sh
index 914007ac87..466f643c75 100755
--- a/gui/scripts/crowdin.sh
+++ b/gui/scripts/crowdin.sh
@@ -19,6 +19,7 @@ mode=$1
 function upload_pot {
     curl \
         -F "files[/messages.pot]=@$LOCALE_DIR/messages.pot" \
+        -F "files[/relay-locations.pot]=@$LOCALE_DIR/relay-locations.pot" \
         $BASE_URL/update-file?key="$CROWDIN_API_KEY"
 }
 
diff --git a/gui/scripts/extract-geo-data.py b/gui/scripts/extract-geo-data.py
new file mode 100644
index 0000000000..85a1613e8b
--- /dev/null
+++ b/gui/scripts/extract-geo-data.py
@@ -0,0 +1,501 @@
+"""
+This module forms a geo json of highly populated cities in the world
+"""
+
+import os
+from os import path
+import json
+import urllib2
+from subprocess import Popen, PIPE
+from polib import POFile, POEntry
+import colorful as c
+from terminaltables import AsciiTable
+
+# import order is important, see https://github.com/Toblerity/Shapely/issues/553
+from shapely.geometry import shape, mapping
+import fiona
+
+SCRIPT_DIR = path.dirname(path.realpath(__file__))
+LOCALE_DIR = path.normpath(path.join(SCRIPT_DIR, "../locales"))
+OUT_DIR = path.join(SCRIPT_DIR, "out")
+LOCALE_OUT_DIR = path.join(OUT_DIR, "locales")
+
+POPULATION_MAX_FILTER = 50000
+
+def extract_cities():
+  input_path = get_shape_path("ne_50m_populated_places")
+  output_path = path.join(OUT_DIR, "cities.json")
+
+  props_to_keep = frozenset(["scalerank", "name", "latitude", "longitude"])
+
+  features = []
+  with fiona.collection(input_path, "r") as source:
+    for feat in source:
+      props = lower_dict_keys(feat["properties"])
+
+      if props["pop_max"] >= POPULATION_MAX_FILTER:
+        for k in frozenset(props) - props_to_keep:
+          del props[k]
+        features.append(feat)
+
+  my_layer = {
+    "type": "FeatureCollection",
+    "features": features
+  }
+
+  with open(output_path, "w") as f:
+    f.write(json.dumps(my_layer))
+
+  print c.green("Extracted data to {}".format(output_path))
+
+
+def extract_countries():
+  input_path = get_shape_path("ne_50m_admin_0_countries")
+  output_path = path.join(OUT_DIR, "countries.json")
+
+  props_to_keep = frozenset(["name"])
+
+  features = []
+  with fiona.open(input_path) as source:
+    for feat in source:
+      geometry = feat["geometry"]
+
+      # convert country polygon to point
+      geometry.update(mapping(shape(geometry).representative_point()))
+
+      props = lower_dict_keys(feat["properties"])
+      for k in frozenset(props) - props_to_keep:
+        del props[k]
+
+      feat["properties"] = props
+
+      features.append(feat)
+
+  my_layer = {
+    "type": "FeatureCollection",
+    "features": features
+  }
+
+  with open(output_path, "w") as f:
+    f.write(json.dumps(my_layer))
+
+  print c.green("Extracted data to {}".format(output_path))
+
+
+def extract_geometry():
+  input_path = get_shape_path("ne_50m_admin_0_countries")
+  output_path = path.join(OUT_DIR, "geometry.json")
+
+  features = []
+  with fiona.open(input_path) as source:
+    for feat in source:
+      del feat["properties"]
+      geometry = feat["geometry"]
+      feat["bbox"] = shape(geometry).bounds
+      features.append(feat)
+
+  my_layer = {
+    "type": "FeatureCollection",
+    "features": features
+  }
+
+  p = Popen(
+    ['geo2topo', '-q', '1e5', 'geometry=-', '-o', output_path],
+    stdin=PIPE, stdout=PIPE, stderr=PIPE
+  )
+  errors = p.communicate(input=json.dumps(my_layer))[1]
+  if p.returncode == 0:
+    print c.green("Extracted data to {}".format(output_path))
+  else:
+    print c.red("geo2topo exited with {}. {}".format(p.returncode, errors.decode('utf-8').strip()))
+
+
+def extract_provinces_and_states_lines():
+  input_path = get_shape_path("ne_50m_admin_1_states_provinces_lines")
+  output_path = path.join(OUT_DIR, "states-provinces-lines.json")
+
+  features = []
+  with fiona.open(input_path) as source:
+    for feat in source:
+      del feat["properties"]
+      geometry = feat["geometry"]
+      feat["bbox"] = shape(geometry).bounds
+      features.append(feat)
+
+  my_layer = {
+    "type": "FeatureCollection",
+    "features": features
+  }
+
+  p = Popen(
+    ['geo2topo', '-q', '1e5', 'geometry=-', '-o', output_path],
+    stdin=PIPE, stdout=PIPE, stderr=PIPE
+  )
+  errors = p.communicate(input=json.dumps(my_layer))[1]
+  if p.returncode == 0:
+    print c.green("Extracted data to {}".format(output_path))
+  else:
+    print c.red("geo2topo exited with {}. {}".format(p.returncode, errors.decode('utf-8').strip()))
+
+
+def extract_countries_po():
+  input_path = get_shape_path("ne_50m_admin_0_countries")
+
+  for locale in os.listdir(LOCALE_DIR):
+    locale_dir = path.join(LOCALE_DIR, locale)
+    locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
+
+    if os.path.isdir(locale_dir):
+      with fiona.open(input_path) as source:
+        po = POFile(encoding='UTF-8')
+        po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
+        output_path = path.join(locale_out_dir, "countries.po")
+
+        if not path.exists(locale_out_dir):
+          os.makedirs(locale_out_dir)
+
+        print "Generating {}/countries.po".format(locale)
+
+        for feat in source:
+          props = lower_dict_keys(feat["properties"])
+          name_key = "_".join(("name", get_locale_language(locale)))
+          name_alt_key = "_".join(("name", convert_locale_ident(locale)))
+          name_fallback = "name"
+
+          if props.get(name_key) is not None:
+            translated_name = props.get(name_key)
+          elif props.get(name_alt_key) is not None:
+            translated_name = props.get(name_alt_key)
+          elif props.get(name_fallback) is not None:
+            translated_name = props.get(name_fallback)
+            print c.orange(u" Missing translation for {}".format(translated_name))
+          else:
+            raise ValueError(
+              "Cannot find the translation for {}. Probe keys: {}"
+              .format(locale, (name_key, name_alt_key))
+              )
+
+          entry = POEntry(
+            msgid=props["name"],
+            msgstr=translated_name
+          )
+          po.append(entry)
+
+        po.save(output_path)
+        print c.green("Extracted {} countries for {} to {}".format(len(po), locale, output_path))
+
+
+def extract_cities_po():
+  input_path = get_shape_path("ne_50m_populated_places")
+  stats = []
+
+  for locale in os.listdir(LOCALE_DIR):
+    locale_dir = path.join(LOCALE_DIR, locale)
+    locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
+
+    if os.path.isdir(locale_dir):
+      po = POFile(encoding='UTF-8')
+      po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
+      output_path = path.join(locale_out_dir, "cities.po")
+      hits = 0
+      misses = 0
+
+      if not path.exists(locale_out_dir):
+        os.makedirs(locale_out_dir)
+
+      print "Generating {}/cities.po".format(locale)
+
+      with fiona.open(input_path) as source:
+        for feat in source:
+          props = lower_dict_keys(feat["properties"])
+
+          if props["pop_max"] >= POPULATION_MAX_FILTER:
+            name_key = "_".join(("name", get_locale_language(locale)))
+            name_alt_key = "_".join(("name", convert_locale_ident(locale)))
+            name_fallback = "name"
+
+            if props.get(name_key) is not None:
+              translated_name = props.get(name_key)
+              hits += 1
+            elif props.get(name_alt_key) is not None:
+              translated_name = props.get(name_alt_key)
+              hits += 1
+            elif props.get(name_fallback) is not None:
+              translated_name = props.get(name_fallback)
+              print c.orange(u"  Missing translation for {}".format(translated_name))
+              misses += 1
+            else:
+              raise ValueError(
+                "Cannot find the translation for {}. Probe keys: {}"
+                .format(locale, (name_key, name_alt_key))
+                )
+
+            entry = POEntry(
+              msgid=props["name"],
+              msgstr=translated_name
+            )
+            po.append(entry)
+
+      po.save(output_path)
+      print c.green("Extracted {} cities to {}".format(len(po), output_path))
+
+      stats.append((locale, hits, misses))
+
+  print_stats_table("Cities translations", stats)
+
+
+def extract_relay_translations():
+  try:
+    response = request_relays()
+  except Exception as e:
+    print c.red("Failed to fetch the relays list: {}".format(e))
+    raise
+
+  result = response.get("result")
+  if result is not None:
+    countries = result.get("countries")
+    if countries is None:
+      raise Exception("Missing the countries field.")
+  else:
+    raise Exception("Missing the result field.")
+
+  extract_relay_locations_pot(countries)
+  translate_relay_locations_pot(countries)
+
+
+def extract_relay_locations_pot(countries):
+  pot = POFile(encoding='UTF-8')
+  pot.metadata = {"Content-Type": "text/plain; charset=utf-8"}
+  output_path = path.join(LOCALE_OUT_DIR, "relay-locations.pot")
+
+  print "Generating relay-locations.pot"
+
+  for country in countries:
+    cities = country.get("cities")
+    if cities is not None:
+      for city in cities:
+        city_name = city.get("name")
+        if city_name is not None:
+          entry = POEntry(
+            msgid=city_name,
+            msgstr=u"",
+            comment=u"{} {}".format(country.get("code").upper(), city.get("code").upper())
+          )
+          pot.append(entry)
+          print u"  {} ({})".format(city["name"], city["code"]).encode('utf-8')
+
+  pot.save(output_path)
+
+
+def prepare_stats_table_column(item):
+  (locale, hits, misses) = item
+  total = hits + misses
+  hits_ratio = round(float(hits) / total * 100, 2) if total > 0 else 0
+
+  misses_column = c.orange(str(misses)) if misses > 0 else c.green(str(misses))
+  hits_column = c.green(str(hits))
+  ratio_column = c.green(str(hits_ratio) + "%") if hits_ratio >= 80 else c.orange(str(hits_ratio))
+  total_column = str(total)
+
+  return (locale, hits_column, misses_column, ratio_column, total_column)
+
+def print_stats_table(title, data):
+  header = ("Locale", "Hits", "Misses", "% translated", "Total")
+  color_data = map(prepare_stats_table_column, data)
+
+  table = AsciiTable([header] + color_data)
+  table.title = title
+
+  for i in range(1, 5):
+    table.justify_columns[i] = 'center'
+
+  print ""
+  print table.table
+  print ""
+
+
+def translate_relay_locations_pot(countries):
+  place_translator = PlaceTranslator()
+  stats = []
+
+  for locale in os.listdir(LOCALE_DIR):
+    locale_dir = path.join(LOCALE_DIR, locale)
+    if path.isdir(locale_dir):
+      print "Generating {}/relay-locations.po".format(locale)
+      (hits, misses) = translate_relay_locations(place_translator, countries, locale)
+      stats.append((locale, hits, misses))
+
+  print_stats_table("Relay location translations", stats)
+
+
+def translate_relay_locations(place_translator, countries, locale):
+  po = POFile(encoding='UTF-8')
+  po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
+  locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
+  output_path = path.join(locale_out_dir, "relay-locations.po")
+
+  hits = 0
+  misses = 0
+
+  if not path.exists(locale_out_dir):
+    os.makedirs(locale_out_dir)
+
+  for country in countries:
+    country_name = country.get("name")
+    country_code = country.get("code")
+    cities = country.get("cities")
+
+    if cities is None:
+      print c.orange(u"Skip {} ({}) because no cities were found.".format(
+        country_name, country_code))
+      continue
+
+    for city in cities:
+      city_name = city.get("name")
+      city_code = city.get("code")
+      if city_name is None:
+        raise ValueError("Missing the name field in city record.")
+
+      # Make sure to append the US state back to the translated name of the city
+      if country_code == "us":
+        split = city_name.rsplit(",", 2)
+        translated_name = place_translator.translate(locale, split[0].strip())
+
+        if translated_name is not None and len(split) > 1:
+          translated_name = u"{}, {}".format(translated_name, split[1].strip())
+      else:
+        translated_name = place_translator.translate(locale, city_name)
+
+      # Default to empty string if no translation was found
+      found_translation = translated_name is not None
+      if found_translation:
+        hits += 1
+      else:
+        translated_name = ""
+        misses += 1
+
+      log_message = u"  {} ({}) -> \"{}\"".format(
+        city_name, city_code, translated_name).encode('utf-8')
+      if found_translation:
+        print c.green(log_message)
+      else:
+        print c.orange(log_message)
+
+      entry = POEntry(
+        msgid=city_name,
+        msgstr=translated_name,
+        comment=u"{} {}".format(country.get("code").upper(), city.get("code").upper())
+      )
+      po.append(entry)
+
+  po.save(output_path)
+
+  return (hits, misses)
+
+
+### HELPERS ###
+
+class PlaceTranslator(object):
+  """
+  This class provides facilities for translating places from one language to the other.
+  It supports both English and
+  """
+
+  def __init__(self):
+    super(PlaceTranslator, self).__init__()
+    shape_path = get_shape_path("ne_10m_populated_places")
+    self.source = fiona.open(shape_path, "r")
+
+  def __del__(self):
+    self.source.close()
+
+  def translate(self, locale, english_city_name):
+    """
+    Lookup the populated places dataset for the city matching by name, par name or
+    name representation in ASCII.
+
+    When there is a match, the function looks for the translation using the given locale or using
+    the language component of it.
+
+    Returns None when either there is no match or there is no translation for the matched city.
+    """
+    preferred_locales = (get_locale_language(locale), convert_locale_ident(locale))
+    match_prop_keys = ("name_" + x for x in preferred_locales)
+
+    for feat in self.source:
+      props = lower_dict_keys(feat["properties"])
+
+      # namepar works for "Wien"
+      # use nameascii to match "Sao Paolo"
+      if props.get("name") == english_city_name or \
+         props.get("namepar") == english_city_name or \
+         props.get("nameascii") == english_city_name:
+        for key in match_prop_keys:
+          value = props.get(key)
+
+          if value is not None:
+            return value
+
+        print c.orange(u"Missing translation for {} ({}). Probe keys: {}".format(
+          english_city_name, locale, match_prop_keys).encode('utf-8'))
+
+    return None
+
+
+def get_shape_path(dataset_name):
+  return path.join(SCRIPT_DIR, dataset_name, dataset_name + ".shp")
+
+
+def lower_dict_keys(input_dict):
+  return dict((k.lower(), v) for k, v in input_dict.iteritems())
+
+
+def convert_locale_ident(locale_ident):
+  """
+  Return the locale identifie converting dashes to underscores.
+
+  Example: en-US becomes en_US
+  """
+  return locale_ident.replace("-", "_")
+
+
+def get_locale_language(locale_ident):
+  """
+  Return a langauge code from locale identifier.
+
+  Example #1: en-US, the function returns en
+  Example #2: en, the function returns en
+  """
+  return locale_ident.split("-")[0]
+
+
+def request_relays():
+  data = json.dumps({"jsonrpc": "2.0", "id": "0", "method": "relay_list_v2"})
+  headers = {"Content-Type": "application/json"}
+  request = urllib2.Request("https://api.mullvad.net/rpc/", data, headers)
+  return json.load(urllib2.urlopen(request))
+
+
+# Program main()
+
+def main():
+  # ensure output path exists
+  if not path.exists(OUT_DIR):
+    os.makedirs(OUT_DIR)
+
+  # ensure locales output path exists
+  if not path.exists(LOCALE_OUT_DIR):
+    os.makedirs(LOCALE_OUT_DIR)
+
+  # extract geo data
+  extract_cities()
+  extract_countries()
+  extract_geometry()
+  extract_provinces_and_states_lines()
+
+  # extract translations
+  extract_countries_po()
+  extract_cities_po()
+  extract_relay_translations()
+
+main()
diff --git a/gui/scripts/extract-translations.js b/gui/scripts/extract-translations.js
index 07383ae553..b9a895d37c 100644
--- a/gui/scripts/extract-translations.js
+++ b/gui/scripts/extract-translations.js
@@ -11,27 +11,27 @@ const comments = {
 
 extractor
   .createJsParser([
-    JsExtractors.callExpression('gettext', {
+    JsExtractors.callExpression('messages.gettext', {
       arguments: {
         text: 0,
       },
       comments,
     }),
-    JsExtractors.callExpression('pgettext', {
+    JsExtractors.callExpression('messages.pgettext', {
       arguments: {
         context: 0,
         text: 1,
       },
       comments,
     }),
-    JsExtractors.callExpression('ngettext', {
+    JsExtractors.callExpression('messages.ngettext', {
       arguments: {
         text: 0,
         textPlural: 1,
       },
       comments,
     }),
-    JsExtractors.callExpression('npgettext', {
+    JsExtractors.callExpression('messages.npgettext', {
       arguments: {
         context: 0,
         text: 1,
diff --git a/gui/scripts/integrate-into-app.py b/gui/scripts/integrate-into-app.py
new file mode 100644
index 0000000000..9a853573c7
--- /dev/null
+++ b/gui/scripts/integrate-into-app.py
@@ -0,0 +1,102 @@
+"""
+A helper script to integrate the generated geo data into the app.
+"""
+
+import os
+from os import path
+from subprocess import Popen, PIPE
+import shutil
+import colorful as c
+
+SCRIPT_DIR = path.dirname(path.realpath(__file__))
+SOURCE_DIR = path.join(SCRIPT_DIR, "out")
+GEO_ASSETS_DEST_DIR = path.realpath(path.join(SCRIPT_DIR, "../assets/geo"))
+TRANSLATIONS_SOURCE_DIR = path.join(SOURCE_DIR, "locales")
+TRANSLATIONS_DEST_DIR = path.realpath(path.join(SCRIPT_DIR, "../locales"))
+
+GEO_ASSETS_TO_COPY = [
+  "cities.rbush.json",
+  "countries.rbush.json",
+  "geometry.json",
+  "geometry.rbush.json",
+  "states-provinces-lines.json",
+  "states-provinces-lines.rbush.json",
+]
+
+TRANSLATIONS_TO_COPY = [
+  "cities.po",
+  "countries.po"
+]
+
+TRANSLATIONS_TO_MERGE = [
+  "relay-locations.po"
+]
+
+def remove_common_prefix(source, destination):
+  prefix_len = len(path.commonprefix((source, destination)))
+  return (source[prefix_len:], destination[prefix_len:])
+
+def run_program(args):
+  p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+  errors = p.communicate()[1]
+  return (p.returncode, errors)
+
+def copy_geo_assets():
+  for f in GEO_ASSETS_TO_COPY:
+    src = path.join(SOURCE_DIR, f)
+    dst = path.join(GEO_ASSETS_DEST_DIR, f)
+
+    print "Copying {} to {}".format(*remove_common_prefix(src, dst))
+
+    shutil.copyfile(src, dst)
+
+def copy_and_merge_translations():
+  for f in os.listdir(TRANSLATIONS_SOURCE_DIR):
+    src = path.join(TRANSLATIONS_SOURCE_DIR, f)
+    dst = path.join(TRANSLATIONS_DEST_DIR, f)
+
+    if path.isdir(src):
+      merge_single_locale_folder(src, dst)
+    else:
+      print "Copying {} to {}".format(*remove_common_prefix(src, dst))
+      shutil.copyfile(src, dst)
+
+def merge_single_locale_folder(src, dst):
+  for f in os.listdir(src):
+    src_po = path.join(src, f)
+    dst_po = path.join(dst, f)
+
+    if f in TRANSLATIONS_TO_COPY:
+      print "Copying {} to {}".format(*remove_common_prefix(src_po, dst_po))
+      shutil.copyfile(src_po, dst_po)
+    elif f in TRANSLATIONS_TO_MERGE:
+      if path.exists(dst_po):
+        pot_basename = path.basename(path.splitext(dst_po)[0])
+        pot_path = path.join(TRANSLATIONS_DEST_DIR, pot_basename + ".pot")
+
+        (msgmerge_code, msgmerge_errors) = run_program([
+          "msgmerge", "--update", "--no-fuzzy-matching", dst_po, pot_path])
+
+        if msgmerge_code == 0:
+          (msgcat_code, msgcat_errors) = run_program([
+            "msgcat", src_po, dst_po, "--output-file", dst_po])
+
+          if msgcat_code == 0:
+            print c.green("Merged and concatenated the catalogues.")
+          else:
+            print c.red("msgcat exited with {}: {}".format(
+              msgcat_code, msgcat_errors.decode('utf-8').strip()))
+        else:
+          print c.red("msgmerge exited with {}: {}".format(
+            msgmerge_code, msgmerge_errors.decode('utf-8').strip()))
+      else:
+        shutil.copy(src_po, dst_po)
+    else:
+      print c.orange("Unexpected file: {}".format(src_po))
+
+
+if not path.exists(GEO_ASSETS_DEST_DIR):
+  os.makedirs(GEO_ASSETS_DEST_DIR)
+
+copy_geo_assets()
+copy_and_merge_translations()
diff --git a/gui/scripts/prepare-rtree.ts b/gui/scripts/prepare-rtree.ts
new file mode 100644
index 0000000000..3f2519d0c0
--- /dev/null
+++ b/gui/scripts/prepare-rtree.ts
@@ -0,0 +1,104 @@
+//
+// Script that generates r-trees for geo data.
+// run with `npx babel-node geo-data/prepare-rtree.js`
+//
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { Topology, GeometryCollection } from 'topojson-specification';
+import { GeoJSON } from 'geojson';
+import rbush from 'rbush';
+
+interface GeometryTopologyObjects {
+  [key: string]: any;
+  geometry: GeometryCollection;
+}
+
+function main() {
+  const GEOMETRY_DATA_FILES = ['geometry', 'states-provinces-lines'];
+  const POINT_DATA_FILES = ['countries', 'cities'];
+  const OUTPUT_DIR = path.join(__dirname, 'out');
+
+  for (const name of GEOMETRY_DATA_FILES) {
+    const source = path.join(OUTPUT_DIR, `${name}.json`);
+    const destination = path.join(OUTPUT_DIR, `${name}.rbush.json`);
+
+    try {
+      processGeometry(source, destination);
+    } catch (error) {
+      console.error(`Failed to process ${name}: ${error.message}`);
+    }
+  }
+
+  for (const name of POINT_DATA_FILES) {
+    const source = path.join(OUTPUT_DIR, `${name}.json`);
+    const destination = path.join(OUTPUT_DIR, `${name}.rbush.json`);
+
+    try {
+      processPoints(source, destination);
+    } catch (error) {
+      console.error(`Failed to process ${name}: ${error.message}`);
+    }
+  }
+}
+
+function processGeometry(source: string, destination: string) {
+  const collection = JSON.parse(fs.readFileSync(source, { encoding: 'utf8' })) as Topology<
+    GeometryTopologyObjects
+  >;
+
+  const { geometry } = collection.objects;
+  const treeData = geometry.geometries.map((object, i) => {
+    if (!object.bbox) {
+      throw new Error(`Expected a geometry at index ${i} to have a bbox property.`);
+    }
+
+    const [minX, minY, maxX, maxY] = object.bbox;
+    return {
+      ...object,
+      minX,
+      minY,
+      maxX,
+      maxY,
+    };
+  });
+
+  const tree = rbush();
+  tree.load(treeData);
+  fs.writeFileSync(destination, JSON.stringify(tree.toJSON()));
+
+  console.log(`Saved a rbush to ${destination}`);
+}
+
+function processPoints(source: string, destination: string) {
+  const collection = JSON.parse(fs.readFileSync(source, { encoding: 'utf8' })) as GeoJSON;
+
+  if (collection.type !== 'FeatureCollection') {
+    throw new Error(
+      `Invalid collection type ${collection.type} in ${source}. Expected FeatureCollection`,
+    );
+  }
+
+  const treeData = collection.features.map((feat) => {
+    if (feat.geometry.type !== 'Point') {
+      throw new Error(`Invalid geometry in ${source}. Expected "Point", got ${feat.geometry.type}`);
+    }
+
+    const { coordinates } = feat.geometry;
+    return {
+      ...feat,
+      minX: coordinates[0],
+      minY: coordinates[1],
+      maxX: coordinates[0],
+      maxY: coordinates[1],
+    };
+  });
+
+  const tree = rbush();
+  tree.load(treeData);
+  fs.writeFileSync(destination, JSON.stringify(tree.toJSON()));
+
+  console.log(`Saved a rbush to ${destination}`);
+}
+
+main();
diff --git a/gui/scripts/pylintrc b/gui/scripts/pylintrc
new file mode 100644
index 0000000000..7f79439857
--- /dev/null
+++ b/gui/scripts/pylintrc
@@ -0,0 +1,15 @@
+[FORMAT]
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='  '
+
+# Number of spaces of indent required inside a hanging  or continued line.
+indent-after-paren=2
+
+[MESSAGES CONTROL]
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+disable=invalid-name
diff --git a/gui/scripts/requirements.txt b/gui/scripts/requirements.txt
new file mode 100644
index 0000000000..4693c81bf8
--- /dev/null
+++ b/gui/scripts/requirements.txt
@@ -0,0 +1,33 @@
+Fiona==1.8.6 \
+    --hash=sha256:1353a0c7d03f96b0755d13bc0277b0b6a79585f94981e1ae6452c55958431b2b \
+    --hash=sha256:1783c3abceccca318fac3d5bfd4305bc31c94fbb5abdc60e96b0ccedaa263857 \
+    --hash=sha256:22dfc9d82adae78567b2a242f297915aeb7727cde06a3d762252e5393af959c5 \
+    --hash=sha256:2e1ba6e87f9a63962af4c4c44152f9f49245c29fd90ce1c4bb19bddbb31e22f4 \
+    --hash=sha256:6c2b665ad369768cde426def664a9af46263152731b4a9493380a337f6c5a79e \
+    --hash=sha256:7ebfdce0be685aa44c31529d57cdea232c2386ad305c11a329f91c7613dbd2f4 \
+    --hash=sha256:8e361b3cf00a472570fdd25ad39c722f1b3fda03fc4de1d35aac2b5da39bcdb3 \
+    --hash=sha256:cb9081003e8585a284365b28cff61343541405263fd4e1c79f3baec7321b8b3f \
+    --hash=sha256:e3bfabe62c4081c88c47aa10c6165ecfefaa7c9aa5406496b75ff364757a0bbe \
+    --hash=sha256:ee2bbe5640f68342b6746a0ba733e25ee221106368558be6cd73dd08fc6fbe04 \
+    --hash=sha256:fa31dfe8855b9cd0b128b47a4df558f1b8eda90d2181bff1dd9854e5556efb3e
+Shapely==1.6.4.post2 \
+    --hash=sha256:0378964902f89b8dbc332e5bdfa08e0bc2f7ab39fecaeb17fbb2a7699a44fe71 \
+    --hash=sha256:34e7c6f41fb27906ccdf2514ee44a5774b90b39a256b6511a6a57d11ffe64999 \
+    --hash=sha256:3ca69d4b12e2b05b549465822744b6a3a1095d8488cc27b2728a06d3c07d0eee \
+    --hash=sha256:3e9388f29bd81fcd4fa5c35125e1fbd4975ee36971a87a90c093f032d0e9de24 \
+    --hash=sha256:3ef28e3f20a1c37f5b99ea8cf8dcb58e2f1a8762d65ed2d21fd92bf1d4811182 \
+    --hash=sha256:523c94403047eb6cacd7fc1863ebef06e26c04d8a4e7f8f182d49cd206fe787e \
+    --hash=sha256:5d22a1a705c2f70f61ccadc696e33d922c1a92e00df8e1d58a6ade14dd7e3b4f \
+    --hash=sha256:714b6680215554731389a1bbdae4cec61741aa4726921fa2b2b96a6f578a2534 \
+    --hash=sha256:7dfe1528650c3f0dc82f41a74cf4f72018288db9bfb75dcd08f6f04233ec7e78 \
+    --hash=sha256:ba58b21b9cf3c33725f7f530febff9ed6a6846f9d0bf8a120fc74683ff919f89 \
+    --hash=sha256:c4b87bb61fc3de59fc1f85e71a79b0c709dc68364d9584473697aad4aa13240f \
+    --hash=sha256:ebb4d2bee7fac3f6c891fcdafaa17f72ab9c6480f6d00de0b2dc9a5137dfe342
+polib==1.1.0 \
+    --hash=sha256:93b730477c16380c9a96726c54016822ff81acfa553977fdd131f2b90ba858d7 \
+    --hash=sha256:fad87d13696127ffb27ea0882d6182f1a9cf8a5e2b37a587751166c51e5a332a
+colorful==0.5.0 \
+    --hash=sha256:74bbaeb90fb5b3d31fccdc6e562bdce721f19dc6d566f72a31e43f5503ac2841 \
+    --hash=sha256:86db34ba9b41106d7c69a394049e9d47be4cf15f863d04b8a0b21509ea9df6bc
+terminaltables==3.1.0 \
+    --hash=sha256:f3eb0eb92e3833972ac36796293ca0906e998dc3be91fbe1f8615b331b853b81