Merge branch 'update-map-and-translation-scripts'

3 files changed, 82 insertions, 122 deletions

diff --git a/gui/scripts/extract-geo-data.py b/gui/scripts/extract-geo-data.py
index 0619ed375b..2198a6f7e0 100755
--- a/gui/scripts/extract-geo-data.py
+++ b/gui/scripts/extract-geo-data.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 """
 This module forms a geo json of highly populated cities in the world
 """
@@ -108,18 +107,48 @@ def extract_relay_translations():
     print(c.red("Failed to fetch the relays list: {}".format(e)))
     raise
 
-  result = response.get("result")
-  if result is not None:
-    countries = result.get("countries")
-    if countries is None:
-      raise Exception("Missing the countries field.")
-  else:
-    raise Exception("Missing the result field.")
+  locations = response.get("locations")
+  countries = structure_locations(locations)
 
   extract_relay_locations_pot(countries)
   translate_relay_locations(countries)
 
 
+def structure_locations(locations):
+  countries = {}
+
+  for location_key in locations:
+    location = locations.get(location_key)
+    country_name = location.get("country")
+    city_name = location.get("city")
+
+    if not "-" in location_key:
+      print("Location key incorrectly formatted: {}".format(location_key))
+      continue
+
+    country_code, city_code = location_key.split("-")
+
+    if country_name is None:
+      print("Country name missing for {}".format(location_key))
+      continue
+
+    if city_name is None:
+      print("City name missing for {}".format(location_key))
+      continue
+
+    if country_code not in countries:
+      countries[country_code] = {"name": country_name, "cities": {}}
+
+    country = countries[country_code]
+    cities = country["cities"]
+    if city_code not in cities:
+      cities[city_code] = city_name
+    else:
+      print("There are multiple entries for {} in {}".format(city_name, country_name))
+
+  return countries
+
+
 def extract_relay_locations_pot(countries):
   pot = POFile(encoding='utf-8', check_for_duplicates=True)
   pot.metadata = {"Content-Type": "text/plain; charset=utf-8"}
@@ -127,34 +156,27 @@ def extract_relay_locations_pot(countries):
 
   print("Generating {}".format(output_path))
 
-  for country in countries:
-    country_name = country.get("name")
-    if country_name is not None:
+  for country_code in countries:
+    country = countries[country_code]
+    entry = POEntry(
+      msgid=country["name"],
+      msgstr="",
+      comment=country_code.upper()
+    )
+    pot.append(entry)
+
+    cities = country["cities"]
+    for city_code in cities:
       entry = POEntry(
-        msgid=country_name,
+        msgid=cities[city_code],
         msgstr="",
-        comment=country.get("code").upper()
+        comment="{} {}".format(country_code.upper(), city_code.upper())
       )
-      pot.append(entry)
-      print("{} ({})".format(country_name, country.get("code")))
-
-    cities = country.get("cities")
-    if cities is not None:
-      for city in cities:
-        city_name = city.get("name")
-        if city_name is not None:
-          entry = POEntry(
-            msgid=city_name,
-            msgstr="",
-            comment="{} {}".format(country.get("code").upper(), city.get("code").upper())
-          )
 
-          try:
-            pot.append(entry)
-          except ValueError as err:
-            print(c.orange("Cannot add an entry: {}".format(err)))
-
-          print("{} ({})".format(city_name, city.get("code")))
+      try:
+        pot.append(entry)
+      except ValueError as err:
+        print(c.orange("Cannot add an entry: {}".format(err)))
 
   pot.save(output_path)
 
@@ -228,25 +250,18 @@ def translate_single_relay_locations(country_translator, city_translator, countr
   if not path.exists(locale_out_dir):
     os.makedirs(locale_out_dir)
 
-  for country in countries:
-    country_name = country.get("name")
-    country_code = country.get("code")
+  for country_code in countries:
+    country = countries[country_code]
+    country_name = country["name"]
 
     translated_country_name = country_translator.translate(locale, country_code)
-    found_country_translation = translated_country_name is not None
     # Default to empty string if no translation was found
-    if found_country_translation:
+    if translated_country_name is not None:
       hits += 1
     else:
       translated_country_name = ""
       misses += 1
 
-    log_message = "{} ({}) -> \"{}\"".format(country_name, country_code, translated_country_name)
-    if found_country_translation:
-      print(c.green(log_message))
-    else:
-      print(c.orange(log_message))
-
     # translate country
     entry = POEntry(
       msgid=country_name,
@@ -256,17 +271,9 @@ def translate_single_relay_locations(country_translator, city_translator, countr
     po.append(entry)
 
     # translate cities
-    cities = country.get("cities")
-    if cities is None:
-      print(c.orange("Skip {} ({}) because no cities were found."
-        .format(country_name, country_code)))
-      continue
-
-    for city in cities:
-      city_name = city.get("name")
-      city_code = city.get("code")
-      if city_name is None:
-        raise ValueError("Missing the name field in city record.")
+    cities = country["cities"]
+    for city_code in cities:
+      city_name = cities[city_code]
 
       # Make sure to append the US state back to the translated name of the city
       if country_code == "us":
@@ -286,12 +293,6 @@ def translate_single_relay_locations(country_translator, city_translator, countr
         translated_name = ""
         misses += 1
 
-      log_message = "{} ({}) -> \"{}\"".format(city_name, city_code, translated_name)
-      if found_translation:
-        print(c.green(log_message))
-      else:
-        print(c.orange(log_message))
-
       entry = POEntry(
         msgid=city_name,
         msgstr=translated_name,
@@ -331,13 +332,7 @@ class CountryTranslator:
 
     if props is not None:
       name_key = "name_" + map_locale(locale)
-      value = props.get(name_key)
-
-      if value is None:
-        print(c.orange("Missing translation for {} ({}) under the {} key"
-          .format(iso_a2, locale, name_key)))
-      else:
-        return value
+      return props.get(name_key)
 
     return None
 
@@ -384,13 +379,7 @@ class CityTranslator:
 
     if props is not None:
       name_key = "name_" + map_locale(locale)
-      value = props.get(name_key)
-
-      if value is None:
-        print(c.orange("Missing translation for {} ({}) under the {} key"
-          .format(english_name, locale, name_key)))
-      else:
-        return value
+      return props.get(name_key)
 
     return None
 
@@ -457,9 +446,7 @@ def map_locale(locale_ident):
 
 
 def request_relays():
-  data = json.dumps({"jsonrpc": "2.0", "id": "0", "method": "relay_list_v3"}).encode()
-  request = urllib.request.Request("https://api.mullvad.net/rpc/", data=data)
-  request.add_header("Content-Type", "application/json")
+  request = urllib.request.Request("https://api.mullvad.net/app/v1/relays")
   with urllib.request.urlopen(request) as connection:
     return json.load(connection)
 
diff --git a/gui/scripts/prepare-rtree.ts b/gui/scripts/prepare-rtree.ts
index f18f772aa0..27fabb2f13 100644
--- a/gui/scripts/prepare-rtree.ts
+++ b/gui/scripts/prepare-rtree.ts
@@ -23,16 +23,17 @@ function main() {
 
     try {
       processGeometry(source, destination);
-    } catch (error) {
+    } catch (e) {
+      const error = e as Error;
       console.error(`Failed to process ${name}: ${error.message}`);
     }
   }
 }
 
 function processGeometry(source: string, destination: string) {
-  const collection = JSON.parse(fs.readFileSync(source, { encoding: 'utf8' })) as Topology<
-    GeometryTopologyObjects
-  >;
+  const collection = JSON.parse(
+    fs.readFileSync(source, { encoding: 'utf8' }),
+  ) as Topology<GeometryTopologyObjects>;
 
   const { geometry } = collection.objects;
   const treeData = geometry.geometries.map((object, i) => {
diff --git a/gui/scripts/requirements.txt b/gui/scripts/requirements.txt
index 22b9af315b..00899fc5d5 100644
--- a/gui/scripts/requirements.txt
+++ b/gui/scripts/requirements.txt
@@ -2,12 +2,12 @@ Fiona==1.8.13.post1 \
     --hash=sha256:1a432bf9fd56f089256c010da009c90d4a795c531a848132c965052185336600 \
     --hash=sha256:79c3b80e00c9d055d20aead5d74319f54cdd1384e0d9e1a9e67446da2d74d89c \
     --hash=sha256:923a64bded457adee795b4f926b8cbb87d58bbafaabded77bc1d47abb2bba5c6
-Shapely==1.7.0 \
-    --hash=sha256:234c5424d61d8b263d6d20045f5f32437819627ca57c1ea0c08368013b49824b \
-    --hash=sha256:29be7767a32df19e2186288cee63e539b386a35139524dc22eeceb244d0b092b \
-    --hash=sha256:9c62a9f7adceaa3110f2ec359c70dddd1640191609e91029e4d307e63fc8a5af \
-    --hash=sha256:ae9a2da2b30c0b42029337854f78c71c28d285d254efd5f3be3700d997bfd18e \
-    --hash=sha256:e21a9fe1a416463ff11ae037766fe410526c95700b9e545372475d2361cc951e
+Shapely==1.7.1 \
+    --hash=sha256:1641724c1055459a7e2b8bbe47ba25bdc89554582e62aec23cb3f3ca25f9b129 \
+    --hash=sha256:182716ffb500d114b5d1b75d7fd9d14b7d3414cef3c38c0490534cc9ce20981a \
+    --hash=sha256:35be1c5d869966569d3dfd4ec31832d7c780e9df760e1fe52131105685941891 \
+    --hash=sha256:4f3c59f6dbf86a9fc293546de492f5e07344e045f9333f3a753f2dda903c45d1 \
+    --hash=sha256:6871acba8fbe744efa4f9f34e726d070bfbf9bffb356a8f6d64557846324232b
 polib==1.1.0 \
     --hash=sha256:93b730477c16380c9a96726c54016822ff81acfa553977fdd131f2b90ba858d7 \
     --hash=sha256:fad87d13696127ffb27ea0882d6182f1a9cf8a5e2b37a587751166c51e5a332a
@@ -50,41 +50,13 @@ defusedxml==0.6.0 \
 munch==2.5.0 \
     --hash=sha256:2d735f6f24d4dba3417fa448cae40c6e896ec1fdab6cdb5e6510999758a4dbd2 \
     --hash=sha256:6f44af89a2ce4ed04ff8de41f70b226b984db10a91dcc7b9ac2efc1c77022fdd
-Pillow==8.2.0 \
-    --hash=sha256:dc38f57d8f20f06dd7c3161c59ca2c86893632623f33a42d592f097b00f720a9 \
-    --hash=sha256:a013cbe25d20c2e0c4e85a9daf438f85121a4d0344ddc76e33fd7e3965d9af4b \
-    --hash=sha256:8bb1e155a74e1bfbacd84555ea62fa21c58e0b4e7e6b20e4447b8d07990ac78b \
-    --hash=sha256:c5236606e8570542ed424849f7852a0ff0bce2c4c8d0ba05cc202a5a9c97dee9 \
-    --hash=sha256:12e5e7471f9b637762453da74e390e56cc43e486a88289995c1f4c1dc0bfe727 \
-    --hash=sha256:5afe6b237a0b81bd54b53f835a153770802f164c5570bab5e005aad693dab87f \
-    --hash=sha256:cb7a09e173903541fa888ba010c345893cd9fc1b5891aaf060f6ca77b6a3722d \
-    --hash=sha256:0d19d70ee7c2ba97631bae1e7d4725cdb2ecf238178096e8c82ee481e189168a \
-    --hash=sha256:083781abd261bdabf090ad07bb69f8f5599943ddb539d64497ed021b2a67e5a9 \
-    --hash=sha256:c6b39294464b03457f9064e98c124e09008b35a62e3189d3513e5148611c9388 \
-    --hash=sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5 \
-    --hash=sha256:3b570f84a6161cf8865c4e08adf629441f56e32f180f7aa4ccbd2e0a5a02cba2 \
-    --hash=sha256:031a6c88c77d08aab84fecc05c3cde8414cd6f8406f4d2b16fed1e97634cc8a4 \
-    --hash=sha256:66cc56579fd91f517290ab02c51e3a80f581aba45fd924fcdee01fa06e635812 \
-    --hash=sha256:6c32cc3145928c4305d142ebec682419a6c0a8ce9e33db900027ddca1ec39178 \
-    --hash=sha256:624b977355cde8b065f6d51b98497d6cd5fbdd4f36405f7a8790e3376125e2bb \
-    --hash=sha256:5cbf3e3b1014dddc45496e8cf38b9f099c95a326275885199f427825c6522232 \
-    --hash=sha256:463822e2f0d81459e113372a168f2ff59723e78528f91f0bd25680ac185cf797 \
-    --hash=sha256:95d5ef984eff897850f3a83883363da64aae1000e79cb3c321915468e8c6add5 \
-    --hash=sha256:b91c36492a4bbb1ee855b7d16fe51379e5f96b85692dc8210831fbb24c43e484 \
-    --hash=sha256:d68cb92c408261f806b15923834203f024110a2e2872ecb0bd2a110f89d3c602 \
-    --hash=sha256:f217c3954ce5fd88303fc0c317af55d5e0204106d86dea17eb8205700d47dec2 \
-    --hash=sha256:5b70110acb39f3aff6b74cf09bb4169b167e2660dabc304c1e25b6555fa781ef \
-    --hash=sha256:a7d5e9fad90eff8f6f6106d3b98b553a88b6f976e51fce287192a5d2d5363713 \
-    --hash=sha256:238c197fc275b475e87c1453b05b467d2d02c2915fdfdd4af126145ff2e4610c \
-    --hash=sha256:0e04d61f0064b545b989126197930807c86bcbd4534d39168f4aa5fda39bb8f9 \
-    --hash=sha256:63728564c1410d99e6d1ae8e3b810fe012bc440952168af0a2877e8ff5ab96b9 \
-    --hash=sha256:c03c07ed32c5324939b19e36ae5f75c660c81461e312a41aea30acdd46f93a7c \
-    --hash=sha256:4d98abdd6b1e3bf1a1cbb14c3895226816e666749ac040c4e2554231068c639b \
-    --hash=sha256:aac00e4bc94d1b7813fe882c28990c1bc2f9d0e1aa765a5f2b516e8a6a16a9e4 \
-    --hash=sha256:22fd0f42ad15dfdde6c581347eaa4adb9a6fc4b865f90b23378aa7914895e120 \
-    --hash=sha256:e98eca29a05913e82177b3ba3d198b1728e164869c613d76d0de4bde6768a50e \
-    --hash=sha256:8b56553c0345ad6dcb2e9b433ae47d67f95fc23fe28a0bde15a120f25257e291 \
-    --hash=sha256:a787ab10d7bb5494e5f76536ac460741788f1fbce851068d73a87ca7c35fc3e1
+Pillow==8.4.0 \
+    --hash=sha256:1394a6ad5abc838c5cd8a92c5a07535648cdf6d09e8e2d6df916dfa9ea86ead8 \
+    --hash=sha256:792e5c12376594bfcb986ebf3855aa4b7c225754e9a9521298e460e92fb4a488 \
+    --hash=sha256:7b7017b61bbcdd7f6363aeceb881e23c46583739cb69a3ab39cb384f6ec82e5b \
+    --hash=sha256:8c803ac3c28bbc53763e6825746f05cc407b20e4a69d0122e526a582e3b5e153 \
+    --hash=sha256:b8e2f83c56e141920c39464b852de3719dfbfb6e3c99a2d8da0edf4fb33176ed \
+    --hash=sha256:d99ec152570e4196772e7a8e4ba5320d2d27bf22fdf11743dd882936ed64305b
 pycparser==2.20 \
     --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
     --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705