diff options
Diffstat (limited to 'gui/scripts')
| -rwxr-xr-x | gui/scripts/extract-geo-data.py | 141 | ||||
| -rw-r--r-- | gui/scripts/prepare-rtree.ts | 9 | ||||
| -rw-r--r-- | gui/scripts/requirements.txt | 54 |
3 files changed, 82 insertions, 122 deletions
diff --git a/gui/scripts/extract-geo-data.py b/gui/scripts/extract-geo-data.py index 0619ed375b..2198a6f7e0 100755 --- a/gui/scripts/extract-geo-data.py +++ b/gui/scripts/extract-geo-data.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - """ This module forms a geo json of highly populated cities in the world """ @@ -108,18 +107,48 @@ def extract_relay_translations(): print(c.red("Failed to fetch the relays list: {}".format(e))) raise - result = response.get("result") - if result is not None: - countries = result.get("countries") - if countries is None: - raise Exception("Missing the countries field.") - else: - raise Exception("Missing the result field.") + locations = response.get("locations") + countries = structure_locations(locations) extract_relay_locations_pot(countries) translate_relay_locations(countries) +def structure_locations(locations): + countries = {} + + for location_key in locations: + location = locations.get(location_key) + country_name = location.get("country") + city_name = location.get("city") + + if not "-" in location_key: + print("Location key incorrectly formatted: {}".format(location_key)) + continue + + country_code, city_code = location_key.split("-") + + if country_name is None: + print("Country name missing for {}".format(location_key)) + continue + + if city_name is None: + print("City name missing for {}".format(location_key)) + continue + + if country_code not in countries: + countries[country_code] = {"name": country_name, "cities": {}} + + country = countries[country_code] + cities = country["cities"] + if city_code not in cities: + cities[city_code] = city_name + else: + print("There are multiple entries for {} in {}".format(city_name, country_name)) + + return countries + + def extract_relay_locations_pot(countries): pot = POFile(encoding='utf-8', check_for_duplicates=True) pot.metadata = {"Content-Type": "text/plain; charset=utf-8"} @@ -127,34 +156,27 @@ def extract_relay_locations_pot(countries): print("Generating {}".format(output_path)) - for country in countries: - country_name = country.get("name") - if country_name is not None: + for country_code in countries: + country = countries[country_code] + entry = POEntry( + msgid=country["name"], + msgstr="", + comment=country_code.upper() + ) + pot.append(entry) + + cities = country["cities"] + for city_code in cities: entry = POEntry( - msgid=country_name, + msgid=cities[city_code], msgstr="", - comment=country.get("code").upper() + comment="{} {}".format(country_code.upper(), city_code.upper()) ) - pot.append(entry) - print("{} ({})".format(country_name, country.get("code"))) - - cities = country.get("cities") - if cities is not None: - for city in cities: - city_name = city.get("name") - if city_name is not None: - entry = POEntry( - msgid=city_name, - msgstr="", - comment="{} {}".format(country.get("code").upper(), city.get("code").upper()) - ) - try: - pot.append(entry) - except ValueError as err: - print(c.orange("Cannot add an entry: {}".format(err))) - - print("{} ({})".format(city_name, city.get("code"))) + try: + pot.append(entry) + except ValueError as err: + print(c.orange("Cannot add an entry: {}".format(err))) pot.save(output_path) @@ -228,25 +250,18 @@ def translate_single_relay_locations(country_translator, city_translator, countr if not path.exists(locale_out_dir): os.makedirs(locale_out_dir) - for country in countries: - country_name = country.get("name") - country_code = country.get("code") + for country_code in countries: + country = countries[country_code] + country_name = country["name"] translated_country_name = country_translator.translate(locale, country_code) - found_country_translation = translated_country_name is not None # Default to empty string if no translation was found - if found_country_translation: + if translated_country_name is not None: hits += 1 else: translated_country_name = "" misses += 1 - log_message = "{} ({}) -> \"{}\"".format(country_name, country_code, translated_country_name) - if found_country_translation: - print(c.green(log_message)) - else: - print(c.orange(log_message)) - # translate country entry = POEntry( msgid=country_name, @@ -256,17 +271,9 @@ def translate_single_relay_locations(country_translator, city_translator, countr po.append(entry) # translate cities - cities = country.get("cities") - if cities is None: - print(c.orange("Skip {} ({}) because no cities were found." - .format(country_name, country_code))) - continue - - for city in cities: - city_name = city.get("name") - city_code = city.get("code") - if city_name is None: - raise ValueError("Missing the name field in city record.") + cities = country["cities"] + for city_code in cities: + city_name = cities[city_code] # Make sure to append the US state back to the translated name of the city if country_code == "us": @@ -286,12 +293,6 @@ def translate_single_relay_locations(country_translator, city_translator, countr translated_name = "" misses += 1 - log_message = "{} ({}) -> \"{}\"".format(city_name, city_code, translated_name) - if found_translation: - print(c.green(log_message)) - else: - print(c.orange(log_message)) - entry = POEntry( msgid=city_name, msgstr=translated_name, @@ -331,13 +332,7 @@ class CountryTranslator: if props is not None: name_key = "name_" + map_locale(locale) - value = props.get(name_key) - - if value is None: - print(c.orange("Missing translation for {} ({}) under the {} key" - .format(iso_a2, locale, name_key))) - else: - return value + return props.get(name_key) return None @@ -384,13 +379,7 @@ class CityTranslator: if props is not None: name_key = "name_" + map_locale(locale) - value = props.get(name_key) - - if value is None: - print(c.orange("Missing translation for {} ({}) under the {} key" - .format(english_name, locale, name_key))) - else: - return value + return props.get(name_key) return None @@ -457,9 +446,7 @@ def map_locale(locale_ident): def request_relays(): - data = json.dumps({"jsonrpc": "2.0", "id": "0", "method": "relay_list_v3"}).encode() - request = urllib.request.Request("https://api.mullvad.net/rpc/", data=data) - request.add_header("Content-Type", "application/json") + request = urllib.request.Request("https://api.mullvad.net/app/v1/relays") with urllib.request.urlopen(request) as connection: return json.load(connection) diff --git a/gui/scripts/prepare-rtree.ts b/gui/scripts/prepare-rtree.ts index f18f772aa0..27fabb2f13 100644 --- a/gui/scripts/prepare-rtree.ts +++ b/gui/scripts/prepare-rtree.ts @@ -23,16 +23,17 @@ function main() { try { processGeometry(source, destination); - } catch (error) { + } catch (e) { + const error = e as Error; console.error(`Failed to process ${name}: ${error.message}`); } } } function processGeometry(source: string, destination: string) { - const collection = JSON.parse(fs.readFileSync(source, { encoding: 'utf8' })) as Topology< - GeometryTopologyObjects - >; + const collection = JSON.parse( + fs.readFileSync(source, { encoding: 'utf8' }), + ) as Topology<GeometryTopologyObjects>; const { geometry } = collection.objects; const treeData = geometry.geometries.map((object, i) => { diff --git a/gui/scripts/requirements.txt b/gui/scripts/requirements.txt index 22b9af315b..00899fc5d5 100644 --- a/gui/scripts/requirements.txt +++ b/gui/scripts/requirements.txt @@ -2,12 +2,12 @@ Fiona==1.8.13.post1 \ --hash=sha256:1a432bf9fd56f089256c010da009c90d4a795c531a848132c965052185336600 \ --hash=sha256:79c3b80e00c9d055d20aead5d74319f54cdd1384e0d9e1a9e67446da2d74d89c \ --hash=sha256:923a64bded457adee795b4f926b8cbb87d58bbafaabded77bc1d47abb2bba5c6 -Shapely==1.7.0 \ - --hash=sha256:234c5424d61d8b263d6d20045f5f32437819627ca57c1ea0c08368013b49824b \ - --hash=sha256:29be7767a32df19e2186288cee63e539b386a35139524dc22eeceb244d0b092b \ - --hash=sha256:9c62a9f7adceaa3110f2ec359c70dddd1640191609e91029e4d307e63fc8a5af \ - --hash=sha256:ae9a2da2b30c0b42029337854f78c71c28d285d254efd5f3be3700d997bfd18e \ - --hash=sha256:e21a9fe1a416463ff11ae037766fe410526c95700b9e545372475d2361cc951e +Shapely==1.7.1 \ + --hash=sha256:1641724c1055459a7e2b8bbe47ba25bdc89554582e62aec23cb3f3ca25f9b129 \ + --hash=sha256:182716ffb500d114b5d1b75d7fd9d14b7d3414cef3c38c0490534cc9ce20981a \ + --hash=sha256:35be1c5d869966569d3dfd4ec31832d7c780e9df760e1fe52131105685941891 \ + --hash=sha256:4f3c59f6dbf86a9fc293546de492f5e07344e045f9333f3a753f2dda903c45d1 \ + --hash=sha256:6871acba8fbe744efa4f9f34e726d070bfbf9bffb356a8f6d64557846324232b polib==1.1.0 \ --hash=sha256:93b730477c16380c9a96726c54016822ff81acfa553977fdd131f2b90ba858d7 \ --hash=sha256:fad87d13696127ffb27ea0882d6182f1a9cf8a5e2b37a587751166c51e5a332a @@ -50,41 +50,13 @@ defusedxml==0.6.0 \ munch==2.5.0 \ --hash=sha256:2d735f6f24d4dba3417fa448cae40c6e896ec1fdab6cdb5e6510999758a4dbd2 \ --hash=sha256:6f44af89a2ce4ed04ff8de41f70b226b984db10a91dcc7b9ac2efc1c77022fdd -Pillow==8.2.0 \ - --hash=sha256:dc38f57d8f20f06dd7c3161c59ca2c86893632623f33a42d592f097b00f720a9 \ - --hash=sha256:a013cbe25d20c2e0c4e85a9daf438f85121a4d0344ddc76e33fd7e3965d9af4b \ - --hash=sha256:8bb1e155a74e1bfbacd84555ea62fa21c58e0b4e7e6b20e4447b8d07990ac78b \ - --hash=sha256:c5236606e8570542ed424849f7852a0ff0bce2c4c8d0ba05cc202a5a9c97dee9 \ - --hash=sha256:12e5e7471f9b637762453da74e390e56cc43e486a88289995c1f4c1dc0bfe727 \ - --hash=sha256:5afe6b237a0b81bd54b53f835a153770802f164c5570bab5e005aad693dab87f \ - --hash=sha256:cb7a09e173903541fa888ba010c345893cd9fc1b5891aaf060f6ca77b6a3722d \ - --hash=sha256:0d19d70ee7c2ba97631bae1e7d4725cdb2ecf238178096e8c82ee481e189168a \ - --hash=sha256:083781abd261bdabf090ad07bb69f8f5599943ddb539d64497ed021b2a67e5a9 \ - --hash=sha256:c6b39294464b03457f9064e98c124e09008b35a62e3189d3513e5148611c9388 \ - --hash=sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5 \ - --hash=sha256:3b570f84a6161cf8865c4e08adf629441f56e32f180f7aa4ccbd2e0a5a02cba2 \ - --hash=sha256:031a6c88c77d08aab84fecc05c3cde8414cd6f8406f4d2b16fed1e97634cc8a4 \ - --hash=sha256:66cc56579fd91f517290ab02c51e3a80f581aba45fd924fcdee01fa06e635812 \ - --hash=sha256:6c32cc3145928c4305d142ebec682419a6c0a8ce9e33db900027ddca1ec39178 \ - --hash=sha256:624b977355cde8b065f6d51b98497d6cd5fbdd4f36405f7a8790e3376125e2bb \ - --hash=sha256:5cbf3e3b1014dddc45496e8cf38b9f099c95a326275885199f427825c6522232 \ - --hash=sha256:463822e2f0d81459e113372a168f2ff59723e78528f91f0bd25680ac185cf797 \ - --hash=sha256:95d5ef984eff897850f3a83883363da64aae1000e79cb3c321915468e8c6add5 \ - --hash=sha256:b91c36492a4bbb1ee855b7d16fe51379e5f96b85692dc8210831fbb24c43e484 \ - --hash=sha256:d68cb92c408261f806b15923834203f024110a2e2872ecb0bd2a110f89d3c602 \ - --hash=sha256:f217c3954ce5fd88303fc0c317af55d5e0204106d86dea17eb8205700d47dec2 \ - --hash=sha256:5b70110acb39f3aff6b74cf09bb4169b167e2660dabc304c1e25b6555fa781ef \ - --hash=sha256:a7d5e9fad90eff8f6f6106d3b98b553a88b6f976e51fce287192a5d2d5363713 \ - --hash=sha256:238c197fc275b475e87c1453b05b467d2d02c2915fdfdd4af126145ff2e4610c \ - --hash=sha256:0e04d61f0064b545b989126197930807c86bcbd4534d39168f4aa5fda39bb8f9 \ - --hash=sha256:63728564c1410d99e6d1ae8e3b810fe012bc440952168af0a2877e8ff5ab96b9 \ - --hash=sha256:c03c07ed32c5324939b19e36ae5f75c660c81461e312a41aea30acdd46f93a7c \ - --hash=sha256:4d98abdd6b1e3bf1a1cbb14c3895226816e666749ac040c4e2554231068c639b \ - --hash=sha256:aac00e4bc94d1b7813fe882c28990c1bc2f9d0e1aa765a5f2b516e8a6a16a9e4 \ - --hash=sha256:22fd0f42ad15dfdde6c581347eaa4adb9a6fc4b865f90b23378aa7914895e120 \ - --hash=sha256:e98eca29a05913e82177b3ba3d198b1728e164869c613d76d0de4bde6768a50e \ - --hash=sha256:8b56553c0345ad6dcb2e9b433ae47d67f95fc23fe28a0bde15a120f25257e291 \ - --hash=sha256:a787ab10d7bb5494e5f76536ac460741788f1fbce851068d73a87ca7c35fc3e1 +Pillow==8.4.0 \ + --hash=sha256:1394a6ad5abc838c5cd8a92c5a07535648cdf6d09e8e2d6df916dfa9ea86ead8 \ + --hash=sha256:792e5c12376594bfcb986ebf3855aa4b7c225754e9a9521298e460e92fb4a488 \ + --hash=sha256:7b7017b61bbcdd7f6363aeceb881e23c46583739cb69a3ab39cb384f6ec82e5b \ + --hash=sha256:8c803ac3c28bbc53763e6825746f05cc407b20e4a69d0122e526a582e3b5e153 \ + --hash=sha256:b8e2f83c56e141920c39464b852de3719dfbfb6e3c99a2d8da0edf4fb33176ed \ + --hash=sha256:d99ec152570e4196772e7a8e4ba5320d2d27bf22fdf11743dd882936ed64305b pycparser==2.20 \ --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 |