diff options
| author | Andrej Mihajlov <and@mullvad.net> | 2018-07-18 15:07:37 +0200 |
|---|---|---|
| committer | Andrej Mihajlov <and@mullvad.net> | 2018-08-15 17:39:38 +0200 |
| commit | 71592249b2dd669b6f24f37bfb7b0f4e43b74998 (patch) | |
| tree | a6097dc7e5d94d06e99c65fdfe160e824395f50c /geo-data | |
| parent | e84e87f4ce5a8c242f756566cdc6fb59a45f7bea (diff) | |
| download | mullvadvpn-71592249b2dd669b6f24f37bfb7b0f4e43b74998.tar.xz mullvadvpn-71592249b2dd669b6f24f37bfb7b0f4e43b74998.zip | |
Add workspaces
Diffstat (limited to 'geo-data')
| -rw-r--r-- | geo-data/README.md | 73 | ||||
| -rw-r--r-- | geo-data/extract-geo-data.py | 144 | ||||
| -rw-r--r-- | geo-data/integrate-into-app.py | 31 | ||||
| -rw-r--r-- | geo-data/prepare-rtree.js | 57 | ||||
| -rw-r--r-- | geo-data/pylintrc | 15 | ||||
| -rw-r--r-- | geo-data/requirements.txt | 10 |
6 files changed, 0 insertions, 330 deletions
diff --git a/geo-data/README.md b/geo-data/README.md deleted file mode 100644 index 22cdc5ee52..0000000000 --- a/geo-data/README.md +++ /dev/null @@ -1,73 +0,0 @@ -This is a folder with python 2 and node scripts to produce geographical -data for the Mullvad VPN app. - - -## Dependency installation notes - -1. Run the following command in terminal to install python dependencies: -`pip install -r requirements.txt` - -2. Run `npm install -g topojson-server` to install `geo2topo` tool which is -used by python scripts to convert GeoJSON to TopoJSON - - -## Geo data installation notes - -Go to http://www.naturalearthdata.com/downloads/50m-cultural-vectors/ and -download ZIP files with the following shapes: - -- Admin 0 – Countries -- Admin 1 – States, provinces - boundary lines -- Populated Places - simple dataset is enough - -or use cURL to download all ZIPs: - -``` -curl -L -O http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_countries.zip -curl -L -O http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip -curl -L -O http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_populated_places_simple.zip -``` - -Extract the downloaded ZIP files into geo-data. -Make sure the following folders exist after extraction: - -- ne_50m_admin_0_countries -- ne_50m_admin_1_states_provinces_lines -- ne_50m_populated_places_simple - -or use the following script: - -``` -mkdir ne_50m_admin_1_states_provinces_lines -mkdir ne_50m_populated_places_simple -mkdir ne_50m_admin_0_countries - -unzip ne_50m_admin_0_countries.zip -d ne_50m_admin_0_countries -unzip ne_50m_admin_1_states_provinces_lines.zip -d ne_50m_admin_1_states_provinces_lines -unzip ne_50m_populated_places_simple.zip -d ne_50m_populated_places_simple -``` - -## Geo data extraction notes - -Run the following script to produce a TopoJSON data used by the app: - -``` -python extract-geo-data.py -``` - -and finally generate the R-Tree cache: - -``` -npx babel-node prepare-rtree.js -``` - -At this point all of the data should be saved in `geo-data/out` folder. - -## App integration notes - -Once you've extracted all the geo data, run the integration script that will -copy all files ignoring intermediate ones into the `app/assets/geo` folder: - -``` -python integrate-into-app.py -``` diff --git a/geo-data/extract-geo-data.py b/geo-data/extract-geo-data.py deleted file mode 100644 index 77c16259dd..0000000000 --- a/geo-data/extract-geo-data.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -This module forms a geo json of highly populated cities in the world -""" - -import os -import json -from subprocess import Popen, PIPE - -# import order is important, see https://github.com/Toblerity/Shapely/issues/553 -from shapely.geometry import shape, mapping -import fiona - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -OUT_DIR = os.path.join(SCRIPT_DIR, "out") - -def get_shape_path(dataset_name): - return os.path.join(SCRIPT_DIR, dataset_name, dataset_name + ".shp") - -def extract_cites(): - input_path = get_shape_path("ne_50m_populated_places_simple") - output_path = os.path.join(OUT_DIR, "cities.json") - - props_to_keep = frozenset(["scalerank", "name", "latitude", "longitude"]) - - features = [] - with fiona.collection(input_path, "r") as source: - for feat in source: - props = feat["properties"] - if props["scalerank"] < 8: - for k in frozenset(props) - props_to_keep: - del props[k] - features.append(feat) - - my_layer = { - "type": "FeatureCollection", - "features": features - } - - with open(output_path, "w") as f: - f.write(json.dumps(my_layer)) - - print "Extracted data to {}".format(output_path) - - -def extract_countries(): - input_path = get_shape_path("ne_50m_admin_0_countries") - output_path = os.path.join(OUT_DIR, "countries.json") - - props_to_keep = frozenset(["name"]) - - features = [] - with fiona.open(input_path) as source: - for feat in source: - geometry = feat["geometry"] - - # convert country polygon to point - geometry.update(mapping(shape(geometry).representative_point())) - - # lowercase all keys - props = dict((k.lower(), v) for k, v in feat["properties"].iteritems()) - - for k in frozenset(props) - props_to_keep: - del props[k] - - feat["properties"] = props - - features.append(feat) - - my_layer = { - "type": "FeatureCollection", - "features": features - } - - with open(output_path, "w") as f: - f.write(json.dumps(my_layer)) - - print "Extracted data to {}".format(output_path) - - -def extract_geometry(): - input_path = get_shape_path("ne_50m_admin_0_countries") - output_path = os.path.join(OUT_DIR, "geometry.json") - - features = [] - with fiona.open(input_path) as source: - for feat in source: - del feat["properties"] - geometry = feat["geometry"] - feat["bbox"] = shape(geometry).bounds - features.append(feat) - - my_layer = { - "type": "FeatureCollection", - "features": features - } - - p = Popen( - ['geo2topo', '-q', '1e5', 'geometry=-', '-o', output_path], - stdin=PIPE, stdout=PIPE, stderr=PIPE - ) - errors = p.communicate(input=json.dumps(my_layer))[1] - if p.returncode == 0: - print "Extracted data to {}".format(output_path) - else: - print "geo2topo exited with {}. {}".format(p.returncode, errors.decode('utf-8').strip()) - - -def extract_provinces_and_states_lines(): - input_path = get_shape_path("ne_50m_admin_1_states_provinces_lines") - output_path = os.path.join(OUT_DIR, "states-provinces-lines.json") - - features = [] - with fiona.open(input_path) as source: - for feat in source: - del feat["properties"] - geometry = feat["geometry"] - feat["bbox"] = shape(geometry).bounds - features.append(feat) - - my_layer = { - "type": "FeatureCollection", - "features": features - } - - p = Popen( - ['geo2topo', '-q', '1e5', 'geometry=-', '-o', output_path], - stdin=PIPE, stdout=PIPE, stderr=PIPE - ) - errors = p.communicate(input=json.dumps(my_layer))[1] - if p.returncode == 0: - print "Extracted data to {}".format(output_path) - else: - print "geo2topo exited with {}. {}".format(p.returncode, errors.decode('utf-8').strip()) - - -# ensure output path exists -if not os.path.exists(OUT_DIR): - os.makedirs(OUT_DIR) - -# extract all data -extract_cites() -extract_countries() -extract_geometry() -extract_provinces_and_states_lines()
\ No newline at end of file diff --git a/geo-data/integrate-into-app.py b/geo-data/integrate-into-app.py deleted file mode 100644 index 6209f6367b..0000000000 --- a/geo-data/integrate-into-app.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -A helper script to integrate the generated geo data into the app. -""" - -import os -import shutil - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -SOURCE_DIR = os.path.join(SCRIPT_DIR, "out") -DESTINATION_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, "../app/assets/geo")) - -FILES_TO_COPY = [ - "cities.rbush.json", - "countries.rbush.json", - "geometry.json", - "geometry.rbush.json", - "states-provinces-lines.json", - "states-provinces-lines.rbush.json" -] - -if not os.path.exists(DESTINATION_DIR): - os.makedirs(DESTINATION_DIR) - -for f in FILES_TO_COPY: - src = os.path.join(SOURCE_DIR, f) - dst = os.path.join(DESTINATION_DIR, f) - prefix_len = len(os.path.commonprefix((src, dst))) - - print "Copying {} to {}".format(src[prefix_len:], dst[prefix_len:]) - - shutil.copyfile(src, dst) diff --git a/geo-data/prepare-rtree.js b/geo-data/prepare-rtree.js deleted file mode 100644 index b98942eee2..0000000000 --- a/geo-data/prepare-rtree.js +++ /dev/null @@ -1,57 +0,0 @@ -// -// Script that generates r-trees for geo data. -// run with `yarn babel-node geo-data/prepare-rtree.js` -// - -import fs from 'fs'; -import path from 'path'; -import rbush from 'rbush'; - -const geometryData = ['geometry', 'states-provinces-lines']; -const pointData = ['countries', 'cities']; - -const output_dir = path.join(__dirname, 'out'); - -for(const name of geometryData) { - const source = path.join(output_dir, `${name}.json`); - const destination = path.join(output_dir, `${name}.rbush.json`); - const collection = JSON.parse(fs.readFileSync(source)); - - const { geometry } = collection.objects; - const treeData = geometry.geometries.map(object => { - const [minX, minY, maxX, maxY] = object.bbox; - return { - ...object, - minX, minY, maxX, maxY - }; - }); - - const tree = rbush(); - tree.load(treeData); - fs.writeFileSync(destination, JSON.stringify(tree.toJSON())); - - console.log(`Saved a rbush to ${destination}`); -} - -for(const name of pointData) { - const source = path.join(output_dir, `${name}.json`); - const destination = path.join(output_dir, `${name}.rbush.json`); - const collection = JSON.parse(fs.readFileSync(source)); - - const treeData = collection.features.map((feat) => { - const { coordinates } = feat.geometry; - return { ...feat, - minX: coordinates[0], - minY: coordinates[1], - maxX: coordinates[0], - maxY: coordinates[1], - }; - }); - - const tree = rbush(); - tree.load(treeData); - fs.writeFileSync(destination, JSON.stringify(tree.toJSON())); - - console.log(`Saved a rbush to ${destination}`); -} - diff --git a/geo-data/pylintrc b/geo-data/pylintrc deleted file mode 100644 index 7f79439857..0000000000 --- a/geo-data/pylintrc +++ /dev/null @@ -1,15 +0,0 @@ -[FORMAT] -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=2 - -[MESSAGES CONTROL] - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). -disable=invalid-name diff --git a/geo-data/requirements.txt b/geo-data/requirements.txt deleted file mode 100644 index 6804e323c4..0000000000 --- a/geo-data/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -Fiona==1.7.11 \ - --hash=sha256:c9becd2b450599ed59bd21418325ea0a388690822765de6ce2813049a53cd50e \ - --hash=sha256:7e438a7f9e63ecac81074136cb821a76b8c9e160bd5f03fb4fea78651f78036f \ - --hash=sha256:c2d9afb8b6cdc80b7e6e77872041dc61fb96a938007a6ee51703893d93ab76a6 \ - --hash=sha256:5e9c68ea71e9d79fcfb68c9a101c0b133301e233c9bcca7b7c65f33cc7636ef5 -Shapely==1.6.3 \ - --hash=sha256:db8e5e512824c58084092a644a6256b404f5e6f24b48f635ed86bc9c6e6299fa \ - --hash=sha256:32e2e45994c3d757fa09e40538aa59d1d070375599ac149ba30d80ce2dd2c5af \ - --hash=sha256:f0edcea8ae2f3cc17ea5f7bbe466250681498045b6e99bb570e619eb8bbc82f7 \ - --hash=sha256:14152f111c7711fc6756fd538ec12fc8cdde7419f869b244922f71f61b2a6c6b |
