summaryrefslogtreecommitdiffhomepage
path: root/geo-data
diff options
context:
space:
mode:
Diffstat (limited to 'geo-data')
-rw-r--r--geo-data/README.md73
-rw-r--r--geo-data/extract-geo-data.py144
-rw-r--r--geo-data/integrate-into-app.py31
-rw-r--r--geo-data/prepare-rtree.js57
-rw-r--r--geo-data/pylintrc15
-rw-r--r--geo-data/requirements.txt10
6 files changed, 0 insertions, 330 deletions
diff --git a/geo-data/README.md b/geo-data/README.md
deleted file mode 100644
index 22cdc5ee52..0000000000
--- a/geo-data/README.md
+++ /dev/null
@@ -1,73 +0,0 @@
-This is a folder with python 2 and node scripts to produce geographical
-data for the Mullvad VPN app.
-
-
-## Dependency installation notes
-
-1. Run the following command in terminal to install python dependencies:
-`pip install -r requirements.txt`
-
-2. Run `npm install -g topojson-server` to install `geo2topo` tool which is
-used by python scripts to convert GeoJSON to TopoJSON
-
-
-## Geo data installation notes
-
-Go to http://www.naturalearthdata.com/downloads/50m-cultural-vectors/ and
-download ZIP files with the following shapes:
-
-- Admin 0 – Countries
-- Admin 1 – States, provinces - boundary lines
-- Populated Places - simple dataset is enough
-
-or use cURL to download all ZIPs:
-
-```
-curl -L -O http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_countries.zip
-curl -L -O http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip
-curl -L -O http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_populated_places_simple.zip
-```
-
-Extract the downloaded ZIP files into geo-data.
-Make sure the following folders exist after extraction:
-
-- ne_50m_admin_0_countries
-- ne_50m_admin_1_states_provinces_lines
-- ne_50m_populated_places_simple
-
-or use the following script:
-
-```
-mkdir ne_50m_admin_1_states_provinces_lines
-mkdir ne_50m_populated_places_simple
-mkdir ne_50m_admin_0_countries
-
-unzip ne_50m_admin_0_countries.zip -d ne_50m_admin_0_countries
-unzip ne_50m_admin_1_states_provinces_lines.zip -d ne_50m_admin_1_states_provinces_lines
-unzip ne_50m_populated_places_simple.zip -d ne_50m_populated_places_simple
-```
-
-## Geo data extraction notes
-
-Run the following script to produce a TopoJSON data used by the app:
-
-```
-python extract-geo-data.py
-```
-
-and finally generate the R-Tree cache:
-
-```
-npx babel-node prepare-rtree.js
-```
-
-At this point all of the data should be saved in `geo-data/out` folder.
-
-## App integration notes
-
-Once you've extracted all the geo data, run the integration script that will
-copy all files ignoring intermediate ones into the `app/assets/geo` folder:
-
-```
-python integrate-into-app.py
-```
diff --git a/geo-data/extract-geo-data.py b/geo-data/extract-geo-data.py
deleted file mode 100644
index 77c16259dd..0000000000
--- a/geo-data/extract-geo-data.py
+++ /dev/null
@@ -1,144 +0,0 @@
-"""
-This module forms a geo json of highly populated cities in the world
-"""
-
-import os
-import json
-from subprocess import Popen, PIPE
-
-# import order is important, see https://github.com/Toblerity/Shapely/issues/553
-from shapely.geometry import shape, mapping
-import fiona
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-OUT_DIR = os.path.join(SCRIPT_DIR, "out")
-
-def get_shape_path(dataset_name):
- return os.path.join(SCRIPT_DIR, dataset_name, dataset_name + ".shp")
-
-def extract_cites():
- input_path = get_shape_path("ne_50m_populated_places_simple")
- output_path = os.path.join(OUT_DIR, "cities.json")
-
- props_to_keep = frozenset(["scalerank", "name", "latitude", "longitude"])
-
- features = []
- with fiona.collection(input_path, "r") as source:
- for feat in source:
- props = feat["properties"]
- if props["scalerank"] < 8:
- for k in frozenset(props) - props_to_keep:
- del props[k]
- features.append(feat)
-
- my_layer = {
- "type": "FeatureCollection",
- "features": features
- }
-
- with open(output_path, "w") as f:
- f.write(json.dumps(my_layer))
-
- print "Extracted data to {}".format(output_path)
-
-
-def extract_countries():
- input_path = get_shape_path("ne_50m_admin_0_countries")
- output_path = os.path.join(OUT_DIR, "countries.json")
-
- props_to_keep = frozenset(["name"])
-
- features = []
- with fiona.open(input_path) as source:
- for feat in source:
- geometry = feat["geometry"]
-
- # convert country polygon to point
- geometry.update(mapping(shape(geometry).representative_point()))
-
- # lowercase all keys
- props = dict((k.lower(), v) for k, v in feat["properties"].iteritems())
-
- for k in frozenset(props) - props_to_keep:
- del props[k]
-
- feat["properties"] = props
-
- features.append(feat)
-
- my_layer = {
- "type": "FeatureCollection",
- "features": features
- }
-
- with open(output_path, "w") as f:
- f.write(json.dumps(my_layer))
-
- print "Extracted data to {}".format(output_path)
-
-
-def extract_geometry():
- input_path = get_shape_path("ne_50m_admin_0_countries")
- output_path = os.path.join(OUT_DIR, "geometry.json")
-
- features = []
- with fiona.open(input_path) as source:
- for feat in source:
- del feat["properties"]
- geometry = feat["geometry"]
- feat["bbox"] = shape(geometry).bounds
- features.append(feat)
-
- my_layer = {
- "type": "FeatureCollection",
- "features": features
- }
-
- p = Popen(
- ['geo2topo', '-q', '1e5', 'geometry=-', '-o', output_path],
- stdin=PIPE, stdout=PIPE, stderr=PIPE
- )
- errors = p.communicate(input=json.dumps(my_layer))[1]
- if p.returncode == 0:
- print "Extracted data to {}".format(output_path)
- else:
- print "geo2topo exited with {}. {}".format(p.returncode, errors.decode('utf-8').strip())
-
-
-def extract_provinces_and_states_lines():
- input_path = get_shape_path("ne_50m_admin_1_states_provinces_lines")
- output_path = os.path.join(OUT_DIR, "states-provinces-lines.json")
-
- features = []
- with fiona.open(input_path) as source:
- for feat in source:
- del feat["properties"]
- geometry = feat["geometry"]
- feat["bbox"] = shape(geometry).bounds
- features.append(feat)
-
- my_layer = {
- "type": "FeatureCollection",
- "features": features
- }
-
- p = Popen(
- ['geo2topo', '-q', '1e5', 'geometry=-', '-o', output_path],
- stdin=PIPE, stdout=PIPE, stderr=PIPE
- )
- errors = p.communicate(input=json.dumps(my_layer))[1]
- if p.returncode == 0:
- print "Extracted data to {}".format(output_path)
- else:
- print "geo2topo exited with {}. {}".format(p.returncode, errors.decode('utf-8').strip())
-
-
-# ensure output path exists
-if not os.path.exists(OUT_DIR):
- os.makedirs(OUT_DIR)
-
-# extract all data
-extract_cites()
-extract_countries()
-extract_geometry()
-extract_provinces_and_states_lines() \ No newline at end of file
diff --git a/geo-data/integrate-into-app.py b/geo-data/integrate-into-app.py
deleted file mode 100644
index 6209f6367b..0000000000
--- a/geo-data/integrate-into-app.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-A helper script to integrate the generated geo data into the app.
-"""
-
-import os
-import shutil
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-SOURCE_DIR = os.path.join(SCRIPT_DIR, "out")
-DESTINATION_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, "../app/assets/geo"))
-
-FILES_TO_COPY = [
- "cities.rbush.json",
- "countries.rbush.json",
- "geometry.json",
- "geometry.rbush.json",
- "states-provinces-lines.json",
- "states-provinces-lines.rbush.json"
-]
-
-if not os.path.exists(DESTINATION_DIR):
- os.makedirs(DESTINATION_DIR)
-
-for f in FILES_TO_COPY:
- src = os.path.join(SOURCE_DIR, f)
- dst = os.path.join(DESTINATION_DIR, f)
- prefix_len = len(os.path.commonprefix((src, dst)))
-
- print "Copying {} to {}".format(src[prefix_len:], dst[prefix_len:])
-
- shutil.copyfile(src, dst)
diff --git a/geo-data/prepare-rtree.js b/geo-data/prepare-rtree.js
deleted file mode 100644
index b98942eee2..0000000000
--- a/geo-data/prepare-rtree.js
+++ /dev/null
@@ -1,57 +0,0 @@
-//
-// Script that generates r-trees for geo data.
-// run with `yarn babel-node geo-data/prepare-rtree.js`
-//
-
-import fs from 'fs';
-import path from 'path';
-import rbush from 'rbush';
-
-const geometryData = ['geometry', 'states-provinces-lines'];
-const pointData = ['countries', 'cities'];
-
-const output_dir = path.join(__dirname, 'out');
-
-for(const name of geometryData) {
- const source = path.join(output_dir, `${name}.json`);
- const destination = path.join(output_dir, `${name}.rbush.json`);
- const collection = JSON.parse(fs.readFileSync(source));
-
- const { geometry } = collection.objects;
- const treeData = geometry.geometries.map(object => {
- const [minX, minY, maxX, maxY] = object.bbox;
- return {
- ...object,
- minX, minY, maxX, maxY
- };
- });
-
- const tree = rbush();
- tree.load(treeData);
- fs.writeFileSync(destination, JSON.stringify(tree.toJSON()));
-
- console.log(`Saved a rbush to ${destination}`);
-}
-
-for(const name of pointData) {
- const source = path.join(output_dir, `${name}.json`);
- const destination = path.join(output_dir, `${name}.rbush.json`);
- const collection = JSON.parse(fs.readFileSync(source));
-
- const treeData = collection.features.map((feat) => {
- const { coordinates } = feat.geometry;
- return { ...feat,
- minX: coordinates[0],
- minY: coordinates[1],
- maxX: coordinates[0],
- maxY: coordinates[1],
- };
- });
-
- const tree = rbush();
- tree.load(treeData);
- fs.writeFileSync(destination, JSON.stringify(tree.toJSON()));
-
- console.log(`Saved a rbush to ${destination}`);
-}
-
diff --git a/geo-data/pylintrc b/geo-data/pylintrc
deleted file mode 100644
index 7f79439857..0000000000
--- a/geo-data/pylintrc
+++ /dev/null
@@ -1,15 +0,0 @@
-[FORMAT]
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-indent-string=' '
-
-# Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=2
-
-[MESSAGES CONTROL]
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once).
-disable=invalid-name
diff --git a/geo-data/requirements.txt b/geo-data/requirements.txt
deleted file mode 100644
index 6804e323c4..0000000000
--- a/geo-data/requirements.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-Fiona==1.7.11 \
- --hash=sha256:c9becd2b450599ed59bd21418325ea0a388690822765de6ce2813049a53cd50e \
- --hash=sha256:7e438a7f9e63ecac81074136cb821a76b8c9e160bd5f03fb4fea78651f78036f \
- --hash=sha256:c2d9afb8b6cdc80b7e6e77872041dc61fb96a938007a6ee51703893d93ab76a6 \
- --hash=sha256:5e9c68ea71e9d79fcfb68c9a101c0b133301e233c9bcca7b7c65f33cc7636ef5
-Shapely==1.6.3 \
- --hash=sha256:db8e5e512824c58084092a644a6256b404f5e6f24b48f635ed86bc9c6e6299fa \
- --hash=sha256:32e2e45994c3d757fa09e40538aa59d1d070375599ac149ba30d80ce2dd2c5af \
- --hash=sha256:f0edcea8ae2f3cc17ea5f7bbe466250681498045b6e99bb570e619eb8bbc82f7 \
- --hash=sha256:14152f111c7711fc6756fd538ec12fc8cdde7419f869b244922f71f61b2a6c6b
Copyright © 2025 - 2026 Wayne Cole. WTFPL. Attribution appreciated. No warranty. Not liable for bodily damages.