diff --git a/osmaxx/__init__.py b/osmaxx/__init__.py
index 9e9401fc84f018637c313ead07f3c6cb1f4fc911..83b50415afdb1912382336599f81b2e270be7c96 100644
--- a/osmaxx/__init__.py
+++ b/osmaxx/__init__.py
@@ -1,4 +1,4 @@
-__version__ = 'v3.5.0'
+__version__ = 'v3.6.0'
__all__ = [
'__version__',
diff --git a/osmaxx/conversion/converters/converter.py b/osmaxx/conversion/converters/converter.py
index c8af7b046eec6701c75917904bf29b39cf84df3d..d7b433c62b8a8134cd2c5e2911cb295baff0ec99 100644
--- a/osmaxx/conversion/converters/converter.py
+++ b/osmaxx/conversion/converters/converter.py
@@ -1,7 +1,8 @@
from osmaxx.conversion.converters.converter_garmin.garmin import Garmin
from osmaxx.conversion.converters.converter_gis.gis import GISConverter
+from osmaxx.conversion.converters.converter_pbf.to_pbf import produce_pbf
from osmaxx.conversion.job_dispatcher.rq_dispatcher import rq_enqueue_with_settings
-from osmaxx.conversion_api.formats import FGDB, SHAPEFILE, GPKG, SPATIALITE, GARMIN
+from osmaxx.conversion_api.formats import FGDB, SHAPEFILE, GPKG, SPATIALITE, GARMIN, PBF
class Conversion(object):
@@ -26,6 +27,7 @@ class Conversion(object):
_format_process = {
GARMIN: self._create_garmin_export,
+ PBF: self._create_pbf,
FGDB: self._extract_postgis_format,
SHAPEFILE: self._extract_postgis_format,
GPKG: self._extract_postgis_format,
@@ -55,6 +57,13 @@ class Conversion(object):
)
garmin.create_garmin_export()
+ def _create_pbf(self):
+ produce_pbf(
+ out_zip_file_path=self._output_zip_file_path,
+ area_name=self._area_name,
+ polyfile_string=self._polyfile_string,
+ )
+
def convert(
*, conversion_format, area_name, osmosis_polygon_file_string, output_zip_file_path, filename_prefix,
diff --git a/osmaxx/conversion/converters/converter_garmin/garmin.py b/osmaxx/conversion/converters/converter_garmin/garmin.py
index 052e7ea660d5ac05ba2a90b7d3dc1f3021f8fe05..851a605666a484303c1492c75f9026fe4d50a545 100644
--- a/osmaxx/conversion/converters/converter_garmin/garmin.py
+++ b/osmaxx/conversion/converters/converter_garmin/garmin.py
@@ -1,5 +1,4 @@
import shutil
-import subprocess
import os
import tempfile
@@ -8,7 +7,7 @@ from rq import get_current_job
from osmaxx.conversion._settings import CONVERSION_SETTINGS, odb_license, copying_notice, creative_commons_license
-from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize
+from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize, logged_check_call
_path_to_commandline_utils = os.path.join(os.path.dirname(__file__), 'command_line_utils')
_path_to_bounds_zip = os.path.join(CONVERSION_SETTINGS['SEA_AND_BOUNDS_ZIP_DIRECTORY'], 'bounds.zip')
@@ -47,7 +46,7 @@ class Garmin:
def _split(self, workdir):
memory_option = '-Xmx7000m'
_splitter_path = os.path.abspath(os.path.join(_path_to_commandline_utils, 'splitter', 'splitter.jar'))
- subprocess.check_call([
+ logged_check_call([
'java',
memory_option,
'-jar', _splitter_path,
@@ -79,7 +78,7 @@ class Garmin:
'--route',
]
- subprocess.check_call(
+ logged_check_call(
mkg_map_command +
output_dir +
config
diff --git a/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py b/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py
index 5316fd7a302b7879c2a902c7dc7623dd626fa3cc..5cb75530a1da8b3979b24d9a3c4152ee5dc8aa3a 100644
--- a/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py
+++ b/osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py
@@ -1,31 +1,30 @@
import glob
import os
-import subprocess
-from osmaxx.conversion._settings import CONVERSION_SETTINGS
+from memoize import mproperty
+
from osmaxx.conversion.converters.converter_gis.helper.default_postgres import get_default_postgres_wrapper
from osmaxx.conversion.converters.converter_gis.helper.osm_boundaries_importer import OSMBoundariesImporter
from osmaxx.conversion.converters import detail_levels
+from osmaxx.conversion.converters.converter_pbf.to_pbf import cut_pbf_along_polyfile
from osmaxx.conversion.converters.detail_levels import DETAIL_LEVEL_TABLES
+from osmaxx.conversion.converters.utils import logged_check_call
from osmaxx.utils import polyfile_helpers
class BootStrapper:
def __init__(self, area_polyfile_string, *, detail_level=detail_levels.DETAIL_LEVEL_ALL):
+ self.area_polyfile_string = area_polyfile_string
self._postgres = get_default_postgres_wrapper()
self._script_base_dir = os.path.abspath(os.path.dirname(__file__))
self._terminal_style_path = os.path.join(self._script_base_dir, 'styles', 'terminal.style')
self._style_path = os.path.join(self._script_base_dir, 'styles', 'style.lua')
- self._extent = polyfile_helpers.parse_poly_string(area_polyfile_string)
- self._extent_polyfile_path = os.path.join('/tmp', 'polyfile_extent.poly')
self._pbf_file_path = os.path.join('/tmp', 'pbf_cutted.pbf')
self._detail_level = DETAIL_LEVEL_TABLES[detail_level]
- with open(self._extent_polyfile_path, 'w') as f:
- f.write(area_polyfile_string)
def bootstrap(self):
self._reset_database()
- self._cut_area_from_pbf()
+ cut_pbf_along_polyfile(self.area_polyfile_string, self._pbf_file_path)
self._import_boundaries()
self._import_pbf()
self._setup_db_functions()
@@ -33,9 +32,9 @@ class BootStrapper:
self._filter_data()
self._create_views()
- @property
+ @mproperty
def geom(self):
- return self._extent
+ return polyfile_helpers.parse_poly_string(self.area_polyfile_string)
def _reset_database(self):
self._postgres.drop_db()
@@ -50,7 +49,7 @@ class BootStrapper:
def _import_boundaries(self):
osm_importer = OSMBoundariesImporter()
- osm_importer.load_area_specific_data(extent=self._extent)
+ osm_importer.load_area_specific_data(extent=self.geom)
def _setup_db_functions(self):
self._execute_sql_scripts_in_folder(os.path.join(self._script_base_dir, 'sql', 'functions'))
@@ -112,18 +111,6 @@ class BootStrapper:
script_path = self._level_adapted_script_path(script_path)
self._postgres.execute_sql_file(script_path)
- def _cut_area_from_pbf(self):
- command = [
- "osmconvert",
- "--out-pbf",
- "--complete-ways",
- "--complex-ways",
- "-o={}".format(self._pbf_file_path),
- "-B={}".format(self._extent_polyfile_path),
- "{}".format(CONVERSION_SETTINGS["PBF_PLANET_FILE_PATH"]),
- ]
- subprocess.check_call(command)
-
def _import_pbf(self):
db_name = self._postgres.get_db_name()
postgres_user = self._postgres.get_user()
@@ -144,4 +131,4 @@ class BootStrapper:
'--input-reader', 'pbf',
self._pbf_file_path,
]
- subprocess.check_call(osm_2_pgsql_command)
+ logged_check_call(osm_2_pgsql_command)
diff --git a/osmaxx/conversion/converters/converter_pbf/__init__.py b/osmaxx/conversion/converters/converter_pbf/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/osmaxx/conversion/converters/converter_pbf/to_pbf.py b/osmaxx/conversion/converters/converter_pbf/to_pbf.py
new file mode 100644
index 0000000000000000000000000000000000000000..0326a1daff30609ea6d2e7f03fe1cdf1b3cd442e
--- /dev/null
+++ b/osmaxx/conversion/converters/converter_pbf/to_pbf.py
@@ -0,0 +1,54 @@
+import os
+import shutil
+import tempfile
+
+from django.utils import timezone
+
+from rq import get_current_job
+
+from osmaxx.conversion._settings import CONVERSION_SETTINGS, odb_license
+from osmaxx.conversion.converters.utils import zip_folders_relative, recursive_getsize, logged_check_call
+
+
+def cut_area_from_pbf(pbf_result_file_path, extent_polyfile_path):
+ command = [
+ "osmconvert",
+ "--out-pbf",
+ "--complete-ways",
+ "--complex-ways",
+ "-o={}".format(pbf_result_file_path),
+ "-B={}".format(extent_polyfile_path),
+ "{}".format(CONVERSION_SETTINGS["PBF_PLANET_FILE_PATH"]),
+ ]
+ logged_check_call(command)
+
+
+def cut_pbf_along_polyfile(polyfile_string, pbf_out_path):
+ with tempfile.NamedTemporaryFile('w') as polyfile:
+ polyfile.write(polyfile_string)
+ polyfile.flush()
+ os.fsync(polyfile)
+ cut_area_from_pbf(pbf_out_path, polyfile.name)
+
+
+def produce_pbf(*, out_zip_file_path, area_name, polyfile_string):
+ _start_time = timezone.now()
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ out_dir = os.path.join(tmp_dir, 'pbf')
+ os.makedirs(out_dir, exist_ok=True)
+ pbf_out_path = os.path.join(out_dir, area_name + '.pbf')
+
+ shutil.copy(odb_license, out_dir)
+
+ cut_pbf_along_polyfile(polyfile_string, pbf_out_path)
+
+ unzipped_result_size = recursive_getsize(out_dir)
+
+ zip_folders_relative([tmp_dir], out_zip_file_path)
+
+ job = get_current_job()
+ if job:
+ job.meta['duration'] = timezone.now() - _start_time
+ job.meta['unzipped_result_size'] = unzipped_result_size
+ job.save()
diff --git a/osmaxx/conversion/converters/utils.py b/osmaxx/conversion/converters/utils.py
index ec79c7ea444fc0c63e57f6ee6dcfe13d6b71d6e8..64e1d24dc8c3ff40afa949789eb612b45c6a02f3 100644
--- a/osmaxx/conversion/converters/utils.py
+++ b/osmaxx/conversion/converters/utils.py
@@ -1,8 +1,9 @@
+import logging
+import os
+import subprocess
import uuid
import zipfile
-import os
-
# Use the built-in version of scandir if possible, otherwise
# use the scandir module version
try:
@@ -10,6 +11,8 @@ try:
except ImportError:
from scandir import scandir
+logger = logging.getLogger(__name__)
+
def zip_folders_relative(folder_list, zip_out_file_path=None):
"""
@@ -42,3 +45,11 @@ def recursive_getsize(path):
elif entry.is_dir(follow_symlinks=False):
size += recursive_getsize(os.path.join(path, entry.path))
return size
+
+
+def logged_check_call(*args, **kwargs):
+ try:
+ subprocess.check_call(*args, **kwargs)
+ except subprocess.CalledProcessError as e:
+ logger.error('Command `{}` exited with return value {}\nOutput:\n{}'.format(e.cmd, e.returncode, e.output))
+ raise
diff --git a/osmaxx/conversion/size_estimator.py b/osmaxx/conversion/size_estimator.py
index 947781c7afdf05118bebff7609b6b44cbf6db781..57c9cbeff350ae59d8230e005c74c49b82e64de8 100644
--- a/osmaxx/conversion/size_estimator.py
+++ b/osmaxx/conversion/size_estimator.py
@@ -11,6 +11,11 @@ PRE_DATA = {
detail_levels.DETAIL_LEVEL_ALL: [11000, 18000, 42000, 95000],
detail_levels.DETAIL_LEVEL_REDUCED: [11000, 18000, 42000, 95000],
},
+ formats.PBF: {
+ 'pbf_predicted': [25000, 44000, 96000, 390000],
+ detail_levels.DETAIL_LEVEL_ALL: [25000, 44000, 96000, 390000],
+ detail_levels.DETAIL_LEVEL_REDUCED: [25000, 44000, 96000, 390000],
+ },
formats.FGDB: {
'pbf_predicted': [25000, 44000, 96000, 390000],
detail_levels.DETAIL_LEVEL_ALL: [46000, 101000, 309000, 676000],
diff --git a/osmaxx/conversion_api/formats.py b/osmaxx/conversion_api/formats.py
index 1c65e0281333b30191f1508e3ddb0bfc3c8850c4..92d01b69ccccd254217bd15cb14cc62260c34b00 100644
--- a/osmaxx/conversion_api/formats.py
+++ b/osmaxx/conversion_api/formats.py
@@ -3,7 +3,7 @@ from collections import OrderedDict
from django.utils.translation import gettext as _
-FGDB, SHAPEFILE, GPKG, SPATIALITE, GARMIN = 'fgdb', 'shapefile', 'gpkg', 'spatialite', 'garmin'
+FGDB, SHAPEFILE, GPKG, SPATIALITE, GARMIN, PBF = 'fgdb', 'shapefile', 'gpkg', 'spatialite', 'garmin', 'pbf'
class OutputFormat:
@@ -93,6 +93,13 @@ FORMAT_DEFINITIONS = OrderedDict([
abbreviations=[],
is_white_box=False,
)),
+ (PBF, OutputFormat(
+ long_identifier='OSM Protocolbuffer Binary Format',
+ verbose_name=_('OSM Protocolbuffer Binary Format'),
+ archive_file_name_identifier='pbf',
+ abbreviations=[],
+ is_white_box=False,
+ )),
])
FORMAT_CHOICES = tuple((key, definition.verbose_name) for key, definition in FORMAT_DEFINITIONS.items())
diff --git a/osmaxx/core/static/osmaxx/stylesheets/main.css b/osmaxx/core/static/osmaxx/stylesheets/main.css
index 37728f0b29e58a3b3605137e0d55983c7ed0ea98..06cff7c95f56d28820c4a97785f5fdf806695a16 100644
--- a/osmaxx/core/static/osmaxx/stylesheets/main.css
+++ b/osmaxx/core/static/osmaxx/stylesheets/main.css
@@ -282,4 +282,4 @@ footer a:link,
footer a:visited,
footer a:hover,
footer a:focus,
-footer a:active { color: white; background-color: rgb(50, 110, 160); padding: 4px 4px; }
+footer a:active { color: white; }
diff --git a/osmaxx/excerptexport/forms/order_options_mixin.py b/osmaxx/excerptexport/forms/order_options_mixin.py
index ce46ec1213887080549064134f12bfe3d151a714..23f27d7b1125e41d13231a7359310f64d19e09b6 100644
--- a/osmaxx/excerptexport/forms/order_options_mixin.py
+++ b/osmaxx/excerptexport/forms/order_options_mixin.py
@@ -41,7 +41,7 @@ class OrderOptionsMixin(forms.Form):
),
Div(
Fieldset(
- _('GIS options (ignored for Garmin)'),
+ _('GIS options (ignored for Garmin and PBF)'),
'coordinate_reference_system',
'detail_level',
),
diff --git a/osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_emblem.svg b/osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_icon.svg
similarity index 100%
rename from osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_emblem.svg
rename to osmaxx/excerptexport/static/excerptexport/images/define_and_export_new_excerpt_icon.svg
diff --git a/osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_emblem.svg b/osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_icon.svg
similarity index 100%
rename from osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_emblem.svg
rename to osmaxx/excerptexport/static/excerptexport/images/export_existing_excerpt_or_country_or_admin_area_icon.svg
diff --git a/osmaxx/excerptexport/templates/excerptexport/templates/index.html b/osmaxx/excerptexport/templates/excerptexport/templates/index.html
index d18dce7b7ab6db1f263547fd7593b37a024be133..0ead0672f588d93992554890cfe2285134910874 100644
--- a/osmaxx/excerptexport/templates/excerptexport/templates/index.html
+++ b/osmaxx/excerptexport/templates/excerptexport/templates/index.html
@@ -18,17 +18,17 @@
</h1>
<div class="container-fluid">
<div class="text-center col-sm-6">
- <img src="{% static 'excerptexport/images/export_existing_excerpt_or_country_or_admin_area_emblem.svg' %}" width="70%" style="max-width: 550px;" /><br />
- <a class="btn btn-primary btn-lg" href="{% url 'excerptexport:order_existing_excerpt' %}">
- <strong>{% trans 'Export an existing excerpt' %}</strong>
+ <img src="{% static 'excerptexport/images/export_existing_excerpt_or_country_or_admin_area_icon.svg' %}" width="70%" style="max-width: 550px;" /><br />
+ <a class="btn btn-primary btn-lg" href="{% url 'excerptexport:order_existing_excerpt' %}"
+ title="{% trans 'Export a predefined country or other administrative area,<br />or one of your previously defined excerpts' %}">
+ <strong>{% trans 'Existing Excerpt / Country' %}</strong>
</a>
- <br />
- {% trans '(a predefined country or other administrative area,<br />or one of your previously defined excerpts)' %}
</div>
<div class="text-center col-sm-6">
- <img src="{% static 'excerptexport/images/define_and_export_new_excerpt_emblem.svg' %}" width="70%" style="max-width: 550px;" /><br />
- <a class="btn btn-primary btn-lg" href="{% url 'excerptexport:order_new_excerpt' %}">
- <strong>{% trans 'Define & export a new excerpt' %}</strong>
+ <img src="{% static 'excerptexport/images/define_and_export_new_excerpt_icon.svg' %}" width="70%" style="max-width: 550px;" /><br />
+ <a class="btn btn-primary btn-lg" href="{% url 'excerptexport:order_new_excerpt' %}"
+ title="{% trans 'Define & export a new excerpt' %}">
+ <strong>{% trans 'New Excerpt' %}</strong>
</a>
</div>
{% if user.is_authenticated %}
@@ -45,17 +45,14 @@
{% block copyright %}
{{ block.super }},
<small><small>
- <a href="{% static 'excerptexport/images/export_existing_excerpt_or_country_or_admin_area_emblem.svg' %}" target="_blank">
- "folder with map"
- </a>
+ <a href="{% static 'excerptexport/images/export_existing_excerpt_or_country_or_admin_area_icon.svg' %}" target="_blank">
+ "folder with map"</a>
and
- <a href="{% static 'excerptexport/images/define_and_export_new_excerpt_emblem.svg' %}" target="_blank">
- "cut-out from planet" emblems
- </a>
+ <a href="{% static 'excerptexport/images/define_and_export_new_excerpt_icon.svg' %}" target="_blank">
+ "cut-out from planet" icons</a>
derived from <span title="see SVG source code for individual attributions">CC BY-SA and public domain works</span>
and licensed under
<a href="https://creativecommons.org/licenses/by-sa/4.0/" title="Creative Commons Attribution-ShareAlike 4.0 International Public License" target="_blank">
- CC BY-SA 4.0 international
- </a>
+ CC BY-SA 4.0 international</a>
</small></small>
{% endblock %}
diff --git a/osmaxx/locale/en/LC_MESSAGES/django.mo b/osmaxx/locale/en/LC_MESSAGES/django.mo
index 4d0943b6266485dd7637991fe9a9b9a97e48d3c7..3dfefd568a5f924d4ed293096931edf49a395c91 100644
Binary files a/osmaxx/locale/en/LC_MESSAGES/django.mo and b/osmaxx/locale/en/LC_MESSAGES/django.mo differ
diff --git a/osmaxx/locale/en/LC_MESSAGES/django.po b/osmaxx/locale/en/LC_MESSAGES/django.po
index dc73cc063cdd486ea8afa22517b95b593644cd7d..0fb56ee28d611ada642b929a414de68ce4efafc0 100644
--- a/osmaxx/locale/en/LC_MESSAGES/django.po
+++ b/osmaxx/locale/en/LC_MESSAGES/django.po
@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2017-02-02 09:37+0100\n"
+"POT-Creation-Date: 2017-02-08 16:03+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -665,6 +665,10 @@ msgstr ""
msgid "Garmin navigation & map data"
msgstr ""
+#: osmaxx/conversion_api/formats.py:98
+msgid "OSM Protocolbuffer Binary Format"
+msgstr ""
+
#: osmaxx/conversion_api/statuses.py:12
msgid "received"
msgstr ""
@@ -731,10 +735,12 @@ msgid "Home"
msgstr ""
#: osmaxx/core/templates/osmaxx/navigation.html:7
+#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:24
msgid "Existing Excerpt / Country"
msgstr ""
#: osmaxx/core/templates/osmaxx/navigation.html:12
+#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:31
msgid "New Excerpt"
msgstr ""
@@ -804,7 +810,7 @@ msgid "Export options"
msgstr ""
#: osmaxx/excerptexport/forms/order_options_mixin.py:44
-msgid "GIS options (ignored for Garmin)"
+msgid "GIS options (ignored for Garmin and PBF)"
msgstr ""
#: osmaxx/excerptexport/models/excerpt.py:20
@@ -1047,16 +1053,12 @@ msgid "Get the OpenStreetMap data you want in the file format you need"
msgstr ""
#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:23
-msgid "Export an existing excerpt"
-msgstr ""
-
-#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:26
msgid ""
-"(a predefined country or other administrative area,<br />or one of your "
-"previously defined excerpts)"
+"Export a predefined country or other administrative area,<br />or one of "
+"your previously defined excerpts"
msgstr ""
-#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:31
+#: osmaxx/excerptexport/templates/excerptexport/templates/index.html:30
msgid "Define & export a new excerpt"
msgstr ""
diff --git a/requirements-all.txt b/requirements-all.txt
index 994e58e5b3eed4ca98f4b84d50cecf84fb83dc0a..e9986033235c6cf984610f414234dc8bb19ed304 100644
--- a/requirements-all.txt
+++ b/requirements-all.txt
@@ -4,17 +4,18 @@
#
# pip-compile --output-file requirements-all.txt requirements-all.in
#
+appdirs==1.4.0 # via setuptools
astroid==1.4.9 # via pylint, pylint-celery, pylint-flask, pylint-plugin-utils, requirements-detector
click==6.7 # via mkdocs, rq
contextlib2==0.5.4 # via raven
coverage==4.3.4
decorator==4.0.11 # via ipython, traitlets
-defusedxml==0.4.1 # via python3-openid
+defusedxml==0.5.0rc1 # via python3-openid, social-auth-core
django-crispy-forms==1.6.1
django-debug-toolbar==1.6
django-downloadview==1.9
django-environ==0.4.1
-django-extensions==1.7.5
+django-extensions==1.7.6
django-filter==1.0.1
django-model-utils==2.6.1
django-rq==0.9.4
@@ -27,38 +28,40 @@ djangorestframework==3.4.7
docutils==0.13.1 # via pyroma
dodgy==0.1.9 # via prospector
drf-extensions==0.3.1
-flake8==3.2.1
+flake8==3.3.0
GeoAlchemy2==0.4.0
geometalab.drf-utm-zone-info==0.2.0
geometalab.osm-pbf-file-size-estimation-service==1.1.0
gevent==1.2.1
-greenlet==0.4.11 # via gevent
+greenlet==0.4.12 # via gevent
gunicorn==19.6.0
ipython-genutils==0.1.0 # via traitlets
-ipython==5.1.0
+ipython==5.2.2
isort==4.2.5 # via pylint
-Jinja2==2.9.4 # via mkdocs
+Jinja2==2.9.5 # via mkdocs
lazy-object-proxy==1.2.2 # via astroid
livereload==2.5.1 # via mkdocs
-markdown==2.6.7
+markdown==2.6.8
MarkupSafe==0.23 # via jinja2
-mccabe==0.5.3 # via flake8, prospector, pylint
+mccabe==0.6.1 # via flake8, prospector, pylint
+memoize==1.0.0
mkdocs==0.16.1
multidict==2.1.4 # via yarl
numpy==1.12.0
oauthlib==2.0.1 # via requests-oauthlib, social-auth-core
+packaging==16.8 # via setuptools
pep8-naming==0.4.1 # via prospector
pexpect==4.2.1 # via ipython
pickleshare==0.7.4 # via ipython
-prompt-toolkit==1.0.9 # via ipython
+prompt-toolkit==1.0.13 # via ipython
prospector==0.12.4
psycopg2==2.6.2
ptyprocess==0.5.1 # via pexpect
py==1.4.32 # via pytest
pycodestyle==2.0.0 # via flake8, prospector
pydocstyle==1.0.0 # via prospector
-pyflakes==1.3.0 # via flake8, prospector
-pygments==2.1.3 # via ipython
+pyflakes==1.5.0 # via flake8, prospector
+pygments==2.2.0 # via ipython
pyhamcrest==1.9.0
PyJWT==1.4.2 # via djangorestframework-jwt, social-auth-core
pylint-celery==0.3 # via prospector
@@ -66,36 +69,37 @@ pylint-common==0.2.2 # via prospector
pylint-django==0.7.2 # via prospector
pylint-flask==0.5 # via prospector
pylint-plugin-utils==0.2.4 # via prospector, pylint-celery, pylint-django, pylint-flask
-pylint==1.6.4 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils
+pylint==1.6.5 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils
pypandoc==1.3.3
+pyparsing==2.1.10 # via packaging
pyroma==2.2
pytest-base-url==1.2.0 # via pytest-selenium
pytest-cov==2.4.0
pytest-django==3.1.2
pytest-html==1.13.0 # via pytest-selenium
pytest-mock==1.5.0
-pytest-selenium==1.7.0
+pytest-selenium==1.8.0
pytest-variables==1.4 # via pytest-selenium
-pytest==3.0.5 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-mock, pytest-selenium, pytest-variables
-python-social-auth[django]==0.3.3
+pytest==3.0.6 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-mock, pytest-selenium, pytest-variables
+python-social-auth[django]==0.3.6
python3-openid==3.0.10 # via social-auth-core
PyYAML==3.12 # via mkdocs, prospector, vcrpy
raven==5.32.0
redis==2.10.5 # via rq
-requests-mock==1.2.0
+requests-mock==1.3.0
requests-oauthlib==0.7.0 # via social-auth-core
-requests==2.12.5
+requests==2.13.0
requirements-detector==0.5.2 # via prospector
rq==0.7.1
-ruamel.yaml==0.13.9
+ruamel.yaml==0.13.13
scandir==1.4
scipy==0.18.1
selenium==3.0.2 # via pytest-selenium
setoptconf==0.2.0 # via prospector
simplegeneric==0.8.1 # via ipython
-six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, livereload, prompt-toolkit, pyhamcrest, pylint, requests-mock, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy
-social-auth-app-django==0.1.0 # via python-social-auth
-social-auth-core==0.2.1 # via python-social-auth, social-auth-app-django
+six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, livereload, packaging, prompt-toolkit, pyhamcrest, pylint, requests-mock, setuptools, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy
+social-auth-app-django==1.0.1 # via python-social-auth
+social-auth-core==1.1.0 # via python-social-auth, social-auth-app-django
SQLAlchemy-Utils==0.32.12
sqlalchemy-views==0.2.1
SQLAlchemy==1.1.5 # via geoalchemy2, sqlalchemy-utils, sqlalchemy-views
@@ -107,11 +111,11 @@ vulture==0.12
wcwidth==0.1.7 # via prompt-toolkit
Werkzeug==0.11.15
wheel==0.29.0
-whitenoise==3.2.3
+whitenoise==3.3.0
wrapt==1.10.8 # via astroid, vcrpy
-yarl==0.8.1 # via vcrpy
+yarl==0.9.1 # via vcrpy
# The following packages are commented out because they are
# considered to be unsafe in a requirements file:
# pip # via pypandoc
-# setuptools # via django-downloadview, ipython, pyhamcrest, pypandoc, pyroma
+# setuptools # via django-downloadview, ipython, pyhamcrest, pypandoc, pyroma, pytest
diff --git a/requirements.in b/requirements.in
index 2ca4870e80cb5ddf1f0e2acb2e1440a2eb6d1b9f..ea024f5b918d6588e9a97ad02629cef891f5afa0 100644
--- a/requirements.in
+++ b/requirements.in
@@ -11,6 +11,7 @@ geometalab.drf-utm-zone-info>=0.1.0,<1.0.0
djangorestframework-jwt
requests
markdown
+memoize
django-filter
rq
django-rq
diff --git a/requirements.txt b/requirements.txt
index 3861a2d55a94151b6cd2de4c0bab3c42727b66e8..046260b26002e2f00340a1404dbc0b5589663fcb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,17 +4,18 @@
#
# pip-compile --output-file requirements.txt requirements.in
#
+appdirs==1.4.0 # via setuptools
astroid==1.4.9 # via pylint, pylint-celery, pylint-flask, pylint-plugin-utils, requirements-detector
click==6.7 # via mkdocs, rq
contextlib2==0.5.4 # via raven
coverage==4.3.4
decorator==4.0.11 # via ipython, traitlets
-defusedxml==0.4.1 # via python3-openid
+defusedxml==0.5.0rc1 # via python3-openid, social-auth-core
django-crispy-forms==1.6.1
django-debug-toolbar==1.6
django-downloadview==1.9
django-environ==0.4.1
-django-extensions==1.7.5
+django-extensions==1.7.6
django-filter==1.0.1
django-model-utils==2.6.1
django-rq==0.9.4
@@ -27,38 +28,40 @@ djangorestframework==3.4.7
docutils==0.13.1 # via pyroma
dodgy==0.1.9 # via prospector
drf-extensions==0.3.1
-flake8==3.2.1
+flake8==3.3.0
GeoAlchemy2==0.4.0
geometalab.drf-utm-zone-info==0.2.0
geometalab.osm-pbf-file-size-estimation-service==1.1.0
gevent==1.2.1
-greenlet==0.4.11 # via gevent
+greenlet==0.4.12 # via gevent
gunicorn==19.6.0
ipython-genutils==0.1.0 # via traitlets
-ipython==5.1.0
+ipython==5.2.2
isort==4.2.5 # via pylint
-Jinja2==2.9.4 # via mkdocs
+Jinja2==2.9.5 # via mkdocs
lazy-object-proxy==1.2.2 # via astroid
livereload==2.5.1 # via mkdocs
-markdown==2.6.7
+markdown==2.6.8
MarkupSafe==0.23 # via jinja2
-mccabe==0.5.3 # via flake8, prospector, pylint
+mccabe==0.6.1 # via flake8, prospector, pylint
+memoize==1.0.0
mkdocs==0.16.1
multidict==2.1.4 # via yarl
numpy==1.12.0
oauthlib==2.0.1 # via requests-oauthlib, social-auth-core
+packaging==16.8 # via setuptools
pep8-naming==0.4.1 # via prospector
pexpect==4.2.1 # via ipython
pickleshare==0.7.4 # via ipython
-prompt-toolkit==1.0.9 # via ipython
+prompt-toolkit==1.0.13 # via ipython
prospector==0.12.4
psycopg2==2.6.2
ptyprocess==0.5.1 # via pexpect
py==1.4.32 # via pytest
pycodestyle==2.0.0 # via flake8, prospector
pydocstyle==1.0.0 # via prospector
-pyflakes==1.3.0 # via flake8, prospector
-pygments==2.1.3 # via ipython
+pyflakes==1.5.0 # via flake8, prospector
+pygments==2.2.0 # via ipython
pyhamcrest==1.9.0
PyJWT==1.4.2 # via djangorestframework-jwt, social-auth-core
pylint-celery==0.3 # via prospector
@@ -66,24 +69,25 @@ pylint-common==0.2.2 # via prospector
pylint-django==0.7.2 # via prospector
pylint-flask==0.5 # via prospector
pylint-plugin-utils==0.2.4 # via prospector, pylint-celery, pylint-django, pylint-flask
-pylint==1.6.4 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils
+pylint==1.6.5 # via prospector, pylint-celery, pylint-common, pylint-django, pylint-flask, pylint-plugin-utils
+pyparsing==2.1.10 # via packaging
pyroma==2.2
pytest-base-url==1.2.0 # via pytest-selenium
pytest-cov==2.4.0
pytest-django==3.1.2
pytest-html==1.13.0 # via pytest-selenium
pytest-mock==1.5.0
-pytest-selenium==1.7.0
+pytest-selenium==1.8.0
pytest-variables==1.4 # via pytest-selenium
-pytest==3.0.5 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-mock, pytest-selenium, pytest-variables
-python-social-auth[django]==0.3.3
+pytest==3.0.6 # via pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-mock, pytest-selenium, pytest-variables
+python-social-auth[django]==0.3.6
python3-openid==3.0.10 # via social-auth-core
PyYAML==3.12 # via mkdocs, prospector, vcrpy
raven==5.32.0
redis==2.10.5 # via rq
-requests-mock==1.2.0
+requests-mock==1.3.0
requests-oauthlib==0.7.0 # via social-auth-core
-requests==2.12.5
+requests==2.13.0
requirements-detector==0.5.2 # via prospector
rq==0.7.1
scandir==1.4
@@ -91,9 +95,9 @@ scipy==0.18.1
selenium==3.0.2 # via pytest-selenium
setoptconf==0.2.0 # via prospector
simplegeneric==0.8.1 # via ipython
-six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, livereload, prompt-toolkit, pyhamcrest, pylint, requests-mock, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy
-social-auth-app-django==0.1.0 # via python-social-auth
-social-auth-core==0.2.1 # via python-social-auth, social-auth-app-django
+six==1.10.0 # via astroid, django-downloadview, django-environ, django-extensions, livereload, packaging, prompt-toolkit, pyhamcrest, pylint, requests-mock, setuptools, social-auth-app-django, social-auth-core, sqlalchemy-utils, traitlets, vcrpy
+social-auth-app-django==1.0.1 # via python-social-auth
+social-auth-core==1.1.0 # via python-social-auth, social-auth-app-django
SQLAlchemy-Utils==0.32.12
sqlalchemy-views==0.2.1
SQLAlchemy==1.1.5 # via geoalchemy2, sqlalchemy-utils, sqlalchemy-views
@@ -105,10 +109,10 @@ vulture==0.12
wcwidth==0.1.7 # via prompt-toolkit
Werkzeug==0.11.15
wheel==0.29.0
-whitenoise==3.2.3
+whitenoise==3.3.0
wrapt==1.10.8 # via astroid, vcrpy
-yarl==0.8.1 # via vcrpy
+yarl==0.9.1 # via vcrpy
# The following packages are commented out because they are
# considered to be unsafe in a requirements file:
-# setuptools # via django-downloadview, ipython, pyhamcrest, pyroma
+# setuptools # via django-downloadview, ipython, pyhamcrest, pyroma, pytest
diff --git a/tests/conversion/converters/converter_test.py b/tests/conversion/converters/converter_test.py
index 4014d30e015ad2ead1bf79812f058d26320ff043..9121ce96319e9ab3aa401af0a50c8ecdd1ea2d40 100644
--- a/tests/conversion/converters/converter_test.py
+++ b/tests/conversion/converters/converter_test.py
@@ -4,6 +4,7 @@ from osmaxx.conversion.converters.converter import Conversion, convert
def test_start_format_extraction(conversion_format, area_name, simple_osmosis_line_string, output_zip_file_path, filename_prefix, detail_level, out_srs, mocker):
gis_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter_gis.gis.GISConverter.create_gis_export')
garmin_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter_garmin.garmin.Garmin.create_garmin_export')
+ pbf_converter_mock_create = mocker.patch('osmaxx.conversion.converters.converter.produce_pbf')
conversion = Conversion(
conversion_format=conversion_format,
area_name=area_name,
@@ -14,7 +15,7 @@ def test_start_format_extraction(conversion_format, area_name, simple_osmosis_li
out_srs='EPSG:{}'.format(out_srs),
)
conversion.start_format_extraction()
- assert gis_converter_mock_create.call_count + garmin_converter_mock_create.call_count == 1
+ assert gis_converter_mock_create.call_count + garmin_converter_mock_create.call_count + pbf_converter_mock_create.call_count == 1
def test_convert_returns_id_when_use_worker_is_true(conversion_format, area_name, simple_osmosis_line_string, output_zip_file_path, filename_prefix, detail_level, out_srs, rq_mock_return, mocker, monkeypatch):
diff --git a/tests/conversion/converters/inside_worker_test/conftest.py b/tests/conversion/converters/inside_worker_test/conftest.py
index d837276c81a8def1bc982e986c0feca9087572a5..ae9f48d42b2a5ae13c6f44b435678c36569fd06e 100644
--- a/tests/conversion/converters/inside_worker_test/conftest.py
+++ b/tests/conversion/converters/inside_worker_test/conftest.py
@@ -142,7 +142,7 @@ def sql_from_bootstrap_relative_location(file_name):
@pytest.fixture()
-def data_import(osmaxx_schemas, clean_osm_tables, monkeypatch):
+def data_import(osmaxx_schemas, clean_osm_tables, monkeypatch, mocker):
from tests.conversion.converters.inside_worker_test.conftest import cleanup_osmaxx_schemas
from osmaxx.conversion.converters.converter_gis.bootstrap.bootstrap import BootStrapper
@@ -159,9 +159,6 @@ def data_import(osmaxx_schemas, clean_osm_tables, monkeypatch):
def _reset_database(self):
pass # Already taken care of by clean_osm_tables fixture.
- def _cut_area_from_pbf(self):
- pass
-
def _import_pbf(self):
pass
@@ -174,6 +171,8 @@ def data_import(osmaxx_schemas, clean_osm_tables, monkeypatch):
@contextmanager
def import_data(data):
+ from osmaxx.conversion.converters.converter_pbf import to_pbf
+ mocker.patch.object(to_pbf, 'cut_area_from_pbf', return_value=None)
bootstrapper = _BootStrapperWithoutPbfFile(data)
try:
bootstrapper.bootstrap()
diff --git a/tests/conversion/converters/inside_worker_test/osm_cutter_test.py b/tests/conversion/converters/inside_worker_test/osm_cutter_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..46d353275c79e0284a61e96f232d1430aac9eae1
--- /dev/null
+++ b/tests/conversion/converters/inside_worker_test/osm_cutter_test.py
@@ -0,0 +1,22 @@
+from osmaxx.conversion.converters.converter_pbf.to_pbf import cut_area_from_pbf
+
+
+def test_cut_area_from_pbf(mocker):
+ import subprocess
+ from osmaxx.conversion._settings import CONVERSION_SETTINGS
+
+ check_call_mock = mocker.patch.object(subprocess, 'check_call')
+ pbf_result_file_path = mocker.Mock()
+ extent_polyfile_path = mocker.Mock()
+
+ cut_area_from_pbf(pbf_result_file_path, extent_polyfile_path)
+ command = [
+ "osmconvert",
+ "--out-pbf",
+ "--complete-ways",
+ "--complex-ways",
+ "-o={}".format(pbf_result_file_path),
+ "-B={}".format(extent_polyfile_path),
+ "{}".format(CONVERSION_SETTINGS["PBF_PLANET_FILE_PATH"]),
+ ]
+ check_call_mock.assert_called_with(command)
diff --git a/tests/conversion/size_estimator_test.py b/tests/conversion/size_estimator_test.py
index e95437e2ad2b094a5d723e7c83f30cf6f1bfd09c..90875626c1c931d868cdc81c78fea25b2b51d43b 100644
--- a/tests/conversion/size_estimator_test.py
+++ b/tests/conversion/size_estimator_test.py
@@ -9,6 +9,10 @@ range_for_format_and_level = {
detail_levels.DETAIL_LEVEL_ALL: {'upper': 1.4, 'lower': 0.2},
detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 1.4, 'lower': 0.2},
},
+ formats.PBF: {
+ detail_levels.DETAIL_LEVEL_ALL: {'upper': 1.000001, 'lower': 0.99999},
+ detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 1.000001, 'lower': 0.99999},
+ },
formats.FGDB: {
detail_levels.DETAIL_LEVEL_ALL: {'upper': 7.4, 'lower': 1.6},
detail_levels.DETAIL_LEVEL_REDUCED: {'upper': 2.3, 'lower': 0.6},
diff --git a/tests/utils/test_logged_subprocess_wrapper.py b/tests/utils/test_logged_subprocess_wrapper.py
new file mode 100644
index 0000000000000000000000000000000000000000..cbdbe652647eba834d2dff53b84d5414c86a05bf
--- /dev/null
+++ b/tests/utils/test_logged_subprocess_wrapper.py
@@ -0,0 +1,44 @@
+import pytest
+import subprocess
+
+from osmaxx.conversion.converters import utils
+
+
+@pytest.fixture
+def subprocess_check_call_mock(mocker):
+ return mocker.patch.object(subprocess, 'check_call')
+
+
+def test_logged_check_call_forwards_args(subprocess_check_call_mock):
+ test_call = ['echo', '0']
+ utils.logged_check_call(test_call)
+ subprocess_check_call_mock.assert_called_with(test_call)
+
+
+def test_logged_check_call_forwards_kwargs(subprocess_check_call_mock):
+ test_call = "echo 0"
+ utils.logged_check_call(test_call, shell=True)
+ subprocess_check_call_mock.assert_called_with(test_call, shell=True)
+
+
+def test_logged_check_call_raises():
+ with pytest.raises(subprocess.CalledProcessError) as excinfo:
+ utils.logged_check_call(['false'])
+ assert excinfo.value.output is None
+ assert excinfo.value.returncode == 1
+
+
+def test_logged_check_call_logs_error_content(mocker, subprocess_check_call_mock):
+ error_output = 'example output'
+ error_return_code = 17
+ command = 'cmd'
+
+ error = subprocess.CalledProcessError(cmd=command, output=error_output, returncode=error_return_code)
+ subprocess_check_call_mock.side_effect = error
+ logger_mock = mocker.patch.object(utils.logger, 'error')
+
+ with pytest.raises(subprocess.CalledProcessError):
+ utils.logged_check_call(command)
+
+ expected_output = 'Command `cmd` exited with return value 17\nOutput:\nexample output'
+ logger_mock.assert_called_with(expected_output)