updated database usage in django and bootstrapper

osmaxx/conversion/conversion_helper.py

@@ -61,7 +61,7 @@ class ConversionHelper:
             self._cut_pbf_along_polyfile(polyfile.name)
 
     def create_export(self, export_id):
-        # this looks funny, but because processing tales so long,
+        # this looks funny, but because processing takes so long,
         # we need to refetch the objects otherwise the database
         # connection is already closed
         export = Export.objects.get(pk=export_id)

osmaxx/conversion/converters/converter_gis/bootstrap/bootstrap.py

+import random
+import string
 import glob
 import os
 from time import sleep
@@ -21,6 +23,13 @@ from osmaxx.utils import polyfile_helpers
 from osmaxx.conversion.conversion_settings import DBConfig
 
 
+def get_random_string(length):
+    # choose from all lowercase letter
+    letters = string.ascii_lowercase
+    result_str = ''.join(random.choice(letters) for i in range(length))
+    return result_str
+
+
 class BootStrapper:
     def __init__(
         self, area_polyfile_string, *, cutted_pbf_file, detail_level=DETAIL_LEVEL_ALL
@@ -29,7 +38,7 @@ class BootStrapper:
         self.area_polyfile_string = area_polyfile_string
         self._pbf_file_path = cutted_pbf_file
         self._detail_level = DETAIL_LEVEL_TABLES[detail_level]
-        tmp_name = str(uuid.uuid4()).replace("-", "_")
+        tmp_name = get_random_string(20)
         self.db_config = DBConfig(db_name=f"osmaxx_{tmp_name}")
         self._postgres = get_default_postgres_wrapper(
             db_name=self.db_config.db_name,
@@ -52,8 +61,10 @@ class BootStrapper:
 
     def bootstrap(self):
         print("Bootstrap", "#" * 30)
-        print("resetting database/views")
-        self._reset_database()
+        # print("resetting database/views")
+        # self._reset_database()
+        print("setup database")
+        self._setup_db()
         print("import boundaries")
         self._import_boundaries()
         print("import pbf")

osmaxx/conversion/converters/converter_gis/helper/postgres_wrapper.py

@@ -5,6 +5,7 @@ from sqlalchemy import create_engine, event
 
 from sqlalchemy.engine.url import URL
 from sqlalchemy_utils import functions as sql_alchemy_utils
+from sqlalchemy.orm.session import close_all_sessions
 
 logger = logging.getLogger()
 
@@ -22,7 +23,14 @@ class Postgres:
         }
         connection_url = URL("postgresql", **self._connection_parameters)
 
-        self._engine = create_engine(connection_url, echo=False)
+        self._engine = create_engine(
+            connection_url,
+            echo=False,
+            pool_size=6,
+            max_overflow=0,
+            pool_pre_ping=True,
+        )
+
         if search_path:
 
             @event.listens_for(self._engine, "connect", insert=True)

osmaxx/excerptexport/static/excerptexport/scripts/maps/excerpt_viewer.js

@@ -6,7 +6,7 @@ window.ExcerptViewer = function(mapElementID, excerptApiUrl) {
 
     L.control.scale().addTo(this.map);
     // add an OpenStreetMap tile layer
-    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
+    L.tileLayer('https://{s}.tile.osm.org/{z}/{x}/{y}.png', {
         attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors'
     }).addTo(this.map);
 

osmaxx/excerptexport/static/excerptexport/scripts/maps/new_excerpt_map.js

@@ -2,7 +2,7 @@
 (function(){
     var map = L.map('map', {worldCopyJump: true}).setView([0, 0], 2);
     // add an OpenStreetMap tile layer
-    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
+    L.tileLayer('https://{s}.tile.osm.org/{z}/{x}/{y}.png', {
         attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors'
     }).addTo(map);
     L.control.scale().addTo(map);

web_frontend/config/settings.py

@@ -144,8 +144,9 @@ DATABASES = {
     ),
 }
 # enable persistent connections:
-# https://docs.djangoproject.com/en/3.2/ref/settings/#conn-max-age
-DATABASES["default"]["CONN_MAX_AGE"] = 0
+# https://docs.djangoproject.com/en/4.0/ref/settings/#conn-max-age
+DATABASES["default"]["CONN_MAX_AGE"] = None
+
 # disble atomic request, don't remember why we enabled it
 # DATABASES["default"]["ATOMIC_REQUESTS"] = True
 
@@ -190,6 +191,7 @@ TEMPLATES = [
         "DIRS": [
             str(APPS_DIR.path("templates")),
         ],
+        "APP_DIRS": True,
         "OPTIONS": {
             "context_processors": [
                 "django.contrib.auth.context_processors.auth",
@@ -204,10 +206,6 @@ TEMPLATES = [
                 "social_django.context_processors.login_redirect",
                 "django.template.context_processors.request",
             ],
-            "loaders": [
-                "django.template.loaders.filesystem.Loader",
-                "django.template.loaders.app_directories.Loader",
-            ],
         },
     },
 ]