From b30d0336ae959133662e1f1f2ed367e9dd7f51bf Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 12 Jun 2026 14:45:06 +0200 Subject: [PATCH 01/10] Upgrade GeoNode to Ubuntu 26.04 LTS --- .env.sample | 2 +- .env_dev | 2 +- .env_local | 2 +- .env_test | 2 +- .github/workflows/tests.yml | 2 +- Dockerfile | 2 +- docker-compose-test.yml | 2 +- docker-compose.yml | 2 +- docker/base/ubuntu/Dockerfile | 45 +++++++++++----------- geonode/settings.py | 2 +- geonode/upload/handlers/common/vector.py | 2 +- geonode/upload/handlers/kml/handler.py | 2 +- geonode/upload/handlers/tiles3d/handler.py | 2 +- geonode/upload/handlers/xlsx/handler.py | 1 + pyproject.toml | 4 +- 15 files changed, 38 insertions(+), 36 deletions(-) diff --git a/.env.sample b/.env.sample index 21422dc8bd9..d6d915b077e 100644 --- a/.env.sample +++ b/.env.sample @@ -168,7 +168,7 @@ ASSETS_ROOT=/mnt/volumes/statics/assets/ CACHE_BUSTING_STATIC_ENABLED=False MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache MEMCACHED_LOCATION=memcached:11211 MEMCACHED_LOCK_EXPIRE=3600 MEMCACHED_LOCK_TIMEOUT=10 diff --git a/.env_dev b/.env_dev index 755e6c3b39a..e2b1d2e729a 100644 --- a/.env_dev +++ b/.env_dev @@ -166,7 +166,7 @@ SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' CACHE_BUSTING_STATIC_ENABLED=False MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache MEMCACHED_LOCATION=127.0.0.1:11211 MEMCACHED_LOCK_EXPIRE=3600 MEMCACHED_LOCK_TIMEOUT=10 diff --git a/.env_local b/.env_local index 61a65461676..69dbb9e0cbf 100644 --- a/.env_local +++ b/.env_local @@ -169,7 +169,7 @@ SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' CACHE_BUSTING_STATIC_ENABLED=False MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache MEMCACHED_LOCATION=127.0.0.1:11211 MEMCACHED_LOCK_EXPIRE=3600 MEMCACHED_LOCK_TIMEOUT=10 diff --git a/.env_test b/.env_test index ae540ec6956..a75d720319b 100644 --- a/.env_test +++ b/.env_test @@ -180,7 +180,7 @@ MEDIA_ROOT=/mnt/volumes/statics/uploaded/ CACHE_BUSTING_STATIC_ENABLED=False MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache MEMCACHED_LOCATION=memcached:11211 MEMCACHED_LOCK_EXPIRE=3600 MEMCACHED_LOCK_TIMEOUT=10 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 516d8ed8ac4..c416fa4889b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: - name: Save built Docker images run: | mkdir -p docker_images - docker save -o docker_images/django.tar geonode/geonode:latest-ubuntu-24.04 + docker save -o docker_images/django.tar geonode/geonode:latest-ubuntu-26.04 - name: Upload Docker images as artifact uses: actions/upload-artifact@v6 diff --git a/Dockerfile b/Dockerfile index 25cfe784252..88aef325e63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM geonode/geonode-base:latest-ubuntu-24.04 +FROM geonode/geonode-base:latest-ubuntu-26.04 LABEL GeoNode development team RUN mkdir -p /usr/src/geonode diff --git a/docker-compose-test.yml b/docker-compose-test.yml index c1d018a3d64..0d28c7de9ae 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -3,7 +3,7 @@ version: '3.9' # Common Django template for GeoNode and Celery services below x-common-django: &default-common-django - image: geonode/geonode:latest-ubuntu-24.04 + image: geonode/geonode:latest-ubuntu-26.04 build: context: ./ dockerfile: Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index a0d35841bba..500870dae95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.9' # Common Django template for GeoNode and Celery services below x-common-django: &default-common-django - image: geonode/geonode:latest-ubuntu-24.04 + image: geonode/geonode:latest-ubuntu-26.04 build: context: ./ dockerfile: Dockerfile diff --git a/docker/base/ubuntu/Dockerfile b/docker/base/ubuntu/Dockerfile index d34907376f4..283762cade0 100644 --- a/docker/base/ubuntu/Dockerfile +++ b/docker/base/ubuntu/Dockerfile @@ -1,19 +1,25 @@ -FROM ubuntu:24.04 +FROM ubuntu:26.04 -## Enable postgresql-client-15 -RUN apt-get update -y && apt-get install curl wget unzip gnupg2 -y -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -# will install python3.10 -RUN apt-get install lsb-release -y -RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |tee /etc/apt/sources.list.d/pgdg.list +ENV DEBIAN_FRONTEND=noninteractive -# Prepraing dependencies +RUN mkdir -p /usr/src/geonode + +## Enable PostgreSQL apt repo (PGDG) +RUN apt-get update -y && apt-get install -y curl wget unzip gnupg2 lsb-release ca-certificates +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/postgresql.gpg +# PGDG has no repo for 26.04's codename yet; pin to a supported one. +RUN echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt/ noble-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list +RUN apt-get update -y + +# Preparing dependencies RUN apt-get install -y \ libgdal-dev libpq-dev libxml2-dev \ - libxml2 libxslt1-dev zlib1g-dev libjpeg-dev \ - libmemcached-dev libldap2-dev libsasl2-dev libffi-dev + libxslt1-dev zlib1g-dev libjpeg-dev \ + libmemcached-dev libldap2-dev libsasl2-dev libffi-dev git -RUN apt-get update -y && apt-get install -y --no-install-recommends \ +RUN apt-get install -y --no-install-recommends \ gcc vim zip gettext geoip-bin cron \ postgresql-client-15 \ python3-all-dev python3-dev \ @@ -21,28 +27,23 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \ python3-pip python3-lxml \ uwsgi uwsgi-plugin-python3 python3-gdbm python3-venv python-is-python3 gdal-bin -RUN apt-get install -y devscripts build-essential debhelper pkg-kde-tools sharutils -# RUN git clone https://salsa.debian.org/debian-gis-team/proj.git /tmp/proj -# RUN cd /tmp/proj && debuild -i -us -uc -b && dpkg -i ../*.deb +RUN apt-get install -y devscripts build-essential debhelper pkg-kde-tools sharutils libcrypt-dev RUN python -m venv /usr/src/venv ENV PATH="/usr/src/venv/bin:$PATH" -RUN . /usr/src/venv/bin/activate # Install pip packages -RUN pip3 install uwsgi \ +RUN pip install uwsgi \ && pip install pip --upgrade \ && pip install wheel==0.38.1 \ && pip install invoke==2.2.0 \ - && pip install GDAL==3.8.4 + && pip install GDAL==3.12.2 -# Create symlink for "invoke" command -# Required after installing via pip inside a virtualenv and avoid changing the -# entrypoint.sh in existing GeoNode Projects where /usr/local/bin/invoke is used +# Symlink for "invoke" so existing entrypoint.sh paths (/usr/local/bin/invoke) keep working RUN ln -s $(which invoke) /usr/local/bin/invoke -# Install "sherlock" to be used with "memcached" +# Install "sherlock" for use with "memcached" RUN pip install sherlock -# Cleanup apt update lists +# Cleanup apt lists RUN rm -rf /var/lib/apt/lists/* diff --git a/geonode/settings.py b/geonode/settings.py index 232905eb328..970811578e8 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -336,7 +336,7 @@ ) MEMCACHED_ENABLED = ast.literal_eval(os.getenv("MEMCACHED_ENABLED", "False")) -MEMCACHED_BACKEND = os.getenv("MEMCACHED_BACKEND", "django.core.cache.backends.memcached.PyLibMCCache") +MEMCACHED_BACKEND = os.getenv("MEMCACHED_BACKEND", "django.core.cache.backends.memcached.PyMemcacheCache") MEMCACHED_LOCATION = os.getenv("MEMCACHED_LOCATION", "127.0.0.1:11211") MEMCACHED_LOCK_EXPIRE = int(os.getenv("MEMCACHED_LOCK_EXPIRE", 3600)) MEMCACHED_LOCK_TIMEOUT = int(os.getenv("MEMCACHED_LOCK_TIMEOUT", 10)) diff --git a/geonode/upload/handlers/common/vector.py b/geonode/upload/handlers/common/vector.py index b0b2cb9df15..31c0fbcb407 100644 --- a/geonode/upload/handlers/common/vector.py +++ b/geonode/upload/handlers/common/vector.py @@ -583,7 +583,7 @@ def open_source_file(self, files): gdal_proxy = gdal.OpenEx( files.get("base_file"), nOpenFlags=gdal.OF_VECTOR, - allowed_drivers=[self.get_ogr2ogr_driver().name], + allowed_drivers=[self.get_ogr2ogr_driver().ShortName], **self._gdal_open_options(), ) return [gdal_proxy] diff --git a/geonode/upload/handlers/kml/handler.py b/geonode/upload/handlers/kml/handler.py index 8f323238b54..ee96e34505a 100644 --- a/geonode/upload/handlers/kml/handler.py +++ b/geonode/upload/handlers/kml/handler.py @@ -84,7 +84,7 @@ def upload_validation_config(self): }, }, "kmz": { - "mimes": {"application/zip", "application/vnd.google-earth.kmz"}, + "mimes": {"application/zip", "application/vnd.google-earth.kmz", "application/octet-stream"}, }, } diff --git a/geonode/upload/handlers/tiles3d/handler.py b/geonode/upload/handlers/tiles3d/handler.py index 2c7b3929294..b6452436dd4 100755 --- a/geonode/upload/handlers/tiles3d/handler.py +++ b/geonode/upload/handlers/tiles3d/handler.py @@ -81,7 +81,7 @@ def supported_file_extension_config(self): @property def upload_validation_config(self): return { - "zip": {"mimes": {"application/zip"}}, + "zip": {"mimes": {"application/zip", "application/octet-stream"}}, } @staticmethod diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 384a37b324e..ae9e1ccd4d1 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -91,6 +91,7 @@ def upload_validation_config(self): "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/zip", + "application/octet-stream" }, }, "xls": { diff --git a/pyproject.toml b/pyproject.toml index e487a0e082c..866e8d4446e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,9 +129,9 @@ dependencies = [ "pytz==2024.1", "requests==2.34.2", "timeout-decorator==0.5.0", - "pylibmc==1.6.3", + "pymemcache==4.0.0", "sherlock==0.4.1", - "GDAL==3.8.4", + "GDAL==3.12.2", "psutil==5.9.8", "django-cors-headers==4.9.0", "django-user-agents==0.4.0", From 67ed133fa8e7723a94a26d11c7e6272570dee0a3 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 12 Jun 2026 15:27:05 +0200 Subject: [PATCH 02/10] Upgrade GeoNode to Ubuntu 26.04 LTS --- geonode/upload/handlers/common/vector.py | 3 ++- geonode/upload/handlers/xlsx/handler.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/geonode/upload/handlers/common/vector.py b/geonode/upload/handlers/common/vector.py index 31c0fbcb407..e2e5bc5910d 100644 --- a/geonode/upload/handlers/common/vector.py +++ b/geonode/upload/handlers/common/vector.py @@ -580,10 +580,11 @@ def open_source_file(self, files): while the ogr library does not allow that. For example we can call the AUTODETECT_TYPE even in python """ + driver = self.get_ogr2ogr_driver() gdal_proxy = gdal.OpenEx( files.get("base_file"), nOpenFlags=gdal.OF_VECTOR, - allowed_drivers=[self.get_ogr2ogr_driver().ShortName], + allowed_drivers=[driver.ShortName] if driver else None, **self._gdal_open_options(), ) return [gdal_proxy] diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index ae9e1ccd4d1..20c164ec51f 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -91,7 +91,7 @@ def upload_validation_config(self): "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/zip", - "application/octet-stream" + "application/octet-stream", }, }, "xls": { From 558238f793123973cd9876ca8655e5f07625d741 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 12 Jun 2026 15:28:42 +0200 Subject: [PATCH 03/10] Upgrade GeoNode to Ubuntu 26.04 LTS --- geonode/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/settings.py b/geonode/settings.py index 970811578e8..b5453bd8272 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -336,7 +336,7 @@ ) MEMCACHED_ENABLED = ast.literal_eval(os.getenv("MEMCACHED_ENABLED", "False")) -MEMCACHED_BACKEND = os.getenv("MEMCACHED_BACKEND", "django.core.cache.backends.memcached.PyMemcacheCache") +MEMCACHED_BACKEND = "django.core.cache.backends.memcached.PyMemcacheCache" MEMCACHED_LOCATION = os.getenv("MEMCACHED_LOCATION", "127.0.0.1:11211") MEMCACHED_LOCK_EXPIRE = int(os.getenv("MEMCACHED_LOCK_EXPIRE", 3600)) MEMCACHED_LOCK_TIMEOUT = int(os.getenv("MEMCACHED_LOCK_TIMEOUT", 10)) From 89efebf9d7e56bf21ac5334598bc4e7073e9b6b2 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 12 Jun 2026 18:08:27 +0200 Subject: [PATCH 04/10] Upgrade GeoNode to Ubuntu 26.04 LTS --- .github/workflows/tests.yml | 14 ++++++------ Makefile | 4 ++-- geonode/settings.py | 10 ++++++++- geonode/tasks/tasks.py | 16 +++++++------- geonode/upload/api/tests.py | 22 +++++++++---------- .../upload/handlers/common/tests_vector.py | 6 ++--- 6 files changed, 40 insertions(+), 32 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c416fa4889b..d6af526cfa1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,7 +43,7 @@ jobs: suite_name: "Smoke Tests" test_command: > ./tests/test.sh geonode.tests.smoke geonode.tests.test_rest_api geonode.tests.test_search - geonode.tests.test_utils geonode.tests.test_headers --duration=30 --failfast + geonode.tests.test_utils geonode.tests.test_headers --duration=30 main: uses: ./.github/workflows/run-test-suite.yml @@ -53,7 +53,7 @@ jobs: test_command: > TESTS=$(python -c 'import sys; from geonode import settings; print(" ".join([a + ".tests" for a in settings.GEONODE_APPS if "security" not in a and "geoserver" not in a and "upload" not in a]))') && - ./tests/test.sh $TESTS geonode.thumbs.tests geonode.people.tests geonode.people.socialaccount.providers.geonode_openid_connect.tests --duration=30 --failfast + ./tests/test.sh $TESTS geonode.thumbs.tests geonode.people.tests geonode.people.socialaccount.providers.geonode_openid_connect.tests --duration=30 security: uses: ./.github/workflows/run-test-suite.yml @@ -63,7 +63,7 @@ jobs: test_command: > TESTS=$(python -c 'import sys; from geonode import settings; print(" ".join([a + ".tests" for a in settings.GEONODE_APPS if "security" in a]))') && - ./tests/test.sh $TESTS --duration=30 --failfast + ./tests/test.sh $TESTS --duration=30 gis_backend: uses: ./.github/workflows/run-test-suite.yml @@ -73,7 +73,7 @@ jobs: test_command: > TESTS=$(python -c 'import sys; from geonode import settings; print(" ".join([a + ".tests" for a in settings.GEONODE_APPS if "geoserver" in a]))') && - ./tests/test.sh $TESTS --duration=30 --failfast + ./tests/test.sh $TESTS --duration=30 rest_apis: uses: ./.github/workflows/run-test-suite.yml @@ -82,7 +82,7 @@ jobs: suite_name: "REST API Tests" test_command: > ./tests/test.sh geonode.api.tests geonode.base.api.tests geonode.layers.api.tests - geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests --duration=30 --failfast + geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests --duration=30 csw: uses: ./.github/workflows/run-test-suite.yml @@ -90,7 +90,7 @@ jobs: with: suite_name: "CSW Tests" test_command: > - ./tests/test.sh geonode.tests.csw geonode.catalogue.backends.tests --duration=30 --failfast + ./tests/test.sh geonode.tests.csw geonode.catalogue.backends.tests --duration=30 upload: uses: ./.github/workflows/run-test-suite.yml @@ -98,7 +98,7 @@ jobs: with: suite_name: "Upload Tests" test_command: > - ./tests/test.sh geonode.upload --duration=30 --failfast + ./tests/test.sh geonode.upload --duration=30 # ------------------------------------------------- # CLEANUP JOB: REMOVE ALL ARTIFACTS AT THE END diff --git a/Makefile b/Makefile index e51e278e303..91809027d2a 100644 --- a/Makefile +++ b/Makefile @@ -46,10 +46,10 @@ pull: docker-compose pull smoketest: up - docker-compose exec django python manage.py test geonode.tests.smoke --noinput --nocapture --detailed-errors --verbosity=1 --failfast + docker-compose exec django python manage.py test geonode.tests.smoke --noinput --nocapture --detailed-errors --verbosity=1 unittest: up - docker-compose exec django python manage.py test geonode.people.tests geonode.base.tests geonode.layers.tests geonode.maps.tests geonode.proxy.tests geonode.security.tests geonode.social.tests geonode.catalogue.tests geonode.documents.tests geonode.api.tests geonode.groups.tests geonode.services.tests geonode.geoserver.tests geonode.upload.tests geonode.tasks.tests --noinput --failfast + docker-compose exec django python manage.py test geonode.people.tests geonode.base.tests geonode.layers.tests geonode.maps.tests geonode.proxy.tests geonode.security.tests geonode.social.tests geonode.catalogue.tests geonode.documents.tests geonode.api.tests geonode.groups.tests geonode.services.tests geonode.geoserver.tests geonode.upload.tests geonode.tasks.tests --noinput test: smoketest unittest diff --git a/geonode/settings.py b/geonode/settings.py index b5453bd8272..2fc86e24e84 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -348,7 +348,15 @@ "TIMEOUT": 600, "OPTIONS": {"MAX_ENTRIES": 10000}, }, - "memcached": {"BACKEND": MEMCACHED_BACKEND, "LOCATION": MEMCACHED_LOCATION}, + "memcached": { + "BACKEND": MEMCACHED_BACKEND, + "LOCATION": MEMCACHED_LOCATION, + "OPTIONS": { + "connect_timeout": 5, + "timeout": 5, + "no_delay": True, + }, + }, # MEMCACHED EXAMPLE # 'default': { # 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', diff --git a/geonode/tasks/tasks.py b/geonode/tasks/tasks.py index 1a7763d359d..4e8576690bf 100644 --- a/geonode/tasks/tasks.py +++ b/geonode/tasks/tasks.py @@ -28,19 +28,19 @@ from importlib import import_module try: - import pylibmc import sherlock - from sherlock import MCLock as Lock + from sherlock import RedisLock as Lock + import redis sherlock.configure(expire=settings.MEMCACHED_LOCK_EXPIRE, timeout=settings.MEMCACHED_LOCK_TIMEOUT) - memcache_client = pylibmc.Client([settings.MEMCACHED_LOCATION], binary=True) - lock_type = "MEMCACHED" + redis_client = redis.Redis.from_url(settings.CELERY_BROKER_URL) + lock_type = "REDIS" except Exception: from django.core.cache import cache from contextlib import contextmanager lock_type = "MEMCACHED-LOCAL-CONTEXT" - memcache_client = None + redis_client = None """ ref. @@ -76,9 +76,9 @@ def release(self): logger = get_task_logger(__name__) -def memcache_lock(lock_id): +def redis_lock(lock_id): logger.info(f"Using '{lock_type}' lock type.") - lock = Lock(lock_id, client=memcache_client) + lock = Lock(lock_id, client=redis_client) return lock @@ -88,7 +88,7 @@ def __init__(self, lock_id, blocking=False): self.blocking = blocking def __enter__(self): - self.lock = memcache_lock(self.lock_id) + self.lock = redis_lock(self.lock_id) return self def __exit__(self, exc_type, exc_value, exc_traceback): diff --git a/geonode/upload/api/tests.py b/geonode/upload/api/tests.py index 81b7701aad0..f7009647f45 100644 --- a/geonode/upload/api/tests.py +++ b/geonode/upload/api/tests.py @@ -350,22 +350,22 @@ def test_remote_upload_rejects_unsafe_url(self): response = self.client.post(self.url, data=payload) self.assertEqual(400, response.status_code) - @patch("geonode.utils.socket.getaddrinfo") @override_settings(SAFE_URL_CHECK_ENABLED=True) - def test_remote_upload_rejects_dns_resolving_to_private_ip(self, mock_dns): + def test_remote_upload_rejects_dns_resolving_to_private_ip(self): self.client.force_login(get_user_model().objects.get(username="admin")) - mock_dns.return_value = [(None, None, None, None, ("127.0.0.1", 0))] - payload = { - "url": "http://example.com/tileset.json", - "title": "Remote Title", - "type": "3dtiles", - "action": "upload", - } + with patch("geonode.utils.socket.getaddrinfo") as mock_dns: + mock_dns.return_value = [(None, None, None, None, ("127.0.0.1", 0))] + payload = { + "url": "http://example.com/tileset.json", + "title": "Remote Title", + "type": "3dtiles", + "action": "upload", + } - response = self.client.post(self.url, data=payload) + response = self.client.post(self.url, data=payload) - self.assertEqual(400, response.status_code) + self.assertEqual(400, response.status_code) def test_copy_method_not_allowed(self): self.client.force_login(get_user_model().objects.get(username="admin")) diff --git a/geonode/upload/handlers/common/tests_vector.py b/geonode/upload/handlers/common/tests_vector.py index a0b8b214dce..88b0499649b 100644 --- a/geonode/upload/handlers/common/tests_vector.py +++ b/geonode/upload/handlers/common/tests_vector.py @@ -299,7 +299,7 @@ def test_import_resource_should_not_be_imported(self, celery_chord, ogr2ogr_driv is not going to be called and the layer is skipped """ mocked_obj = MagicMock() - mocked_obj.name = "CSV" + mocked_obj.ShortName = "DUAIAI" ogr2ogr_driver.return_value = mocked_obj exec_id = None try: @@ -315,9 +315,9 @@ def test_import_resource_should_not_be_imported(self, celery_chord, ogr2ogr_driv # start the resource import self.handler.import_resource(files=self.valid_files, execution_id=str(exec_id)) self.assertIn( - "not recognized as a supported file format.", + "not recognized as being in a supported file format", exception.exception.args[0], - "not recognized as a supported file format.", + "not recognized as being in a supported file format", ) celery_chord.assert_not_called() From 719f17f16fa2af5e23ccef93aad9a54b8f96d55a Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Tue, 16 Jun 2026 11:30:55 +0200 Subject: [PATCH 05/10] Upgrade GeoNode to Ubuntu 26.04 LTS --- docker-compose-test.yml | 2 +- geonode/upload/handlers/csv/handler.py | 9 ++++++--- geonode/upload/handlers/kml/handler.py | 8 ++++++-- tests/test.sh | 3 +-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 0d28c7de9ae..6e7fb7d9220 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -11,7 +11,7 @@ x-common-django: env_file: - .env_test volumes: - # - '.:/usr/src/geonode' + - '.:/usr/src/geonode' - statics:/mnt/volumes/statics - geoserver-data-dir:/geoserver_data/data - backup-restore:/backup_restore diff --git a/geonode/upload/handlers/csv/handler.py b/geonode/upload/handlers/csv/handler.py index d025432d2b5..c8cd0b30dae 100644 --- a/geonode/upload/handlers/csv/handler.py +++ b/geonode/upload/handlers/csv/handler.py @@ -93,10 +93,13 @@ def is_valid(files, user, **kwargs): upload_validator.validate_parallelism_limit_per_user() actual_upload = upload_validator._get_parallel_uploads_count() max_upload = upload_validator._get_max_parallel_uploads() + try: + layers = CSVFileHandler().get_ogr2ogr_driver().Open(files.get("base_file")) - layers = CSVFileHandler().get_ogr2ogr_driver().Open(files.get("base_file")) - - if not layers: + if not layers: + raise InvalidCSVException("The CSV provided is invalid, no layers found") + except Exception as e: + logger.exception(e) raise InvalidCSVException("The CSV provided is invalid, no layers found") layers_count = len(layers) diff --git a/geonode/upload/handlers/kml/handler.py b/geonode/upload/handlers/kml/handler.py index ee96e34505a..b64b2017bd3 100644 --- a/geonode/upload/handlers/kml/handler.py +++ b/geonode/upload/handlers/kml/handler.py @@ -129,9 +129,13 @@ def is_valid(files, user, **kwargs): actual_upload = upload_validator._get_parallel_uploads_count() max_upload = upload_validator._get_max_parallel_uploads() - layers = KMLFileHandler().get_ogr2ogr_driver().Open(files.get("base_file")) + try: + layers = KMLFileHandler().get_ogr2ogr_driver().Open(files.get("base_file")) - if not layers: + if not layers: + raise InvalidKmlException("The kml provided is invalid") + except Exception as e: + logger.exception(e) raise InvalidKmlException("The kml provided is invalid") layers_count = len(layers) diff --git a/tests/test.sh b/tests/test.sh index d9987fd1849..2775e5f7481 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -4,5 +4,4 @@ set -a . ./.env_test set +a -paver setup_data -coverage run --branch --source=geonode manage.py test -v 3 --keepdb $@ \ No newline at end of file +coverage run --branch --source=geonode manage.py test -v 3 --keepdb $@ From 7a162ec6a55b4c451af6ee8ef292ac001284866a Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Tue, 16 Jun 2026 17:39:32 +0200 Subject: [PATCH 06/10] Upgrade GeoNode to Ubuntu 26.04 LTS --- tests/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test.sh b/tests/test.sh index 2775e5f7481..433d4f6390b 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -4,4 +4,5 @@ set -a . ./.env_test set +a +paver setup_data coverage run --branch --source=geonode manage.py test -v 3 --keepdb $@ From edfef2ec58994e542b688970356a5133693f1644 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Wed, 17 Jun 2026 16:13:41 +0200 Subject: [PATCH 07/10] move cache and locks in redis --- .env.sample | 6 - .env_dev | 7 -- .env_local | 5 - .env_test | 5 - docker-compose-dev.yml | 6 + docker-compose-test.yml | 13 --- docker-compose.yml | 21 ++-- docker/base/ubuntu/Dockerfile | 4 +- docs/src/setup/bare/prerequisites.md | 2 +- docs/src/setup/configuration/settings.md | 14 --- geonode/harvesting/admin.py | 6 + geonode/harvesting/apps.py | 2 +- geonode/harvesting/models.py | 3 +- geonode/resource/api/tasks.py | 142 ++++++++++++----------- geonode/settings.py | 119 ++++++++----------- geonode/tasks/tasks.py | 18 +-- pyproject.toml | 2 +- 17 files changed, 159 insertions(+), 216 deletions(-) diff --git a/.env.sample b/.env.sample index d6d915b077e..6578d4e5cfd 100644 --- a/.env.sample +++ b/.env.sample @@ -167,12 +167,6 @@ ASSETS_ROOT=/mnt/volumes/statics/assets/ CACHE_BUSTING_STATIC_ENABLED=False -MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache -MEMCACHED_LOCATION=memcached:11211 -MEMCACHED_LOCK_EXPIRE=3600 -MEMCACHED_LOCK_TIMEOUT=10 -# # Options for memcached binary, e.g. -vvv to log all requests and cache hits # MEMCACHED_OPTIONS= diff --git a/.env_dev b/.env_dev index e2b1d2e729a..03762bceee0 100644 --- a/.env_dev +++ b/.env_dev @@ -165,17 +165,10 @@ SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' CACHE_BUSTING_STATIC_ENABLED=False -MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache -MEMCACHED_LOCATION=127.0.0.1:11211 -MEMCACHED_LOCK_EXPIRE=3600 -MEMCACHED_LOCK_TIMEOUT=10 PERMISSION_CACHE_EXPIRATION_TIME=604800 # # Options for memcached binary, e.g. -vvv to log all requests and cache hits # -MEMCACHED_OPTIONS= - MAX_DOCUMENT_SIZE=200 CLIENT_RESULTS_LIMIT=5 API_LIMIT_PER_PAGE=1000 diff --git a/.env_local b/.env_local index 69dbb9e0cbf..06b3015d8bf 100644 --- a/.env_local +++ b/.env_local @@ -168,11 +168,6 @@ SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' CACHE_BUSTING_STATIC_ENABLED=False -MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache -MEMCACHED_LOCATION=127.0.0.1:11211 -MEMCACHED_LOCK_EXPIRE=3600 -MEMCACHED_LOCK_TIMEOUT=10 PERMISSION_CACHE_EXPIRATION_TIME=604800 # # Options for memcached binary, e.g. -vvv to log all requests and cache hits diff --git a/.env_test b/.env_test index a75d720319b..cb730acec35 100644 --- a/.env_test +++ b/.env_test @@ -179,11 +179,6 @@ MEDIA_ROOT=/mnt/volumes/statics/uploaded/ CACHE_BUSTING_STATIC_ENABLED=False -MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache -MEMCACHED_LOCATION=memcached:11211 -MEMCACHED_LOCK_EXPIRE=3600 -MEMCACHED_LOCK_TIMEOUT=10 PERMISSION_CACHE_EXPIRATION_TIME=604800 # # Options for memcached binary, e.g. -vvv to log all requests and cache hits diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index ab2398edafc..334eb2a4c45 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -139,6 +139,12 @@ services: volumes: - redisdata:/data restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 20s + timeout: 3s + retries: 3 + start_period: 5s volumes: statics: diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 6e7fb7d9220..eaf9d4162f0 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -55,19 +55,6 @@ services: - statics:/mnt/volumes/statics restart: unless-stopped - # memcached service - memcached: - image: memcached:alpine - container_name: memcached4${COMPOSE_PROJECT_NAME} - command: memcached ${MEMCACHED_OPTIONS} - restart: unless-stopped - healthcheck: - test: nc -z 127.0.0.1 11211 - interval: 30s - timeout: 30s - retries: 5 - start_period: 30s - # # Gets and installs letsencrypt certificates # letsencrypt: # image: geonode/letsencrypt:2.6.0-latest diff --git a/docker-compose.yml b/docker-compose.yml index 500870dae95..e6fed504474 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,8 @@ x-common-django: depends_on: db: condition: service_healthy + redis: + condition: service_healthy services: @@ -50,19 +52,6 @@ services: entrypoint: ["/usr/src/geonode/entrypoint.sh"] command: "celery-cmd" - # memcached service - memcached: - image: memcached:alpine - container_name: memcached4${COMPOSE_PROJECT_NAME} - command: memcached ${MEMCACHED_OPTIONS} - restart: unless-stopped - healthcheck: - test: nc -z 127.0.0.1 11211 - interval: 30s - timeout: 30s - retries: 5 - start_period: 30s - # Nginx is serving django static and media files and proxies to django and geonode geonode: image: geonode/nginx:1.31.0-latest @@ -152,6 +141,12 @@ services: volumes: - redisdata:/data restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 20s + timeout: 3s + retries: 3 + start_period: 5s volumes: statics: diff --git a/docker/base/ubuntu/Dockerfile b/docker/base/ubuntu/Dockerfile index 283762cade0..ad9e5faaae8 100644 --- a/docker/base/ubuntu/Dockerfile +++ b/docker/base/ubuntu/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update -y RUN apt-get install -y \ libgdal-dev libpq-dev libxml2-dev \ libxslt1-dev zlib1g-dev libjpeg-dev \ - libmemcached-dev libldap2-dev libsasl2-dev libffi-dev git + libldap2-dev libsasl2-dev libffi-dev git RUN apt-get install -y --no-install-recommends \ gcc vim zip gettext geoip-bin cron \ @@ -42,7 +42,7 @@ RUN pip install uwsgi \ # Symlink for "invoke" so existing entrypoint.sh paths (/usr/local/bin/invoke) keep working RUN ln -s $(which invoke) /usr/local/bin/invoke -# Install "sherlock" for use with "memcached" +# Install "sherlock" for use with "redis cache" RUN pip install sherlock # Cleanup apt lists diff --git a/docs/src/setup/bare/prerequisites.md b/docs/src/setup/bare/prerequisites.md index 73cd0686e37..ed1295c3482 100644 --- a/docs/src/setup/bare/prerequisites.md +++ b/docs/src/setup/bare/prerequisites.md @@ -52,7 +52,7 @@ sudo apt install -y --allow-downgrades \ build-essential \ python3-gdal=3.8.4+dfsg-3ubuntu3 gdal-bin=3.8.4+dfsg-3ubuntu3 libgdal-dev=3.8.4+dfsg-3ubuntu3 \ python3-all-dev python3.12-dev python3.12-venv \ - libxml2 libxml2-dev gettext libmemcached-dev zlib1g-dev \ + libxml2 libxml2-dev gettext zlib1g-dev \ libxslt1-dev libjpeg-dev libpng-dev libpq-dev \ software-properties-common \ git unzip gcc zlib1g-dev libgeos-dev libproj-dev \ diff --git a/docs/src/setup/configuration/settings.md b/docs/src/setup/configuration/settings.md index 5c245b4b90c..d37292c20e2 100644 --- a/docs/src/setup/configuration/settings.md +++ b/docs/src/setup/configuration/settings.md @@ -1196,20 +1196,6 @@ For more information, please rely to `TestMetadataStorers` which contain a smoke The path to an image used as thumbnail placeholder. -**MEMCACHED_BACKEND** - -: Default: ``django.core.cache.backends.memcached.PyMemcacheCache`` - -Define which backend of memcached will be used - - -**MEMCACHED_ENABLED** - -: Default: ``False`` - -If True, will use MEMCACHED_BACKEND as default backend in CACHES - - **MODIFY_TOPICCATEGORY** diff --git a/geonode/harvesting/admin.py b/geonode/harvesting/admin.py index e992bc4e6bc..3e536fc80e2 100644 --- a/geonode/harvesting/admin.py +++ b/geonode/harvesting/admin.py @@ -37,6 +37,9 @@ models, ) +from django.core.cache import caches +service_cache = caches["services"] + logger = logging.getLogger(__name__) @@ -85,6 +88,7 @@ class HarvesterAdmin(admin.ModelAdmin): ] def save_model(self, request, harvester: models.Harvester, form, change): + service_cache.delete(harvester.remote_url) super().save_model(request, harvester, form, change) if _worker_config_changed(form): self.message_user( @@ -190,6 +194,7 @@ def initiate_abort_perform_harvesting(self, request, queryset): if harvester.update_availability(): harvester.initiate_abort_perform_harvesting() being_aborted.append(harvester) + service_cache.delete(harvester.remote_url) else: raise RuntimeError(f"Harvester {harvester!r} is not available") except RuntimeError as exc: @@ -208,6 +213,7 @@ def initiate_abort_perform_harvesting(self, request, queryset): def reset_harvester_status(self, request, queryset): for harvester in queryset: if harvester.status != models.Harvester.STATUS_READY: + service_cache.delete(harvester.remote_url) harvester.status = models.Harvester.STATUS_READY harvester.save() self.message_user(request, _("Resetting status for harvester %(name)s...") % {"name": harvester.name}) diff --git a/geonode/harvesting/apps.py b/geonode/harvesting/apps.py index 97b886165ad..eee03c89a34 100644 --- a/geonode/harvesting/apps.py +++ b/geonode/harvesting/apps.py @@ -34,5 +34,5 @@ def ready(self): urlpatterns += [re_path(r"^api/v2/", include("geonode.harvesting.api.urls"))] settings.CELERY_BEAT_SCHEDULE["harvesting-scheduler"] = { "task": "geonode.harvesting.tasks.harvesting_scheduler", - "schedule": config.get_setting("HARVESTER_SCHEDULER_FREQUENCY_MINUTES") * 60, + "schedule": config.get_setting("HARVESTER_SCHEDULER_FREQUENCY_MINUTES") * 0.5, } diff --git a/geonode/harvesting/models.py b/geonode/harvesting/models.py index 8afd24534ea..04fb5954612 100644 --- a/geonode/harvesting/models.py +++ b/geonode/harvesting/models.py @@ -30,13 +30,14 @@ from django.utils import timezone from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ - +from django.core.cache import caches from geonode import celery_app from .config import get_setting logger = logging.getLogger(__name__) +service_cache = caches["services"] class Harvester(models.Model): STATUS_READY = "ready" diff --git a/geonode/resource/api/tasks.py b/geonode/resource/api/tasks.py index d7f632929c6..ce792702953 100644 --- a/geonode/resource/api/tasks.py +++ b/geonode/resource/api/tasks.py @@ -92,71 +92,79 @@ def resouce_service_dispatcher(self, execution_id: str): A client is able to query the `status_url` endpoint in order to get the current `status` other than the `output_params`. """ - with AcquireLock(execution_id) as lock: - if lock.acquire() is True: - try: - _exec_request = ExecutionRequest.objects.filter(exec_id=execution_id) - if _exec_request.exists(): - _request = _exec_request.get() - if _request.status == ExecutionRequest.STATUS_READY: - _exec_request.update(status=ExecutionRequest.STATUS_RUNNING) - _request.refresh_from_db() - if _request.geonode_resource: - _manager = resource_manager_registry.get_for_instance(_request.geonode_resource) - else: - resource_type = _request.input_params.get("resource_type") - _manager = resource_manager_registry.get_for_type(resource_type) - if hasattr(_manager, _request.func_name): - try: - _signature = signature(getattr(_manager, _request.func_name)) - _args = [] - _kwargs = {} - for _param_name in _signature.parameters: - if _request.input_params and _request.input_params.get(_param_name, None): - _param = _signature.parameters.get(_param_name) - _param_value = _get_param_value(_param, _request.input_params.get(_param_name)) - if _param.kind == Parameter.POSITIONAL_ONLY: - _args.append(_param_value) - else: - _kwargs[_param_name] = _param_value - - _bindings = _signature.bind(*_args, **_kwargs) - _bindings.apply_defaults() - - _output = getattr(_manager, _request.func_name)(*_bindings.args, **_bindings.kwargs) - _output_params = {} - if _output is not None and _signature.return_annotation != Signature.empty: - if _signature.return_annotation.__module__ == "builtins": - _output_params = {"output": _output} - elif _signature.return_annotation == ResourceBase or isinstance( - _output, ResourceBase - ): - _output_params = {"output": {"uuid": _output.uuid}} - else: - _output_params = {"output": None} - _exec_request.update( - status=ExecutionRequest.STATUS_FINISHED, - finished=datetime.now(), - output_params=_output_params, - ) - _request.refresh_from_db() - except Exception as e: - logger.exception(e) - _exec_request.update( - status=ExecutionRequest.STATUS_FAILED, - finished=datetime.now(), - output_params={ - "error": _( - f"Error occurred while executin the operation: '{_request.func_name}'" - ), - "exception": str(e), - }, - ) - _request.refresh_from_db() - else: - logger.warning(_(f"Could not find the operation name: '{_request.func_name}'")) + _exec_request = None + try: + with AcquireLock(execution_id) as lock: + if lock.acquire() is True: + try: + _exec_request = ExecutionRequest.objects.filter(exec_id=execution_id) + if _exec_request.exists(): + _request = _exec_request.get() + if _request.status == ExecutionRequest.STATUS_READY: + _exec_request.update(status=ExecutionRequest.STATUS_RUNNING) _request.refresh_from_db() - finally: - lock.release() - - logger.debug(f"WARNING: The requested ExecutionRequest with 'exec_id'={execution_id} was not found!") + if _request.geonode_resource: + _manager = resource_manager_registry.get_for_instance(_request.geonode_resource) + else: + resource_type = _request.input_params.get("resource_type") + _manager = resource_manager_registry.get_for_type(resource_type) + if hasattr(_manager, _request.func_name): + try: + _signature = signature(getattr(_manager, _request.func_name)) + _args = [] + _kwargs = {} + for _param_name in _signature.parameters: + if _request.input_params and _request.input_params.get(_param_name, None): + _param = _signature.parameters.get(_param_name) + _param_value = _get_param_value(_param, _request.input_params.get(_param_name)) + if _param.kind == Parameter.POSITIONAL_ONLY: + _args.append(_param_value) + else: + _kwargs[_param_name] = _param_value + + _bindings = _signature.bind(*_args, **_kwargs) + _bindings.apply_defaults() + + _output = getattr(_manager, _request.func_name)(*_bindings.args, **_bindings.kwargs) + _output_params = {} + if _output is not None and _signature.return_annotation != Signature.empty: + if _signature.return_annotation.__module__ == "builtins": + _output_params = {"output": _output} + elif _signature.return_annotation == ResourceBase or isinstance( + _output, ResourceBase + ): + _output_params = {"output": {"uuid": _output.uuid}} + else: + _output_params = {"output": None} + _exec_request.update( + status=ExecutionRequest.STATUS_FINISHED, + finished=datetime.now(), + output_params=_output_params, + ) + _request.refresh_from_db() + except Exception as e: + logger.exception(e) + _exec_request.update( + status=ExecutionRequest.STATUS_FAILED, + finished=datetime.now(), + output_params={ + "error": _( + f"Error occurred while executin the operation: '{_request.func_name}'" + ), + "exception": str(e), + }, + ) + _request.refresh_from_db() + else: + logger.warning(_(f"Could not find the operation name: '{_request.func_name}'")) + _request.refresh_from_db() + finally: + lock.release() + + logger.debug(f"WARNING: The requested ExecutionRequest with 'exec_id'={execution_id} was not found!") + except Exception as e: + logger.exception(e) + ExecutionRequest.objects.filter(exec_id=execution_id)\ + .update(status=ExecutionRequest.STATUS_FAILED, log=str(e)) + + raise "Error durng the resource dispatcher run, please verify the log" \ No newline at end of file diff --git a/geonode/settings.py b/geonode/settings.py index 2fc86e24e84..fea750dc0f2 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -335,76 +335,6 @@ # 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) -MEMCACHED_ENABLED = ast.literal_eval(os.getenv("MEMCACHED_ENABLED", "False")) -MEMCACHED_BACKEND = "django.core.cache.backends.memcached.PyMemcacheCache" -MEMCACHED_LOCATION = os.getenv("MEMCACHED_LOCATION", "127.0.0.1:11211") -MEMCACHED_LOCK_EXPIRE = int(os.getenv("MEMCACHED_LOCK_EXPIRE", 3600)) -MEMCACHED_LOCK_TIMEOUT = int(os.getenv("MEMCACHED_LOCK_TIMEOUT", 10)) - -CACHES = { - # Local Memory CACHE FOR DEVELOPMENT - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "TIMEOUT": 600, - "OPTIONS": {"MAX_ENTRIES": 10000}, - }, - "memcached": { - "BACKEND": MEMCACHED_BACKEND, - "LOCATION": MEMCACHED_LOCATION, - "OPTIONS": { - "connect_timeout": 5, - "timeout": 5, - "no_delay": True, - }, - }, - # MEMCACHED EXAMPLE - # 'default': { - # 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - # 'LOCATION': '127.0.0.1:11211', - # }, - # FILECACHE EXAMPLE - # 'default': { - # 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - # 'LOCATION': '/tmp/django_cache', - # }, - # DATABASE EXAMPLE -> python manage.py createcachetable - # 'default': { - # 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', - # 'LOCATION': 'my_cache_table', - # }, - # LOCAL-MEMORY CACHING - # 'default': { - # 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - # 'LOCATION': 'geonode-cache', - # 'TIMEOUT': 10, - # 'OPTIONS': { - # 'MAX_ENTRIES': 10000 - # } - # }, - "resources": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "TIMEOUT": 600, - "OPTIONS": {"MAX_ENTRIES": 10000}, - }, - "services": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "TIMEOUT": 600, - "OPTIONS": {"MAX_ENTRIES": 10000}, - }, -} - -PERMISSION_CACHE_EXPIRATION_TIME = int(os.getenv("PERMISSION_CACHE_EXPIRATION_TIME", 60 * 60 * 24 * 7)) # 7 days - -# define service cache timeout -SERVICE_CACHE_EXPIRATION_TIME = int(os.getenv("SERVICE_CACHE_EXPIRATION_TIME", 600)) - -if MEMCACHED_ENABLED: - CACHES["default"] = { - "BACKEND": MEMCACHED_BACKEND, - "LOCATION": MEMCACHED_LOCATION, - } - CACHES["services"] = CACHES["default"].copy() | {"TIMEOUT": SERVICE_CACHE_EXPIRATION_TIME} - # Whitenoise Settings - ref.: http://whitenoise.evans.io/en/stable/django.html WHITENOISE_MANIFEST_STRICT = ast.literal_eval(os.getenv("WHITENOISE_MANIFEST_STRICT", "False")) COMPRESS_STATIC_FILES = ast.literal_eval(os.getenv("COMPRESS_STATIC_FILES", "False")) @@ -809,7 +739,7 @@ SESSION_SERIALIZER = "django.contrib.sessions.serializers.JSONSerializer" SESSION_ENGINE = os.environ.get("SESSION_ENGINE", "django.contrib.sessions.backends.db") if SESSION_ENGINE in ("django.contrib.sessions.backends.cached_db", "django.contrib.sessions.backends.cache"): - SESSION_CACHE_ALIAS = "memcached" # use memcached cache if a cached backend is requested + SESSION_CACHE_ALIAS = "default" # use redis cache if a cached backend is requested # Add additional paths (as regular expressions) that don't require # authentication. @@ -1736,6 +1666,53 @@ def get_geonode_catalogue_service(): # Disabled by default and I like it, because we use Sentry for this. CELERY_SEND_TASK_ERROR_EMAILS = ast.literal_eval(os.environ.get("CELERY_SEND_TASK_ERROR_EMAILS", "False")) + +# ########################################################################### # +# REDIS CACHE SETTINGS +# ########################################################################### # + + +REDIS_LOCK_EXPIRE = int(os.getenv("REDIS_LOCK_EXPIRE", 3600)) +REDIS_LOCK_TIMEOUT = int(os.getenv("REDIS_LOCK_TIMEOUT", 10)) + +REDIS_CACHE_URL = REDIS_SIGNALS_BROKER_URL.replace("/0", "/3") + +CACHES = { + # Local Memory CACHE FOR DEVELOPMENT + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": 600, + "OPTIONS": {"MAX_ENTRIES": 10000}, + }, + "resources": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": 600, + "OPTIONS": {"MAX_ENTRIES": 10000}, + }, + "services": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": 600, + "OPTIONS": {"MAX_ENTRIES": 10000}, + }, +} + +PERMISSION_CACHE_EXPIRATION_TIME = int(os.getenv("PERMISSION_CACHE_EXPIRATION_TIME", 60 * 60 * 24 * 7)) # 7 days + +# define service cache timeout +SERVICE_CACHE_EXPIRATION_TIME = int(os.getenv("SERVICE_CACHE_EXPIRATION_TIME", 600)) + +if not TESTING: + CACHES["default"] = { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": REDIS_CACHE_URL, + "TIMEOUT": 300, + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + } + } + CACHES["services"] = CACHES["default"].copy() | {"TIMEOUT": SERVICE_CACHE_EXPIRATION_TIME} + CACHES["resources"] = CACHES["default"].copy() + # ########################################################################### # # NOTIFICATIONS SETTINGS # ########################################################################### # diff --git a/geonode/tasks/tasks.py b/geonode/tasks/tasks.py index 4e8576690bf..901d7e4ddb0 100644 --- a/geonode/tasks/tasks.py +++ b/geonode/tasks/tasks.py @@ -29,18 +29,18 @@ try: import sherlock - from sherlock import RedisLock as Lock + from sherlock import RedisLock as SherlockRedisLock import redis - - sherlock.configure(expire=settings.MEMCACHED_LOCK_EXPIRE, timeout=settings.MEMCACHED_LOCK_TIMEOUT) - redis_client = redis.Redis.from_url(settings.CELERY_BROKER_URL) - lock_type = "REDIS" -except Exception: - from django.core.cache import cache from contextlib import contextmanager +except ImportError: + from django.core.cache import cache - lock_type = "MEMCACHED-LOCAL-CONTEXT" + lock_type = "LOCAL-CONTEXT" redis_client = None +else: + sherlock.configure(expire=settings.REDIS_LOCK_EXPIRE, timeout=settings.REDIS_LOCK_TIMEOUT) + redis_client = redis.Redis.from_url(settings.REDIS_CACHE_URL) + lock_type = "REDIS" """ ref. @@ -78,7 +78,7 @@ def release(self): def redis_lock(lock_id): logger.info(f"Using '{lock_type}' lock type.") - lock = Lock(lock_id, client=redis_client) + lock = SherlockRedisLock(lock_id, client=redis_client) return lock diff --git a/pyproject.toml b/pyproject.toml index 866e8d4446e..24d2940dac3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,7 @@ dependencies = [ "pytz==2024.1", "requests==2.34.2", "timeout-decorator==0.5.0", - "pymemcache==4.0.0", + "django-redis>=5.4", "sherlock==0.4.1", "GDAL==3.12.2", "psutil==5.9.8", From 774bd69d56847a170a5bb235608872890863036d Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Wed, 17 Jun 2026 16:47:44 +0200 Subject: [PATCH 08/10] move cache and locks in redis --- docker-compose-test.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index eaf9d4162f0..405d211c215 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -20,6 +20,8 @@ x-common-django: depends_on: db: condition: service_healthy + redis: + condition: service_healthy services: @@ -120,6 +122,20 @@ services: #ports: # - "5432:5432" + # Vanilla Redis service. This is needed by celery + redis: + image: redis:7-alpine + container_name: redis4${COMPOSE_PROJECT_NAME} + volumes: + - redisdata:/data + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 20s + timeout: 3s + retries: 3 + start_period: 5s + volumes: statics: name: ${COMPOSE_PROJECT_NAME}-statics @@ -139,3 +155,5 @@ volumes: name: ${COMPOSE_PROJECT_NAME}-data tmp: name: ${COMPOSE_PROJECT_NAME}-tmp + redisdata: + name: ${COMPOSE_PROJECT_NAME}-redisdata From d581be1c0d06269db37bb0f02d27f62c5de64396 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Wed, 17 Jun 2026 18:33:07 +0200 Subject: [PATCH 09/10] merge with master --- geonode/harvesting/admin.py | 1 + geonode/harvesting/models.py | 1 + geonode/resource/api/tasks.py | 11 ++++++----- geonode/settings.py | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/geonode/harvesting/admin.py b/geonode/harvesting/admin.py index 3e536fc80e2..7d2f375c780 100644 --- a/geonode/harvesting/admin.py +++ b/geonode/harvesting/admin.py @@ -38,6 +38,7 @@ ) from django.core.cache import caches + service_cache = caches["services"] logger = logging.getLogger(__name__) diff --git a/geonode/harvesting/models.py b/geonode/harvesting/models.py index 04fb5954612..8bd117463a9 100644 --- a/geonode/harvesting/models.py +++ b/geonode/harvesting/models.py @@ -39,6 +39,7 @@ service_cache = caches["services"] + class Harvester(models.Model): STATUS_READY = "ready" STATUS_UPDATING_HARVESTABLE_RESOURCES = "updating-harvestable-resources" diff --git a/geonode/resource/api/tasks.py b/geonode/resource/api/tasks.py index ce792702953..653cf5d0a0d 100644 --- a/geonode/resource/api/tasks.py +++ b/geonode/resource/api/tasks.py @@ -116,7 +116,9 @@ def resouce_service_dispatcher(self, execution_id: str): for _param_name in _signature.parameters: if _request.input_params and _request.input_params.get(_param_name, None): _param = _signature.parameters.get(_param_name) - _param_value = _get_param_value(_param, _request.input_params.get(_param_name)) + _param_value = _get_param_value( + _param, _request.input_params.get(_param_name) + ) if _param.kind == Parameter.POSITIONAL_ONLY: _args.append(_param_value) else: @@ -164,7 +166,6 @@ def resouce_service_dispatcher(self, execution_id: str): logger.debug(f"WARNING: The requested ExecutionRequest with 'exec_id'={execution_id} was not found!") except Exception as e: logger.exception(e) - ExecutionRequest.objects.filter(exec_id=execution_id)\ - .update(status=ExecutionRequest.STATUS_FAILED, log=str(e)) - - raise "Error durng the resource dispatcher run, please verify the log" \ No newline at end of file + ExecutionRequest.objects.filter(exec_id=execution_id).update(status=ExecutionRequest.STATUS_FAILED, log=str(e)) + + raise "Error durng the resource dispatcher run, please verify the log" diff --git a/geonode/settings.py b/geonode/settings.py index 867a62bd031..7bb3b66a2f6 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -1688,7 +1688,7 @@ def get_geonode_catalogue_service(): "TIMEOUT": 600, "OPTIONS": {"MAX_ENTRIES": 10000}, }, - "services": { + "services": { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", "TIMEOUT": 600, "OPTIONS": {"MAX_ENTRIES": 10000}, @@ -1707,7 +1707,7 @@ def get_geonode_catalogue_service(): "TIMEOUT": 300, "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", - } + }, } CACHES["services"] = CACHES["default"].copy() | {"TIMEOUT": SERVICE_CACHE_EXPIRATION_TIME} CACHES["resources"] = CACHES["default"].copy() From 473cec1e1510055f3d795f9c2f2f232c88a2e9be Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Thu, 18 Jun 2026 11:20:03 +0200 Subject: [PATCH 10/10] Fix worng merge --- .github/workflows/tests.yml | 2 +- Dockerfile | 4 ---- docker-compose.yml | 8 -------- geonode/settings.py | 2 +- 4 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index daa7a0c47ef..ef261afd99c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: ENVIRONMENT=test docker compose build db geoserver django nginx data-dir-conf --progress plain + run: ENVIRONMENT=test docker compose build db geoserver django nginx data-dir-conf - name: Save built Docker images run: | diff --git a/Dockerfile b/Dockerfile index 20c91c8c01a..62c112be0d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,5 @@ -<<<<<<< HEAD FROM geonode/geonode-base:latest-ubuntu-26.04 LABEL GeoNode development team -======= -FROM geonode/geonode-base:latest-ubuntu-24.04 ->>>>>>> a9ade4357dd3aaecd37654627c6d694a283679a1 ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 diff --git a/docker-compose.yml b/docker-compose.yml index 16fec1c8125..19918029de9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,11 +19,6 @@ x-common-django: condition: service_healthy redis: condition: service_healthy -<<<<<<< HEAD -======= - memcached: - condition: service_healthy ->>>>>>> a9ade4357dd3aaecd37654627c6d694a283679a1 services: @@ -53,15 +48,12 @@ services: - IS_CELERY=True entrypoint: ["/usr/src/geonode/entrypoint.sh"] command: "celery-cmd" -<<<<<<< HEAD -======= healthcheck: test: "celery -b ${BROKER_URL} inspect ping" start_period: 30s interval: 30s timeout: 10s retries: 2 ->>>>>>> a9ade4357dd3aaecd37654627c6d694a283679a1 # Nginx is serving django static and media files and proxies to django and geonode nginx: diff --git a/geonode/settings.py b/geonode/settings.py index 7bb3b66a2f6..1d198fea73d 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -1700,7 +1700,7 @@ def get_geonode_catalogue_service(): # define service cache timeout SERVICE_CACHE_EXPIRATION_TIME = int(os.getenv("SERVICE_CACHE_EXPIRATION_TIME", 600)) -if not TESTING: +if ASYNC_SIGNALS: CACHES["default"] = { "BACKEND": "django_redis.cache.RedisCache", "LOCATION": REDIS_CACHE_URL,