From a0684922db764626d9e2df5f99f3c0c12341259d Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Wed, 15 Jul 2026 10:39:45 +0200 Subject: [PATCH 1/2] fix(api): support emoji and punycode domains in analysis tasks WebPage validation stores URLs as Unicode, which breaks requests for emoji domains like xn--3s8h30f.ws. Use AnyHttpUrl for punycode conversion and pass the encoded URL to the worker queue. Fixes cnumr/EcoIndex#416 Co-authored-by: Cursor --- bases/ecoindex/backend/routers/tasks.py | 42 +++-------------------- test/bases/ecoindex/backend/test_tasks.py | 16 +++++++++ 2 files changed, 20 insertions(+), 38 deletions(-) create mode 100644 test/bases/ecoindex/backend/test_tasks.py diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 48454fc..3a19ea2 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -1,8 +1,8 @@ from typing import Annotated -from urllib.parse import urlparse, urlunparse -import idna import requests +from pydantic import TypeAdapter +from pydantic.networks import AnyHttpUrl from ecoindex.backend.dependencies.validation import validate_api_key_batch from ecoindex.backend.models.dependencies_parameters.id import IdParameter from ecoindex.backend.utils import check_quota @@ -42,42 +42,8 @@ def convert_url_to_punycode(url: str) -> str: """ Convert an URL with emoji domain (or any Unicode domain) to Punycode. This makes the URL compatible with requests library. - - Args: - url: The URL string that may contain Unicode characters in the domain - - Returns: - The URL with the domain converted to Punycode """ - parsed = urlparse(url) - - # Extract the hostname (netloc may contain port, so we need to handle that) - hostname = parsed.hostname - if not hostname: - return url - - try: - # Convert the hostname to Punycode - hostname_punycode = idna.encode(hostname).decode("ascii") - - # Reconstruct the netloc with the converted hostname - if parsed.port: - netloc = f"{hostname_punycode}:{parsed.port}" - else: - netloc = hostname_punycode - - # Reconstruct the URL with the converted hostname - return urlunparse(( - parsed.scheme, - netloc, - parsed.path, - parsed.params, - parsed.query, - parsed.fragment, - )) - except (idna.IDNAError, UnicodeError): - # If conversion fails, return the original URL - return url + return str(TypeAdapter(AnyHttpUrl).validate_python(url)) def _enqueue_settings(*, with_retry: bool = True) -> dict[str, object]: @@ -166,7 +132,7 @@ async def add_ecoindex_analysis_task( job = ecoindex_queue.enqueue( ecoindex_task, - url=str(web_page.url), + url=url_for_request, width=web_page.width, height=web_page.height, custom_headers=headers, diff --git a/test/bases/ecoindex/backend/test_tasks.py b/test/bases/ecoindex/backend/test_tasks.py new file mode 100644 index 0000000..4d6f76d --- /dev/null +++ b/test/bases/ecoindex/backend/test_tasks.py @@ -0,0 +1,16 @@ +from ecoindex.backend.routers.tasks import convert_url_to_punycode +from ecoindex.models import WebPage + + +def test_convert_url_to_punycode_from_idna_url() -> None: + assert convert_url_to_punycode("https://xn--3s8h30f.ws/") == "https://xn--3s8h30f.ws/" + + +def test_convert_url_to_punycode_from_unicode_domain() -> None: + assert convert_url_to_punycode("https://🦊💻.ws/") == "https://xn--3s8h30f.ws/" + + +def test_convert_url_to_punycode_from_webpage_validation() -> None: + web_page = WebPage(url="https://xn--3s8h30f.ws/") + + assert convert_url_to_punycode(web_page.url) == "https://xn--3s8h30f.ws/" From 0c924c93e4780c6656b2838e11de00c088706b60 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Wed, 15 Jul 2026 10:56:13 +0200 Subject: [PATCH 2/2] fix(api): include request error details when URL pre-check fails Expose SSL, timeout, and DNS errors in the unreachable URL response instead of empty parentheses. Co-authored-by: Cursor --- bases/ecoindex/backend/routers/tasks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 3a19ea2..3497a1b 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -123,11 +123,16 @@ async def add_ecoindex_analysis_task( ) r.raise_for_status() except requests.exceptions.RequestException as e: + error_detail = ( + str(e.response.status_code) + if e.response is not None + else str(e) + ) raise HTTPException( status_code=e.response.status_code - if e.response + if e.response is not None else status.HTTP_400_BAD_REQUEST, - detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔 ({e.response.status_code if e.response else ''})", + detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔 ({error_detail})", ) job = ecoindex_queue.enqueue(