From 2c041bf0f82e6649b04dd2046d393a08487307de Mon Sep 17 00:00:00 2001 From: TomNicholas Date: Sat, 18 Jul 2026 17:18:16 -0700 Subject: [PATCH] fix: preserve ';', '?', '#' in object keys during resolve() ObjectStoreRegistry.resolve() derived the object key via urlparse(url).path, which silently drops anything urlparse treats as URL params (';'), query ('?') or fragment ('#'). These characters are valid in S3/GCS/Azure object keys (e.g. 'DC_DCAM#1_CAM3.tif' in the Cell Painting Gallery), so the key was truncated and lookups failed with a spurious FileNotFound. Reattach the params/query/fragment to reconstruct the full key. For HTTP-backed stores this also preserves query strings (presigned URLs, Azure SAS tokens), which were previously dropped. --- src/obspec_utils/registry.py | 10 ++++++++++ tests/test_registry.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/obspec_utils/registry.py b/src/obspec_utils/registry.py index 0267bc1..b995eb1 100644 --- a/src/obspec_utils/registry.py +++ b/src/obspec_utils/registry.py @@ -318,7 +318,17 @@ def resolve(self, url: Url) -> tuple[T, Path]: ``` """ parsed = urlparse(url) + # Object-store keys can contain ";", "?" and "#", which urlparse peels off + # into the URL params/query/fragment. These characters are valid in + # S3/GCS/Azure keys (e.g. "DC_DCAM#1_CAM3.tif"), so reattach them to + # reconstruct the full path; otherwise the key would be silently truncated. path = parsed.path + if parsed.params: + path = f"{path};{parsed.params}" + if parsed.query: + path = f"{path}?{parsed.query}" + if parsed.fragment: + path = f"{path}#{parsed.fragment}" key = UrlKey(parsed.scheme, parsed.netloc) diff --git a/tests/test_registry.py b/tests/test_registry.py index a515df8..f866dcc 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -29,6 +29,26 @@ def test_registry(): assert ret is memstore +@pytest.mark.parametrize( + ("url", "expected"), + [ + # "#" is valid in object-store keys but urlparse treats it as a fragment. + ("s3://bucket1/path/to/DC_DCAM#1_CAM3.tif", "path/to/DC_DCAM#1_CAM3.tif"), + # "?" is valid too and urlparse treats it as a query string. + ("s3://bucket1/path/to/obj?v=1.tif", "path/to/obj?v=1.tif"), + # ";" in the final segment is treated as params by urlparse. + ("s3://bucket1/path/to/obj;a=b.tif", "path/to/obj;a=b.tif"), + ], +) +def test_resolve_preserves_special_chars_in_key(url, expected): + registry = ObjectStoreRegistry() + memstore = MemoryStore() + registry.register("s3://bucket1", memstore) + ret, path = registry.resolve(url) + assert path == expected + assert ret is memstore + + def test_register_raises(): registry = ObjectStoreRegistry() with pytest.raises(