From bd6b268bd6ee4fc3032daf74aff4330493abfeb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9on=20Kuchenbecker?= Date: Tue, 19 Nov 2024 08:12:08 +0000 Subject: [PATCH 1/3] Allow colon characters in bucket names --- src/hexkit/protocols/objstorage.py | 2 +- src/hexkit/providers/s3/provider.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/hexkit/protocols/objstorage.py b/src/hexkit/protocols/objstorage.py index b976b939..39757065 100644 --- a/src/hexkit/protocols/objstorage.py +++ b/src/hexkit/protocols/objstorage.py @@ -490,7 +490,7 @@ async def _delete_object(self, *, bucket_id: str, object_id: str) -> None: # (is typically only used by the protocol but may also be used in # provider-specific code or overwritten by the provider) - _re_bucket_id = re.compile(r"^[a-z0-9\-]{3,63}$") + _re_bucket_id = re.compile(r"^[:a-z0-9\-]{3,63}$") _re_bucket_id_msg = "must consist of 3-63 lowercase letters, digits or hyphens" @classmethod diff --git a/src/hexkit/providers/s3/provider.py b/src/hexkit/providers/s3/provider.py index 393fb4e9..7159a24e 100644 --- a/src/hexkit/providers/s3/provider.py +++ b/src/hexkit/providers/s3/provider.py @@ -22,6 +22,7 @@ # ruff: noqa: PLR0913 import asyncio +import re from functools import lru_cache from pathlib import Path from typing import Any, Optional @@ -31,6 +32,7 @@ import botocore.config import botocore.configloader import botocore.exceptions +import botocore.handlers from boto3.s3.transfer import TransferConfig from pydantic import Field, SecretStr from pydantic_settings import BaseSettings @@ -39,6 +41,8 @@ from hexkit.utils import calc_part_size __all__ = ["ObjectStorageProtocol", "PresignedPostURL"] +# Allow colon character in bucket names to accommodate Ceph multi tenancy S3 +botocore.handlers.VALID_BUCKET = re.compile(r"^[:a-zA-Z0-9.\-_]{1,255}$") class S3Config(BaseSettings): From 79647a76cdbdf321261f4074f2fcbce05f52b5cb Mon Sep 17 00:00:00 2001 From: Christoph Zwerschke Date: Tue, 19 Nov 2024 11:25:11 +0000 Subject: [PATCH 2/3] Change bucket id validation in s3 only and add test --- src/hexkit/protocols/objstorage.py | 2 +- src/hexkit/providers/s3/provider.py | 6 +++++- tests/integration/test_s3.py | 13 +++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/hexkit/protocols/objstorage.py b/src/hexkit/protocols/objstorage.py index 39757065..b976b939 100644 --- a/src/hexkit/protocols/objstorage.py +++ b/src/hexkit/protocols/objstorage.py @@ -490,7 +490,7 @@ async def _delete_object(self, *, bucket_id: str, object_id: str) -> None: # (is typically only used by the protocol but may also be used in # provider-specific code or overwritten by the provider) - _re_bucket_id = re.compile(r"^[:a-z0-9\-]{3,63}$") + _re_bucket_id = re.compile(r"^[a-z0-9\-]{3,63}$") _re_bucket_id_msg = "must consist of 3-63 lowercase letters, digits or hyphens" @classmethod diff --git a/src/hexkit/providers/s3/provider.py b/src/hexkit/providers/s3/provider.py index 7159a24e..fbdfe897 100644 --- a/src/hexkit/providers/s3/provider.py +++ b/src/hexkit/providers/s3/provider.py @@ -42,7 +42,9 @@ __all__ = ["ObjectStorageProtocol", "PresignedPostURL"] # Allow colon character in bucket names to accommodate Ceph multi tenancy S3 -botocore.handlers.VALID_BUCKET = re.compile(r"^[:a-zA-Z0-9.\-_]{1,255}$") +botocore.handlers.VALID_BUCKET = re.compile( + r"^(?:[a-zA-Z0-9_]{1,191}:)?[a-z0-9\-]{3,63}$" +) class S3Config(BaseSettings): @@ -120,6 +122,8 @@ def read_aws_config_ini(aws_config_ini: Path) -> botocore.config.Config: class S3ObjectStorage(ObjectStorageProtocol): """S3-based provider implementing the ObjectStorageProtocol.""" + _re_bucket_id = botocore.handlers.VALID_BUCKET + def __init__( self, *, diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py index ea87d9dc..e0a1b262 100644 --- a/tests/integration/test_s3.py +++ b/tests/integration/test_s3.py @@ -105,6 +105,19 @@ async def test_object_existence_checks(s3: S3Fixture, tmp_file: FileObject): # ) +async def test_bucket_name_with_tenant(s3: S3Fixture): + """Test if bucket names containing a tenant work correctly.""" + check_bucket = s3.storage.does_bucket_exist + assert not await check_bucket("non-existing-bucket") + assert not await check_bucket("tenant:non-existing-bucket") + with pytest.raises(ObjectStorageProtocol.BucketIdValidationError): + assert not await check_bucket("tenant:invalid:bucket") + with pytest.raises(ObjectStorageProtocol.BucketIdValidationError): + assert not await check_bucket("tenant-invalid:bucket-valid") + with pytest.raises(ObjectStorageProtocol.BucketIdValidationError): + assert not await check_bucket("tenant_valid:bucket_invalid") + + async def test_get_object_etag(s3: S3Fixture, tmp_file: FileObject): # noqa: F811 """Test ETag retrieval.""" await s3.populate_file_objects([tmp_file]) From 52dc2529f56e5b03060dfdcb004b4672cff896f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9on=20Kuchenbecker?= Date: Tue, 19 Nov 2024 12:43:14 +0000 Subject: [PATCH 3/3] Version bump --- .pyproject_generation/pyproject_custom.toml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pyproject_generation/pyproject_custom.toml b/.pyproject_generation/pyproject_custom.toml index 7cad0e66..697c0586 100644 --- a/.pyproject_generation/pyproject_custom.toml +++ b/.pyproject_generation/pyproject_custom.toml @@ -1,6 +1,6 @@ [project] name = "hexkit" -version = "3.7.0" +version = "3.8.0" description = "A Toolkit for Building Microservices using the Hexagonal Architecture" requires-python = ">=3.9" classifiers = [ diff --git a/pyproject.toml b/pyproject.toml index 8bbb2cb6..becaa8d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ "Intended Audience :: Developers", ] name = "hexkit" -version = "3.7.0" +version = "3.8.0" description = "A Toolkit for Building Microservices using the Hexagonal Architecture" dependencies = [ "pydantic >=2, <3",