diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 014524ab..8d770866 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,6 +78,7 @@ jobs: pip install --upgrade pip pip install . pip install -r requirements-dev.txt + pip install tzdata - name: Run tests with coverage run: pytest --cov=mashumaro --cov=tests - name: Upload Coverage diff --git a/README.md b/README.md index bc0f1836..3acfff58 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,11 @@ Table of contents * [Usage example](#usage-example) * [How does it work?](#how-does-it-work) * [Benchmark](#benchmark) -* [API](#api) +* [Serialization mixins](#serialization-mixins) + * [`DataClassDictMixin`](#dataclassdictmixin) + * [`DataClassJSONMixin`](#dataclassjsonmixin) + * [`DataClassMessagePackMixin`](#dataclassmessagepackmixin) + * [`DataClassYAMLMixin`](#dataclassyamlmixin) * [Customization](#customization) * [`SerializableType` interface](#serializabletype-interface) * [Field options](#field-options) @@ -39,6 +43,7 @@ Table of contents * [`namedtuple_as_dict` config option](#namedtuple_as_dict-config-option) * [`allow_postponed_evaluation` config option](#allow_postponed_evaluation-config-option) * [`dialect` config option](#dialect-config-option) + * [Passing field values as is](#passing-field-values-as-is) * [Dialects](#dialects) * [`serialization_strategy` dialect option](#serialization_strategy-dialect-option) * [Changing the default dialect](#changing-the-default-dialect) @@ -120,6 +125,9 @@ for special primitives from the [`typing`](https://docs.python.org/3/library/typ * [`Optional`](https://docs.python.org/3/library/typing.html#typing.Optional) * [`Union`](https://docs.python.org/3/library/typing.html#typing.Union) * [`TypeVar`](https://docs.python.org/3/library/typing.html#typing.TypeVar) +* [`NewType`](https://docs.python.org/3/library/typing.html#newtype) +* [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) +* [`Literal`](https://docs.python.org/3/library/typing.html#typing.Literal) for standard interpreter types from [`types`](https://docs.python.org/3/library/types.html#standard-interpreter-types) module: * [`NoneType`](https://docs.python.org/3/library/types.html#types.NoneType) @@ -145,6 +153,7 @@ for built-in datetime oriented types (see [more](#deserialize-option) details): * [`time`](https://docs.python.org/3/library/datetime.html#datetime.time) * [`timedelta`](https://docs.python.org/3/library/datetime.html#datetime.timedelta) * [`timezone`](https://docs.python.org/3/library/datetime.html#datetime.timezone) +* [`ZoneInfo`](https://docs.python.org/3/library/zoneinfo.html#zoneinfo.ZoneInfo) for pathlike types: * [`PurePath`](https://docs.python.org/3/library/pathlib.html#pathlib.PurePath) @@ -167,6 +176,12 @@ for other less popular built-in types: * [`ipaddress.IPv4Interface`](https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv4Interface) * [`ipaddress.IPv6Interface`](https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv6Interface) +for backported types from [`typing-extensions`](https://github.com/python/typing/blob/master/typing_extensions/README.rst): +* [`OrderedDict`](https://docs.python.org/3/library/typing.html#typing.OrderedDict) +* [`TypedDict`](https://docs.python.org/3/library/typing.html#typing.TypedDict) +* [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) +* [`Literal`](https://docs.python.org/3/library/typing.html#typing.Literal) + for arbitrary types: * [user-defined classes](#serializabletype-interface) * [user-defined generic types](#user-defined-generic-types) @@ -178,7 +193,7 @@ Usage example from enum import Enum from typing import List from dataclasses import dataclass -from mashumaro import DataClassJSONMixin +from mashumaro.mixins.json import DataClassJSONMixin class Currency(Enum): USD = "USD" @@ -305,89 +320,65 @@ pip install -r requirements-dev.txt python benchmark/run.py ``` -API +Serialization mixins -------------------------------------------------------------------------------- -Mashumaro provides a couple of mixins for each format. +Mashumaro provides mixins for each serialization format. -#### `DataClassDictMixin.to_dict(use_bytes: bool, use_enum: bool, use_datetime: bool)` +#### [`DataClassDictMixin`](https://github.com/Fatal1ty/mashumaro/blob/master/mashumaro/mixins/dict.py#L9) -Make a dictionary from dataclass object based on the dataclass schema provided. -Options include: +Can be imported in two ways: ```python -use_bytes: False # False - convert bytes/bytearray objects to base64 encoded string, True - keep untouched -use_enum: False # False - convert enum objects to enum values, True - keep untouched -use_datetime: False # False - convert datetime oriented objects to ISO 8601 formatted string, True - keep untouched +from mashumaro import DataClassDictMixin +from mashumaro.mixins.dict import DataClassDictMixin ``` -#### `DataClassDictMixin.from_dict(data: Mapping, use_bytes: bool, use_enum: bool, use_datetime: bool)` +The core mixin that adds serialization functionality to a dataclass. +This mixin is a base class for all other serialization format mixins. +It adds methods `from_dict` and `to_dict`. -Make a new object from dict object based on the dataclass schema provided. -Options include: +#### [`DataClassJSONMixin`](https://github.com/Fatal1ty/mashumaro/blob/master/mashumaro/mixins/json.py#L22) + +Can be imported as: ```python -use_bytes: False # False - load bytes/bytearray objects from base64 encoded string, True - keep untouched -use_enum: False # False - load enum objects from enum values, True - keep untouched -use_datetime: False # False - load datetime oriented objects from ISO 8601 formatted string, True - keep untouched +from mashumaro.mixins.json import DataClassJSONMixin ``` -#### `DataClassJSONMixin.to_json(encoder: Optional[Encoder], dict_params: Optional[Mapping], **encoder_kwargs)` - -Make a JSON formatted string from dataclass object based on the dataclass -schema provided. Options include: -``` -encoder # function called for json encoding, defaults to json.dumps -dict_params # dictionary of parameter values passed underhood to `to_dict` function -encoder_kwargs # keyword arguments for encoder function -``` +This mixins adds json serialization functionality to a dataclass. +It adds methods `from_json` and `to_json`. -#### `DataClassJSONMixin.from_json(data: Union[str, bytes, bytearray], decoder: Optional[Decoder], dict_params: Optional[Mapping], **decoder_kwargs)` +#### [`DataClassMessagePackMixin`](https://github.com/Fatal1ty/mashumaro/blob/master/mashumaro/mixins/msgpack.py#L26) -Make a new object from JSON formatted string based on the dataclass schema -provided. Options include: -``` -decoder # function called for json decoding, defaults to json.loads -dict_params # dictionary of parameter values passed underhood to `from_dict` function -decoder_kwargs # keyword arguments for decoder function +Can be imported as: +```python +from mashumaro.mixins.msgpack import DataClassMessagePackMixin ``` -#### `DataClassMessagePackMixin.to_msgpack(encoder: Optional[Encoder], dict_params: Optional[Mapping], **encoder_kwargs)` +This mixins adds MessagePack serialization functionality to a dataclass. +It adds methods `from_msgpack` and `to_msgpack`. -Make a MessagePack formatted bytes object from dataclass object based on the -dataclass schema provided. Options include: -``` -encoder # function called for MessagePack encoding, defaults to msgpack.packb -dict_params # dictionary of parameter values passed underhood to `to_dict` function -encoder_kwargs # keyword arguments for encoder function -``` +In order to use this mixin, the [`msgpack`](https://pypi.org/project/msgpack/) package must be installed. +You can install it manually or using an extra option for mashumaro: -#### `DataClassMessagePackMixin.from_msgpack(data: Union[str, bytes, bytearray], decoder: Optional[Decoder], dict_params: Optional[Mapping], **decoder_kwargs)` - -Make a new object from MessagePack formatted data based on the -dataclass schema provided. Options include: -``` -decoder # function called for MessagePack decoding, defaults to msgpack.unpackb -dict_params # dictionary of parameter values passed underhood to `from_dict` function -decoder_kwargs # keyword arguments for decoder function +```shell +pip install mashumaro[msgpack] ``` -#### `DataClassYAMLMixin.to_yaml(encoder: Optional[Encoder], dict_params: Optional[Mapping], **encoder_kwargs)` +#### [`DataClassYAMLMixin`](https://github.com/Fatal1ty/mashumaro/blob/master/mashumaro/mixins/yaml.py#L34) -Make an YAML formatted bytes object from dataclass object based on the -dataclass schema provided. Options include: -``` -encoder # function called for YAML encoding, defaults to yaml.dump -dict_params # dictionary of parameter values passed underhood to `to_dict` function -encoder_kwargs # keyword arguments for encoder function +Can be imported as: +```python +from mashumaro.mixins.yaml import DataClassYAMLMixin ``` -#### `DataClassYAMLMixin.from_yaml(data: Union[str, bytes], decoder: Optional[Decoder], dict_params: Optional[Mapping], **decoder_kwargs)` +This mixins adds YAML serialization functionality to a dataclass. +It adds methods `from_yaml` and `to_yaml`. -Make a new object from YAML formatted data based on the -dataclass schema provided. Options include: -``` -decoder # function called for YAML decoding, defaults to yaml.safe_load -dict_params # dictionary of parameter values passed underhood to `from_dict` function -decoder_kwargs # keyword arguments for decoder function +In order to use this mixin, the [`pyyaml`](https://pypi.org/project/PyYAML/) package must be installed. +You can install it manually or using an extra option for mashumaro: + +```shell +pip install mashumaro[yaml] ``` Customization @@ -466,9 +457,12 @@ A value of type `str` sets a specific engine for serialization. Keep in mind that all possible engines depend on the field type that this option is used with. At this moment there are next serialization engines to choose from: -| Applicable field types | Supported engines | Description -|:-------------------------- |:-------------------------|:------------------------------| -| `NamedTuple`, `namedtuple` | `as_list`, `as_dict` | How to pack named tuples. By default `as_list` engine is used that means your named tuple class instance will be packed into a list of its values. You can pack it into a dictionary using `as_dict` engine. +| Applicable field types | Supported engines | Description | +|:-------------------------- |:-------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `NamedTuple`, `namedtuple` | `as_list`, `as_dict` | How to pack named tuples. By default `as_list` engine is used that means your named tuple class instance will be packed into a list of its values. You can pack it into a dictionary using `as_dict` engine. | + +In addition, you can pass a field value as is without changes using +[`pass_through`](#passing-field-values-as-is). Example: @@ -506,10 +500,13 @@ A value of type `str` sets a specific engine for deserialization. Keep in mind that all possible engines depend on the field type that this option is used with. At this moment there are next deserialization engines to choose from: -| Applicable field types | Supported engines | Description -|:-------------------------- |:-------------------------|:------------------------------| +| Applicable field types | Supported engines | Description | +|:---------------------------|:------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `datetime`, `date`, `time` | [`ciso8601`](https://github.com/closeio/ciso8601#supported-subset-of-iso-8601), [`pendulum`](https://github.com/sdispater/pendulum) | How to parse datetime string. By default native [`fromisoformat`](https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat) of corresponding class will be used for `datetime`, `date` and `time` fields. It's the fastest way in most cases, but you can choose an alternative. | -| `NamedTuple`, `namedtuple` | `as_list`, `as_dict` | How to unpack named tuples. By default `as_list` engine is used that means your named tuple class instance will be created from a list of its values. You can unpack it from a dictionary using `as_dict` engine. +| `NamedTuple`, `namedtuple` | `as_list`, `as_dict` | How to unpack named tuples. By default `as_list` engine is used that means your named tuple class instance will be created from a list of its values. You can unpack it from a dictionary using `as_dict` engine. | + +In addition, you can pass a field value as is without changes using +[`pass_through`](#passing-field-values-as-is). Example: @@ -597,6 +594,9 @@ dictionary = formats.to_dict() assert DateTimeFormats.from_dict(dictionary) == formats ``` +In addition, you can pass a field value as is without changes using `pass_through` +as a value for `serialize` option. + #### `alias` option In some cases it's better to have different names for a field in your class and @@ -699,8 +699,8 @@ so the fastest basic behavior of the library will always remain by default. The following table provides a brief overview of all the available constants described below. -| Constant | Description -|:--------------------------------------------------------------- |:---------------------------------------------------------------------------| +| Constant | Description | +|:----------------------------------------------------------------|:---------------------------------------------------------------------------| | [`TO_DICT_ADD_OMIT_NONE_FLAG`](#add-omit_none-keyword-argument) | Adds `omit_none` keyword-only argument to `to_dict` method. | | [`TO_DICT_ADD_BY_ALIAS_FLAG`](#add-by_alias-keyword-argument) | Adds `by_alias` keyword-only argument to `to_dict` method. | | [`ADD_DIALECT_SUPPORT`](#add-dialect-keyword-argument) | Adds `dialect` keyword-only argument to `from_dict` and `to_dict` methods. | @@ -913,6 +913,75 @@ class B is declared below or not. This option is described [below](#changing-the-default-dialect) in the Dialects section. +### Passing field values as is + +In some cases it's needed to pass a field value as is without any changes +during serialization / deserialization. There is a predefined +[`pass_through`](https://github.com/Fatal1ty/mashumaro/blob/master/mashumaro/helper.py#L46) +object that can be used as `serialization_strategy` or +`serialize` / `deserialize` options: + +```python +from dataclasses import dataclass, field +from mashumaro import DataClassDictMixin, pass_through + +class MyClass: + def __init__(self, some_value): + self.some_value = some_value + +@dataclass +class A1(DataClassDictMixin): + x: MyClass = field( + metadata={ + "serialize": pass_through, + "deserialize": pass_through, + } + ) + +@dataclass +class A2(DataClassDictMixin): + x: MyClass = field( + metadata={ + "serialization_strategy": pass_through, + } + ) + +@dataclass +class A3(DataClassDictMixin): + x: MyClass + + class Config: + serialization_strategy = { + MyClass: pass_through, + } + +@dataclass +class A4(DataClassDictMixin): + x: MyClass + + class Config: + serialization_strategy = { + MyClass: { + "serialize": pass_through, + "deserialize": pass_through, + } + } + +my_class_instance = MyClass(42) + +assert A1.from_dict({'x': my_class_instance}).x == my_class_instance +assert A2.from_dict({'x': my_class_instance}).x == my_class_instance +assert A3.from_dict({'x': my_class_instance}).x == my_class_instance +assert A4.from_dict({'x': my_class_instance}).x == my_class_instance + +a1_dict = A1(my_class_instance).to_dict() +a2_dict = A2(my_class_instance).to_dict() +a3_dict = A3(my_class_instance).to_dict() +a4_dict = A4(my_class_instance).to_dict() + +assert a1_dict == a2_dict == a3_dict == a4_dict == {"x": my_class_instance} +``` + ### Dialects Sometimes it's needed to have different serialization and deserialization @@ -1155,8 +1224,7 @@ called `GenericSerializableType`. It makes it possible to serialize and deserial instances of generic types depending on the types provided: ```python -from typing import Dict, TypeVar, Iterator -from datetime import datetime +from typing import Dict, TypeVar from dataclasses import dataclass from mashumaro import DataClassDictMixin from mashumaro.types import GenericSerializableType diff --git a/docs/2to3.md b/docs/2to3.md new file mode 100644 index 00000000..68be0acd --- /dev/null +++ b/docs/2to3.md @@ -0,0 +1,217 @@ +Migration from version 2 to version 3 +-------------------------------------------------------------------------------- + +* [Moving serialization format mixins](#moving-serialization-format-mixins) +* [Removing `use_bytes` parameter](#removing-use_bytes-parameter) +* [Removing `use_enum` parameter](#removing-use_enum-parameter) +* [Removing `use_datetime` parameter](#removing-use_datetime-parameter) +* [Changing `from_json`, `from_msgpack`, `from_yaml` signature](#changing-from_json-from_msgpack-from_yaml-signature) +* [Changing `to_json`, `to_msgpack`, `to_yaml` signature](#changing-to_json-to_msgpack-to_yaml-signature) + +### Moving serialization format mixins + +You might need to alter your imports if you've used the following mixins: +* `DataClassJSONMixin` +* `DataClassMessagePackMixin` +* `DataClassYAMLMixin` + +Tne new imports will look like this: + +```python +from mashumaro.mixins.json import DataClassJSONMixin +from mashumaro.mixins.msgpack import DataClassMessagePackMixin +from mashumaro.mixins.yaml import DataClassYAMLMixin +``` + +### Removing `use_bytes` parameter + +Parameter `use_bytes` was removed from `from_dict` / `to_dict` methods. +If you've used it to pass bytes or bytearray values as is, you can do the same +with [dialect](https://github.com/Fatal1ty/mashumaro#dialects) and +[pass_through](https://github.com/Fatal1ty/mashumaro/tree/master#passing-field-values-as-is) +features: + +```python +from dataclasses import dataclass + +from mashumaro import DataClassDictMixin, pass_through +from mashumaro.config import BaseConfig, ADD_DIALECT_SUPPORT +from mashumaro.dialect import Dialect + +class BytesDialect(Dialect): + serialization_strategy = { + bytes: pass_through, + bytearray: pass_through, + } + +@dataclass +class A(DataClassDictMixin): + bytes: bytes + bytearray: bytearray + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + +obj = A(b"\x00", bytearray(b"\x00")) +dct = {"bytes": b"\x00", "bytearray": bytearray(b"\x00")} + +assert A.from_dict(dct, dialect=BytesDialect) == obj +assert obj.to_dict(dialect=BytesDialect) == dct +``` + +### Removing `use_enum` parameter + +Parameter `use_enum` was removed from `from_dict` / `to_dict` methods. +If you've used it to pass enum values as is, you can do the same +with [dialect](https://github.com/Fatal1ty/mashumaro#dialects) and +[pass_through](https://github.com/Fatal1ty/mashumaro/tree/master#passing-field-values-as-is) +features: + +```python +from dataclasses import dataclass +from enum import Enum + +from mashumaro import DataClassDictMixin, pass_through +from mashumaro.config import BaseConfig, ADD_DIALECT_SUPPORT +from mashumaro.dialect import Dialect + +class MyEnum(Enum): + a = 1 + b = 2 + +class EnumDialect(Dialect): + serialization_strategy = { + MyEnum: pass_through, + } + +@dataclass +class A(DataClassDictMixin): + my_enum: MyEnum + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + +obj = A(MyEnum.a) +dct = {"my_enum": MyEnum.a} + +assert A.from_dict(dct, dialect=EnumDialect) == obj +assert obj.to_dict(dialect=EnumDialect) == dct +``` + +### Removing `use_datetime` parameter + +Parameter `use_datetime` was removed from `from_dict` / `to_dict` methods. +If you've used it to pass datetime, date and time values as is, you can do +the same with [dialect](https://github.com/Fatal1ty/mashumaro#dialects) and +[pass_through](https://github.com/Fatal1ty/mashumaro/tree/master#passing-field-values-as-is) +features: + +```python +from dataclasses import dataclass +from datetime import date, datetime, time + +from mashumaro import DataClassDictMixin, pass_through +from mashumaro.config import BaseConfig, ADD_DIALECT_SUPPORT +from mashumaro.dialect import Dialect + +class DatetimeDialect(Dialect): + serialization_strategy = { + date: pass_through, + datetime: pass_through, + time: pass_through, + } + +@dataclass +class A(DataClassDictMixin): + datetime: datetime + date: date + time: time + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + +obj = A( + datetime=datetime(2022, 2, 9, 12, 0), + date=date(2022, 2, 9), + time=time(12, 0), +) +dct = { + "datetime": datetime(2022, 2, 9, 12, 0), + "date": date(2022, 2, 9), + "time": time(12, 0), +} + +assert A.from_dict(dct, dialect=DatetimeDialect) == obj +assert obj.to_dict(dialect=DatetimeDialect) == dct +``` + +### Changing `from_json`, `from_msgpack`, `from_yaml` signature + +In version 2 methods `from_json`, `from_msgpack`, `from_yaml` had the following +signature: +```python +@classmethod +def from_*( # where * is json, msgpack, yaml + cls, + data: EncodedData, + decoder: Decoder = ..., + dict_params: Mapping = ..., + **decoder_kwargs, +) +``` + +In version 3 these methods have a slightly different signature: +```python +@classmethod +def from_*( # where * is json, msgpack, yaml + cls, + data: EncodedData, + decoder: Decoder = ..., + **from_dict_kwargs, +) +``` + +As you can see, the `dict_params` positional argument was removed in order +to pass keyword arguments to underlying `from_dict` method. Decoder parameters +were removed because they can be easily passed to decoder using +a lambda function, a partial object or something else: + +```python +A.from_json( + data, + decoder=lambda data: json.loads(data, parse_float=decimal.Decimal), +) +``` + +### Changing `to_json`, `to_msgpack`, `to_yaml` signature + +In version 2 methods `to_json`, `to_msgpack`, `to_yaml` had the following +signature: +```python +def to_*( # where * is json, msgpack, yaml + self, + encoder: Encoder = ... + dict_params: Mapping = ..., + **encoder_kwargs, +) +``` + +In version 3 these methods have a slightly different signature: +```python +def to_*( # where * is json, msgpack, yaml + self, + encoder: Encoder = ..., + **to_dict_kwargs, +) +``` + +As you can see, the `dict_params` positional argument was removed in order +to pass keyword arguments to underlying `to_dict` method. Encoder parameters +were removed because they can be easily passed to encoder using +a lambda function, a partial object or something else: + +```python +dataclass_obj.to_json( + encoder=lambda data: json.dumps(data, ensure_ascii=False), +) +``` diff --git a/mashumaro/__init__.py b/mashumaro/__init__.py index 5be4086c..3950db2c 100644 --- a/mashumaro/__init__.py +++ b/mashumaro/__init__.py @@ -1,15 +1,10 @@ from mashumaro.exceptions import MissingField -from mashumaro.helper import field_options -from mashumaro.serializer.base.dict import DataClassDictMixin -from mashumaro.serializer.json import DataClassJSONMixin -from mashumaro.serializer.msgpack import DataClassMessagePackMixin -from mashumaro.serializer.yaml import DataClassYAMLMixin +from mashumaro.helper import field_options, pass_through +from mashumaro.mixins.dict import DataClassDictMixin __all__ = [ "MissingField", "DataClassDictMixin", - "DataClassJSONMixin", - "DataClassMessagePackMixin", - "DataClassYAMLMixin", "field_options", + "pass_through", ] diff --git a/mashumaro/config.py b/mashumaro/config.py index 113895d2..7b7df0fa 100644 --- a/mashumaro/config.py +++ b/mashumaro/config.py @@ -1,7 +1,7 @@ from typing import Any, Callable, Dict, List, Optional, Type, Union +from mashumaro.core.const import PEP_586_COMPATIBLE from mashumaro.dialect import Dialect -from mashumaro.meta.macros import PEP_586_COMPATIBLE from mashumaro.types import SerializationStrategy if PEP_586_COMPATIBLE: diff --git a/mashumaro/meta/__init__.py b/mashumaro/core/__init__.py similarity index 100% rename from mashumaro/meta/__init__.py rename to mashumaro/core/__init__.py diff --git a/mashumaro/meta/macros.py b/mashumaro/core/const.py similarity index 100% rename from mashumaro/meta/macros.py rename to mashumaro/core/const.py diff --git a/mashumaro/serializer/base/helpers.py b/mashumaro/core/helpers.py similarity index 92% rename from mashumaro/serializer/base/helpers.py rename to mashumaro/core/helpers.py index 3bee0cf1..67c2fed9 100644 --- a/mashumaro/serializer/base/helpers.py +++ b/mashumaro/core/helpers.py @@ -2,7 +2,7 @@ import re -def parse_timezone(s: str): +def parse_timezone(s: str) -> datetime.timezone: regexp = re.compile(r"^UTC(([+-][0-2][0-9]):([0-5][0-9]))?$") match = regexp.match(s) if not match: diff --git a/mashumaro/serializer/__init__.py b/mashumaro/core/meta/__init__.py similarity index 100% rename from mashumaro/serializer/__init__.py rename to mashumaro/core/meta/__init__.py diff --git a/mashumaro/serializer/base/metaprogramming.py b/mashumaro/core/meta/builder.py similarity index 71% rename from mashumaro/serializer/base/metaprogramming.py rename to mashumaro/core/meta/builder.py index ac305470..98776308 100644 --- a/mashumaro/serializer/base/metaprogramming.py +++ b/mashumaro/core/meta/builder.py @@ -18,32 +18,25 @@ from dataclasses import _FIELDS, MISSING, Field, is_dataclass # type: ignore from decimal import Decimal from fractions import Fraction +from functools import lru_cache from hashlib import md5 from types import MappingProxyType +import typing_extensions + from mashumaro.config import ( ADD_DIALECT_SUPPORT, TO_DICT_ADD_BY_ALIAS_FLAG, TO_DICT_ADD_OMIT_NONE_FLAG, BaseConfig, ) -from mashumaro.dialect import Dialect -from mashumaro.exceptions import ( # noqa - BadDialect, - BadHookSignature, - InvalidFieldValue, - MissingField, - ThirdPartyModuleNotFoundError, - UnresolvedTypeReferenceError, - UnserializableDataError, - UnserializableField, - UnsupportedDeserializationEngine, - UnsupportedSerializationEngine, -) -from mashumaro.meta.helpers import ( +from mashumaro.core.const import PY_39_MIN +from mashumaro.core.helpers import * # noqa +from mashumaro.core.meta.helpers import ( get_args, get_class_that_defines_field, get_class_that_defines_method, + get_literal_values, get_name_error_name, get_type_origin, is_class_var, @@ -52,7 +45,9 @@ is_dialect_subclass, is_generic, is_init_var, + is_literal, is_named_tuple, + is_new_type, is_optional, is_special_typing_primitive, is_type_var, @@ -63,15 +58,30 @@ resolve_type_vars, type_name, ) -from mashumaro.meta.macros import PY_37_MIN -from mashumaro.meta.patch import patch_fromisoformat -from mashumaro.serializer.base.helpers import * # noqa +from mashumaro.core.meta.patch import patch_fromisoformat +from mashumaro.dialect import Dialect +from mashumaro.exceptions import ( # noqa + BadDialect, + BadHookSignature, + InvalidFieldValue, + MissingField, + ThirdPartyModuleNotFoundError, + UnresolvedTypeReferenceError, + UnserializableDataError, + UnserializableField, + UnsupportedDeserializationEngine, + UnsupportedSerializationEngine, +) +from mashumaro.helper import pass_through from mashumaro.types import ( GenericSerializableType, SerializableType, SerializationStrategy, ) +if PY_39_MIN: + import zoneinfo + try: import ciso8601 except ImportError: # pragma no cover @@ -98,7 +108,7 @@ def __init__(self): self._lines: typing.List[str] = [] self._current_indent: str = "" - def append(self, line: str): + def append(self, line: str) -> None: self._lines.append(f"{self._current_indent}{line}") @contextmanager @@ -112,7 +122,7 @@ def indent(self) -> typing.Generator[None, None, None]: def as_text(self) -> str: return "\n".join(self._lines) - def reset(self): + def reset(self) -> None: self._lines = [] self._current_indent = "" @@ -124,6 +134,7 @@ def __init__( arg_types: typing.Tuple = (), dialect: typing.Optional[typing.Type[Dialect]] = None, first_method: str = "from_dict", + allow_postponed_evaluation: bool = True, ): self.cls = cls self.lines: CodeLines = CodeLines() @@ -137,6 +148,7 @@ def __init__( f"in {type_name(self.cls)}.{first_method}" ) self.dialect = dialect + self.allow_postponed_evaluation = allow_postponed_evaluation def reset(self) -> None: self.lines.reset() @@ -176,7 +188,7 @@ def __get_field_types( fields[fname] = ftype return fields - def _get_field_class(self, field_name): + def _get_field_class(self, field_name) -> typing.Any: try: cls = self.field_classes[field_name] except KeyError: @@ -184,11 +196,11 @@ def _get_field_class(self, field_name): self.field_classes[field_name] = cls return cls - def __get_real_type(self, field_name, field_type): + def __get_real_type(self, field_name, field_type) -> typing.Any: cls = self._get_field_class(field_name) return self.type_vars[cls].get(field_type, field_type) - def _get_field_type_vars(self, field_name): + def _get_field_type_vars(self, field_name) -> typing.Dict[str, typing.Any]: cls = self._get_field_class(field_name) return self.type_vars[cls] @@ -196,50 +208,58 @@ def _get_field_type_vars(self, field_name): def field_types(self) -> typing.Dict[str, typing.Any]: return self.__get_field_types() - @property - def defaults(self) -> typing.Dict[str, typing.Any]: + @property # type: ignore + @lru_cache() + def dataclass_fields(self) -> typing.Dict[str, Field]: d = {} for ancestor in self.cls.__mro__[-1:0:-1]: if is_dataclass(ancestor): for field in getattr(ancestor, _FIELDS).values(): - if field.default is not MISSING: - d[field.name] = field.default - else: - d[field.name] = field.default_factory + d[field.name] = field for name in self.__get_field_types(recursive=False): field = self.namespace.get(name, MISSING) if isinstance(field, Field): - if field.default is not MISSING: - d[name] = field.default - else: - # https://github.com/python/mypy/issues/6910 - d[name] = field.default_factory # type: ignore - else: d[name] = field + else: + field = self.namespace.get(_FIELDS, {}).get(name, MISSING) + if isinstance(field, Field): + d[name] = field + else: + d.pop(name, None) return d @property def metadatas(self) -> typing.Dict[str, typing.Mapping[str, typing.Any]]: - d = {} - for ancestor in self.cls.__mro__[-1:0:-1]: - if is_dataclass(ancestor): - for field in getattr(ancestor, _FIELDS).values(): - d[field.name] = field.metadata - for name in self.__get_field_types(recursive=False): - field = self.namespace.get(name, MISSING) - if isinstance(field, Field): - d[name] = field.metadata - return d + return { + name: field.metadata + for name, field in self.dataclass_fields.items() # type: ignore + # https://github.com/python/mypy/issues/1362 + } + + def get_field_default(self, name: str) -> typing.Any: + field = self.dataclass_fields.get(name) # type: ignore + # https://github.com/python/mypy/issues/1362 + if field: + if field.default is not MISSING: + return field.default + else: + return field.default_factory + else: + return self.namespace.get(name, MISSING) def _add_type_modules(self, *types_) -> None: for t in types_: module = inspect.getmodule(t) if not module: - return + continue self.ensure_module_imported(module) - args = get_args(t) - if args: + if is_literal(t): + args = get_literal_values(t) self._add_type_modules(*args) + else: + args = get_args(t) + if args: + self._add_type_modules(*args) constraints = getattr(t, "__constraints__", ()) if constraints: self._add_type_modules(*constraints) @@ -270,7 +290,7 @@ def compile(self) -> None: print(code) exec(code, self.globals, self.__dict__) - def get_declared_hook(self, method_name: str): + def get_declared_hook(self, method_name: str) -> typing.Any: if not hasattr(self.cls, method_name): return cls = get_class_that_defines_method(method_name, self.cls) @@ -283,10 +303,11 @@ def _add_from_dict(self) -> None: method_name += f"_{self._hash_arg_types(self.initial_arg_types)}" if self.dialect is None: self.add_line("@classmethod") - self.add_line( - f"def {method_name}(cls, d, " - f"{self.get_from_dict_default_flag_values()}):" - ) + default_kwargs = self.get_from_dict_default_flag_values() + if default_kwargs: + self.add_line(f"def {method_name}(cls, d, {default_kwargs}):") + else: + self.add_line(f"def {method_name}(cls, d):") with self.indent(): self._add_from_dict_lines() if self.dialect is None: @@ -297,70 +318,88 @@ def _add_from_dict(self) -> None: ) self.compile() - def _add_from_dict_lines(self): + def _add_from_dict_lines(self) -> None: config = self.get_config() - pre_deserialize = self.get_declared_hook(__PRE_DESERIALIZE__) - if pre_deserialize: - if not isinstance(pre_deserialize, classmethod): - raise BadHookSignature( - f"`{__PRE_DESERIALIZE__}` must be a class method with " - f"Callable[[Dict[Any, Any]], Dict[Any, Any]] signature" - ) - else: - self.add_line(f"d = cls.{__PRE_DESERIALIZE__}(d)") - self.add_line("try:") - with self.indent(): - self.add_line("kwargs = {}") - for fname, ftype in self.field_types.items(): - self._add_type_modules(ftype) - metadata = self.metadatas.get(fname, {}) - alias = metadata.get("alias") - if alias is None: - alias = config.aliases.get(fname) - self._from_dict_set_value(fname, ftype, metadata, alias) - self.add_line("except AttributeError:") - with self.indent(): - self.add_line("if not isinstance(d, dict):") + try: + field_types = self.field_types + except UnresolvedTypeReferenceError: + if ( + not self.allow_postponed_evaluation + or not config.allow_postponed_evaluation + ): + raise + self.add_line( + "builder = CodeBuilder(cls, allow_postponed_evaluation=False)" + ) + self.add_line("builder.add_from_dict()") + from_dict_args = ", ".join( + filter(None, ("d", self.get_from_dict_flags())) + ) + self.add_line(f"return cls.from_dict({from_dict_args})") + else: + pre_deserialize = self.get_declared_hook(__PRE_DESERIALIZE__) + if pre_deserialize: + if not isinstance(pre_deserialize, classmethod): + raise BadHookSignature( + f"`{__PRE_DESERIALIZE__}` must be a class method with " + f"Callable[[Dict[Any, Any]], Dict[Any, Any]] signature" + ) + else: + self.add_line(f"d = cls.{__PRE_DESERIALIZE__}(d)") + self.add_line("try:") with self.indent(): - self.add_line( - f"raise ValueError('Argument for " - f"{type_name(self.cls)}.from_dict method " - f"should be a dict instance') from None" - ) - self.add_line("else:") + self.add_line("kwargs = {}") + for fname, ftype in field_types.items(): + self._add_type_modules(ftype) + metadata = self.metadatas.get(fname, {}) + alias = metadata.get("alias") + if alias is None: + alias = config.aliases.get(fname) + self._from_dict_set_value(fname, ftype, metadata, alias) + self.add_line("except AttributeError:") with self.indent(): - self.add_line("raise") - post_deserialize = self.get_declared_hook(__POST_DESERIALIZE__) - if post_deserialize: - if not isinstance(post_deserialize, classmethod): - raise BadHookSignature( - f"`{__POST_DESERIALIZE__}` must be a class method " - f"with Callable[[{type_name(self.cls)}], " - f"{type_name(self.cls)}] signature" - ) + self.add_line("if not isinstance(d, dict):") + with self.indent(): + self.add_line( + f"raise ValueError('Argument for " + f"{type_name(self.cls)}.from_dict method " + f"should be a dict instance') from None" + ) + self.add_line("else:") + with self.indent(): + self.add_line("raise") + post_deserialize = self.get_declared_hook(__POST_DESERIALIZE__) + if post_deserialize: + if not isinstance(post_deserialize, classmethod): + raise BadHookSignature( + f"`{__POST_DESERIALIZE__}` must be a class method " + f"with Callable[[{type_name(self.cls)}], " + f"{type_name(self.cls)}] signature" + ) + else: + self.add_line( + f"return cls.{__POST_DESERIALIZE__}(cls(**kwargs))" + ) else: - self.add_line( - f"return cls.{__POST_DESERIALIZE__}(cls(**kwargs))" - ) - else: - self.add_line("return cls(**kwargs)") + self.add_line("return cls(**kwargs)") def _add_from_dict_with_dialect_lines(self) -> None: + from_dict_args = ", ".join( + filter(None, ("cls", "d", self.get_from_dict_flags())) + ) self.add_line( "from_dict = cls.__dialect_from_dict_cache__.get(dialect)" ) self.add_line("if from_dict is not None:") with self.indent(): - self.add_line( - f"return from_dict(cls,d,{self.get_from_dict_flags()})" - ) + self.add_line(f"return from_dict({from_dict_args})") self.add_line( "CodeBuilder(cls,dialect=dialect," "first_method='from_dict').add_from_dict()" ) self.add_line( f"return cls.__dialect_from_dict_cache__[dialect]" - f"(cls,d,{self.get_from_dict_flags()})" + f"({from_dict_args})" ) def add_from_dict(self) -> None: @@ -381,10 +420,12 @@ def add_from_dict(self) -> None: if self.initial_arg_types: method_name += f"_{self._hash_arg_types(self.initial_arg_types)}" self.add_line("@classmethod") - self.add_line( - f"def {method_name}(cls, d, " - f"{self.get_to_dict_default_flag_values()}):" - ) + default_kwargs = self.get_from_dict_default_flag_values() + if default_kwargs: + self.add_line(f"def {method_name}(cls, d, {default_kwargs}):") + else: # pragma no cover + # there will be at least a dialect parameter + self.add_line(f"def {method_name}(cls, d):") with self.indent(): self.add_line("if dialect is None:") with self.indent(): @@ -395,7 +436,7 @@ def add_from_dict(self) -> None: self.add_line(f"setattr(cls, '{method_name}', {method_name})") self.compile() - def _from_dict_set_value(self, fname, ftype, metadata, alias=None): + def _from_dict_set_value(self, fname, ftype, metadata, alias=None) -> None: unpacked_value = self._unpack_field_value( fname=fname, ftype=ftype, @@ -406,7 +447,7 @@ def _from_dict_set_value(self, fname, ftype, metadata, alias=None): self.add_line("if value is None:") with self.indent(): self.add_line(f"kwargs['{fname}'] = None") - if self.defaults[fname] is MISSING: + if self.get_field_default(fname) is MISSING: self.add_line("elif value is MISSING:") with self.indent(): field_type = type_name( @@ -417,34 +458,41 @@ def _from_dict_set_value(self, fname, ftype, metadata, alias=None): ) self.add_line("else:") with self.indent(): - self.add_line("try:") - with self.indent(): + if unpacked_value == "value": self.add_line(f"kwargs['{fname}'] = {unpacked_value}") - self.add_line("except Exception as e:") - with self.indent(): - field_type = type_name( - ftype, type_vars=self._get_field_type_vars(fname) - ) - self.add_line( - f"raise InvalidFieldValue('{fname}'," - f"{field_type},value,cls)" - ) + else: + self.add_line("try:") + with self.indent(): + self.add_line(f"kwargs['{fname}'] = {unpacked_value}") + self.add_line("except Exception as e:") + with self.indent(): + field_type = type_name( + ftype, type_vars=self._get_field_type_vars(fname) + ) + self.add_line( + f"raise InvalidFieldValue('{fname}'," + f"{field_type},value,cls)" + ) else: self.add_line("elif value is not MISSING:") with self.indent(): - self.add_line("try:") - with self.indent(): + if unpacked_value == "value": self.add_line(f"kwargs['{fname}'] = {unpacked_value}") - self.add_line("except Exception as e:") - with self.indent(): - field_type = type_name( - ftype, type_vars=self._get_field_type_vars(fname) - ) - self.add_line( - f"raise InvalidFieldValue('{fname}'," - f"{field_type},value,cls)" - ) + else: + self.add_line("try:") + with self.indent(): + self.add_line(f"kwargs['{fname}'] = {unpacked_value}") + self.add_line("except Exception as e:") + with self.indent(): + field_type = type_name( + ftype, type_vars=self._get_field_type_vars(fname) + ) + self.add_line( + f"raise InvalidFieldValue('{fname}'," + f"{field_type},value,cls)" + ) + @lru_cache() def get_config(self, cls=None) -> typing.Type[BaseConfig]: if cls is None: cls = self.cls @@ -458,36 +506,24 @@ def get_config(self, cls=None) -> typing.Type[BaseConfig]: return config_cls def get_to_dict_flags(self, cls=None) -> str: - config = self.get_config(cls) - code_generation_options = config.code_generation_options - parent_config = self.get_config() - parent_code_generation_options = parent_config.code_generation_options pluggable_flags = [] for option, flag in ( (TO_DICT_ADD_OMIT_NONE_FLAG, "omit_none"), (TO_DICT_ADD_BY_ALIAS_FLAG, "by_alias"), (ADD_DIALECT_SUPPORT, "dialect"), ): - if option in code_generation_options: - if option in parent_code_generation_options: + if self.is_code_generation_option_enabled(option, cls): + if self.is_code_generation_option_enabled(option): pluggable_flags.append(f"{flag}={flag}") - return ",".join( - ["use_bytes", "use_enum", "use_datetime", *pluggable_flags] - ) + return ", ".join(pluggable_flags) def get_from_dict_flags(self, cls=None) -> str: - config = self.get_config(cls) - code_generation_options = config.code_generation_options - parent_config = self.get_config() - parent_code_generation_options = parent_config.code_generation_options pluggable_flags = [] for option, flag in ((ADD_DIALECT_SUPPORT, "dialect"),): - if option in code_generation_options: - if option in parent_code_generation_options: + if self.is_code_generation_option_enabled(option, cls): + if self.is_code_generation_option_enabled(option): pluggable_flags.append(f"{flag}={flag}") - return ",".join( - ["use_bytes", "use_enum", "use_datetime", *pluggable_flags] - ) + return ", ".join(pluggable_flags) def get_to_dict_default_flag_values(self, cls=None) -> str: flag_names = [] @@ -512,15 +548,12 @@ def get_to_dict_default_flag_values(self, cls=None) -> str: flag_names.append("dialect") flag_values.append("None") if flag_names: - pluggable_flags_str = ", *, " + ", ".join( + pluggable_flags_str = "*, " + ", ".join( [f"{n}={v}" for n, v in zip(flag_names, flag_values)] ) else: pluggable_flags_str = "" - return ( - f"use_bytes=False, use_enum=False, use_datetime=False" - f"{pluggable_flags_str}" - ) + return pluggable_flags_str def get_from_dict_default_flag_values(self, cls=None) -> str: flag_names = [] @@ -532,17 +565,22 @@ def get_from_dict_default_flag_values(self, cls=None) -> str: flag_names.append("dialect") flag_values.append("None") if flag_names: - pluggable_flags_str = ", *, " + ", ".join( + pluggable_flags_str = "*, " + ", ".join( [f"{n}={v}" for n, v in zip(flag_names, flag_values)] ) else: pluggable_flags_str = "" - return ( - f"use_bytes=False, use_enum=False, use_datetime=False" - f"{pluggable_flags_str}" - ) - - def is_code_generation_option_enabled(self, option: str, cls=None): + return pluggable_flags_str + + def is_code_generation_option_enabled(self, option: str, cls=None) -> bool: + if option == ADD_DIALECT_SUPPORT: + # TODO: make inheritance for code_generation_options + for ancestor in self.cls.__mro__[-1:0:-1]: + if ( + type_name(ancestor) + == "mashumaro.mixins.msgpack.DataClassMessagePackMixin" + ): + return True return option in self.get_config(cls).code_generation_options def _add_to_dict(self) -> None: @@ -550,10 +588,11 @@ def _add_to_dict(self) -> None: if self.initial_arg_types: method_name += f"_{self._hash_arg_types(self.initial_arg_types)}" self.reset() - self.add_line( - f"def {method_name}" - f"(self, {self.get_to_dict_default_flag_values()}):" - ) + default_kwargs = self.get_to_dict_default_flag_values() + if default_kwargs: + self.add_line(f"def {method_name}(self, {default_kwargs}):") + else: + self.add_line(f"def {method_name}(self):") with self.indent(): self._add_to_dict_lines() if self.dialect is None: @@ -565,34 +604,53 @@ def _add_to_dict(self) -> None: self.compile() def _add_to_dict_lines(self) -> None: - pre_serialize = self.get_declared_hook(__PRE_SERIALIZE__) - if pre_serialize: - self.add_line(f"self = self.{__PRE_SERIALIZE__}()") - self.add_line("kwargs = {}") - for fname, ftype in self.field_types.items(): - metadata = self.metadatas.get(fname, {}) - self._to_dict_set_value(fname, ftype, metadata) - post_serialize = self.get_declared_hook(__POST_SERIALIZE__) - if post_serialize: - self.add_line(f"return self.{__POST_SERIALIZE__}(kwargs)") + config = self.get_config() + try: + field_types = self.field_types + except UnresolvedTypeReferenceError: + if ( + not self.allow_postponed_evaluation + or not config.allow_postponed_evaluation + ): + raise + self.add_line( + "builder = CodeBuilder(self.__class__, " + "allow_postponed_evaluation=False)" + ) + self.add_line("builder.add_to_dict()") + self.add_line(f"return self.to_dict({self.get_to_dict_flags()})") else: - self.add_line("return kwargs") + pre_serialize = self.get_declared_hook(__PRE_SERIALIZE__) + if pre_serialize: + self.add_line(f"self = self.{__PRE_SERIALIZE__}()") + self.add_line("kwargs = {}") + for fname, ftype in field_types.items(): + metadata = self.metadatas.get(fname, {}) + self._to_dict_set_value(fname, ftype, metadata) + post_serialize = self.get_declared_hook(__POST_SERIALIZE__) + if post_serialize: + self.add_line(f"return self.{__POST_SERIALIZE__}(kwargs)") + else: + self.add_line("return kwargs") def _add_to_dict_with_dialect_lines(self) -> None: + to_dict_args = ", ".join( + filter(None, ("self", self.get_to_dict_flags())) + ) self.add_line( "to_dict = self.__class__." "__dialect_to_dict_cache__.get(dialect)" ) self.add_line("if to_dict is not None:") with self.indent(): - self.add_line(f"return to_dict(self,{self.get_to_dict_flags()})") + self.add_line(f"return to_dict({to_dict_args})") self.add_line( "CodeBuilder(self.__class__,dialect=dialect," "first_method='to_dict').add_to_dict()" ) self.add_line( f"return self.__class__.__dialect_to_dict_cache__[dialect]" - f"(self,{self.get_to_dict_flags()})" + f"({to_dict_args})" ) def add_to_dict(self) -> None: @@ -610,10 +668,12 @@ def add_to_dict(self) -> None: method_name = "to_dict" if self.initial_arg_types: method_name += f"_{self._hash_arg_types(self.initial_arg_types)}" - self.add_line( - f"def {method_name}" - f"(self, {self.get_to_dict_default_flag_values()}):" - ) + default_kwargs = self.get_to_dict_default_flag_values() + if default_kwargs: + self.add_line(f"def {method_name}(self, {default_kwargs}):") + else: # pragma no cover + # there will be at least a dialect parameter + self.add_line(f"def {method_name}(self):") with self.indent(): self.add_line("if dialect is None:") with self.indent(): @@ -624,7 +684,7 @@ def add_to_dict(self) -> None: self.add_line(f"setattr(cls, '{method_name}', {method_name})") self.compile() - def _to_dict_set_value(self, fname, ftype, metadata): + def _to_dict_set_value(self, fname, ftype, metadata) -> None: omit_none_feature = self.is_code_generation_option_enabled( TO_DICT_ADD_OMIT_NONE_FLAG ) @@ -697,7 +757,7 @@ def _pack_value( ftype = self.__get_real_type(fname, ftype) origin_type = get_type_origin(ftype) - overridden: typing.Optional[str] = None + strategy: typing.Optional[SerializationStrategy] = None serialize_option = metadata.get("serialize") overridden_fn_suffix = str(uuid.uuid4().hex) if serialize_option is None: @@ -730,23 +790,22 @@ def _pack_value( serialize_option = strategy.get("serialize") elif isinstance(strategy, SerializationStrategy): serialize_option = strategy.serialize - if callable(serialize_option): + if pass_through in (strategy, serialize_option): + return value_name + elif callable(serialize_option): overridden_fn = f"__{fname}_serialize_{overridden_fn_suffix}" setattr(self.cls, overridden_fn, staticmethod(serialize_option)) - overridden = f"self.{overridden_fn}({value_name})" + return f"self.{overridden_fn}({value_name})" with suppress(TypeError): - if issubclass(ftype, SerializableType): - return overridden or f"{value_name}._serialize()" + if issubclass(origin_type, SerializableType): + return f"{value_name}._serialize()" with suppress(TypeError): if issubclass(origin_type, GenericSerializableType): arg_type_names = ", ".join( list(map(type_name, get_args(ftype))) ) - return ( - overridden - or f"{value_name}._serialize([{arg_type_names}])" - ) + return f"{value_name}._serialize([{arg_type_names}])" if is_dataclass_dict_mixin_subclass(origin_type): arg_types = get_args(ftype) @@ -759,14 +818,12 @@ def _pack_value( else: method_name = "to_dict" flags = self.get_to_dict_flags(ftype) - return overridden or f"{value_name}.{method_name}({flags})" + return f"{value_name}.{method_name}({flags})" if is_special_typing_primitive(origin_type): if origin_type is typing.Any: - return overridden or value_name + return value_name elif is_union(ftype): - if overridden: - return overridden args = get_args(ftype) field_type_vars = self._get_field_type_vars(fname) if is_optional(ftype, field_type_vars): @@ -779,36 +836,27 @@ def _pack_value( else: return pv else: - method_name = self._add_pack_union( - fname, ftype, args, parent, metadata - ) - return ( - f"self.{method_name}({value_name}," - f"{self.get_to_dict_flags()})" + return self._pack_union( + fname, ftype, value_name, args, parent, metadata ) elif origin_type is typing.AnyStr: raise UnserializableDataError( "AnyStr is not supported by mashumaro" ) elif is_type_var_any(ftype): - return overridden or value_name + return value_name elif is_type_var(ftype): - if overridden: - return overridden constraints = getattr(ftype, "__constraints__") if constraints: - method_name = self._add_pack_union( + return self._pack_union( fname=fname, ftype=ftype, + value_name=value_name, args=constraints, parent=parent, metadata=metadata, prefix="type_var", ) - return ( - f"self.{method_name}({value_name}," - f"{self.get_to_dict_flags()})" - ) else: bound = getattr(ftype, "__bound__") # act as if it was Optional[bound] @@ -819,28 +867,39 @@ def _pack_value( return f"{pv} if {value_name} is not None else None" else: return pv + elif is_new_type(ftype): + return self._pack_value( + fname=fname, + ftype=ftype.__supertype__, + parent=parent, + value_name=value_name, + metadata=metadata, + could_be_none=could_be_none, + ) + elif is_literal(ftype): + return self._pack_literal( + fname, ftype, value_name, parent, metadata + ) else: raise UnserializableDataError( f"{ftype} as a field type is not supported by mashumaro" ) elif origin_type is int: - return overridden or f"int({value_name})" + return f"int({value_name})" elif origin_type is float: - return overridden or f"float({value_name})" + return f"float({value_name})" elif origin_type in (bool, NoneType, None): - return overridden or value_name + return value_name elif origin_type in (datetime.datetime, datetime.date, datetime.time): - if overridden: - return f"{value_name} if use_datetime else {overridden}" - return ( - f"{value_name} if use_datetime else {value_name}.isoformat()" - ) + return f"{value_name}.isoformat()" elif origin_type is datetime.timedelta: - return overridden or f"{value_name}.total_seconds()" + return f"{value_name}.total_seconds()" elif origin_type is datetime.timezone: - return overridden or f"{value_name}.tzname(None)" + return f"{value_name}.tzname(None)" + elif PY_39_MIN and origin_type is zoneinfo.ZoneInfo: + return f"str({value_name})" elif origin_type is uuid.UUID: - return overridden or f"str({value_name})" + return f"str({value_name})" elif origin_type in [ ipaddress.IPv4Address, ipaddress.IPv6Address, @@ -849,11 +908,11 @@ def _pack_value( ipaddress.IPv4Interface, ipaddress.IPv6Interface, ]: - return overridden or f"str({value_name})" + return f"str({value_name})" elif origin_type is Decimal: - return overridden or f"str({value_name})" + return f"str({value_name})" elif origin_type is Fraction: - return overridden or f"str({value_name})" + return f"str({value_name})" elif issubclass(origin_type, typing.Collection) and not issubclass( origin_type, enum.Enum ): @@ -872,15 +931,12 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) if issubclass(origin_type, typing.ByteString): - specific = f"encodebytes({value_name}).decode()" - return ( - f"{value_name} if use_bytes else {overridden or specific}" - ) + return f"encodebytes({value_name}).decode()" elif issubclass(origin_type, str): - return overridden or value_name + return value_name elif issubclass(origin_type, typing.Tuple): if is_named_tuple(ftype): - return overridden or self._pack_named_tuple( + return self._pack_named_tuple( fname, ftype, value_name, @@ -889,7 +945,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): serialize_option, ) elif is_generic(ftype): - return overridden or self._pack_tuple( + return self._pack_tuple( fname, value_name, args, parent, metadata ) elif ftype is tuple: @@ -900,10 +956,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): origin_type, (typing.List, typing.Deque, typing.AbstractSet) ): if is_generic(ftype): - return ( - overridden - or f"[{inner_expr()} for value in {value_name}]" - ) + return f"[{inner_expr()} for value in {value_name}]" elif ftype is list: raise UnserializableField( fname, ftype, parent, "Use typing.List[T] instead" @@ -929,8 +982,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f'[{{{inner_expr(0,"key")}:{inner_expr(1)} ' + f'[{{{inner_expr(0,"key")}:{inner_expr(1)} ' f"for key,value in m.items()}} " f"for m in {value_name}.maps]" ) @@ -941,7 +993,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): parent, "Use typing.ChainMap[KT,VT] instead", ) - elif PY_37_MIN and issubclass(origin_type, typing.OrderedDict): + elif issubclass(origin_type, typing_extensions.OrderedDict): if is_generic(ftype): if args and is_dataclass(args[0]): raise UnserializableDataError( @@ -950,8 +1002,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f'{{{inner_expr(0, "key")}: {inner_expr(1)} ' + f'{{{inner_expr(0, "key")}: {inner_expr(1)} ' f"for key, value in {value_name}.items()}}" ) elif ftype is collections.OrderedDict: @@ -970,8 +1021,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f'{{{inner_expr(0, "key")}: ' + f'{{{inner_expr(0, "key")}: ' f"{inner_expr(1, v_type=int)} " f"for key, value in {value_name}.items()}}" ) @@ -984,16 +1034,9 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) elif issubclass(origin_type, typing.Mapping): if is_typed_dict(ftype): - if overridden: - return overridden - else: - method_name = self._add_pack_typed_dict( - fname, ftype, value_name, parent, metadata - ) - return ( - f"self.{method_name}({value_name}," - f"{self.get_to_dict_flags()})" - ) + return self._pack_typed_dict( + fname, ftype, value_name, parent, metadata + ) elif is_generic(ftype): if args and is_dataclass(args[0]): raise UnserializableDataError( @@ -1002,8 +1045,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f'{{{inner_expr(0,"key")}: {inner_expr(1)} ' + f'{{{inner_expr(0,"key")}: {inner_expr(1)} ' f"for key, value in {value_name}.items()}}" ) elif ftype is dict: @@ -1015,17 +1057,11 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) elif issubclass(origin_type, typing.Sequence): if is_generic(ftype): - return ( - overridden - or f"[{inner_expr()} for value in {value_name}]" - ) + return f"[{inner_expr()} for value in {value_name}]" elif issubclass(origin_type, os.PathLike): - return overridden or f"{value_name}.__fspath__()" + return f"{value_name}.__fspath__()" elif issubclass(origin_type, enum.Enum): - specific = f"{value_name}.value" - return f"{value_name} if use_enum else {overridden or specific}" - if overridden: - return overridden + return f"{value_name}.value" raise UnserializableField(fname, ftype, parent) @@ -1042,7 +1078,7 @@ def _unpack_field_value( ftype = self.__get_real_type(fname, ftype) origin_type = get_type_origin(ftype) - overridden: typing.Optional[str] = None + strategy: typing.Optional[SerializationStrategy] = None deserialize_option = metadata.get("deserialize") overridden_fn_suffix = str(uuid.uuid4().hex) if deserialize_option is None: @@ -1075,25 +1111,23 @@ def _unpack_field_value( deserialize_option = strategy.get("deserialize") elif isinstance(strategy, SerializationStrategy): deserialize_option = strategy.deserialize - if callable(deserialize_option): + if pass_through in (strategy, deserialize_option): + return value_name + elif callable(deserialize_option): overridden_fn = f"__{fname}_deserialize_{overridden_fn_suffix}" setattr(self.cls, overridden_fn, deserialize_option) - overridden = f"cls.{overridden_fn}({value_name})" + return f"cls.{overridden_fn}({value_name})" with suppress(TypeError): - if issubclass(ftype, SerializableType): - return ( - overridden - or f"{type_name(ftype)}._deserialize({value_name})" - ) + if issubclass(origin_type, SerializableType): + return f"{type_name(ftype)}._deserialize({value_name})" with suppress(TypeError): if issubclass(origin_type, GenericSerializableType): arg_type_names = ", ".join( list(map(type_name, get_args(ftype))) ) return ( - overridden - or f"{type_name(ftype)}._deserialize({value_name}, " + f"{type_name(ftype)}._deserialize({value_name}, " f"[{arg_type_names}])" ) @@ -1107,17 +1141,15 @@ def _unpack_field_value( builder.add_to_dict() else: method_name = "from_dict" - return overridden or ( - f"{type_name(ftype)}.{method_name}({value_name}, " - f"{self.get_from_dict_flags(ftype)})" + method_args = ", ".join( + filter(None, (value_name, self.get_from_dict_flags(ftype))) ) + return f"{type_name(origin_type)}.{method_name}({method_args})" if is_special_typing_primitive(origin_type): if origin_type is typing.Any: - return overridden or value_name + return value_name elif is_union(ftype): - if overridden: - return overridden args = get_args(ftype) field_type_vars = self._get_field_type_vars(fname) if is_optional(ftype, field_type_vars): @@ -1130,36 +1162,27 @@ def _unpack_field_value( else: return ufv else: - method_name = self._add_unpack_union( - fname, ftype, args, parent, metadata - ) - return ( - f"cls.{method_name}({value_name}," - f"use_bytes,use_enum,use_datetime)" + return self._unpack_union( + fname, ftype, value_name, args, parent, metadata ) elif origin_type is typing.AnyStr: raise UnserializableDataError( "AnyStr is not supported by mashumaro" ) elif is_type_var_any(ftype): - return overridden or value_name + return value_name elif is_type_var(ftype): - if overridden: - return overridden constraints = getattr(ftype, "__constraints__") if constraints: - method_name = self._add_unpack_union( + return self._unpack_union( fname=fname, ftype=ftype, + value_name=value_name, args=constraints, parent=parent, metadata=metadata, prefix="type_var", ) - return ( - f"cls.{method_name}({value_name}," - f"use_bytes,use_enum,use_datetime)" - ) else: bound = getattr(ftype, "__bound__") # act as if it was Optional[bound] @@ -1170,20 +1193,31 @@ def _unpack_field_value( return f"{ufv} if {value_name} is not None else None" else: return ufv + elif is_new_type(ftype): + return self._unpack_field_value( + fname=fname, + ftype=ftype.__supertype__, + parent=parent, + value_name=value_name, + metadata=metadata, + could_be_none=could_be_none, + ) + elif is_literal(ftype): + return self._unpack_literal( + fname, ftype, value_name, parent, metadata + ) else: raise UnserializableDataError( f"{ftype} as a field type is not supported by mashumaro" ) elif origin_type is int: - return overridden or f"int({value_name})" + return f"int({value_name})" elif origin_type is float: - return overridden or f"float({value_name})" + return f"float({value_name})" elif origin_type in (bool, NoneType, None): - return overridden or value_name + return value_name elif origin_type in (datetime.datetime, datetime.date, datetime.time): - if overridden: - return f"{value_name} if use_datetime else {overridden}" - elif deserialize_option is not None: + if deserialize_option is not None: if deserialize_option == "ciso8601": if ciso8601: self.ensure_module_imported(ciso8601) @@ -1209,37 +1243,34 @@ def _unpack_field_value( suffix = ".date()" elif origin_type is datetime.time: suffix = ".time()" - return ( - f"{value_name} if use_datetime else " - f"{datetime_parser}({value_name}){suffix}" - ) + return f"{datetime_parser}({value_name}){suffix}" return ( - f"{value_name} if use_datetime else " - f"datetime.{origin_type.__name__}." - f"fromisoformat({value_name})" + f"datetime.{origin_type.__name__}.fromisoformat({value_name})" ) elif origin_type is datetime.timedelta: - return overridden or f"datetime.timedelta(seconds={value_name})" + return f"datetime.timedelta(seconds={value_name})" elif origin_type is datetime.timezone: - return overridden or f"parse_timezone({value_name})" + return f"parse_timezone({value_name})" + elif PY_39_MIN and origin_type is zoneinfo.ZoneInfo: + return f"zoneinfo.ZoneInfo({value_name})" elif origin_type is uuid.UUID: - return overridden or f"uuid.UUID({value_name})" + return f"uuid.UUID({value_name})" elif origin_type is ipaddress.IPv4Address: - return overridden or f"ipaddress.IPv4Address({value_name})" + return f"ipaddress.IPv4Address({value_name})" elif origin_type is ipaddress.IPv6Address: - return overridden or f"ipaddress.IPv6Address({value_name})" + return f"ipaddress.IPv6Address({value_name})" elif origin_type is ipaddress.IPv4Network: - return overridden or f"ipaddress.IPv4Network({value_name})" + return f"ipaddress.IPv4Network({value_name})" elif origin_type is ipaddress.IPv6Network: - return overridden or f"ipaddress.IPv6Network({value_name})" + return f"ipaddress.IPv6Network({value_name})" elif origin_type is ipaddress.IPv4Interface: - return overridden or f"ipaddress.IPv4Interface({value_name})" + return f"ipaddress.IPv4Interface({value_name})" elif origin_type is ipaddress.IPv6Interface: - return overridden or f"ipaddress.IPv6Interface({value_name})" + return f"ipaddress.IPv6Interface({value_name})" elif origin_type is Decimal: - return overridden or f"Decimal({value_name})" + return f"Decimal({value_name})" elif origin_type is Fraction: - return overridden or f"Fraction({value_name})" + return f"Fraction({value_name})" elif issubclass(origin_type, typing.Collection) and not issubclass( origin_type, enum.Enum ): @@ -1261,30 +1292,14 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): if issubclass(origin_type, typing.ByteString): if origin_type is bytes: - specific = f"decodebytes({value_name}.encode())" - return ( - f"{value_name} if use_bytes else " - f"{overridden or specific}" - ) + return f"decodebytes({value_name}.encode())" elif origin_type is bytearray: - if overridden: - overridden = ( - f"bytearray({value_name}) if use_bytes else " - f"{overridden}" - ) - specific = ( - f"bytearray({value_name} if use_bytes else " - f"decodebytes({value_name}.encode()))" - ) - return overridden or specific + return f"bytearray(decodebytes({value_name}.encode()))" elif issubclass(origin_type, str): - return overridden or value_name + return value_name elif issubclass(origin_type, typing.List): if is_generic(ftype): - return ( - overridden - or f"[{inner_expr()} for value in {value_name}]" - ) + return f"[{inner_expr()} for value in {value_name}]" elif ftype is list: raise UnserializableField( fname, ftype, parent, "Use typing.List[T] instead" @@ -1292,8 +1307,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): elif issubclass(origin_type, typing.Deque): if is_generic(ftype): return ( - overridden - or f"collections.deque([{inner_expr()} " + f"collections.deque([{inner_expr()} " f"for value in {value_name}])" ) elif ftype is collections.deque: @@ -1302,7 +1316,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) elif issubclass(origin_type, typing.Tuple): if is_named_tuple(ftype): - return overridden or self._unpack_named_tuple( + return self._unpack_named_tuple( fname, ftype, value_name, @@ -1311,7 +1325,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): deserialize_option, ) elif is_generic(ftype): - return overridden or self._unpack_tuple( + return self._unpack_tuple( fname, value_name, args, parent, metadata ) elif ftype is tuple: @@ -1321,8 +1335,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): elif issubclass(origin_type, typing.FrozenSet): if is_generic(ftype): return ( - overridden - or f"frozenset([{inner_expr()} " + f"frozenset([{inner_expr()} " f"for value in {value_name}])" ) elif ftype is frozenset: @@ -1331,10 +1344,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) elif issubclass(origin_type, typing.AbstractSet): if is_generic(ftype): - return ( - overridden - or f"set([{inner_expr()} for value in {value_name}])" - ) + return f"set([{inner_expr()} for value in {value_name}])" elif ftype is set: raise UnserializableField( fname, ftype, parent, "Use typing.Set[T] instead" @@ -1348,8 +1358,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f"collections.ChainMap(" + f"collections.ChainMap(" f'*[{{{inner_expr(0,"key")}:{inner_expr(1)} ' f"for key, value in m.items()}} " f"for m in {value_name}])" @@ -1361,7 +1370,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): parent, "Use typing.ChainMap[KT,VT] instead", ) - elif PY_37_MIN and issubclass(origin_type, typing.OrderedDict): + elif issubclass(origin_type, typing_extensions.OrderedDict): if is_generic(ftype): if args and is_dataclass(args[0]): raise UnserializableDataError( @@ -1370,8 +1379,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f"collections.OrderedDict(" + f"collections.OrderedDict(" f'{{{inner_expr(0,"key")}: {inner_expr(1)} ' f"for key, value in {value_name}.items()}})" ) @@ -1391,8 +1399,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f"collections.Counter(" + f"collections.Counter(" f'{{{inner_expr(0,"key")}: ' f"{inner_expr(1, v_type=int)} " f"for key, value in {value_name}.items()}})" @@ -1406,16 +1413,9 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) elif issubclass(origin_type, typing.Mapping): if is_typed_dict(ftype): - if overridden: - return overridden - else: - method_name = self._add_unpack_typed_dict( - fname, ftype, value_name, parent, metadata - ) - return ( - f"cls.{method_name}({value_name}," - f"use_bytes,use_enum,use_datetime)" - ) + return self._unpack_typed_dict( + fname, ftype, value_name, parent, metadata + ) elif is_generic(ftype): if args and is_dataclass(args[0]): raise UnserializableDataError( @@ -1424,8 +1424,7 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) else: return ( - overridden - or f'{{{inner_expr(0,"key")}: {inner_expr(1)} ' + f'{{{inner_expr(0,"key")}: {inner_expr(1)} ' f"for key, value in {value_name}.items()}}" ) elif ftype is dict: @@ -1437,14 +1436,9 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): ) elif issubclass(origin_type, typing.Sequence): if is_generic(ftype): - return ( - overridden - or f"[{inner_expr()} for value in {value_name}]" - ) + return f"[{inner_expr()} for value in {value_name}]" elif issubclass(origin_type, os.PathLike): - if overridden: - return overridden - elif issubclass(origin_type, pathlib.PosixPath): + if issubclass(origin_type, pathlib.PosixPath): return f"pathlib.PosixPath({value_name})" elif issubclass(origin_type, pathlib.WindowsPath): return f"pathlib.WindowsPath({value_name})" @@ -1461,25 +1455,23 @@ def inner_expr(arg_num=0, v_name="value", v_type=None): else: return f"{type_name(origin_type)}({value_name})" elif issubclass(origin_type, enum.Enum): - specific = f"{type_name(origin_type)}({value_name})" - return f"{value_name} if use_enum else {overridden or specific}" - if overridden: - return overridden + return f"{type_name(origin_type)}({value_name})" raise UnserializableField(fname, ftype, parent) - def _add_pack_union( - self, fname, ftype, args, parent, metadata, prefix="union" + def _pack_union( + self, fname, ftype, value_name, args, parent, metadata, prefix="union" ) -> str: lines = CodeLines() method_name = ( f"__pack_{prefix}_{parent.__name__}_{fname}__" f"{str(uuid.uuid4().hex)}" ) - lines.append( - f"def {method_name}" - f"(self,value, {self.get_to_dict_default_flag_values()}):" - ) + default_kwargs = self.get_to_dict_default_flag_values() + if default_kwargs: + lines.append(f"def {method_name}(self, value, {default_kwargs}):") + else: + lines.append(f"def {method_name}(self, value):") with lines.indent(): for packer in ( self._pack_value(fname, arg_type, parent, metadata=metadata) @@ -1502,21 +1494,25 @@ def _add_pack_union( print(f"{type_name(self.cls)}:") print(lines.as_text()) exec(lines.as_text(), self.globals, self.__dict__) - return method_name + method_args = ", ".join( + filter(None, (value_name, self.get_to_dict_flags())) + ) + return f"self.{method_name}({method_args})" - def _add_unpack_union( - self, fname, ftype, args, parent, metadata, prefix="union" + def _unpack_union( + self, fname, ftype, value_name, args, parent, metadata, prefix="union" ) -> str: lines = CodeLines() method_name = ( f"__unpack_{prefix}_{parent.__name__}_{fname}__" f"{str(uuid.uuid4().hex)}" ) + default_kwargs = self.get_from_dict_default_flag_values() lines.append("@classmethod") - lines.append( - f"def {method_name}" - f"(cls,value,{self.get_from_dict_default_flag_values()}):" - ) + if default_kwargs: + lines.append(f"def {method_name}(cls, value, {default_kwargs}):") + else: + lines.append(f"def {method_name}(cls, value):") with lines.indent(): for unpacker in ( self._unpack_field_value( @@ -1541,7 +1537,10 @@ def _add_unpack_union( print(f"{type_name(self.cls)}:") print(lines.as_text()) exec(lines.as_text(), self.globals, self.__dict__) - return method_name + method_args = ", ".join( + filter(None, (value_name, self.get_from_dict_flags())) + ) + return f"cls.{method_name}({method_args})" def _pack_tuple(self, fname, value_name, args, parent, metadata) -> str: if not args: @@ -1597,7 +1596,7 @@ def _unpack_tuple(self, fname, value_name, args, parent, metadata) -> str: ] return f"tuple([{', '.join(unpackers)}])" - def _add_pack_typed_dict( + def _pack_typed_dict( self, fname, ftype, value_name, parent, metadata ) -> str: annotations = ftype.__annotations__ @@ -1609,10 +1608,11 @@ def _add_pack_typed_dict( f"__pack_typed_dict_{parent.__name__}_{fname}__" f"{str(uuid.uuid4().hex)}" ) - lines.append( - f"def {method_name}" - f"(self,value, {self.get_to_dict_default_flag_values()}):" - ) + default_kwargs = self.get_to_dict_default_flag_values() + if default_kwargs: + lines.append(f"def {method_name}(self, value, {default_kwargs}):") + else: + lines.append(f"def {method_name}(self, value):") with lines.indent(): lines.append("d = {}") for key in sorted(required_keys, key=all_keys.index): @@ -1642,9 +1642,12 @@ def _add_pack_typed_dict( print(f"{type_name(self.cls)}:") print(lines.as_text()) exec(lines.as_text(), self.globals, self.__dict__) - return method_name + method_args = ", ".join( + filter(None, (value_name, self.get_to_dict_flags())) + ) + return f"self.{method_name}({method_args})" - def _add_unpack_typed_dict( + def _unpack_typed_dict( self, fname, ftype, value_name, parent, metadata ) -> str: annotations = ftype.__annotations__ @@ -1656,11 +1659,12 @@ def _add_unpack_typed_dict( f"__unpack_typed_dict_{parent.__name__}_{fname}__" f"{str(uuid.uuid4().hex)}" ) + default_kwargs = self.get_from_dict_default_flag_values() lines.append("@classmethod") - lines.append( - f"def {method_name}" - f"(cls,value,use_bytes=False,use_enum=False,use_datetime=False):" - ) + if default_kwargs: + lines.append(f"def {method_name}(cls, value, {default_kwargs}):") + else: + lines.append(f"def {method_name}(cls, value):") with lines.indent(): lines.append("d = {}") for key in sorted(required_keys, key=all_keys.index): @@ -1690,7 +1694,10 @@ def _add_unpack_typed_dict( print(f"{type_name(self.cls)}:") print(lines.as_text()) exec(lines.as_text(), self.globals, self.__dict__) - return method_name + method_args = ", ".join( + filter(None, (value_name, self.get_from_dict_flags())) + ) + return f"cls.{method_name}({method_args})" def _pack_named_tuple( self, fname, ftype, value_name, parent, metadata, serialize_option @@ -1763,10 +1770,11 @@ def _unpack_named_tuple( f"{str(uuid.uuid4().hex)}" ) lines.append("@classmethod") - lines.append( - f"def {method_name}" - f"(cls,value,{self.get_from_dict_default_flag_values()}):" - ) + default_kwargs = self.get_from_dict_default_flag_values() + if default_kwargs: + lines.append(f"def {method_name}(cls, value, {default_kwargs}):") + else: + lines.append(f"def {method_name}(cls, value):") with lines.indent(): lines.append("fields = []") lines.append("try:") @@ -1782,9 +1790,120 @@ def _unpack_named_tuple( print(f"{type_name(self.cls)}:") print(lines.as_text()) exec(lines.as_text(), self.globals, self.__dict__) - return ( - f"cls.{method_name}({value_name},use_bytes,use_enum,use_datetime)" + method_args = ", ".join( + filter(None, (value_name, self.get_from_dict_flags())) + ) + return f"cls.{method_name}({method_args})" + + def _unpack_literal( + self, fname, ftype, value_name, parent, metadata + ) -> str: + lines = CodeLines() + method_name = ( + f"__unpack_literal_{parent.__name__}_{fname}__" + f"{str(uuid.uuid4().hex)}" + ) + default_kwargs = self.get_from_dict_default_flag_values() + lines.append("@classmethod") + if default_kwargs: + lines.append(f"def {method_name}(cls, value, {default_kwargs}):") + else: + lines.append(f"def {method_name}(cls, value):") + with lines.indent(): + for literal_value in get_literal_values(ftype): + if isinstance(literal_value, enum.Enum): + enum_type_name = type_name(type(literal_value)) + lines.append( + f"if {value_name} == {enum_type_name}." + f"{literal_value.name}.value:" + ) + with lines.indent(): + lines.append( + f"return {enum_type_name}.{literal_value.name}" + ) + elif isinstance(literal_value, bytes): + unpacker = self._unpack_field_value( + fname, bytes, parent, value_name, metadata + ) + lines.append("try:") + with lines.indent(): + lines.append(f"if {unpacker} == {literal_value!r}:") + with lines.indent(): + lines.append(f"return {literal_value!r}") + lines.append("except:") + with lines.indent(): + lines.append("pass") + elif isinstance( # type: ignore + literal_value, (int, str, bool, NoneType) # type: ignore + ): + lines.append(f"if {value_name} == {literal_value!r}:") + with lines.indent(): + lines.append(f"return {literal_value!r}") + lines.append(f"raise ValueError({value_name})") + lines.append(f"setattr(cls, '{method_name}', {method_name})") + if self.get_config().debug: + print(f"{type_name(self.cls)}:") + print(lines.as_text()) + exec(lines.as_text(), self.globals, self.__dict__) + method_args = ", ".join( + filter(None, (value_name, self.get_from_dict_flags())) + ) + return f"cls.{method_name}({method_args})" + + def _pack_literal(self, fname, ftype, value_name, parent, metadata) -> str: + self._add_type_modules(ftype) + lines = CodeLines() + method_name = ( + f"__pack_literal_{parent.__name__}_{fname}__" + f"{str(uuid.uuid4().hex)}" + ) + default_kwargs = self.get_to_dict_default_flag_values() + if default_kwargs: + lines.append(f"def {method_name}(self, value, {default_kwargs}):") + else: + lines.append(f"def {method_name}(self, value):") + with lines.indent(): + for literal_value in get_literal_values(ftype): + value_type = type(literal_value) + if isinstance(literal_value, enum.Enum): + enum_type_name = type_name( + value_type, type_vars=self._get_field_type_vars(fname) + ) + lines.append( + f"if {value_name} == {enum_type_name}." + f"{literal_value.name}:" + ) + with lines.indent(): + packer = self._pack_value( + fname, value_type, parent, value_name, metadata + ) + lines.append(f"return {packer}") + elif isinstance( # type: ignore + literal_value, + (int, str, bytes, bool, NoneType), # type: ignore + ): + lines.append(f"if {value_name} == {literal_value!r}:") + with lines.indent(): + packer = self._pack_value( + fname, value_type, parent, value_name, metadata + ) + lines.append(f"return {packer}") + field_type = type_name( + ftype, type_vars=self._get_field_type_vars(fname) + ) + lines.append( + f"raise InvalidFieldValue('{fname}'," + f"{field_type},{value_name},type(self))" + ) + lines.append(f"setattr(cls, '{method_name}', {method_name})") + if self.get_config().debug: + print(f"{type_name(self.cls)}:") + print(lines.as_text()) + exec(lines.as_text(), self.globals, self.__dict__) + method_args = ", ".join( + filter(None, (value_name, self.get_to_dict_flags())) ) + return f"self.{method_name}({method_args})" @classmethod def _hash_arg_types(cls, arg_types) -> str: diff --git a/mashumaro/meta/helpers.py b/mashumaro/core/meta/helpers.py similarity index 69% rename from mashumaro/meta/helpers.py rename to mashumaro/core/meta/helpers.py index c673a3f4..4992eea0 100644 --- a/mashumaro/meta/helpers.py +++ b/mashumaro/core/meta/helpers.py @@ -1,4 +1,5 @@ import dataclasses +import enum import inspect import re import types @@ -10,9 +11,7 @@ import typing_extensions -from mashumaro.dialect import Dialect - -from .macros import ( +from mashumaro.core.const import ( PY_36, PY_37, PY_37_MIN, @@ -21,11 +20,11 @@ PY_39_MIN, PY_310_MIN, ) +from mashumaro.dialect import Dialect NoneType = type(None) DataClassDictMixinPath = ( - f"{__name__.rsplit('.', 2)[:-2][0]}" - f".serializer.base.dict.DataClassDictMixin" + f"{__name__.rsplit('.', 3)[:-3][0]}" f".mixins.dict.DataClassDictMixin" ) @@ -33,6 +32,8 @@ def get_type_origin(t): origin = None try: if PY_36: + if is_annotated(t): + return get_type_origin(t.__args__[0]) origin = t.__extra__ or t.__origin__ elif PY_37_MIN: origin = t.__origin__ @@ -41,27 +42,31 @@ def get_type_origin(t): return origin or t -def is_builtin_type(t): +def is_builtin_type(t) -> bool: try: return t.__module__ == "builtins" except AttributeError: return False -def get_generic_name(t, short: bool = False): +def get_generic_name(t, short: bool = False) -> str: if PY_36: name = getattr(t, "__name__") elif PY_37_MIN: name = getattr(t, "_name", None) if name is None: - return type_name(get_type_origin(t), short, is_type_origin=True) + origin = get_type_origin(t) + if origin is t: + return type_name(origin, short, is_type_origin=True) + else: + return get_generic_name(origin, short) if short: return name else: return f"{t.__module__}.{name}" -def get_args(t: typing.Any): +def get_args(t: typing.Any) -> typing.Tuple[typing.Any, ...]: return getattr(t, "__args__", None) or () @@ -72,7 +77,7 @@ def _get_args_str( limit: typing.Optional[int] = None, none_type_as_none: bool = False, sep: str = ", ", -): +) -> str: args = get_args(t)[:limit] return sep.join( type_name(arg, short, type_vars, none_type_as_none=none_type_as_none) @@ -80,8 +85,40 @@ def _get_args_str( ) -def _typing_name(t: str, short: bool = False): - return t if short else f"typing.{t}" +def get_literal_values(t: typing.Any): + if PY_36: + values = t.__values__ or () + elif PY_37_MIN: + values = t.__args__ + else: + raise NotImplementedError + result = [] + for value in values: + if is_literal(value): + result.extend(get_literal_values(value)) + else: + result.append(value) + return tuple(result) + + +def _get_literal_values_str(t: typing.Any, short: bool): + values_str = [] + for value in get_literal_values(t): + if isinstance(value, enum.Enum): + values_str.append(f"{type_name(type(value), short)}.{value.name}") + elif isinstance( # type: ignore + value, (int, str, bytes, bool, NoneType) # type: ignore + ): + values_str.append(repr(value)) + return ", ".join(values_str) + + +def _typing_name( + t: str, + short: bool = False, + module_name: str = "typing", +) -> str: + return t if short else f"{module_name}.{t}" def type_name( @@ -98,11 +135,18 @@ def type_name( elif t is typing.Any: return _typing_name("Any", short) elif is_optional(t, type_vars): - args_str = type_name(not_none_type_arg(get_args(t), type_vars), short) + args_str = type_name( + not_none_type_arg(get_args(t), type_vars), short, type_vars + ) return f"{_typing_name('Optional', short)}[{args_str}]" elif is_union(t): args_str = _get_args_str(t, short, type_vars, none_type_as_none=True) return f"{_typing_name('Union', short)}[{args_str}]" + elif is_annotated(t): + return type_name(get_args(t)[0], short, type_vars) + elif is_literal(t): + args_str = _get_literal_values_str(t, short) + return f"{_typing_name('Literal', short, t.__module__)}[{args_str}]" elif is_generic(t) and not is_type_origin: args_str = _get_args_str(t, short, type_vars) if not args_str: @@ -125,17 +169,19 @@ def type_name( else: bound = getattr(t, "__bound__") return type_name(bound, short, type_vars) - else: - try: - if short: - return t.__qualname__ - else: - return f"{t.__module__}.{t.__qualname__}" - except AttributeError: - return str(t) + elif is_new_type(t) and not PY_310_MIN: + # because __qualname__ and __module__ are messed up + t = t.__supertype__ + try: + if short: + return t.__qualname__ + else: + return f"{t.__module__}.{t.__qualname__}" + except AttributeError: + return str(t) -def is_special_typing_primitive(t): +def is_special_typing_primitive(t) -> bool: try: issubclass(t, object) return False @@ -150,10 +196,7 @@ def is_generic(t): elif PY_37 or PY_38: # noinspection PyProtectedMember # noinspection PyUnresolvedReferences - return t.__class__ in ( - typing._GenericAlias, - typing._VariadicGenericAlias, - ) + return issubclass(t.__class__, typing._GenericAlias) elif PY_39_MIN: # noinspection PyProtectedMember # noinspection PyUnresolvedReferences @@ -174,7 +217,7 @@ def is_generic(t): raise NotImplementedError -def is_typed_dict(t): +def is_typed_dict(t) -> bool: for module in (typing, typing_extensions): with suppress(AttributeError): if type(t) is getattr(module, "_TypedDictMeta"): @@ -182,13 +225,19 @@ def is_typed_dict(t): return False -def is_named_tuple(t): +def is_named_tuple(t) -> bool: try: - return issubclass(t, typing.Tuple) and hasattr(t, "_fields") + return issubclass(t, typing.Tuple) and hasattr( # type: ignore + t, "_fields" + ) except TypeError: return False +def is_new_type(t) -> bool: + return hasattr(t, "__supertype__") + + def is_union(t): try: if PY_310_MIN and isinstance(t, types.UnionType): @@ -198,7 +247,7 @@ def is_union(t): return False -def is_optional(t, type_vars: typing.Dict[str, typing.Any] = None): +def is_optional(t, type_vars: typing.Dict[str, typing.Any] = None) -> bool: if type_vars is None: type_vars = {} if not is_union(t): @@ -212,6 +261,38 @@ def is_optional(t, type_vars: typing.Dict[str, typing.Any] = None): return False +def is_annotated(t) -> bool: + for module in (typing, typing_extensions): + with suppress(AttributeError): + if type(t) is getattr(module, "_AnnotatedAlias"): + return True + with suppress(AttributeError): + if type(t) is getattr(module, "AnnotatedMeta"): + # Annotated from typing-extensions on Python 3.6 + return True + return False + + +def is_literal(t) -> bool: + if PY_36: + with suppress(AttributeError): + # noinspection PyProtectedMember + # noinspection PyUnresolvedReferences + return ( + isinstance(t, typing_extensions._Literal) # type: ignore + and len(get_literal_values(t)) > 0 + ) + elif PY_37 or PY_38: + with suppress(AttributeError): + return is_generic(t) and get_generic_name(t, True) == "Literal" + elif PY_39_MIN: + with suppress(AttributeError): + # noinspection PyProtectedMember + # noinspection PyUnresolvedReferences + return type(t) is typing._LiteralGenericAlias # type: ignore + return False + + def not_none_type_arg( args: typing.Tuple[typing.Any, ...], type_vars: typing.Dict[str, typing.Any] = None, @@ -223,11 +304,11 @@ def not_none_type_arg( return arg -def is_type_var(t): +def is_type_var(t) -> bool: return hasattr(t, "__constraints__") -def is_type_var_any(t): +def is_type_var_any(t) -> bool: if not is_type_var(t): return False elif t.__constraints__ != (): @@ -238,7 +319,7 @@ def is_type_var_any(t): return True -def is_class_var(t): +def is_class_var(t) -> bool: if PY_36: return ( is_special_typing_primitive(t) and type(t).__name__ == "_ClassVar" @@ -249,7 +330,7 @@ def is_class_var(t): raise NotImplementedError -def is_init_var(t): +def is_init_var(t) -> bool: if PY_36 or PY_37: return get_type_origin(t) is dataclasses.InitVar elif PY_38_MIN: @@ -276,11 +357,11 @@ def get_class_that_defines_field(field_name, cls): return prev_cls or cls -def is_dataclass_dict_mixin(t): +def is_dataclass_dict_mixin(t) -> bool: return type_name(t) == DataClassDictMixinPath -def is_dataclass_dict_mixin_subclass(t): +def is_dataclass_dict_mixin_subclass(t) -> bool: with suppress(AttributeError): for cls in t.__mro__: if is_dataclass_dict_mixin(cls): @@ -370,4 +451,8 @@ def is_dialect_subclass(t) -> bool: "get_generic_name", "get_name_error_name", "is_dialect_subclass", + "is_new_type", + "is_annotated", + "is_literal", + "get_literal_values", ] diff --git a/mashumaro/meta/patch.py b/mashumaro/core/meta/patch.py similarity index 87% rename from mashumaro/meta/patch.py rename to mashumaro/core/meta/patch.py index a6ee7605..2214c4ad 100644 --- a/mashumaro/meta/patch.py +++ b/mashumaro/core/meta/patch.py @@ -1,4 +1,4 @@ -from .macros import PY_36 +from mashumaro.core.const import PY_36 def patch_fromisoformat(): diff --git a/mashumaro/dialects/__init__.py b/mashumaro/dialects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mashumaro/dialects/msgpack.py b/mashumaro/dialects/msgpack.py new file mode 100644 index 00000000..b066f378 --- /dev/null +++ b/mashumaro/dialects/msgpack.py @@ -0,0 +1,15 @@ +from mashumaro.dialect import Dialect +from mashumaro.helper import pass_through + + +class MessagePackDialect(Dialect): + serialization_strategy = { + bytes: pass_through, # type: ignore + bytearray: { + "deserialize": bytearray, + "serialize": pass_through, + }, + } + + +__all__ = ["MessagePackDialect"] diff --git a/mashumaro/exceptions.py b/mashumaro/exceptions.py index 419c1286..7ad9c5e3 100644 --- a/mashumaro/exceptions.py +++ b/mashumaro/exceptions.py @@ -1,4 +1,4 @@ -from mashumaro.meta.helpers import type_name +from mashumaro.core.meta.helpers import type_name class MissingField(LookupError): diff --git a/mashumaro/helper.py b/mashumaro/helper.py index 9331f77d..2eb3444f 100644 --- a/mashumaro/helper.py +++ b/mashumaro/helper.py @@ -1,7 +1,17 @@ from typing import Any, Callable, Optional, Union +from typing_extensions import Literal + from mashumaro.types import SerializationStrategy -from mashumaro.typing import AnyDeserializationEngine, AnySerializationEngine + +NamedTupleDeserializationEngine = Literal["as_dict", "as_list"] +DateTimeDeserializationEngine = Literal["ciso8601", "pendulum"] +AnyDeserializationEngine = Literal[ + NamedTupleDeserializationEngine, DateTimeDeserializationEngine +] + +NamedTupleSerializationEngine = Literal["as_dict", "as_list"] +AnySerializationEngine = NamedTupleSerializationEngine def field_options( @@ -22,4 +32,18 @@ def field_options( } -__all__ = ["field_options"] +class _PassThrough(SerializationStrategy): + def __call__(self, *args, **kwargs): + raise NotImplementedError + + def serialize(self, value): + return value + + def deserialize(self, value): + return value + + +pass_through = _PassThrough() + + +__all__ = ["field_options", "pass_through"] diff --git a/mashumaro/mixins/__init__.py b/mashumaro/mixins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mashumaro/serializer/base/dict.py b/mashumaro/mixins/dict.py similarity index 73% rename from mashumaro/serializer/base/dict.py rename to mashumaro/mixins/dict.py index cf6e8c82..4265b2f3 100644 --- a/mashumaro/serializer/base/dict.py +++ b/mashumaro/mixins/dict.py @@ -1,7 +1,7 @@ from typing import Any, Dict, Mapping, Type, TypeVar +from mashumaro.core.meta.builder import CodeBuilder from mashumaro.exceptions import UnresolvedTypeReferenceError -from mashumaro.serializer.base.metaprogramming import CodeBuilder T = TypeVar("T", bound="DataClassDictMixin") @@ -25,9 +25,6 @@ def __init_subclass__(cls: Type[T], **kwargs): def to_dict( self: T, - use_bytes: bool = False, - use_enum: bool = False, - use_datetime: bool = False, # * # keyword-only arguments that exist with the code generation options: # omit_none: bool = False @@ -35,25 +32,18 @@ def to_dict( # dialect: Type[Dialect] = None **kwargs, ) -> dict: - builder = CodeBuilder(self.__class__) - builder.add_to_dict() - return self.to_dict(use_bytes, use_enum, use_datetime, **kwargs) + ... @classmethod def from_dict( cls: Type[T], d: Mapping, - use_bytes: bool = False, - use_enum: bool = False, - use_datetime: bool = False, # * # keyword-only arguments that exist with the code generation options: # dialect: Type[Dialect] = None **kwargs, ) -> T: - builder = CodeBuilder(cls) - builder.add_from_dict() - return cls.from_dict(d, use_bytes, use_enum, use_datetime, **kwargs) + ... @classmethod def __pre_deserialize__(cls: Type[T], d: Dict[Any, Any]) -> Dict[Any, Any]: diff --git a/mashumaro/mixins/json.py b/mashumaro/mixins/json.py new file mode 100644 index 00000000..4b63878a --- /dev/null +++ b/mashumaro/mixins/json.py @@ -0,0 +1,39 @@ +import json +from typing import Any, Dict, Type, TypeVar, Union + +from typing_extensions import Protocol + +from mashumaro.mixins.dict import DataClassDictMixin + +EncodedData = Union[str, bytes, bytearray] +T = TypeVar("T", bound="DataClassJSONMixin") + + +class Encoder(Protocol): # pragma no cover + def __call__(self, obj, **kwargs) -> EncodedData: + ... + + +class Decoder(Protocol): # pragma no cover + def __call__(self, s: EncodedData, **kwargs) -> Dict[Any, Any]: + ... + + +class DataClassJSONMixin(DataClassDictMixin): + __slots__ = () + + def to_json( + self: T, + encoder: Encoder = json.dumps, + **to_dict_kwargs, + ) -> EncodedData: + return encoder(self.to_dict(**to_dict_kwargs)) + + @classmethod + def from_json( + cls: Type[T], + data: EncodedData, + decoder: Decoder = json.loads, + **from_dict_kwargs, + ) -> T: + return cls.from_dict(decoder(data), **from_dict_kwargs) diff --git a/mashumaro/serializer/msgpack.py b/mashumaro/mixins/msgpack.py similarity index 58% rename from mashumaro/serializer/msgpack.py rename to mashumaro/mixins/msgpack.py index dd59d2e6..94666076 100644 --- a/mashumaro/serializer/msgpack.py +++ b/mashumaro/mixins/msgpack.py @@ -1,20 +1,17 @@ from functools import partial -from types import MappingProxyType -from typing import Any, Dict, Mapping, Type, TypeVar +from typing import Any, Dict, Type, TypeVar import msgpack from typing_extensions import Protocol -from mashumaro.serializer.base import DataClassDictMixin +from mashumaro.dialects.msgpack import MessagePackDialect +from mashumaro.mixins.dict import DataClassDictMixin -DEFAULT_DICT_PARAMS = { - "use_bytes": True, - "use_enum": False, - "use_datetime": False, -} EncodedData = bytes T = TypeVar("T", bound="DataClassMessagePackMixin") +DEFAULT_DICT_PARAMS = {"dialect": MessagePackDialect} + class Encoder(Protocol): # pragma no cover def __call__(self, o, **kwargs) -> EncodedData: @@ -32,13 +29,11 @@ class DataClassMessagePackMixin(DataClassDictMixin): def to_msgpack( self: T, encoder: Encoder = partial(msgpack.packb, use_bin_type=True), - dict_params: Mapping = MappingProxyType({}), - **encoder_kwargs, + **to_dict_kwargs, ) -> EncodedData: return encoder( - self.to_dict(**dict(DEFAULT_DICT_PARAMS, **dict_params)), - **encoder_kwargs, + self.to_dict(**dict(DEFAULT_DICT_PARAMS, **to_dict_kwargs)) ) @classmethod @@ -46,10 +41,9 @@ def from_msgpack( cls: Type[T], data: EncodedData, decoder: Decoder = partial(msgpack.unpackb, raw=False), - dict_params: Mapping = MappingProxyType({}), - **decoder_kwargs, + **from_dict_kwargs, ) -> T: return cls.from_dict( - decoder(data, **decoder_kwargs), - **dict(DEFAULT_DICT_PARAMS, **dict_params), + decoder(data), + **dict(DEFAULT_DICT_PARAMS, **from_dict_kwargs), ) diff --git a/mashumaro/mixins/yaml.py b/mashumaro/mixins/yaml.py new file mode 100644 index 00000000..2c6f3527 --- /dev/null +++ b/mashumaro/mixins/yaml.py @@ -0,0 +1,51 @@ +from typing import Any, Dict, Type, TypeVar, Union + +import yaml +from typing_extensions import Protocol + +from mashumaro.mixins.dict import DataClassDictMixin + +EncodedData = Union[str, bytes] +T = TypeVar("T", bound="DataClassYAMLMixin") + + +class Encoder(Protocol): # pragma no cover + def __call__(self, o, **kwargs) -> EncodedData: + ... + + +class Decoder(Protocol): # pragma no cover + def __call__(self, packed: EncodedData, **kwargs) -> Dict[Any, Any]: + ... + + +DefaultLoader = getattr(yaml, "CSafeLoader", yaml.SafeLoader) +DefaultDumper = getattr(yaml, "CDumper", yaml.Dumper) + + +def default_encoder(data) -> EncodedData: + return yaml.dump(data, Dumper=DefaultDumper) + + +def default_decoder(data: EncodedData) -> Dict[Any, Any]: + return yaml.load(data, DefaultLoader) + + +class DataClassYAMLMixin(DataClassDictMixin): + __slots__ = () + + def to_yaml( + self: T, + encoder: Encoder = default_encoder, # type: ignore + **to_dict_kwargs, + ) -> EncodedData: + return encoder(self.to_dict(**to_dict_kwargs)) + + @classmethod + def from_yaml( + cls: Type[T], + data: EncodedData, + decoder: Decoder = default_decoder, # type: ignore + **from_dict_kwargs, + ) -> T: + return cls.from_dict(decoder(data), **from_dict_kwargs) diff --git a/mashumaro/serializer/base/__init__.py b/mashumaro/serializer/base/__init__.py deleted file mode 100644 index b5d5ce7d..00000000 --- a/mashumaro/serializer/base/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .dict import DataClassDictMixin - -__all__ = ["DataClassDictMixin"] diff --git a/mashumaro/serializer/json.py b/mashumaro/serializer/json.py deleted file mode 100644 index 70111559..00000000 --- a/mashumaro/serializer/json.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -from types import MappingProxyType -from typing import Any, Dict, Mapping, Type, TypeVar, Union - -from typing_extensions import Protocol - -from mashumaro.serializer.base import DataClassDictMixin - -DEFAULT_DICT_PARAMS = { - "use_bytes": False, - "use_enum": False, - "use_datetime": False, -} -EncodedData = Union[str, bytes, bytearray] -T = TypeVar("T", bound="DataClassJSONMixin") - - -class Encoder(Protocol): # pragma no cover - def __call__(self, obj, **kwargs) -> EncodedData: - ... - - -class Decoder(Protocol): # pragma no cover - def __call__(self, s: EncodedData, **kwargs) -> Dict[Any, Any]: - ... - - -class DataClassJSONMixin(DataClassDictMixin): - __slots__ = () - - def to_json( - self: T, - encoder: Encoder = json.dumps, - dict_params: Mapping = MappingProxyType({}), - **encoder_kwargs, - ) -> EncodedData: - - return encoder( - self.to_dict(**dict(DEFAULT_DICT_PARAMS, **dict_params)), - **encoder_kwargs, - ) - - @classmethod - def from_json( - cls: Type[T], - data: EncodedData, - decoder: Decoder = json.loads, - dict_params: Mapping = MappingProxyType({}), - **decoder_kwargs, - ) -> T: - - return cls.from_dict( - decoder(data, **decoder_kwargs), - **dict(DEFAULT_DICT_PARAMS, **dict_params), - ) diff --git a/mashumaro/serializer/yaml.py b/mashumaro/serializer/yaml.py deleted file mode 100644 index 0dafa3d0..00000000 --- a/mashumaro/serializer/yaml.py +++ /dev/null @@ -1,54 +0,0 @@ -from types import MappingProxyType -from typing import Any, Dict, Mapping, Type, TypeVar, Union - -import yaml -from typing_extensions import Protocol - -from mashumaro.serializer.base import DataClassDictMixin - -DEFAULT_DICT_PARAMS = { - "use_bytes": False, - "use_enum": False, - "use_datetime": False, -} -EncodedData = Union[str, bytes] -T = TypeVar("T", bound="DataClassYAMLMixin") - - -class Encoder(Protocol): # pragma no cover - def __call__(self, o, **kwargs) -> EncodedData: - ... - - -class Decoder(Protocol): # pragma no cover - def __call__(self, packed: EncodedData, **kwargs) -> Dict[Any, Any]: - ... - - -class DataClassYAMLMixin(DataClassDictMixin): - __slots__ = () - - def to_yaml( - self: T, - encoder: Encoder = yaml.dump, # type: ignore - dict_params: Mapping = MappingProxyType({}), - **encoder_kwargs, - ) -> EncodedData: - - return encoder( - self.to_dict(**dict(DEFAULT_DICT_PARAMS, **dict_params)), - **encoder_kwargs, - ) - - @classmethod - def from_yaml( - cls: Type[T], - data: EncodedData, - decoder: Decoder = yaml.safe_load, # type: ignore - dict_params: Mapping = MappingProxyType({}), - **decoder_kwargs, - ) -> T: - return cls.from_dict( - decoder(data, **decoder_kwargs), - **dict(DEFAULT_DICT_PARAMS, **dict_params), - ) diff --git a/mashumaro/typing.py b/mashumaro/typing.py deleted file mode 100644 index 7a4d4ba1..00000000 --- a/mashumaro/typing.py +++ /dev/null @@ -1,19 +0,0 @@ -from mashumaro.meta.macros import PEP_586_COMPATIBLE - -if PEP_586_COMPATIBLE: # type: ignore - from typing import Literal # type: ignore -else: - from typing_extensions import Literal # type: ignore - - -NamedTupleDeserializationEngine = Literal["as_dict", "as_list"] -DateTimeDeserializationEngine = Literal["ciso8601", "pendulum"] -AnyDeserializationEngine = Literal[ - NamedTupleDeserializationEngine, DateTimeDeserializationEngine -] - -NamedTupleSerializationEngine = Literal["as_dict", "as_list"] -AnySerializationEngine = NamedTupleSerializationEngine - - -__all__ = ["AnyDeserializationEngine", "AnySerializationEngine"] diff --git a/pyproject.toml b/pyproject.toml index 7c76aa9a..cbc295ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ -[mypy] +[tool.mypy] ignore_missing_imports = true [flake8] diff --git a/requirements-dev.txt b/requirements-dev.txt index 9bdc26cd..f002921e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,8 @@ +# extra +msgpack>=0.5.6 +pyyaml>=3.13 + +# tests mypy>=0.812 flake8>=3.8.4 isort>=5.6.4 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 976ba029..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -ignore_missing_imports = True diff --git a/setup.py b/setup.py index 0ba44d3e..5efb4fc8 100644 --- a/setup.py +++ b/setup.py @@ -29,10 +29,12 @@ python_requires=">=3.6", install_requires=[ "dataclasses;python_version=='3.6'", - "msgpack>=0.5.6", - "pyyaml>=3.13", "backports-datetime-fromisoformat;python_version=='3.6'", "typing_extensions", ], + extras_require={ + "msgpack": ["msgpack>=0.5.6"], + "yaml": ["pyyaml>=3.13"], + }, zip_safe=False, ) diff --git a/tests/conftest.py b/tests/conftest.py index 2c0a2d8a..9004ea8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,12 @@ from unittest.mock import patch -from mashumaro.meta.macros import PY_37_MIN +from mashumaro.core.const import PY_37_MIN if not PY_37_MIN: collect_ignore = ["test_pep_563.py"] fake_add_from_dict = patch( - "mashumaro.serializer.base.metaprogramming." "CodeBuilder.add_from_dict", + "mashumaro.core.meta.builder." "CodeBuilder.add_from_dict", lambda *args, **kwargs: ..., ) diff --git a/tests/entities.py b/tests/entities.py index 3175677f..e27cd29f 100644 --- a/tests/entities.py +++ b/tests/entities.py @@ -1,14 +1,24 @@ from collections import namedtuple from dataclasses import dataclass +from datetime import datetime from enum import Enum, Flag, IntEnum, IntFlag from os import PathLike -from typing import Any, Generic, List, NamedTuple, Optional, TypeVar, Union +from typing import ( + Any, + Generic, + List, + NamedTuple, + NewType, + Optional, + TypeVar, + Union, +) from typing_extensions import TypedDict from mashumaro import DataClassDictMixin from mashumaro.config import TO_DICT_ADD_OMIT_NONE_FLAG, BaseConfig -from mashumaro.meta.macros import PY_37_MIN +from mashumaro.core.const import PY_37_MIN from mashumaro.types import GenericSerializableType, SerializableType T = TypeVar("T") @@ -176,6 +186,21 @@ class MyGenericList(List[T]): pass +class SerializableTypeGenericList(Generic[T], SerializableType): + def __init__(self, value: List[T]): + self.value = value + + def _serialize(self): + return self.value + + @classmethod + def _deserialize(cls, value): + return SerializableTypeGenericList(value) + + def __eq__(self, other): + return self.value == other.value + + TMyDataClass = TypeVar("TMyDataClass", bound=MyDataClass) @@ -212,3 +237,6 @@ class MyNamedTupleWithDefaults(NamedTuple): ("i", "f"), defaults=(1, 2.0), ) + + +MyDatetimeNewType = NewType("MyDatetimeNewType", datetime) diff --git a/tests/test_annotated.py b/tests/test_annotated.py new file mode 100644 index 00000000..2fffc100 --- /dev/null +++ b/tests/test_annotated.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from datetime import date + +from typing_extensions import Annotated + +from mashumaro import DataClassDictMixin + + +def test_annotated(): + @dataclass + class DataClass(DataClassDictMixin): + x: Annotated[date, None] + + obj = DataClass(date(2022, 2, 6)) + assert DataClass.from_dict({"x": "2022-02-06"}) == obj + assert obj.to_dict() == {"x": "2022-02-06"} diff --git a/tests/test_common.py b/tests/test_common.py index 48fb0539..1ac567a7 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -2,12 +2,10 @@ import pytest -from mashumaro import ( - DataClassDictMixin, - DataClassJSONMixin, - DataClassMessagePackMixin, - DataClassYAMLMixin, -) +from mashumaro.mixins.dict import DataClassDictMixin +from mashumaro.mixins.json import DataClassJSONMixin +from mashumaro.mixins.msgpack import DataClassMessagePackMixin +from mashumaro.mixins.yaml import DataClassYAMLMixin def test_slots(): @@ -50,3 +48,11 @@ class YAMLDataClass(DataClassYAMLMixin): str(e.value) == f"'{cls.__name__}' object has no attribute 'new_attribute'" ) + + +def test_data_class_dict_mixin_from_dict(): + assert DataClassDictMixin.from_dict({}) is None + + +def test_data_class_dict_mixin_to_dict(): + assert DataClassDictMixin().to_dict() is None diff --git a/tests/test_config.py b/tests/test_config.py index 3124ccdf..99341f69 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,10 +2,11 @@ from typing import Optional, Union import pytest +from typing_extensions import Literal from mashumaro import DataClassDictMixin from mashumaro.config import TO_DICT_ADD_OMIT_NONE_FLAG, BaseConfig -from mashumaro.meta.macros import PY_37_MIN +from mashumaro.core.const import PY_37_MIN from mashumaro.types import SerializationStrategy from .entities import ( @@ -29,6 +30,7 @@ class _(DataClassDictMixin): union: Union[int, str] typed_dict: TypedDictRequiredKeys named_tuple: MyNamedTupleWithDefaults + literal: Literal[1, 2, 3] class Config(BaseConfig): debug = True diff --git a/tests/test_data_types.py b/tests/test_data_types.py index cbb67557..7285d7fe 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -7,7 +7,6 @@ import uuid from dataclasses import InitVar, dataclass, field from datetime import date, datetime, time, timedelta, timezone -from enum import Enum from pathlib import ( Path, PosixPath, @@ -31,30 +30,25 @@ Mapping, MutableMapping, MutableSet, + NewType, Optional, Sequence, Set, Tuple, ) -from .conftest import fake_add_from_dict - -try: - from typing import OrderedDict # New in version 3.7.2 -except ImportError: - OrderedDict = Dict # type: ignore - import pytest +from typing_extensions import OrderedDict from mashumaro import DataClassDictMixin from mashumaro.config import BaseConfig +from mashumaro.core.const import PEP_585_COMPATIBLE, PY_37_MIN, PY_39_MIN from mashumaro.exceptions import ( InvalidFieldValue, MissingField, UnserializableDataError, UnserializableField, ) -from mashumaro.meta.macros import PEP_585_COMPATIBLE, PY_37_MIN from mashumaro.types import ( GenericSerializableType, RoundedDecimal, @@ -62,6 +56,7 @@ SerializationStrategy, ) +from .conftest import fake_add_from_dict from .entities import ( CustomPath, DataClassWithoutMixin, @@ -70,6 +65,7 @@ MutableString, MyDataClass, MyDataClassWithUnion, + MyDatetimeNewType, MyEnum, MyFlag, MyIntEnum, @@ -93,6 +89,8 @@ if PY_37_MIN: from tests.entities import MyUntypedNamedTupleWithDefaults +if PY_39_MIN: + from zoneinfo import ZoneInfo NoneType = type(None) @@ -130,8 +128,11 @@ class Fixture: DATA_CLASS_WITH_UNION = MyDataClassWithUnion(a=1, b=2) NONE = None DATETIME = datetime(2018, 10, 29, 12, 46, 55, 308495) + DATETIME_STR = "2018-10-29T12:46:55.308495" DATE = DATETIME.date() + DATE_STR = "2018-10-29" TIME = DATETIME.time() + TIME_STR = "12:46:55.308495" TIMEDELTA = timedelta(3.14159265358979323846) TIMEZONE = timezone(timedelta(hours=3)) UUID = uuid.UUID("3c25dd74-f208-46a2-9606-dd3919e975b7") @@ -194,21 +195,21 @@ class Fixture: (MutableMapping, Fixture.DICT, Fixture.DICT), (Sequence[int], Fixture.LIST, Fixture.LIST), (Sequence, Fixture.LIST, Fixture.LIST), - (bytes, Fixture.BYTES, Fixture.BYTES), - (bytearray, Fixture.BYTE_ARRAY, Fixture.BYTE_ARRAY), + (bytes, Fixture.BYTES, Fixture.BYTES_BASE64), + (bytearray, Fixture.BYTE_ARRAY, Fixture.BYTES_BASE64), (str, Fixture.STR, Fixture.STR), - (MyEnum, Fixture.ENUM, Fixture.ENUM), - (MyStrEnum, Fixture.STR_ENUM, Fixture.STR_ENUM), - (MyIntEnum, Fixture.INT_ENUM, Fixture.INT_ENUM), - (MyFlag, Fixture.FLAG, Fixture.FLAG), - (MyIntFlag, Fixture.INT_FLAG, Fixture.INT_FLAG), + (MyEnum, Fixture.ENUM, Fixture.ENUM.value), + (MyStrEnum, Fixture.STR_ENUM, Fixture.STR_ENUM.value), + (MyIntEnum, Fixture.INT_ENUM, Fixture.INT_ENUM.value), + (MyFlag, Fixture.FLAG, Fixture.FLAG.value), + (MyIntFlag, Fixture.INT_FLAG, Fixture.INT_FLAG.value), (MyDataClass, Fixture.DATA_CLASS, Fixture.DICT), (TMyDataClass, Fixture.T_DATA_CLASS, Fixture.DICT), (MyDataClassWithUnion, Fixture.DATA_CLASS_WITH_UNION, Fixture.DICT), (NoneType, Fixture.NONE, Fixture.NONE), - (datetime, Fixture.DATETIME, Fixture.DATETIME), - (date, Fixture.DATE, Fixture.DATE), - (time, Fixture.TIME, Fixture.TIME), + (datetime, Fixture.DATETIME, Fixture.DATETIME_STR), + (date, Fixture.DATE, Fixture.DATE_STR), + (time, Fixture.TIME, Fixture.TIME_STR), (timedelta, Fixture.TIMEDELTA, Fixture.TIMEDELTA.total_seconds()), (timezone, Fixture.TIMEZONE, "UTC+03:00"), (uuid.UUID, Fixture.UUID, Fixture.UUID_STR), @@ -232,6 +233,7 @@ class Fixture: Fixture.GENERIC_SERIALIZABLE_LIST_STR, ["_a", "_b", "_c"], ), + (MyDatetimeNewType, Fixture.DATETIME, Fixture.DATETIME_STR), ] if os.name == "posix": @@ -308,6 +310,9 @@ class Fixture: ] ) +if PY_39_MIN: + inner_values.append((ZoneInfo, ZoneInfo("Europe/Moscow"), "Europe/Moscow")) + hashable_inner_values = [ (type_, value, value_dumped) @@ -404,9 +409,7 @@ class Fixture: # noinspection PyCallingNonCallable -def check_collection_generic( - type_, value_info, use_bytes, use_enum, use_datetime, x_values_number=3 -): +def check_collection_generic(type_, value_info, x_values_number=3): x_type, x_value, x_value_dumped = value_info @dataclass @@ -416,47 +419,19 @@ class DataClass(DataClassDictMixin): x_factory = x_factory_mapping[type_] x = x_factory([x_value for _ in range(x_values_number)]) instance = DataClass(x) - if x_value_dumped is Fixture.BYTES: - v_dumped = Fixture.BYTES if use_bytes else Fixture.BYTES_BASE64 - elif x_value_dumped is Fixture.BYTE_ARRAY: - v_dumped = Fixture.BYTE_ARRAY if use_bytes else Fixture.BYTES_BASE64 - elif isinstance(x_value_dumped, Enum): - v_dumped = x_value_dumped if use_enum else x_value_dumped.value - elif isinstance(x_value_dumped, (datetime, date, time)): - v_dumped = ( - x_value_dumped if use_datetime else x_value_dumped.isoformat() - ) - else: - v_dumped = x_value_dumped - dumped = {"x": list(x_factory([v_dumped for _ in range(x_values_number)]))} - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) + dumped = { + "x": list(x_factory([x_value_dumped for _ in range(x_values_number)])) + } + instance_dumped = instance.to_dict() + instance_loaded = DataClass.from_dict(dumped) assert instance_dumped == dumped assert instance_loaded == instance - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) assert same_types(instance_dumped, dumped) assert same_types(instance_loaded.x, x) # noinspection PyCallingNonCallable -def check_mapping_generic( - type_, key_info, value_info, use_bytes, use_enum, use_datetime -): +def check_mapping_generic(type_, key_info, value_info): k_type, k_value, k_value_dumped = key_info v_type, v_value, v_value_dumped = value_info @@ -475,31 +450,8 @@ class DataClass(DataClassDictMixin): else: x = x_factory([(k_value, v_value) for _ in range(3)]) instance = DataClass(x) - if k_value_dumped is Fixture.BYTES: - k_dumped = Fixture.BYTES if use_bytes else Fixture.BYTES_BASE64 - # Fixture.BYTE_ARRAY is not hashable - # elif k_value_dumped is Fixture.BYTE_ARRAY: - # k_dumped = Fixture.BYTE_ARRAY if use_bytes else Fixture.BYTES_BASE64 - elif isinstance(k_value_dumped, Enum): - k_dumped = k_value_dumped if use_enum else k_value_dumped.value - elif isinstance(k_value_dumped, (datetime, date, time)): - k_dumped = ( - k_value_dumped if use_datetime else k_value_dumped.isoformat() - ) - else: - k_dumped = k_value_dumped - if v_value_dumped is Fixture.BYTES: - v_dumped = Fixture.BYTES if use_bytes else Fixture.BYTES_BASE64 - elif v_value_dumped is Fixture.BYTE_ARRAY: - v_dumped = Fixture.BYTE_ARRAY if use_bytes else Fixture.BYTES_BASE64 - elif isinstance(v_value_dumped, Enum): - v_dumped = v_value_dumped if use_enum else v_value_dumped.value - elif isinstance(v_value_dumped, (datetime, date, time)): - v_dumped = ( - v_value_dumped if use_datetime else v_value_dumped.isoformat() - ) - else: - v_dumped = v_value_dumped + k_dumped = k_value_dumped + v_dumped = v_value_dumped if type_ is ChainMap: dumped = { "x": x_factory([(k_dumped, v_dumped) for _ in range(3)]).maps @@ -508,35 +460,16 @@ class DataClass(DataClassDictMixin): dumped = {"x": x_factory([(k_dumped, 1) for _ in range(3)])} else: dumped = {"x": x_factory([(k_dumped, v_dumped) for _ in range(3)])} - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) + instance_dumped = instance.to_dict() + instance_loaded = DataClass.from_dict(dumped) assert instance_dumped == dumped assert instance_loaded == instance - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) assert same_types(instance_dumped, dumped) assert same_types(instance_loaded.x, x) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) -def test_one_level(value_info, use_bytes, use_enum, use_datetime): +def test_one_level(value_info): x_type, x_value, x_value_dumped = value_info @dataclass @@ -544,181 +477,79 @@ class DataClass(DataClassDictMixin): x: x_type instance = DataClass(x_value) - if x_value_dumped is Fixture.BYTES: - v_dumped = Fixture.BYTES if use_bytes else Fixture.BYTES_BASE64 - elif x_value_dumped is Fixture.BYTE_ARRAY: - v_dumped = Fixture.BYTE_ARRAY if use_bytes else Fixture.BYTES_BASE64 - elif isinstance(x_value_dumped, Enum): - v_dumped = x_value_dumped if use_enum else x_value_dumped.value - elif isinstance(x_value_dumped, (datetime, date, time)): - v_dumped = ( - x_value_dumped if use_datetime else x_value_dumped.isoformat() - ) - else: - v_dumped = x_value_dumped - dumped = {"x": v_dumped} - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) + dumped = {"x": x_value_dumped} + instance_dumped = instance.to_dict() + instance_loaded = DataClass.from_dict(dumped) assert instance_dumped == dumped assert instance_loaded == instance - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) assert same_types(instance_dumped, dumped) assert same_types(instance_loaded.x, x_value) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) -def test_with_generic_list(value_info, use_bytes, use_enum, use_datetime): - check_collection_generic( - List, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_list(value_info): + check_collection_generic(List, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) -def test_with_generic_deque(value_info, use_bytes, use_enum, use_datetime): - check_collection_generic( - Deque, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_deque(value_info): + check_collection_generic(Deque, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) -def test_with_generic_tuple(value_info, use_bytes, use_enum, use_datetime): - check_collection_generic( - Tuple, value_info, use_bytes, use_enum, use_datetime, 1 - ) +def test_with_generic_tuple(value_info): + check_collection_generic(Tuple, value_info, 1) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", hashable_inner_values) -def test_with_generic_set(value_info, use_bytes, use_enum, use_datetime): - check_collection_generic( - Set, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_set(value_info): + check_collection_generic(Set, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", hashable_inner_values) -def test_with_generic_frozenset(value_info, use_bytes, use_enum, use_datetime): - check_collection_generic( - FrozenSet, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_frozenset(value_info): + check_collection_generic(FrozenSet, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", hashable_inner_values) -def test_with_generic_mutable_set( - value_info, use_bytes, use_enum, use_datetime -): - check_collection_generic( - MutableSet, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_mutable_set(value_info): + check_collection_generic(MutableSet, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) @pytest.mark.parametrize("key_info", hashable_inner_values) -def test_with_generic_dict( - key_info, value_info, use_bytes, use_enum, use_datetime -): - check_mapping_generic( - Dict, key_info, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_dict(key_info, value_info): + check_mapping_generic(Dict, key_info, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) @pytest.mark.parametrize("key_info", hashable_inner_values) -def test_with_generic_mapping( - key_info, value_info, use_bytes, use_enum, use_datetime -): - check_mapping_generic( - Mapping, key_info, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_mapping(key_info, value_info): + check_mapping_generic(Mapping, key_info, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) @pytest.mark.parametrize("key_info", hashable_inner_values) -def test_with_generic_ordered_dict( - key_info, value_info, use_bytes, use_enum, use_datetime -): - check_mapping_generic( - OrderedDict, key_info, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_ordered_dict(key_info, value_info): + check_mapping_generic(OrderedDict, key_info, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) @pytest.mark.parametrize("key_info", hashable_inner_values) -def test_with_generic_counter( - key_info, value_info, use_bytes, use_enum, use_datetime -): - check_mapping_generic( - Counter, key_info, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_counter(key_info, value_info): + check_mapping_generic(Counter, key_info, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) @pytest.mark.parametrize("key_info", hashable_inner_values) -def test_with_generic_mutable_mapping( - key_info, value_info, use_bytes, use_enum, use_datetime -): - check_mapping_generic( - MutableMapping, key_info, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_mutable_mapping(key_info, value_info): + check_mapping_generic(MutableMapping, key_info, value_info) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) @pytest.mark.parametrize("key_info", hashable_inner_values) -def test_with_generic_chain_map( - key_info, value_info, use_bytes, use_enum, use_datetime -): - check_mapping_generic( - ChainMap, key_info, value_info, use_bytes, use_enum, use_datetime - ) +def test_with_generic_chain_map(key_info, value_info): + check_mapping_generic(ChainMap, key_info, value_info) @pytest.mark.parametrize("x_type", unsupported_field_types) @@ -849,11 +680,8 @@ class _(DataClassDictMixin): x: ChainMap[Key, int] -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) -def test_with_any(value_info, use_bytes, use_enum, use_datetime): +def test_with_any(value_info): @dataclass class DataClass(DataClassDictMixin): x: Any @@ -861,26 +689,16 @@ class DataClass(DataClassDictMixin): x = value_info[1] dumped = {"x": x} instance = DataClass(x) - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) + instance_dumped = instance.to_dict() + instance_loaded = DataClass.from_dict(dumped) assert instance_dumped == dumped assert instance_loaded == instance assert same_types(instance_dumped, dumped) assert same_types(instance_loaded.x, x) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize("value_info", inner_values) -def test_with_optional(value_info, use_bytes, use_enum, use_datetime): +def test_with_optional(value_info): x_type, x_value, x_value_dumped = value_info @dataclass @@ -890,41 +708,13 @@ class DataClass(DataClassDictMixin): for instance in [DataClass(x_value), DataClass()]: if instance.x is None: v_dumped = None - elif x_value_dumped is Fixture.BYTES: - v_dumped = Fixture.BYTES if use_bytes else Fixture.BYTES_BASE64 - elif x_value_dumped is Fixture.BYTE_ARRAY: - v_dumped = ( - Fixture.BYTE_ARRAY if use_bytes else Fixture.BYTES_BASE64 - ) - elif isinstance(x_value_dumped, Enum): - v_dumped = x_value_dumped if use_enum else x_value_dumped.value - elif isinstance(x_value_dumped, (datetime, date, time)): - v_dumped = ( - x_value_dumped if use_datetime else x_value_dumped.isoformat() - ) else: v_dumped = x_value_dumped dumped = {"x": v_dumped} - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) + instance_dumped = instance.to_dict() + instance_loaded = DataClass.from_dict(dumped) assert instance_dumped == dumped assert instance_loaded == instance - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) assert same_types(instance_dumped, dumped) assert same_types(instance_loaded.x, instance.x) @@ -1174,9 +964,6 @@ class Config(BaseConfig): DataClass.from_dict({"x": "bad_value"}) -@pytest.mark.parametrize("use_datetime", [True, False]) -@pytest.mark.parametrize("use_enum", [True, False]) -@pytest.mark.parametrize("use_bytes", [True, False]) @pytest.mark.parametrize( "value_info", [ @@ -1185,9 +972,7 @@ class Config(BaseConfig): if v[0] not in [MyDataClass, NoneType, MutableString] ], ) -def test_serialize_deserialize_options( - value_info, use_bytes, use_enum, use_datetime -): +def test_serialize_deserialize_options(value_info): x_type, x_value, x_value_dumped = value_info @dataclass @@ -1202,39 +987,12 @@ class DataClass(DataClassDictMixin): ) instance = DataClass(x_value) - if x_value_dumped is Fixture.BYTES: - v_dumped = Fixture.BYTES if use_bytes else Fixture.CUSTOM_SERIALIZE - elif x_value_dumped is Fixture.BYTE_ARRAY: - v_dumped = ( - Fixture.BYTE_ARRAY if use_bytes else Fixture.CUSTOM_SERIALIZE - ) - elif isinstance(x_value_dumped, Enum): - v_dumped = x_value_dumped if use_enum else Fixture.CUSTOM_SERIALIZE - elif isinstance(x_value_dumped, (datetime, date, time)): - v_dumped = x_value_dumped if use_datetime else Fixture.CUSTOM_SERIALIZE - else: - v_dumped = Fixture.CUSTOM_SERIALIZE + v_dumped = Fixture.CUSTOM_SERIALIZE dumped = {"x": v_dumped} - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) + instance_dumped = instance.to_dict() + instance_loaded = DataClass.from_dict(dumped) assert instance_dumped == dumped assert instance_loaded == instance - instance_dumped = instance.to_dict( - use_bytes=use_bytes, use_enum=use_enum, use_datetime=use_datetime - ) - instance_loaded = DataClass.from_dict( - dumped, - use_bytes=use_bytes, - use_enum=use_enum, - use_datetime=use_datetime, - ) assert same_types(instance_dumped, dumped) assert same_types(instance_loaded.x, x_value) @@ -1451,3 +1209,28 @@ class DataClass(DataClassDictMixin): obj = DataClass(x=None, y=None, z=[None]) assert DataClass.from_dict({"x": None, "y": None, "z": [None]}) == obj assert obj.to_dict() == {"x": None, "y": None, "z": [None]} + + +def test_data_class_with_new_type_overridden(): + MyStr = NewType("MyStr", str) + + @dataclass + class DataClass(DataClassDictMixin): + x: str + y: MyStr + + class Config(BaseConfig): + serialization_strategy = { + str: { + "serialize": lambda x: f"str_{x}", + "deserialize": lambda x: x[4:], + }, + MyStr: { + "serialize": lambda x: f"MyStr_{x}", + "deserialize": lambda x: x[6:], + }, + } + + instance = DataClass("a", MyStr("b")) + assert DataClass.from_dict({"x": "str_a", "y": "MyStr_b"}) == instance + assert instance.to_dict() == {"x": "str_a", "y": "MyStr_b"} diff --git a/tests/test_dialect.py b/tests/test_dialect.py index b9dc1d5a..e6b51bbe 100644 --- a/tests/test_dialect.py +++ b/tests/test_dialect.py @@ -1,8 +1,9 @@ from dataclasses import dataclass from datetime import date, datetime -from typing import Generic, List, TypeVar +from typing import Generic, List, NamedTuple, TypeVar, Union import pytest +from typing_extensions import TypedDict from mashumaro import DataClassDictMixin from mashumaro.config import ADD_DIALECT_SUPPORT, BaseConfig @@ -123,6 +124,56 @@ class Config(BaseConfig): dialect = FormattedDialect +class MyNamedTuple(NamedTuple): + x: DataClassWithDialectSupport = DataClassWithDialectSupport( + dt=date(2022, 1, 1), + i=999, + ) + y: DataClassWithoutDialects = DataClassWithoutDialects( + dt=date(2022, 1, 1), + i=999, + ) + + +class MyTypedDict(TypedDict): + x: DataClassWithDialectSupport + y: DataClassWithoutDialects + + +@dataclass +class DataClassWithNamedTupleWithDialectSupport(DataClassDictMixin): + x: MyNamedTuple + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + + +@dataclass +class DataClassWithNamedTupleWithoutDialectSupport(DataClassDictMixin): + x: MyNamedTuple + + +@dataclass +class DataClassWithTypedDictWithDialectSupport(DataClassDictMixin): + x: MyTypedDict + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + + +@dataclass +class DataClassWithTypedDictWithoutDialectSupport(DataClassDictMixin): + x: MyTypedDict + + +@dataclass +class DataClassWithUnionWithDialectSupport(DataClassDictMixin): + x: List[Union[DataClassWithDialectSupport, DataClassWithoutDialects]] + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + + def test_default_dialect(): dt = date.today() ordinal = dt.toordinal() @@ -698,3 +749,95 @@ class Config(BaseConfig): DataClass(date.today()).to_dict(dialect=FormattedDialect) mocked_print.assert_called() assert mocked_print.call_count == 6 + + +def test_dialect_with_named_tuple_with_dialect_support(): + dt = date.today() + ordinal = dt.toordinal() + iso = dt.isoformat() + obj = DataClassWithNamedTupleWithDialectSupport( + x=MyNamedTuple( + x=DataClassWithDialectSupport(dt, 255), + y=DataClassWithoutDialects(dt, 255), + ) + ) + dumped = {"x": [{"dt": ordinal, "i": "0xff"}, {"dt": iso, "i": 255}]} + assert obj.to_dict(dialect=OrdinalDialect) == dumped + assert ( + DataClassWithNamedTupleWithDialectSupport.from_dict( + dumped, dialect=OrdinalDialect + ) + == obj + ) + + +def test_dialect_with_named_tuple_without_dialect_support(): + dt = date.today() + iso = dt.isoformat() + obj = DataClassWithNamedTupleWithoutDialectSupport( + x=MyNamedTuple( + x=DataClassWithDialectSupport(dt, 255), + y=DataClassWithoutDialects(dt, 255), + ) + ) + dumped = {"x": [{"dt": iso, "i": 255}, {"dt": iso, "i": 255}]} + assert obj.to_dict() == dumped + assert ( + DataClassWithNamedTupleWithoutDialectSupport.from_dict(dumped) == obj + ) + + +def test_dialect_with_typed_dict_with_dialect_support(): + dt = date.today() + ordinal = dt.toordinal() + iso = dt.isoformat() + obj = DataClassWithTypedDictWithDialectSupport( + x=MyTypedDict( + x=DataClassWithDialectSupport(dt, 255), + y=DataClassWithoutDialects(dt, 255), + ) + ) + dumped = { + "x": {"x": {"dt": ordinal, "i": "0xff"}, "y": {"dt": iso, "i": 255}} + } + assert obj.to_dict(dialect=OrdinalDialect) == dumped + assert ( + DataClassWithTypedDictWithDialectSupport.from_dict( + dumped, dialect=OrdinalDialect + ) + == obj + ) + + +def test_dialect_with_typed_dict_without_dialect_support(): + dt = date.today() + iso = dt.isoformat() + obj = DataClassWithTypedDictWithoutDialectSupport( + x=MyTypedDict( + x=DataClassWithDialectSupport(dt, 255), + y=DataClassWithoutDialects(dt, 255), + ) + ) + dumped = {"x": {"x": {"dt": iso, "i": 255}, "y": {"dt": iso, "i": 255}}} + assert obj.to_dict() == dumped + assert DataClassWithTypedDictWithoutDialectSupport.from_dict(dumped) == obj + + +def test_dialect_with_union_with_dialect_support(): + dt = date.today() + ordinal = dt.toordinal() + iso = dt.isoformat() + obj = DataClassWithUnionWithDialectSupport( + x=[ + DataClassWithDialectSupport(dt, 255), + DataClassWithoutDialects(dt, 255), + ] + ) + dumped = {"x": [{"dt": ordinal, "i": "0xff"}, {"dt": iso, "i": 255}]} + assert obj.to_dict(dialect=OrdinalDialect) == dumped + assert ( + DataClassWithUnionWithDialectSupport.from_dict( + dumped, dialect=OrdinalDialect + ) + == obj + ) diff --git a/tests/test_forward_refs.py b/tests/test_forward_refs.py index 049f973a..9e88ecef 100644 --- a/tests/test_forward_refs.py +++ b/tests/test_forward_refs.py @@ -1,6 +1,6 @@ import pytest -from mashumaro.meta.macros import PY_37_MIN +from mashumaro.core.const import PY_37_MIN @pytest.mark.skipif(not PY_37_MIN, reason="requires python>=3.7") diff --git a/tests/test_generics.py b/tests/test_generics.py index f07240aa..92d9a0ac 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -3,7 +3,7 @@ from typing import Generic, List, Mapping, TypeVar from mashumaro import DataClassDictMixin -from tests.entities import MyGenericDataClass +from tests.entities import MyGenericDataClass, SerializableTypeGenericList T = TypeVar("T") S = TypeVar("S") @@ -188,3 +188,13 @@ class DataClass(DataClassDictMixin): dictionary = {"date": {"x": "2021-09-14"}, "str": {"x": "2021-09-14"}} assert DataClass.from_dict(dictionary) == obj assert obj.to_dict() == dictionary + + +def test_serializable_type_generic_class(): + @dataclass + class DataClass(DataClassDictMixin): + x: SerializableTypeGenericList[str] + + obj = DataClass(SerializableTypeGenericList(["a", "b", "c"])) + assert DataClass.from_dict({"x": ["a", "b", "c"]}) == obj + assert obj.to_dict() == {"x": ["a", "b", "c"]} diff --git a/tests/test_helper.py b/tests/test_helper.py index f99ca4fb..b1af2859 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -1,4 +1,10 @@ -from mashumaro import field_options +from dataclasses import dataclass, field +from datetime import date, datetime + +import pytest + +from mashumaro import DataClassDictMixin, field_options +from mashumaro.helper import pass_through from mashumaro.types import SerializationStrategy @@ -37,3 +43,30 @@ def serialize(self, value): "serialization_strategy": serialization_strategy, "alias": alias, } + + +def test_pass_through(): + with pytest.raises(NotImplementedError): + pass_through() + assert pass_through.serialize(123) == 123 + assert pass_through.deserialize(123) == 123 + + +def test_dataclass_with_pass_through(): + @dataclass + class DataClass(DataClassDictMixin): + x: datetime = field( + metadata=field_options( + serialize=pass_through, + deserialize=pass_through, + ) + ) + y: date = field( + metadata=field_options(serialization_strategy=pass_through) + ) + + x = datetime.utcnow() + y = x.date() + instance = DataClass(x, y) + assert instance.to_dict() == {"x": x, "y": y} + assert instance.from_dict({"x": x, "y": y}) == instance diff --git a/tests/test_json.py b/tests/test_json.py index f42d0399..7790d1bc 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -1,12 +1,12 @@ import json -from binascii import hexlify from dataclasses import dataclass from datetime import datetime from typing import List import pytest -from mashumaro import DataClassJSONMixin, MissingField +from mashumaro.exceptions import MissingField +from mashumaro.mixins.json import DataClassJSONMixin from .entities import MyEnum @@ -29,17 +29,6 @@ class DataClass(DataClassJSONMixin): assert DataClass.from_json(dumped) == DataClass([1, 2, 3]) -def test_to_json_with_encoder_params(): - @dataclass - class DataClass(DataClassJSONMixin): - x: List[int] - - instance = DataClass(x=[1, 2, 3]) - dumped = json.dumps({"x": [1, 2, 3]}, indent=2) - assert instance.to_json(indent=2) == dumped - assert instance.to_json() != dumped - - def test_to_json_with_custom_encoder(): @dataclass class DataClass(DataClassJSONMixin): @@ -57,20 +46,6 @@ def encoder(d): assert instance.to_json() != dumped -def test_from_json_with_decoder_params(): - @dataclass - class DataClass(DataClassJSONMixin): - x: List[float] - - def multiple_by_ten(s): - return int(s) * 10 - - instance = DataClass(x=[10, 20, 30]) - dumped = json.dumps({"x": [1, 2, 3]}) - assert DataClass.from_json(dumped, parse_int=multiple_by_ten) == instance - assert DataClass.from_json(dumped) != instance - - def test_from_json_with_custom_decoder(): @dataclass class DataClass(DataClassJSONMixin): @@ -89,116 +64,32 @@ def decoder(s): assert DataClass.from_json(dumped) -@pytest.mark.parametrize("use_enum", [True, False]) -def test_json_use_enum(use_enum): +def test_json_enum(): @dataclass class DataClass(DataClassJSONMixin): x: MyEnum - def encode_enum(o): - if isinstance(o, MyEnum): - return str(o.value).upper() - - def decode_enum(d): - dd = {} - for key, value in d.items(): - for enum in MyEnum.__members__.values(): - if value.lower() == enum.value.lower(): - value = enum - break - dd[key] = value - return dd + dumped = '{"x": "letter a"}' + instance = DataClass(MyEnum.a) - dumped_with_enum_normal = '{"x": "letter a"}' - dumped_with_enum_upper = '{"x": "LETTER A"}' + assert instance.to_json() == dumped + assert instance.to_json() == dumped + assert DataClass.from_json(dumped) == instance - instance = DataClass(MyEnum.a) - if not use_enum: - assert instance.to_json() == dumped_with_enum_normal - assert ( - instance.to_json(dict_params={"use_enum": False}) - == dumped_with_enum_normal - ) - assert DataClass.from_json(dumped_with_enum_normal) == instance - assert ( - DataClass.from_json( - data=dumped_with_enum_normal, dict_params={"use_enum": False} - ) - == instance - ) - else: - assert ( - instance.to_json( - default=encode_enum, dict_params={"use_enum": True} - ) - == dumped_with_enum_upper - ) - assert ( - DataClass.from_json( - data=dumped_with_enum_upper, - object_hook=decode_enum, - dict_params={"use_enum": True}, - ) - == instance - ) - - -@pytest.mark.parametrize("use_bytes", [True, False]) -def test_json_use_bytes(use_bytes): +def test_json_bytes(): @dataclass class DataClass(DataClassJSONMixin): x: bytes - def encode_bytes(o): - if isinstance(o, bytes): - return hexlify(o).decode() - - def decode_bytes(d): - dd = {} - for key, value in d.items(): - if value == "313233": - value = b"123" - dd[key] = value - return dd + dumped = r'{"x": "MTIz\n"}' + instance = DataClass(b"123") - dumped_with_bytes_normal = r'{"x": "MTIz\n"}' - dumped_with_bytes_hex = '{"x": "313233"}' + assert instance.to_json() == dumped + assert DataClass.from_json(dumped) == instance - instance = DataClass(b"123") - if not use_bytes: - assert instance.to_json() == dumped_with_bytes_normal - assert ( - instance.to_json(dict_params={"use_bytes": False}) - == dumped_with_bytes_normal - ) - assert DataClass.from_json(dumped_with_bytes_normal) == instance - assert ( - DataClass.from_json( - data=dumped_with_bytes_normal, dict_params={"use_bytes": False} - ) - == instance - ) - else: - assert ( - instance.to_json( - default=encode_bytes, dict_params={"use_bytes": True} - ) - == dumped_with_bytes_hex - ) - assert ( - DataClass.from_json( - data=dumped_with_bytes_hex, - object_hook=decode_bytes, - dict_params={"use_bytes": True}, - ) - == instance - ) - - -@pytest.mark.parametrize("use_datetime", [True, False]) -def test_json_use_datetime(use_datetime): +def test_json_datetime(): dt = datetime(2018, 10, 29, 12, 46, 55, 308495) @@ -206,48 +97,8 @@ def test_json_use_datetime(use_datetime): class DataClass(DataClassJSONMixin): x: datetime - def encode_datetime(o): - if isinstance(o, datetime): - return str(o.timestamp()) - - def decode_datetime(d): - dd = {} - for key, value in d.items(): - if value == str(dt.timestamp()): - value = dt - dd[key] = value - return dd - - dumped_with_dt_normal = json.dumps({"x": dt.isoformat()}) - dumped_with_dt_timestamp = json.dumps({"x": str(dt.timestamp())}) - + dumped = json.dumps({"x": dt.isoformat()}) instance = DataClass(x=dt) - if not use_datetime: - assert instance.to_json() == dumped_with_dt_normal - assert ( - instance.to_json(dict_params={"use_datetime": False}) - == dumped_with_dt_normal - ) - assert DataClass.from_json(dumped_with_dt_normal) == instance - assert ( - DataClass.from_json( - data=dumped_with_dt_normal, dict_params={"use_datetime": False} - ) - == instance - ) - else: - assert ( - instance.to_json( - default=encode_datetime, dict_params={"use_datetime": True} - ) - == dumped_with_dt_timestamp - ) - assert ( - DataClass.from_json( - data=dumped_with_dt_timestamp, - object_hook=decode_datetime, - dict_params={"use_datetime": True}, - ) - == instance - ) + assert instance.to_json() == dumped + assert DataClass.from_json(dumped) == instance diff --git a/tests/test_literal.py b/tests/test_literal.py new file mode 100644 index 00000000..e0783fec --- /dev/null +++ b/tests/test_literal.py @@ -0,0 +1,129 @@ +from dataclasses import dataclass + +import pytest +from typing_extensions import Literal + +from mashumaro import DataClassDictMixin +from mashumaro.config import ADD_DIALECT_SUPPORT, BaseConfig +from mashumaro.dialect import Dialect +from mashumaro.exceptions import InvalidFieldValue +from mashumaro.helper import pass_through +from tests.entities import MyEnum + + +def test_literal_with_str(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal["1", "2", "3"] + + assert DataClass.from_dict({"x": "1"}) == DataClass("1") + assert DataClass.from_dict({"x": "2"}) == DataClass("2") + + assert DataClass("1").to_dict() == {"x": "1"} + assert DataClass("2").to_dict() == {"x": "2"} + + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": 1}) + with pytest.raises(InvalidFieldValue): + DataClass(1).to_dict() + + +def test_literal_with_int(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal[1, 2] + + assert DataClass.from_dict({"x": 1}) == DataClass(1) + assert DataClass.from_dict({"x": 2}) == DataClass(2) + + assert DataClass(1).to_dict() == {"x": 1} + assert DataClass(2).to_dict() == {"x": 2} + + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": "1"}) + with pytest.raises(InvalidFieldValue): + DataClass("1").to_dict() + + +def test_literal_with_bool(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal[True, False] + + assert DataClass.from_dict({"x": True}) == DataClass(True) + assert DataClass.from_dict({"x": False}) == DataClass(False) + + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": "a"}) + with pytest.raises(InvalidFieldValue): + DataClass("a").to_dict() + + +def test_literal_with_none(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal[None] + + assert DataClass.from_dict({"x": None}) == DataClass(None) + assert DataClass(None).to_dict() == {"x": None} + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": "1"}) + with pytest.raises(InvalidFieldValue): + DataClass("1").to_dict() + + +def test_literal_with_bytes(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal[b"\x00"] + + assert DataClass.from_dict({"x": "AA==\n"}) == DataClass(b"\x00") + assert DataClass(b"\x00").to_dict() == {"x": "AA==\n"} + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": "\x00"}) + with pytest.raises(InvalidFieldValue): + DataClass("AA==\n").to_dict() + + +def test_literal_with_bytes_overridden(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal[b"\x00"] + + class Config(BaseConfig): + serialization_strategy = {bytes: pass_through} + + assert DataClass.from_dict({"x": b"\x00"}) == DataClass(b"\x00") + assert DataClass(b"\x00").to_dict() == {"x": b"\x00"} + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": "AA==\n"}) + + +def test_literal_with_enum(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal[MyEnum.a] + + assert DataClass.from_dict({"x": "letter a"}) == DataClass(MyEnum.a) + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": "letter b"}) + with pytest.raises(InvalidFieldValue): + DataClass(MyEnum.b).to_dict() + + +def test_literal_with_dialect(): + @dataclass + class DataClass(DataClassDictMixin): + x: Literal[b"\x00"] + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + + class MyDialect(Dialect): + serialization_strategy = {bytes: pass_through} + + instance = DataClass(b"\x00") + assert DataClass.from_dict({"x": b"\x00"}, dialect=MyDialect) == instance + assert instance.to_dict(dialect=MyDialect) == {"x": b"\x00"} + with pytest.raises(InvalidFieldValue): + DataClass.from_dict({"x": "AA==\n"}, dialect=MyDialect) diff --git a/tests/test_meta.py b/tests/test_meta.py index 871be720..bb93e01d 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -2,24 +2,37 @@ import collections.abc import typing from dataclasses import dataclass -from unittest.mock import patch +from datetime import datetime import pytest +import typing_extensions -from mashumaro import DataClassDictMixin, DataClassJSONMixin -from mashumaro.dialect import Dialect -from mashumaro.meta.helpers import ( +from mashumaro import DataClassDictMixin +from mashumaro.core.const import ( + PEP_585_COMPATIBLE, + PY_37, + PY_37_MIN, + PY_38, + PY_38_MIN, + PY_310_MIN, +) +from mashumaro.core.meta.builder import CodeBuilder +from mashumaro.core.meta.helpers import ( get_args, get_class_that_defines_field, get_class_that_defines_method, get_generic_name, + get_literal_values, get_type_origin, + is_annotated, is_class_var, is_dataclass_dict_mixin, is_dataclass_dict_mixin_subclass, is_dialect_subclass, is_generic, is_init_var, + is_literal, + is_new_type, is_optional, is_type_var_any, is_union, @@ -27,19 +40,19 @@ resolve_type_vars, type_name, ) -from mashumaro.meta.macros import ( - PEP_585_COMPATIBLE, - PY_37, - PY_37_MIN, - PY_38, - PY_310_MIN, -) -from mashumaro.serializer.base.metaprogramming import CodeBuilder +from mashumaro.dialect import Dialect +from mashumaro.mixins.json import DataClassJSONMixin from .entities import ( MyDataClass, + MyDatetimeNewType, + MyEnum, + MyFlag, MyGenericDataClass, MyGenericList, + MyIntEnum, + MyIntFlag, + MyStrEnum, T, TAny, TInt, @@ -51,45 +64,59 @@ TMyDataClass = typing.TypeVar("TMyDataClass", bound=MyDataClass) -def test_is_generic_unsupported_python(): - with patch("mashumaro.meta.helpers.PY_36", False): - with patch("mashumaro.meta.helpers.PY_37", False): - with patch("mashumaro.meta.helpers.PY_38", False): - with patch("mashumaro.meta.helpers.PY_39_MIN", False): - with pytest.raises(NotImplementedError): - is_generic(int) +def test_is_generic_unsupported_python(mocker): + mocker.patch("mashumaro.core.meta.helpers.PY_36", False) + mocker.patch("mashumaro.core.meta.helpers.PY_37", False) + mocker.patch("mashumaro.core.meta.helpers.PY_38", False) + mocker.patch("mashumaro.core.meta.helpers.PY_39_MIN", False) + with pytest.raises(NotImplementedError): + is_generic(int) + + +def test_is_class_var_unsupported_python(mocker): + mocker.patch("mashumaro.core.meta.helpers.PY_36", False) + mocker.patch("mashumaro.core.meta.helpers.PY_37_MIN", False) + with pytest.raises(NotImplementedError): + is_class_var(int) -def test_is_class_var_unsupported_python(): - with patch("mashumaro.meta.helpers.PY_36", False): - with patch("mashumaro.meta.helpers.PY_37_MIN", False): - with pytest.raises(NotImplementedError): - is_class_var(int) +def test_is_init_var_unsupported_python(mocker): + mocker.patch("mashumaro.core.meta.helpers.PY_36", False) + mocker.patch("mashumaro.core.meta.helpers.PY_37", False) + mocker.patch("mashumaro.core.meta.helpers.PY_38_MIN", False) + with pytest.raises(NotImplementedError): + is_init_var(int) -def test_is_init_var_unsupported_python(): - with patch("mashumaro.meta.helpers.PY_36", False): - with patch("mashumaro.meta.helpers.PY_37", False): - with patch("mashumaro.meta.helpers.PY_38_MIN", False): - with pytest.raises(NotImplementedError): - is_init_var(int) +def test_is_literal_unsupported_python(mocker): + mocker.patch("mashumaro.core.meta.helpers.PY_36", False) + mocker.patch("mashumaro.core.meta.helpers.PY_37", False) + mocker.patch("mashumaro.core.meta.helpers.PY_38", False) + mocker.patch("mashumaro.core.meta.helpers.PY_39_MIN", False) + assert not is_literal(typing_extensions.Literal[1]) -def test_no_code_builder(): - with patch( - "mashumaro.serializer.base.dict." - "DataClassDictMixin.__init_subclass__", +def test_get_literal_values_unsupported_python(mocker): + mocker.patch("mashumaro.core.meta.helpers.PY_36", False) + mocker.patch("mashumaro.core.meta.helpers.PY_37_MIN", False) + with pytest.raises(NotImplementedError): + get_literal_values(typing_extensions.Literal[1]) + + +def test_no_code_builder(mocker): + mocker.patch( + "mashumaro.mixins.dict.DataClassDictMixin.__init_subclass__", lambda: ..., - ): + ) - @dataclass - class DataClass(DataClassDictMixin): - pass + @dataclass + class DataClass(DataClassDictMixin): + pass - assert DataClass.__pre_deserialize__({}) is None - assert DataClass.__post_deserialize__(DataClass()) is None - assert DataClass().__pre_serialize__() is None - assert DataClass().__post_serialize__({}) is None + assert DataClass.__pre_deserialize__({}) is None + assert DataClass.__post_deserialize__(DataClass()) is None + assert DataClass().__pre_serialize__() is None + assert DataClass().__post_serialize__({}) is None def test_get_class_that_defines_method(): @@ -185,9 +212,14 @@ def test_type_name(): ) if PY_37_MIN: assert ( - type_name(typing.OrderedDict[int, int]) + type_name(typing_extensions.OrderedDict[int, int]) == "typing.OrderedDict[int, int]" ) + else: + assert ( + type_name(typing_extensions.OrderedDict[int, int]) + == "typing_extensions.OrderedDict[int, int]" + ) assert type_name(typing.Optional[int]) == "typing.Optional[int]" assert type_name(typing.Union[None, int]) == "typing.Optional[int]" assert type_name(typing.Union[int, None]) == "typing.Optional[int]" @@ -205,6 +237,16 @@ def test_type_name(): assert type_name(int | None) == "typing.Optional[int]" assert type_name(None | int) == "typing.Optional[int]" assert type_name(int | str) == "typing.Union[int, str]" + if PY_310_MIN: + assert ( + type_name(MyDatetimeNewType) == "tests.entities.MyDatetimeNewType" + ) + else: + assert type_name(MyDatetimeNewType) == type_name(datetime) + assert ( + type_name(typing_extensions.Annotated[TMyDataClass, None]) + == "tests.entities.MyDataClass" + ) @pytest.mark.skipif(not PEP_585_COMPATIBLE, reason="requires python 3.9+") @@ -272,11 +314,10 @@ def test_type_name_short(): type_name(typing.Union[int, typing.Any], short=True) == "Union[int, Any]" ) - if PY_37_MIN: - assert ( - type_name(typing.OrderedDict[int, int], short=True) - == "OrderedDict[int, int]" - ) + assert ( + type_name(typing_extensions.OrderedDict[int, int], short=True) + == "OrderedDict[int, int]" + ) assert type_name(typing.Optional[int], short=True) == "Optional[int]" assert type_name(typing.Union[None, int], short=True) == "Optional[int]" assert type_name(typing.Union[int, None], short=True) == "Optional[int]" @@ -294,6 +335,16 @@ def test_type_name_short(): assert type_name(int | None, short=True) == "Optional[int]" assert type_name(None | int, short=True) == "Optional[int]" assert type_name(int | str, short=True) == "Union[int, str]" + if PY_310_MIN: + assert type_name(MyDatetimeNewType, short=True) == "MyDatetimeNewType" + else: + assert type_name(MyDatetimeNewType, short=True) == type_name( + datetime, short=True + ) + assert ( + type_name(typing_extensions.Annotated[TMyDataClass, None], short=True) + == "MyDataClass" + ) @pytest.mark.skipif(not PEP_585_COMPATIBLE, reason="requires python 3.9+") @@ -336,6 +387,10 @@ def test_get_type_origin(): assert get_type_origin(typing.List) == list assert get_type_origin(MyGenericDataClass[int]) == MyGenericDataClass assert get_type_origin(MyGenericDataClass) == MyGenericDataClass + assert ( + get_type_origin(typing_extensions.Annotated[datetime, None]) + == datetime + ) def test_resolve_type_vars(): @@ -432,3 +487,57 @@ def test_not_non_type_arg(): assert not_none_type_arg((NoneType, int)) == int assert not_none_type_arg((str, NoneType)) == str assert not_none_type_arg((T, int), {T: NoneType}) == int + + +def test_is_new_type(): + assert is_new_type(typing.NewType("MyNewType", int)) + assert not is_new_type(int) + + +def test_is_annotated(): + assert is_annotated(typing_extensions.Annotated[datetime, None]) + assert not is_annotated(datetime) + + +def test_is_literal(): + assert is_literal(typing_extensions.Literal[1, 2, 3]) + assert not is_literal(typing_extensions.Literal) + assert not is_literal([1, 2, 3]) + + +def test_get_literal_values(): + assert get_literal_values(typing_extensions.Literal[1, 2, 3]) == (1, 2, 3) + assert get_literal_values( + typing_extensions.Literal[ + 1, typing_extensions.Literal[typing_extensions.Literal[2], 3] + ] + ) == (1, 2, 3) + + +def test_type_name_literal(): + if PY_38_MIN: + module_name = "typing" + else: + module_name = "typing_extensions" + assert type_name( + typing_extensions.Literal[ + 1, + "a", + b"\x00", + True, + False, + None, + MyEnum.a, + MyStrEnum.a, + MyIntEnum.a, + MyFlag.a, + MyIntFlag.a, + typing_extensions.Literal[2, 3], + typing_extensions.Literal[typing_extensions.Literal["b", "c"]], + ] + ) == ( + f"{module_name}.Literal[1, 'a', b'\\x00', True, False, None, " + "tests.entities.MyEnum.a, tests.entities.MyStrEnum.a, " + "tests.entities.MyIntEnum.a, tests.entities.MyFlag.a, " + "tests.entities.MyIntFlag.a, 2, 3, 'b', 'c']" + ) diff --git a/tests/test_metadata_options.py b/tests/test_metadata_options.py index 03c66813..9385e4f3 100644 --- a/tests/test_metadata_options.py +++ b/tests/test_metadata_options.py @@ -7,12 +7,12 @@ import pytest from mashumaro import DataClassDictMixin +from mashumaro.core.const import PY_37_MIN from mashumaro.exceptions import ( - UnserializableField, + InvalidFieldValue, UnsupportedDeserializationEngine, UnsupportedSerializationEngine, ) -from mashumaro.meta.macros import PY_37_MIN from mashumaro.types import SerializationStrategy from .entities import ( @@ -197,6 +197,19 @@ class B(A, DataClassDictMixin): assert instance == should_be +def test_derived_dataclass_metadata_deserialize_option_removed(): + @dataclass + class A: + x: datetime = field(metadata={"deserialize": ciso8601.parse_datetime}) + + @dataclass + class B(A, DataClassDictMixin): + x: datetime + + with pytest.raises(InvalidFieldValue): + B.from_dict({"x": "2021-01-02T03:04:05Z", "y": "2021-01-02T03:04:05Z"}) + + def test_bytearray_overridden(): @dataclass class DataClass(DataClassDictMixin): diff --git a/tests/test_msgpack.py b/tests/test_msgpack.py index 7a6b21d9..1ff4cc10 100644 --- a/tests/test_msgpack.py +++ b/tests/test_msgpack.py @@ -4,7 +4,7 @@ import msgpack -from mashumaro import DataClassMessagePackMixin +from mashumaro.mixins.msgpack import DataClassMessagePackMixin def test_to_msgpack(): @@ -33,3 +33,15 @@ class DataClass(DataClassMessagePackMixin): dt = datetime(2018, 10, 29, 12, 46, 55, 308495) dumped = msgpack.packb({"x": dt.isoformat()}) assert DataClass(dt).to_msgpack() == dumped + + +def test_msgpack_with_bytes(): + @dataclass + class DataClass(DataClassMessagePackMixin): + x: bytes + y: bytearray + + instance = DataClass(b"123", bytearray(b"456")) + dumped = msgpack.packb({"x": b"123", "y": bytearray(b"456")}) + assert DataClass.from_msgpack(dumped) == instance + assert instance.to_msgpack() == dumped diff --git a/tests/test_pep_563.py b/tests/test_pep_563.py index 55624ec5..0026d766 100644 --- a/tests/test_pep_563.py +++ b/tests/test_pep_563.py @@ -5,7 +5,8 @@ import pytest from mashumaro import DataClassDictMixin -from mashumaro.config import BaseConfig +from mashumaro.config import ADD_DIALECT_SUPPORT, BaseConfig +from mashumaro.dialect import Dialect from mashumaro.exceptions import UnresolvedTypeReferenceError from .conftest import fake_add_from_dict @@ -21,6 +22,35 @@ class B(DataClassDictMixin): x: int +@dataclass +class Base(DataClassDictMixin): + pass + + +@dataclass +class A1(Base): + a: B1 + + +@dataclass +class A2(Base): + a: B2 + + +@dataclass +class A3(Base): + a: B1 + x: int + + class Config(BaseConfig): + code_generation_options = [ADD_DIALECT_SUPPORT] + + +@dataclass +class B1(Base): + b: int + + def test_postponed_annotation_evaluation(): obj = A(x=B(x=1)) assert obj.to_dict() == {"x": {"x": 1}} @@ -58,3 +88,32 @@ class DataClass(DataClassDictMixin): class Config(BaseConfig): allow_postponed_evaluation = False + + +def test_postponed_annotation_evaluation_with_parent(): + obj = A1(B1(1)) + assert A1.from_dict({"a": {"b": 1}}) == obj + assert obj.to_dict() == {"a": {"b": 1}} + + +def test_postponed_annotation_evaluation_with_parent_and_no_reference(): + with pytest.raises(UnresolvedTypeReferenceError): + A2.from_dict({"a": {"b": 1}}) + with pytest.raises(UnresolvedTypeReferenceError): + A2(None).to_dict() + + +def test_postponed_annotation_evaluation_with_parent_and_dialect(): + class MyDialect(Dialect): + serialization_strategy = { + int: { + "serialize": lambda i: str(int(i * 1000)), + "deserialize": lambda s: int(int(s) / 1000), + } + } + + obj = A3(B1(1), 2) + assert A3.from_dict({"a": {"b": 1}, "x": 2}) == obj + assert A3.from_dict({"a": {"b": 1}, "x": "2000"}, dialect=MyDialect) == obj + assert obj.to_dict() == {"a": {"b": 1}, "x": 2} + assert obj.to_dict(dialect=MyDialect) == {"a": {"b": 1}, "x": "2000"} diff --git a/tests/test_slots.py b/tests/test_slots.py new file mode 100644 index 00000000..a7728901 --- /dev/null +++ b/tests/test_slots.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass, field + +import pytest + +from mashumaro import DataClassDictMixin, field_options +from mashumaro.core.const import PY_310_MIN + +if not PY_310_MIN: + pytest.skip("requires python>=3.10", allow_module_level=True) + + +def test_field_options_in_dataclass_with_slots(): + @dataclass(slots=True) + class DataClass(DataClassDictMixin): + x: int = field(metadata=field_options(serialize=str, alias="alias")) + + instance = DataClass(123) + assert DataClass.from_dict({"alias": 123}) == instance + assert instance.to_dict() == {"x": "123"} + + +def test_field_options_in_inherited_dataclass_with_slots(): + @dataclass + class BaseDataClass(DataClassDictMixin): + y: int + + @dataclass(slots=True) + class DataClass(BaseDataClass): + x: int = field(metadata=field_options(serialize=str, alias="alias")) + + instance = DataClass(x=123, y=456) + assert DataClass.from_dict({"alias": 123, "y": 456}) == instance + assert instance.to_dict() == {"x": "123", "y": 456} + + +def test_no_field_options_in_inherited_dataclass_with_slots(): + @dataclass + class BaseDataClass(DataClassDictMixin): + y: int + + @dataclass(slots=True) + class DataClass(BaseDataClass): + x: int + + instance = DataClass(x=123, y=456) + assert DataClass.from_dict({"x": 123, "y": 456}) == instance + assert instance.to_dict() == {"x": 123, "y": 456} + + +def test_no_field_options_in_inherited_dataclass_with_slots_and_default(): + @dataclass + class BaseDataClass(DataClassDictMixin): + y: int + + @dataclass(slots=True) + class DataClass(BaseDataClass): + x: int = 123 + + instance = DataClass(y=456) + assert DataClass.from_dict({"y": 456}) == instance + assert instance.to_dict() == {"x": 123, "y": 456} diff --git a/tests/test_yaml.py b/tests/test_yaml.py index a4cd8651..70b777c4 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -3,7 +3,7 @@ import yaml -from mashumaro import DataClassYAMLMixin +from mashumaro.mixins.yaml import DataClassYAMLMixin def test_to_yaml():