From f69ec19b5aafca92c110cad20d6379b92bb242da Mon Sep 17 00:00:00 2001 From: Chris Mostert <15890652+chrismostert@users.noreply.github.com> Date: Mon, 11 Mar 2024 10:29:36 +0100 Subject: [PATCH 1/4] Implement custom XmlDateTime converter which keeps full precision of fractional seconds --- pyeml_bindings/__init__.py | 82 ++++++++++++++-------- pyeml_bindings/converters/xml_date_time.py | 51 ++++++++++++++ 2 files changed, 103 insertions(+), 30 deletions(-) create mode 100644 pyeml_bindings/converters/xml_date_time.py diff --git a/pyeml_bindings/__init__.py b/pyeml_bindings/__init__.py index 64e17ee..d3b9579 100644 --- a/pyeml_bindings/__init__.py +++ b/pyeml_bindings/__init__.py @@ -3,6 +3,8 @@ Generator: DataclassGenerator See: https://xsdata.readthedocs.io/ """ + +from pyeml_bindings.converters.xml_date_time import FullPrecisionXmlDateTimeConverter from pyeml_bindings.emlcore_kiesraad_strict import ( Accepted, Affiliation, @@ -23,8 +25,8 @@ BallotIdentifierRange, BallotIdentifierRangeStructure, BallotIdentifierStructure, - BinaryItemStructure, BinaryFormat, + BinaryItemStructure, Candidate, CandidateIdentifier, CandidateIdentifierStructure, @@ -36,19 +38,19 @@ ContactDetailsStructure, ContestIdentifier, ContestIdentifierStructure, + CountingAlgorithm, CountMetric, CountMetricStructure, CountQualifier, CountQualifierStructure, - CountingAlgorithm, DocumentIdentifier, DocumentIdentifierStructure, - Emlstructure, ElectionGroupStructure, ElectionIdentifier, ElectionIdentifierStructure, ElectionStatement, EmailStructure, + Emlstructure, Endorsement, EventIdentifier, EventIdentifierStructure, @@ -63,8 +65,8 @@ ManagingAuthority, ManagingAuthorityStructure, MaxVotes, - MessageType, MessagesStructure, + MessageType, MinVotes, NominatingOfficer, NominatingOfficerStructure, @@ -74,7 +76,6 @@ Period, PeriodStructure, PeriodStructurePermanent, - PersonName as EmlcorePersonName, PollingDistrict, PollingDistrictStructure, PollingPlace, @@ -113,10 +114,6 @@ SupporterStructure, TelephoneStructure, TransactionId, - Vtoken, - VtokenQualified, - VtokenQualifiedStructure, - VtokenStructure, VoterIdentificationStructure, VoterInformationStructure, VoterInformationStructureGender, @@ -125,10 +122,17 @@ VotingChannelType, VotingMethod, VotingMethodType, + Vtoken, + VtokenQualified, + VtokenQualifiedStructure, + VtokenStructure, WriteIn, WriteInType, YesNoType, ) +from pyeml_bindings.emlcore_kiesraad_strict import ( + PersonName as EmlcorePersonName, +) from pyeml_bindings.emlexternals_kiesraad_strict import ( AuthorityAddressStructure, ElectoralAddressStructure, @@ -182,8 +186,8 @@ CandidateStructureKr, ContactDetailsStructureKr, ContestIdentifierStructureKr, - EmlstructureKr, ElectionIdentifierStructureKr, + EmlstructureKr, GenericMailingAddressStructureKr, GenericQualifyingAddressStructureKr, MailingAddressStructureKr, @@ -194,56 +198,66 @@ ) from pyeml_bindings.mod_110a_electionevent_kiesraad_strict import ( ContestIdentifierStructure110A, - Emlstructure110, - Eml as Eml110a, ElectionEvent, ElectionIdentifierStructure110A, + Emlstructure110, PollingPlaceStructure110, PollingPlaceStructure110Channel, ) +from pyeml_bindings.mod_110a_electionevent_kiesraad_strict import ( + Eml as Eml110a, +) from pyeml_bindings.mod_210_nomination_kiesraad_strict import ( AffiliationIdentifierStructure210, AffiliationStructure210, CandidateIdentifierStructure210, CandidateStructure210, ContestIdentifierStructure210, - Emlstructure210, - Eml as Eml210, ElectionIdentifierStructure210, + Emlstructure210, Nomination, ProposerStructureKr, ProposerStructureRestricted, ProposerStructureRestrictedJobTitle, ) +from pyeml_bindings.mod_210_nomination_kiesraad_strict import ( + Eml as Eml210, +) from pyeml_bindings.mod_230_candidatelist_kiesraad_strict import ( CandidateList, + ElectionIdentifierStructure230, Emlstructure230, - Eml as Eml230, Emlstructure230Id, - ElectionIdentifierStructure230, +) +from pyeml_bindings.mod_230_candidatelist_kiesraad_strict import ( + Eml as Eml230, ) from pyeml_bindings.mod_510_count_kiesraad_strict import ( AffiliationIdentifierStructure510, CandidateIdentifierStructure510, CandidateStructure510, Count, - Emlstructure510, - Eml as Eml510, ElectionIdentifierStructure510, + Emlstructure510, RejectedVotesReasonCode, ReportingUnitVotes, UncountedVotesReasonCode, ) +from pyeml_bindings.mod_510_count_kiesraad_strict import ( + Eml as Eml510, +) from pyeml_bindings.mod_520_result_kiesraad_strict import ( AffiliationIdentifierStructure520, CandidateIdentifierStructure520, CandidateStructure520, - Emlstructure520, - Eml as Eml520, ElectionIdentifierStructure520, + Emlstructure520, Result, SelectionRanking, ) +from pyeml_bindings.mod_520_result_kiesraad_strict import ( + Eml as Eml520, +) from pyeml_bindings.x_al_kiesraad_strict import ( Address, AddressDetails, @@ -271,41 +285,41 @@ MailStopType, MinimalCountryType, MinimalLocalityType, + PostalCode, + PostalRouteType, PostBox, PostOffice, PostOfficeNumberIndicatorOccurrence, - PostalCode, - PostalRouteType, Premise, PremiseNameTypeOccurrence, PremiseNumber, + PremiseNumberIndicatorOccurrence, + PremiseNumberNumberType, + PremiseNumberNumberTypeOccurrence, PremiseNumberPrefix, PremiseNumberRangeIndicatorOccurence, PremiseNumberRangeNumberRangeOccurence, PremiseNumberSuffix, - PremiseNumberIndicatorOccurrence, - PremiseNumberNumberType, - PremiseNumberNumberTypeOccurrence, SubPremiseNameTypeOccurrence, SubPremiseNumberIndicatorOccurrence, SubPremiseNumberNumberTypeOccurrence, SubPremiseType, Thoroughfare, + ThoroughfareDependentThoroughfares, ThoroughfareLeadingTypeType, ThoroughfareNameType, ThoroughfareNumber, + ThoroughfareNumberIndicatorOccurrence, + ThoroughfareNumberNumberOccurrence, + ThoroughfareNumberNumberType, ThoroughfareNumberPrefix, ThoroughfareNumberRangeIndicatorOccurrence, ThoroughfareNumberRangeNumberRangeOccurrence, ThoroughfareNumberRangeRangeType, ThoroughfareNumberSuffix, - ThoroughfareNumberIndicatorOccurrence, - ThoroughfareNumberNumberOccurrence, - ThoroughfareNumberNumberType, ThoroughfarePostDirectionType, ThoroughfarePreDirectionType, ThoroughfareTrailingTypeType, - ThoroughfareDependentThoroughfares, XAl, ) from pyeml_bindings.x_nl_kiesraad_strict import ( @@ -316,9 +330,11 @@ NameLineType, OrganisationNameDetails, OrganisationNameDetails1, - PersonName as XNlPersonName, XNl, ) +from pyeml_bindings.x_nl_kiesraad_strict import ( + PersonName as XNlPersonName, +) __all__ = [ "Accepted", @@ -616,3 +632,9 @@ "PollingPlaceStructure110", "PollingPlaceStructure110Channel", ] + +# Register custom converters for serialization +from xsdata.formats.converter import converter +from xsdata.models.datatype import XmlDateTime + +converter.register_converter(XmlDateTime, FullPrecisionXmlDateTimeConverter()) diff --git a/pyeml_bindings/converters/xml_date_time.py b/pyeml_bindings/converters/xml_date_time.py new file mode 100644 index 0000000..7f21898 --- /dev/null +++ b/pyeml_bindings/converters/xml_date_time.py @@ -0,0 +1,51 @@ +from typing import Any, Optional + +from xsdata.formats.converter import Converter +from xsdata.models.datatype import XmlDateTime +from xsdata.utils.dates import format_date, format_offset + + +def custom_format_time( + hour: int, minute: int, second: int, fractional_second: int +) -> str: + """Serializes a time according to ISO 8601. + + Args: + hour (int): The hour to serialize + minute (int): The minute to serialize + second (int): The second to serialize + fractional_second (int): The fractional second to serialize. Can be either nano- micro- or milliseconds. + + Returns: + str: The time formatted according to ISO 8601 (example: 14:14:33.000) + """ + microsecond, nano = divmod(fractional_second, 1000) + if nano: + return f"{hour:02d}:{minute:02d}:{second:02d}.{fractional_second:09d}" + + milli, micro = divmod(microsecond, 1000) + if micro: + return f"{hour:02d}:{minute:02d}:{second:02d}.{microsecond:06d}" + + return f"{hour:02d}:{minute:02d}:{second:02d}.{milli:03d}" + + +class FullPrecisionXmlDateTimeConverter(Converter): + """Override the default XmlDateTimeConverter to preserve full precision when serializing datetimes + + Args: + Converter (xsdata.formats.converter.Converter): Abstract converter class + """ + + def deserialize(self, value: Any, **kwargs: Any) -> Any: + return XmlDateTime.from_string(value) + + def serialize(self, value: Any, **kwargs: Any) -> Optional[str]: + if isinstance(value, XmlDateTime): + return "{}T{}{}".format( + format_date(value.year, value.month, value.day), + custom_format_time( + value.hour, value.minute, value.second, value.fractional_second + ), + format_offset(value.offset), + ) From 934e324a8b95e31c7e61ddab5662ecaa798a5b26 Mon Sep 17 00:00:00 2001 From: Chris Mostert <15890652+chrismostert@users.noreply.github.com> Date: Mon, 11 Mar 2024 15:21:16 +0100 Subject: [PATCH 2/4] Add github action for automated testing --- .github/workflows/test.yml | 38 ++ README.md | 5 + pyeml_bindings_testreport.html | 1091 -------------------------------- pyproject.toml | 4 +- testfiles.txt | 3 + tests/test_roundtrip.py | 22 +- 6 files changed, 62 insertions(+), 1101 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 pyeml_bindings_testreport.html create mode 100644 testfiles.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7286b20 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,38 @@ +name: Build and test + +on: + push: + branches: + - main + + pull_request: + branches: + - main + +jobs: + build_and_test: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Build and install package + run: | + python -m pip install .[tests] + + - name: Download testdata + run: | + wget -i testfiles.txt -P data + + - name: Unzip all testdata + run: | + unzip -d data -o 'data/*.zip' + + - name: Run tests + run: | + pytest \ No newline at end of file diff --git a/README.md b/README.md index d577a8e..9e02434 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,10 @@ To build the package yourself instead of installing the `.whl`, simply clone the python -m build ``` +or you can simply install the package by running +``` +python -m pip install . +``` + ## Codegen These bindings are mostly generated using [xsData](https://xsdata.readthedocs.io) with some minor changes where needed. See commit history for these changes. \ No newline at end of file diff --git a/pyeml_bindings_testreport.html b/pyeml_bindings_testreport.html deleted file mode 100644 index e6c57dc..0000000 --- a/pyeml_bindings_testreport.html +++ /dev/null @@ -1,1091 +0,0 @@ - - - - - pyeml_bindings_testreport.html - - - - -

pyeml_bindings_testreport.html

-

Report generated on 02-Nov-2023 at 15:47:08 by pytest-html - v4.0.2

-
-

Environment

-
-
- - - - - -
-
-

Summary

-
-
-

3084 tests took 01:31:21.

-

(Un)check the boxes to filter the results.

-
- -
-
-
-
- - 4 Failed, - - 3080 Passed, - - 0 Skipped, - - 0 Expected failures, - - 0 Unexpected passes, - - 0 Errors, - - 0 Reruns -
-
-  /  -
-
-
-
-
-
-
-
- - - - - - - - - -
ResultTestDurationLinks
- - - \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index afd0e07..7422ba8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,12 @@ build-backend = "hatchling.build" [project] name = "pyeml_bindings" description = "Data bindings for the EML_NL Standard" -version = "1.0.0" +version = "1.0.1" authors = [{name = "Chris Mostert", email = "chris.mostert@kiesraad.nl"}] readme = "README.md" requires-python = ">=3.10" dependencies = [ - "xsdata == 23.*" + "xsdata == 24.*" ] license = {file = "LICENSE"} classifiers = [ diff --git a/testfiles.txt b/testfiles.txt new file mode 100644 index 0000000..e6f6fdc --- /dev/null +++ b/testfiles.txt @@ -0,0 +1,3 @@ +https://data.overheid.nl/sites/default/files/dataset/e3fe6e42-06ab-4559-a466-a32b04247f68/resources/Verkiezingsuitslag%20Tweede%20Kamer%202023%20%28Deel%201%29.zip +https://data.overheid.nl/sites/default/files/dataset/e3fe6e42-06ab-4559-a466-a32b04247f68/resources/Verkiezingsuitslag%20Tweede%20Kamer%202023%20%28Deel%202%29.zip +https://data.overheid.nl/sites/default/files/dataset/e3fe6e42-06ab-4559-a466-a32b04247f68/resources/Verkiezingsuitslag%20Tweede%20Kamer%202023%20%28Deel%203%29.zip \ No newline at end of file diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index 3d13fde..46b7618 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -1,16 +1,17 @@ +import glob +import itertools +import re +import xml.etree.ElementTree as ET from pathlib import Path + +import pytest +from formencode.doctest_xml_compare import xml_compare from xsdata.formats.dataclass.parsers import XmlParser from xsdata.formats.dataclass.parsers.config import ParserConfig from xsdata.formats.dataclass.serializers import XmlSerializer + from pyeml_bindings import Eml110a, Eml230, Eml510, Eml520 from pyeml_bindings.namespace import NAMESPACE -import xml.etree.ElementTree as ET -from formencode.doctest_xml_compare import xml_compare -from sys import stdout -import pytest -import itertools -import glob -import re def parsing_roundtrip_same(parser, serializer, reporter, path_to_eml, type) -> bool: @@ -28,7 +29,7 @@ def parsing_roundtrip_same(parser, serializer, reporter, path_to_eml, type) -> b parser = XmlParser(ParserConfig(fail_on_unknown_properties=False)) serializer = XmlSerializer() reporter = print -files = glob.glob(f"data/**/*.eml.xml", recursive=True) +files = glob.glob("data/**/*.eml.xml", recursive=True) test_cases = zip( itertools.repeat(parser), @@ -49,6 +50,11 @@ def parsing_roundtrip_same(parser, serializer, reporter, path_to_eml, type) -> b ) def test_roundtrip(parser, serializer, reporter, file): name = Path(file).name + + # Skip known invalid EML file from TK2023 + if name == "Telling_TK2023_NBSB.eml.xml": + pytest.skip(f"Skipping known invalid EML file {name}") + if p_110a.match(name): assert parsing_roundtrip_same(parser, serializer, reporter, file, Eml110a) elif p_230.match(name): From 3217c748636ffeab9080a155b30f925f23a313a6 Mon Sep 17 00:00:00 2001 From: Chris Mostert <15890652+chrismostert@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:53:37 +0100 Subject: [PATCH 3/4] Speedup tests using xdist --- .github/workflows/test.yml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7286b20..746f5db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,4 +35,4 @@ jobs: - name: Run tests run: | - pytest \ No newline at end of file + pytest -n auto \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7422ba8..494b952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ [project.optional-dependencies] tests = [ "pytest", + "pytest-xdist", "formencode" ] From 7c30b36678b2a2dbac66654aa8b3687e1c685b0f Mon Sep 17 00:00:00 2001 From: Chris Mostert <15890652+chrismostert@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:11:16 +0100 Subject: [PATCH 4/4] Update setup-python version --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 746f5db..faab28e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10"