From cadd453ea9d152d8bfd8a87186c37b8c8503812b Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 12 Dec 2021 13:18:10 +0100 Subject: [PATCH 01/10] Add support for viainvest This adds support for parsing viainvest account statements. Withdrawals are currently not supported, only deposits and interest payments. --- README.md | 11 ++++-- config/viainvest.yml | 14 ++++++++ parse-account-statements.py | 1 + src/Statement.py | 17 +++++++-- src/test/test_p2p_account_statement_parser.py | 27 ++++++++++++++ src/test/testdata/viainvest.csv | 7 ++++ src/test_statement.py | 35 +++++++++++++++++++ 7 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 config/viainvest.yml create mode 100644 src/test/testdata/viainvest.csv create mode 100644 src/test_statement.py diff --git a/README.md b/README.md index a89219dc..868f1216 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ List of currently supported providers: - Robocash - Swaper - Debitum Network + - Viainvest Control the way how account statements are processed via the aggregate parameter: - transaction: Currently does not process the input data beyond making it Portfolio Performance compatible. @@ -80,6 +81,7 @@ parse-account-statements.py --type mintos src/test/testdata/mintos.csv * bondora - Supports current account statement format (as of 2019-10-12); exported to csv * bondora go & grow - Supports current account statement format (as of 2019-10-12); exported to csv * debitumnetwork - Supports current account statement format (as of 2020-09-08) exported to csv +* viainvest - Supports current account statement (as of 2021-12-12) exported as csv (Withdrawals do not work yet) ### Alternative solution for Auxmoney @@ -96,9 +98,12 @@ Example: ``` --- type_regex: !!map - deposit: "^Incoming client.*" - withdraw: "^Withdraw application.*" - interest: "(^Delayed interest.*)|(^Late payment.*)|(^Interest income.*)|(^Cashback.*)" + deposit: "(Deposits)|(^Incoming client.*)|(^Incoming currency exchange.*)|(^Affiliate partner bonus$)" + withdraw: "(^Withdraw application.*)|(Outgoing currency.*)|(Withdrawal)" + interest: "(^Delayed interest.*)|(^Late payment.*)|(^Interest income.*)|(^Cashback.*)|(^.*[Ii]nterest received.*)|(^.*late fees received$)" + fee: "(^FX commission.*)|(.*secondary market fee$)" + ignorable_entry: ".*investment in loan.*|.*[Pp]rincipal received.*|.*secondary market transaction.*" + special_entry: "(.*discount/premium.*)" csv_fieldnames: booking_date: 'Date' diff --git a/config/viainvest.yml b/config/viainvest.yml new file mode 100644 index 00000000..92a637e0 --- /dev/null +++ b/config/viainvest.yml @@ -0,0 +1,14 @@ +--- +type_regex: !!map + deposit: "(Amount of funds deposited)" + withdraw: "" + interest: "(Amount of interest payment received)" + ignorable_entry: "(Amount invested in loan)|(Amount of principal repayment received)" + +csv_fieldnames: + booking_date: 'Value date' + booking_date_format: '%m/%d/%Y' + booking_details: 'Loan ID' + booking_id: 'Loan ID' + booking_type: 'Transaction type' + booking_value: 'Credit (€)' diff --git a/parse-account-statements.py b/parse-account-statements.py index 57b3d370..43b7269b 100755 --- a/parse-account-statements.py +++ b/parse-account-statements.py @@ -14,6 +14,7 @@ - Robocash - Swaper - Debitum Network + - Viainvest Control the way how account statements are processed via the aggregate parameter: - transaction: Currently does not process the input data beyond making it Portfolio Performance compatible. diff --git a/src/Statement.py b/src/Statement.py index 9810c2a9..bd0d8e0a 100644 --- a/src/Statement.py +++ b/src/Statement.py @@ -27,7 +27,6 @@ def get_category(self): :return: category of the statement; if ignored on purpose return 'Ignored', if unknown return the empty string """ booking_type = self._statement[self._config.get_booking_type()] - category = "" value = self.get_value() regex_to_category_mappings = [ @@ -39,6 +38,7 @@ def get_category(self): {"regex": self._config.get_ignorable_entry_regex(), "category": "Ignored"}, ] + category = "" for mapping in regex_to_category_mappings: category = self.__match_category(mapping, booking_type, value) if category: @@ -69,7 +69,8 @@ def get_value(self): :return: value of the current statement as float. """ - return float(self._statement[self._config.get_booking_value()].replace(",", ".")) + raw_value = self._statement[self._config.get_booking_value()] + return Statement._parse_value(raw_value) def get_note(self): """ @@ -92,6 +93,18 @@ def get_currency(self): return self._statement[self._config.get_booking_currency()] else: return "EUR" + + @staticmethod + def _parse_value(value): + if value: + # Check order of . and , to replace in right order + if "." in value and "," in value: + if value.find(".") < value.find(","): + value = value.replace(".", "") + else: + value = value.replace(",", "") + return float(value.replace(",", ".")) + return None @staticmethod def __match_category(mapping, booking_type, value): diff --git a/src/test/test_p2p_account_statement_parser.py b/src/test/test_p2p_account_statement_parser.py index 87c5b9ac..7465e30d 100644 --- a/src/test/test_p2p_account_statement_parser.py +++ b/src/test/test_p2p_account_statement_parser.py @@ -473,6 +473,33 @@ def test_mintos_parsing_monthly_aggregation(self): }, ] self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="monthly")) + + def test_viainvest_parsing_transaction_aggregation(self): + """test parse_account_statement for viainvest""" + expected_statement = [ + { + "Buchungswährung": "EUR", + "Datum": datetime.date(2020, 12, 13), + "Notiz": ": ", + "Typ": "Einlage", + "Wert": 1000.0, + }, + { + "Buchungswährung": "EUR", + "Datum": datetime.date(2020, 12, 13), + "Notiz": "05-3233341: 05-3233341", + "Typ": "Zinsen", + "Wert": 0.09, + }, + { + "Buchungswährung": "EUR", + "Datum": datetime.date(2020, 12, 14), + "Notiz": "04-1246342: 04-1246342", + "Typ": "Zinsen", + "Wert": 0.10, + }, + ] + self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="transaction")) @unittest.skip("Currently not checking if infile exists.") def test_no_statement_file(self): diff --git a/src/test/testdata/viainvest.csv b/src/test/testdata/viainvest.csv new file mode 100644 index 00000000..7bf9f551 --- /dev/null +++ b/src/test/testdata/viainvest.csv @@ -0,0 +1,7 @@ +Transaction date;Value date;Transaction type;Country;Loan ID;Loan Type;Credit (€);Debit (€) +12/16/2020;12/16/2020;Amount invested in loan;PL;05-3248349;Short-term loan;;10,00 +12/14/2020;12/14/2020;Amount of principal repayment received;LV;04-1246342;Credit line;0,24; +12/14/2020;12/14/2020;Amount of interest payment received;LV;04-1246342;Credit line;0,10; +12/14/2020;12/14/2020;Amount of principal repayment received;PL;05-3233341;Short-term loan;10,00; +12/14/2020;12/14/2020;Amount of interest payment received;PL;05-3233341;Short-term loan;0,09; +12/13/2020;12/13/2020;Amount of funds deposited;;;;1.000,00; diff --git a/src/test_statement.py b/src/test_statement.py new file mode 100644 index 00000000..ca272f55 --- /dev/null +++ b/src/test_statement.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Unit test for the p2p statement class + +Copyright 2021-12-12 AlexanderLill +""" +import unittest + +from Statement import Statement + + +class TestStatement(unittest.TestCase): + """Test case implementation for Statement""" + + def test_value_parsing(self): + """test parsing of amount value""" + + test_data = [ + ("1.2", 1.2), + ("1,1", 1.1), + ("1.000,30", 1000.3), + ("1,000.30", 1000.3), + ("1000.30", 1000.3), + ] + + for item in test_data: + test_input = item[0] + expected_output = item[1] + self.assertEqual( + expected_output, + Statement._parse_value(test_input) + ) + +if __name__ == '__main__': + unittest.main() From edb19495c22e9ee1a6f903a6152b14d7146b9103 Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 12 Dec 2021 13:23:01 +0100 Subject: [PATCH 02/10] Improve consistency of regex creation --- src/Config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Config.py b/src/Config.py index 4ac6edaf..b4233d1f 100644 --- a/src/Config.py +++ b/src/Config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Module for holding the coniguration of a platform. +Module for holding the configuration of a platform. Copyright 2018-04-29 ChrisRBe """ @@ -18,9 +18,9 @@ def __init__(self, config): Constructor for Config """ logging.info("Config ini") - self._relevant_invest_regex = re.compile(config["type_regex"]["deposit"]) - self._relevant_payment_regex = re.compile(config["type_regex"]["withdraw"]) - self._relevant_income_regex = re.compile(config["type_regex"]["interest"]) + self._relevant_invest_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "deposit"]) + self._relevant_payment_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "withdraw"]) + self._relevant_income_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "interest"]) self._relevant_fee_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "fee"]) self._ignorable_entry_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "ignorable_entry"]) From 28e281a43ce7c106ef4352ec66ca8ae6e80be70a Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 12 Dec 2021 13:33:42 +0100 Subject: [PATCH 03/10] Reduce value parsing complexity --- src/Statement.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Statement.py b/src/Statement.py index bd0d8e0a..7b84d3bb 100644 --- a/src/Statement.py +++ b/src/Statement.py @@ -96,15 +96,20 @@ def get_currency(self): @staticmethod def _parse_value(value): - if value: - # Check order of . and , to replace in right order - if "." in value and "," in value: - if value.find(".") < value.find(","): - value = value.replace(".", "") - else: - value = value.replace(",", "") - return float(value.replace(",", ".")) - return None + if not value: + return None + + dot_pos = value.find(".") + comma_pos = value.find(",") + + # Check position of . and , to replace them in the right order + if dot_pos != -1 and comma_pos != -1: + if dot_pos < comma_pos: + value = value.replace(".", "") + else: + value = value.replace(",", "") + + return float(value.replace(",", ".")) @staticmethod def __match_category(mapping, booking_type, value): From 6daa96042be6126c7b99042f461795764cc7778b Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 19 Dec 2021 12:56:49 +0100 Subject: [PATCH 04/10] Fix formatting --- src/Statement.py | 4 ++-- src/test/test_p2p_account_statement_parser.py | 2 +- src/test_statement.py | 8 +++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Statement.py b/src/Statement.py index 7b84d3bb..9291e5d1 100644 --- a/src/Statement.py +++ b/src/Statement.py @@ -93,7 +93,7 @@ def get_currency(self): return self._statement[self._config.get_booking_currency()] else: return "EUR" - + @staticmethod def _parse_value(value): if not value: @@ -108,7 +108,7 @@ def _parse_value(value): value = value.replace(".", "") else: value = value.replace(",", "") - + return float(value.replace(",", ".")) @staticmethod diff --git a/src/test/test_p2p_account_statement_parser.py b/src/test/test_p2p_account_statement_parser.py index 7465e30d..c98d19cb 100644 --- a/src/test/test_p2p_account_statement_parser.py +++ b/src/test/test_p2p_account_statement_parser.py @@ -473,7 +473,7 @@ def test_mintos_parsing_monthly_aggregation(self): }, ] self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="monthly")) - + def test_viainvest_parsing_transaction_aggregation(self): """test parse_account_statement for viainvest""" expected_statement = [ diff --git a/src/test_statement.py b/src/test_statement.py index ca272f55..5bc8e0eb 100644 --- a/src/test_statement.py +++ b/src/test_statement.py @@ -26,10 +26,8 @@ def test_value_parsing(self): for item in test_data: test_input = item[0] expected_output = item[1] - self.assertEqual( - expected_output, - Statement._parse_value(test_input) - ) + self.assertEqual(expected_output, Statement._parse_value(test_input)) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() From 9556aa25dbc6be48231695d0943e499fda8b823e Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 19 Dec 2021 13:09:06 +0100 Subject: [PATCH 05/10] Add docstring for _parse_value --- src/Statement.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Statement.py b/src/Statement.py index 9291e5d1..7984e254 100644 --- a/src/Statement.py +++ b/src/Statement.py @@ -96,6 +96,13 @@ def get_currency(self): @staticmethod def _parse_value(value): + """ + Parse statement value from string to float. + Includes handling of commas and dots for decimal separators and + digit grouping, such as 1.000,00 and 1,000.00. + + :return: parsed value of the statement as float. + """ if not value: return None From eb902bcecd897e9dba1e94a76f60ef0ed03454f6 Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 19 Dec 2021 13:44:38 +0100 Subject: [PATCH 06/10] Reduce complexity of _parse_value --- src/Statement.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Statement.py b/src/Statement.py index 7984e254..3f99a067 100644 --- a/src/Statement.py +++ b/src/Statement.py @@ -109,14 +109,21 @@ def _parse_value(value): dot_pos = value.find(".") comma_pos = value.find(",") - # Check position of . and , to replace them in the right order - if dot_pos != -1 and comma_pos != -1: - if dot_pos < comma_pos: - value = value.replace(".", "") - else: - value = value.replace(",", "") + if dot_pos == -1 or comma_pos == -1: + # Did not find both comma and dot, just replace comma with dot + value = value.replace(",", ".") + return float(value) - return float(value.replace(",", ".")) + # Check position of . and , to replace them in the right order + if dot_pos < comma_pos: + # dot is used for digit grouping, comma for decimal + value = value.replace(".", "") + value = value.replace(",", ".") + return float(value) + else: + # comma is used for digit grouping, dot for decimal + value = value.replace(",", "") + return float(value) @staticmethod def __match_category(mapping, booking_type, value): From 78964c85cb828da809b3d01a9909579bfb62b020 Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 19 Dec 2021 13:47:01 +0100 Subject: [PATCH 07/10] Explicitly add implicit use of mintos test data --- src/test/test_p2p_account_statement_parser.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/test/test_p2p_account_statement_parser.py b/src/test/test_p2p_account_statement_parser.py index c98d19cb..f8c7239e 100644 --- a/src/test/test_p2p_account_statement_parser.py +++ b/src/test/test_p2p_account_statement_parser.py @@ -183,6 +183,10 @@ def test_estateguru_parsing(self): def test_mintos_parsing(self): """test parse_account_statement for mintos""" + self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "mintos.csv") + self.base_parser.config_file = os.path.join( + os.path.dirname(__file__), os.pardir, os.pardir, "config", "mintos.yml" + ) expected_statement = [ { "Buchungswährung": "EUR", @@ -268,6 +272,10 @@ def test_mintos_parsing(self): def test_mintos_parsing_daily_aggregation(self): """test parse_account_statement for mintos""" + self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "mintos.csv") + self.base_parser.config_file = os.path.join( + os.path.dirname(__file__), os.pardir, os.pardir, "config", "mintos.yml" + ) expected_statement = [ { "Buchungswährung": "EUR", @@ -344,6 +352,10 @@ def test_mintos_parsing_daily_aggregation(self): def test_mintos_parsing_transaction_aggregation(self): """test parse_account_statement for mintos""" + self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "mintos.csv") + self.base_parser.config_file = os.path.join( + os.path.dirname(__file__), os.pardir, os.pardir, "config", "mintos.yml" + ) expected_statement = [ { "Buchungswährung": "EUR", @@ -476,6 +488,10 @@ def test_mintos_parsing_monthly_aggregation(self): def test_viainvest_parsing_transaction_aggregation(self): """test parse_account_statement for viainvest""" + self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "viainvest.csv") + self.base_parser.config_file = os.path.join( + os.path.dirname(__file__), os.pardir, os.pardir, "config", "viainvest.yml" + ) expected_statement = [ { "Buchungswährung": "EUR", From 0a243c384f1ac6ee6782dabd99971397abdb380f Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 19 Dec 2021 13:49:12 +0100 Subject: [PATCH 08/10] Fix order of expected transactions --- src/test/test_p2p_account_statement_parser.py | 10 +++++----- src/test/testdata/viainvest.csv | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/test_p2p_account_statement_parser.py b/src/test/test_p2p_account_statement_parser.py index f8c7239e..e71a2639 100644 --- a/src/test/test_p2p_account_statement_parser.py +++ b/src/test/test_p2p_account_statement_parser.py @@ -502,17 +502,17 @@ def test_viainvest_parsing_transaction_aggregation(self): }, { "Buchungswährung": "EUR", - "Datum": datetime.date(2020, 12, 13), - "Notiz": "05-3233341: 05-3233341", + "Datum": datetime.date(2020, 12, 14), + "Notiz": "04-1246342: 04-1246342", "Typ": "Zinsen", - "Wert": 0.09, + "Wert": 0.10, }, { "Buchungswährung": "EUR", "Datum": datetime.date(2020, 12, 14), - "Notiz": "04-1246342: 04-1246342", + "Notiz": "05-3233341: 05-3233341", "Typ": "Zinsen", - "Wert": 0.10, + "Wert": 0.09, }, ] self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="transaction")) diff --git a/src/test/testdata/viainvest.csv b/src/test/testdata/viainvest.csv index 7bf9f551..976774b8 100644 --- a/src/test/testdata/viainvest.csv +++ b/src/test/testdata/viainvest.csv @@ -1,7 +1,7 @@ Transaction date;Value date;Transaction type;Country;Loan ID;Loan Type;Credit (€);Debit (€) -12/16/2020;12/16/2020;Amount invested in loan;PL;05-3248349;Short-term loan;;10,00 +12/13/2020;12/13/2020;Amount of funds deposited;;;;1.000,00; +12/13/2020;12/13/2020;Amount invested in loan;PL;05-3248349;Short-term loan;;10,00 12/14/2020;12/14/2020;Amount of principal repayment received;LV;04-1246342;Credit line;0,24; 12/14/2020;12/14/2020;Amount of interest payment received;LV;04-1246342;Credit line;0,10; 12/14/2020;12/14/2020;Amount of principal repayment received;PL;05-3233341;Short-term loan;10,00; 12/14/2020;12/14/2020;Amount of interest payment received;PL;05-3233341;Short-term loan;0,09; -12/13/2020;12/13/2020;Amount of funds deposited;;;;1.000,00; From 268ebf1543525d101ede80f996a0d48bbee32032 Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 19 Dec 2021 14:01:45 +0100 Subject: [PATCH 09/10] Move value parse test into right directory --- src/{ => test}/test_statement.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{ => test}/test_statement.py (100%) diff --git a/src/test_statement.py b/src/test/test_statement.py similarity index 100% rename from src/test_statement.py rename to src/test/test_statement.py From b2c0ef77fd3ebfe573e29d355ed81d731da3f972 Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 19 Dec 2021 16:50:07 +0100 Subject: [PATCH 10/10] Add missing param to docstring --- src/Statement.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Statement.py b/src/Statement.py index 3f99a067..7481e1a5 100644 --- a/src/Statement.py +++ b/src/Statement.py @@ -101,6 +101,8 @@ def _parse_value(value): Includes handling of commas and dots for decimal separators and digit grouping, such as 1.000,00 and 1,000.00. + :param value: the statement value as string + :return: parsed value of the statement as float. """ if not value: