Merge #457

457: Add support for viainvest r=ChrisRBe a=AlexanderLill This adds support for parsing viainvest account statements. Withdrawals are currently not supported, only deposits and interest payments. Currently I can't get the tests to run on my environment. `@ChrisRBe` is there a commandline command to run all tests? Edit: I just saw that also there are formatting issues, is there a command to run this locally or a nice way to show how the formatting should look like? :) Co-authored-by: Alexander Lill <[email protected]>
ChrisRBe · Dec 19, 2021 · b6cc99d · b6cc99d
2 parents 90a8ec9 + b2c0ef7
commit b6cc99d
Show file tree

Hide file tree

Showing 8 changed files with 146 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -34,6 +34,7 @@ List of currently supported providers:
     - Robocash
     - Swaper
     - Debitum Network
+    - Viainvest
 
 Control the way how account statements are processed via the aggregate parameter:
     - transaction: Currently does not process the input data beyond making it Portfolio Performance compatible.
@@ -80,6 +81,7 @@ parse-account-statements.py  --type mintos src/test/testdata/mintos.csv
 * bondora - Supports current account statement format (as of 2019-10-12); exported to csv
 * bondora go & grow - Supports current account statement format (as of 2019-10-12); exported to csv
 * debitumnetwork - Supports current account statement format (as of 2020-09-08) exported to csv
+* viainvest - Supports current account statement (as of 2021-12-12) exported as csv (Withdrawals do not work yet)
 
 ### Alternative solution for Auxmoney
 
@@ -96,9 +98,12 @@ Example:
 ```
 ---
 type_regex: !!map
-  deposit: "^Incoming client.*"
-  withdraw: "^Withdraw application.*"
-  interest: "(^Delayed interest.*)|(^Late payment.*)|(^Interest income.*)|(^Cashback.*)"
+  deposit: "(Deposits)|(^Incoming client.*)|(^Incoming currency exchange.*)|(^Affiliate partner bonus$)"
+  withdraw: "(^Withdraw application.*)|(Outgoing currency.*)|(Withdrawal)"
+  interest: "(^Delayed interest.*)|(^Late payment.*)|(^Interest income.*)|(^Cashback.*)|(^.*[Ii]nterest received.*)|(^.*late fees received$)"
+  fee: "(^FX commission.*)|(.*secondary market fee$)"
+  ignorable_entry: ".*investment in loan.*|.*[Pp]rincipal received.*|.*secondary market transaction.*"
+  special_entry: "(.*discount/premium.*)"
 
 csv_fieldnames:
   booking_date: 'Date'

diff --git a/config/viainvest.yml b/config/viainvest.yml
@@ -0,0 +1,14 @@
+---
+type_regex: !!map
+  deposit: "(Amount of funds deposited)"
+  withdraw: ""
+  interest: "(Amount of interest payment received)"
+  ignorable_entry: "(Amount invested in loan)|(Amount of principal repayment received)"
+
+csv_fieldnames:
+  booking_date: 'Value date'
+  booking_date_format: '%m/%d/%Y'
+  booking_details: 'Loan ID'
+  booking_id: 'Loan ID'
+  booking_type: 'Transaction type'
+  booking_value: 'Credit (€)'
diff --git a/parse-account-statements.py b/parse-account-statements.py
@@ -14,6 +14,7 @@
     - Robocash
     - Swaper
     - Debitum Network
+    - Viainvest
 
 Control the way how account statements are processed via the aggregate parameter:
     - transaction: Currently does not process the input data beyond making it Portfolio Performance compatible.

diff --git a/src/Config.py b/src/Config.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 """
-Module for holding the coniguration of a platform.
+Module for holding the configuration of a platform.
 
 Copyright 2018-04-29 ChrisRBe
 """
@@ -18,9 +18,9 @@ def __init__(self, config):
         Constructor for Config
         """
         logging.info("Config ini")
-        self._relevant_invest_regex = re.compile(config["type_regex"]["deposit"])
-        self._relevant_payment_regex = re.compile(config["type_regex"]["withdraw"])
-        self._relevant_income_regex = re.compile(config["type_regex"]["interest"])
+        self._relevant_invest_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "deposit"])
+        self._relevant_payment_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "withdraw"])
+        self._relevant_income_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "interest"])
 
         self._relevant_fee_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "fee"])
         self._ignorable_entry_regex = Config.__get_compiled_regex_or_none(config, ["type_regex", "ignorable_entry"])

diff --git a/src/Statement.py b/src/Statement.py
@@ -27,7 +27,6 @@ def get_category(self):
         :return: category of the statement; if ignored on purpose return 'Ignored', if unknown return the empty string
         """
         booking_type = self._statement[self._config.get_booking_type()]
-        category = ""
         value = self.get_value()
 
         regex_to_category_mappings = [
@@ -39,6 +38,7 @@ def get_category(self):
             {"regex": self._config.get_ignorable_entry_regex(), "category": "Ignored"},
         ]
 
+        category = ""
         for mapping in regex_to_category_mappings:
             category = self.__match_category(mapping, booking_type, value)
             if category:
@@ -69,7 +69,8 @@ def get_value(self):
 
         :return: value of the current statement as float.
         """
-        return float(self._statement[self._config.get_booking_value()].replace(",", "."))
+        raw_value = self._statement[self._config.get_booking_value()]
+        return Statement._parse_value(raw_value)
 
     def get_note(self):
         """
@@ -93,6 +94,39 @@ def get_currency(self):
         else:
             return "EUR"
 
+    @staticmethod
+    def _parse_value(value):
+        """
+        Parse statement value from string to float.
+        Includes handling of commas and dots for decimal separators and
+        digit grouping, such as 1.000,00 and 1,000.00.
+
+        :param value: the statement value as string
+
+        :return: parsed value of the statement as float.
+        """
+        if not value:
+            return None
+
+        dot_pos = value.find(".")
+        comma_pos = value.find(",")
+
+        if dot_pos == -1 or comma_pos == -1:
+            # Did not find both comma and dot, just replace comma with dot
+            value = value.replace(",", ".")
+            return float(value)
+
+        # Check position of . and , to replace them in the right order
+        if dot_pos < comma_pos:
+            # dot is used for digit grouping, comma for decimal
+            value = value.replace(".", "")
+            value = value.replace(",", ".")
+            return float(value)
+        else:
+            # comma is used for digit grouping, dot for decimal
+            value = value.replace(",", "")
+            return float(value)
+
     @staticmethod
     def __match_category(mapping, booking_type, value):
         """

diff --git a/src/test/test_p2p_account_statement_parser.py b/src/test/test_p2p_account_statement_parser.py
@@ -183,6 +183,10 @@ def test_estateguru_parsing(self):
 
     def test_mintos_parsing(self):
         """test parse_account_statement for mintos"""
+        self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "mintos.csv")
+        self.base_parser.config_file = os.path.join(
+            os.path.dirname(__file__), os.pardir, os.pardir, "config", "mintos.yml"
+        )
         expected_statement = [
             {
                 "Buchungswährung": "EUR",
@@ -268,6 +272,10 @@ def test_mintos_parsing(self):
 
     def test_mintos_parsing_daily_aggregation(self):
         """test parse_account_statement for mintos"""
+        self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "mintos.csv")
+        self.base_parser.config_file = os.path.join(
+            os.path.dirname(__file__), os.pardir, os.pardir, "config", "mintos.yml"
+        )
         expected_statement = [
             {
                 "Buchungswährung": "EUR",
@@ -344,6 +352,10 @@ def test_mintos_parsing_daily_aggregation(self):
 
     def test_mintos_parsing_transaction_aggregation(self):
         """test parse_account_statement for mintos"""
+        self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "mintos.csv")
+        self.base_parser.config_file = os.path.join(
+            os.path.dirname(__file__), os.pardir, os.pardir, "config", "mintos.yml"
+        )
         expected_statement = [
             {
                 "Buchungswährung": "EUR",
@@ -474,6 +486,37 @@ def test_mintos_parsing_monthly_aggregation(self):
         ]
         self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="monthly"))
 
+    def test_viainvest_parsing_transaction_aggregation(self):
+        """test parse_account_statement for viainvest"""
+        self.base_parser.account_statement_file = os.path.join(os.path.dirname(__file__), "testdata", "viainvest.csv")
+        self.base_parser.config_file = os.path.join(
+            os.path.dirname(__file__), os.pardir, os.pardir, "config", "viainvest.yml"
+        )
+        expected_statement = [
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2020, 12, 13),
+                "Notiz": ": ",
+                "Typ": "Einlage",
+                "Wert": 1000.0,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2020, 12, 14),
+                "Notiz": "04-1246342: 04-1246342",
+                "Typ": "Zinsen",
+                "Wert": 0.10,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2020, 12, 14),
+                "Notiz": "05-3233341: 05-3233341",
+                "Typ": "Zinsen",
+                "Wert": 0.09,
+            },
+        ]
+        self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="transaction"))
+
     @unittest.skip("Currently not checking if infile exists.")
     def test_no_statement_file(self):
         """test parse_account_statement with non existent file"""

diff --git a/src/test/test_statement.py b/src/test/test_statement.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Unit test for the p2p statement class
+
+Copyright 2021-12-12 AlexanderLill
+"""
+import unittest
+
+from Statement import Statement
+
+
+class TestStatement(unittest.TestCase):
+    """Test case implementation for Statement"""
+
+    def test_value_parsing(self):
+        """test parsing of amount value"""
+
+        test_data = [
+            ("1.2", 1.2),
+            ("1,1", 1.1),
+            ("1.000,30", 1000.3),
+            ("1,000.30", 1000.3),
+            ("1000.30", 1000.3),
+        ]
+
+        for item in test_data:
+            test_input = item[0]
+            expected_output = item[1]
+            self.assertEqual(expected_output, Statement._parse_value(test_input))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/src/test/testdata/viainvest.csv b/src/test/testdata/viainvest.csv
@@ -0,0 +1,7 @@
+Transaction date;Value date;Transaction type;Country;Loan ID;Loan Type;Credit (€);Debit (€)
+12/13/2020;12/13/2020;Amount of funds deposited;;;;1.000,00;
+12/13/2020;12/13/2020;Amount invested in loan;PL;05-3248349;Short-term loan;;10,00
+12/14/2020;12/14/2020;Amount of principal repayment received;LV;04-1246342;Credit line;0,24;
+12/14/2020;12/14/2020;Amount of interest payment received;LV;04-1246342;Credit line;0,10;
+12/14/2020;12/14/2020;Amount of principal repayment received;PL;05-3233341;Short-term loan;10,00;
+12/14/2020;12/14/2020;Amount of interest payment received;PL;05-3233341;Short-term loan;0,09;