Add real daily aggregation, fix small numbers

README.md

@@ -35,11 +35,12 @@ List of currently supported providers:
     - Debitum Network
 
 Control the way how account statements are processed via the aggregate parameter:
-    - daily: Currently does not process the input data beyond making it Portfolio Performance compatible.
+    - transaction: Currently does not process the input data beyond making it Portfolio Performance compatible.
+    - daily: This aggregates all bookings of the same type into one statement per type and day.
     - monthly: This aggregates all bookings of the same type into one statement per type and month. Sets
             the last day of the month as transaction date.
 
-Default behaviour for now is 'daily'.
+Default behaviour for now is 'transaction'.
 
 Copyright 2018-03-17 ChrisRBe
 
@@ -48,7 +49,7 @@ positional arguments:
 
 optional arguments:
   -h, --help            show this help message and exit
-  --aggregate {daily,monthly}
+  --aggregate {transaction,daily,monthly}
                         specify how account statements should be summarized
   --type TYPE           Specifies the p2p lending operator
   --debug               enables debug level logging if set

parse-account-statements.py

@@ -16,11 +16,12 @@
     - Debitum Network
 
 Control the way how account statements are processed via the aggregate parameter:
-    - daily: Currently does not process the input data beyond making it Portfolio Performance compatible.
+    - transaction: Currently does not process the input data beyond making it Portfolio Performance compatible.
+    - daily: This aggregates all bookings of the same type into one statement per type and day.
     - monthly: This aggregates all bookings of the same type into one statement per type and month. Sets
             the last day of the month as transaction date.
 
-Default behaviour for now is 'daily'.
+Default behaviour for now is 'transaction'.
 
 Copyright 2018-03-17 ChrisRBe
 """
@@ -50,14 +51,14 @@ def platform_factory(infile, operator_name="mintos"):
         return None
 
 
-def main(infile, p2p_operator_name="mintos", aggregate="daily"):
+def main(infile, p2p_operator_name="mintos", aggregate="transaction"):
     """
     Processes the provided input file with the rules defined for the given platform.
     Outputs a CSV file readable by Portfolio Performance
 
     :param infile: input file containing the account statements from a supported platform
     :param p2p_operator_name: name of the Peer-to-Peer lending platform, defaults to Mintos
-    :param aggregate: specifies the aggregation period. defaults to daily.
+    :param aggregate: specifies the aggregation period. defaults to transaction.
 
     :return: True, False if an error occurred.
     """
@@ -100,8 +101,8 @@ def main(infile, p2p_operator_name="mintos", aggregate="daily"):
         "--aggregate",
         type=str,
         help="specify how account statements should be summarized",
-        choices=["daily", "monthly"],
-        default="daily",
+        choices=["transaction", "daily", "monthly"],
+        default="transaction",
     )
     ARG_PARSER.add_argument("--type", type=str, help="Specifies the p2p lending operator")
     ARG_PARSER.add_argument("--debug", action="store_true", help="enables debug level logging if set")

src/p2p_account_statement_parser.py

@@ -53,6 +53,26 @@ def config_file(self, value):
         """config file property setter"""
         self._config_file = value
 
+    def __aggregate_statements_daily(self, formatted_account_entry):
+        entry_date = formatted_account_entry[PP_FIELDNAMES[0]]
+        entry_type = formatted_account_entry[PP_FIELDNAMES[3]]
+        logging.debug("entry type is {}. new entry date is {}".format(entry_type, entry_date))
+        if entry_date not in self.aggregation_data:
+            self.aggregation_data[entry_date] = {}
+        if entry_type in self.aggregation_data[entry_date]:
+            logging.debug("add to existing entry")
+            self.aggregation_data[entry_date][entry_type][PP_FIELDNAMES[1]] += formatted_account_entry[
+                PP_FIELDNAMES[1]
+            ]
+        else:
+            self.aggregation_data[entry_date][entry_type] = {
+                PP_FIELDNAMES[0]: entry_date,
+                PP_FIELDNAMES[1]: formatted_account_entry[PP_FIELDNAMES[1]],
+                PP_FIELDNAMES[2]: formatted_account_entry[PP_FIELDNAMES[2]],
+                PP_FIELDNAMES[3]: entry_type,
+                PP_FIELDNAMES[4]: "Tageszusammenfassung",
+            }
+
     def __aggregate_statements_monthly(self, formatted_account_entry):
         entry_date = formatted_account_entry[PP_FIELDNAMES[0]]
         last_day = calendar.monthrange(entry_date.year, entry_date.month)[1]
@@ -115,39 +135,44 @@ def __parse_service_config(self):
             config = safe_load(ymlconfig)
             self.config = Config(config)
 
-    def __process_statement(self, statement, aggregate="daily"):
+    def __process_statement(self, statement, aggregate="transaction"):
         """
         Processes each statement read from the account statement file. First, format in into the dictionary.
         Then check what aggregation should be applied.
 
-            - daily: add directly to the output list.
+            - transaction: add directly to the output list.
+            - daily: add it to intermediate aggregation collection.
             - monthly: add it to intermediate aggregation collection.
 
         :param statement: Contains one line from the account statement file
-        :param aggregate: specify the aggregation format; e.g. daily or monthly. Defaults to daily.
+        :param aggregate: specify the aggregation format; e.g. daily or monthly. Defaults to transaction.
 
         :return:
         """
         formatted_account_entry = self.__format_statement(statement)
         if formatted_account_entry:
-            if aggregate == "daily":
+            if aggregate == "transaction":
                 self.output_list.append(formatted_account_entry)
+            elif aggregate == "daily":
+                self.__aggregate_statements_daily(formatted_account_entry)
             elif aggregate == "monthly":
                 self.__aggregate_statements_monthly(formatted_account_entry)
 
-    def parse_account_statement(self, aggregate="daily"):
+    def parse_account_statement(self, aggregate="transaction"):
         """
         read a platform account statement csv file and filter the content according to the given configuration file.
         If aggregation is selected the output data will be post processed in the following way:
 
-        - aggregate="daily": return the list of processed statements as is.
+        - aggregate="transaction": return the list of processed statements as is.
+        - aggregate="daily": return a list of post-processed statements aggregating on daily basis for each
+          booking type.
         - aggregate="monthly": return a list of post-processed statements aggregating on monthly basis for each
           booking type.
 
         :param aggregate: specifies the aggregation period. defaults to daily.
         :return: list of account statement entries ready for use in Portfolio Performance
         """
-        if aggregate == "daily" or aggregate == "monthly":
+        if aggregate == "transaction" or aggregate == "daily" or aggregate == "monthly":
             logging.info("Aggregating data on a {} basis".format(aggregate))
         else:
             logging.error("Aggregating data on a {} basis not supported.".format(aggregate))
@@ -163,6 +188,6 @@ def parse_account_statement(self, aggregate="daily"):
             for statement in account_statement:
                 self.__process_statement(aggregate=aggregate, statement=statement)
 
-        if aggregate == "monthly":
+        if aggregate == "daily" or aggregate == "monthly":
             self.__migrate_data_to_output()
         return self.output_list

src/portfolio_performance_writer.py

@@ -49,7 +49,7 @@ def update_output(self, statement_dict):
         :return:
         """
         if statement_dict:
-            statement_dict[PP_FIELDNAMES[1]] = str(statement_dict[PP_FIELDNAMES[1]]).replace(".", ",")
+            statement_dict[PP_FIELDNAMES[1]] = "{0:.8f}".format(statement_dict[PP_FIELDNAMES[1]]).replace(".", ",")
             self.out_csv_writer.writerow(statement_dict)
 
     def write_pp_csv_file(self, outfile="portfolio_performance.csv"):

src/test/test_p2p_account_statement_parser.py

@@ -246,6 +246,61 @@ def test_mintos_parsing(self):
         self.assertEqual(expected_statement, self.base_parser.parse_account_statement())
 
     def test_mintos_parsing_daily_aggregation(self):
+        """test parse_account_statement for mintos"""
+        expected_statement = [
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2018, 1, 17),
+                "Notiz": "Tageszusammenfassung",
+                "Typ": "Einlage",
+                "Wert": 20.0,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2018, 1, 18),
+                "Notiz": "Tageszusammenfassung",
+                "Typ": "Zinsen",
+                "Wert": 0.005555556,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2018, 1, 19),
+                "Notiz": "Tageszusammenfassung",
+                "Typ": "Zinsen",
+                "Wert": 0.006861111,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2018, 1, 25),
+                "Notiz": "Tageszusammenfassung",
+                "Typ": "Zinsen",
+                "Wert": 0.001214211,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2018, 1, 29),
+                "Notiz": "Tageszusammenfassung",
+                "Typ": "Zinsen",
+                "Wert": 0.000342077,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2018, 2, 27),
+                "Notiz": "Tageszusammenfassung",
+                "Typ": "Zinsen",
+                "Wert": 0.3,
+            },
+            {
+                "Buchungswährung": "EUR",
+                "Datum": datetime.date(2016, 9, 28),
+                "Notiz": "Tageszusammenfassung",
+                "Typ": "Entnahme",
+                "Wert": -20.0,
+            },
+        ]
+        self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="daily"))
+
+    def test_mintos_parsing_transaction_aggregation(self):
         """test parse_account_statement for mintos"""
         expected_statement = [
             {
@@ -307,7 +362,7 @@ def test_mintos_parsing_daily_aggregation(self):
                 "Wert": -20.0,
             },
         ]
-        self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="daily"))
+        self.assertEqual(expected_statement, self.base_parser.parse_account_statement(aggregate="transaction"))
 
     def test_mintos_parsing_monthly_aggregation(self):
         """test parse_account_statement for mintos"""

src/test/test_portfolio_performance_writer.py

@@ -29,29 +29,29 @@ def test_update_output(self):
         """test update_output"""
         test_entry = {
             PP_FIELDNAMES[0]: "date",
-            PP_FIELDNAMES[1]: "profit",
+            PP_FIELDNAMES[1]: 0,
             PP_FIELDNAMES[2]: "currency",
             PP_FIELDNAMES[3]: "category",
             PP_FIELDNAMES[4]: "note",
         }
         self.pp_writer.update_output(test_entry)
         self.assertEqual(
-            "Datum,Wert,Buchungswährung,Typ,Notiz\r\ndate,profit,currency,category,note",
+            'Datum,Wert,Buchungswährung,Typ,Notiz\r\ndate,"0,00000000",currency,category,note',
             self.pp_writer.out_string_stream.getvalue().strip(),
         )
 
     def test_update_output_umlaut(self):
         """test update_output with umlauts"""
         test_entry = {
             PP_FIELDNAMES[0]: "date",
-            PP_FIELDNAMES[1]: "profit",
+            PP_FIELDNAMES[1]: 0,
             PP_FIELDNAMES[2]: "currency",
             PP_FIELDNAMES[3]: "category",
             PP_FIELDNAMES[4]: "Laiamäe Pärnaõie Užutekio",
         }
         self.pp_writer.update_output(test_entry)
         self.assertEqual(
-            "Datum,Wert,Buchungswährung,Typ,Notiz\r\n" "date,profit,currency,category,Laiamäe Pärnaõie Užutekio",
+            'Datum,Wert,Buchungswährung,Typ,Notiz\r\ndate,"0,00000000",currency,category,Laiamäe Pärnaõie Užutekio',
             self.pp_writer.out_string_stream.getvalue().strip(),
         )