issue #59: Take Stamp from Email Date (#59)

* Add EmailDate parser

* Add new EmailDateParser with testing and make most of HTML provider Combined ones

* Make Stamp mandatory to create a Maintenance

CHANGELOG.md

@@ -13,6 +13,7 @@
   - New `NotificationData` class that enables easier data injection for simple and complex data objects, such as
     emails.
   - Tests refactor to make them more specific to each type of data, mocking interfaces between different classes.
+- #59 - Added a new parser `EmailDateParser` that uses the temail `Date` to get the `Stamp` and use in most of the `Providers` via the `CombinedProcessor`. Also, `Maintenance.stamp` attribute is mandatory.
 - #60 - Added new provider `Seaborn` using `Html` and a new parser for Email Subject: `EmailSubjectParser`
 
 ### Fixed

circuit_maintenance_parser/output.py

@@ -6,7 +6,7 @@
 import json
 from enum import Enum
 
-from typing import List, Optional
+from typing import List
 
 from pydantic import BaseModel, validator, StrictStr, StrictInt, Extra
 
@@ -99,10 +99,10 @@ class Maintenance(BaseModel, extra=Extra.forbid):
         status: defines the overall status or confirmation for the maintenance
         start: timestamp that defines the start date of the maintenance in GMT
         end: timestamp that defines the end date of the maintenance in GMT
+        stamp: timestamp that defines the update date of the maintenance in GMT
         organizer: defines the contact information included in the original notification
 
     Optional attributes:
-        stamp: timestamp that defines the update date of the maintenance in GMT
         summary: description of the maintenace notification
         uid: specific unique identifier for each notification
         sequence: sequence number - initially zero - to serialize updates in case they are received or processed out of
@@ -123,7 +123,7 @@ class Maintenance(BaseModel, extra=Extra.forbid):
         ...     summary="This is a maintenance notification",
         ...     uid="1111",
         ... )
-        Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', circuits=[CircuitImpact(circuit_id='123', impact=<Impact.NO_IMPACT: 'NO-IMPACT'>), CircuitImpact(circuit_id='456', impact=<Impact.OUTAGE: 'OUTAGE'>)], status=<Status.COMPLETED: 'COMPLETED'>, start=1533704400, end=1533712380, organizer='[email protected]', stamp=1533595768, uid='1111', sequence=1, summary='This is a maintenance notification')
+        Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', circuits=[CircuitImpact(circuit_id='123', impact=<Impact.NO_IMPACT: 'NO-IMPACT'>), CircuitImpact(circuit_id='456', impact=<Impact.OUTAGE: 'OUTAGE'>)], status=<Status.COMPLETED: 'COMPLETED'>, start=1533704400, end=1533712380, stamp=1533595768, organizer='[email protected]', uid='1111', sequence=1, summary='This is a maintenance notification')
     """
 
     provider: StrictStr
@@ -133,11 +133,10 @@ class Maintenance(BaseModel, extra=Extra.forbid):
     status: Status
     start: StrictInt
     end: StrictInt
+    stamp: StrictInt
     organizer: StrictStr
 
     # Non mandatory attributes
-
-    stamp: Optional[StrictInt] = None
     uid: StrictStr = "0"
     sequence: StrictInt = 1
     summary: StrictStr = ""

circuit_maintenance_parser/parser.py

@@ -5,6 +5,7 @@
 import datetime
 import quopri
 from typing import Iterable, Union, Dict, List
+from email.utils import parsedate_tz, mktime_tz
 
 import bs4  # type: ignore
 from bs4.element import ResultSet  # type: ignore
@@ -168,6 +169,24 @@ def clean_line(line):
         return line.replace("=C2", "").replace("=A0", "").replace("\r", "").replace("=", "").replace("\n", "")
 
 
+class EmailDateParser(Parser):
+    """Parser for Email Date."""
+
+    _data_types = ["email-header-date"]
+
+    def parse(self, raw: bytes) -> List[Dict]:
+        """Method that returns a list of Maintenance objects."""
+        try:
+            parsed_date = parsedate_tz(raw.decode())
+            if parsed_date:
+                result = [{"stamp": mktime_tz(parsed_date)}]
+                logger.debug("Successful parsing for %s", self.__class__.__name__)
+                return result
+            raise ParserError("Not parsed_date available.")
+        except Exception as exc:
+            raise ParserError from exc
+
+
 class EmailSubjectParser(Parser):
     """Parse data from subject or email."""
 
@@ -187,7 +206,7 @@ def parse(self, raw: bytes) -> List[Dict]:
         except Exception as exc:
             raise ParserError from exc
 
-    def parse_subject(self, subject: ResultSet) -> List[Dict]:
+    def parse_subject(self, subject: str) -> List[Dict]:
         """Custom subject parsing."""
         raise NotImplementedError
 

circuit_maintenance_parser/provider.py

@@ -10,7 +10,7 @@
 
 from circuit_maintenance_parser.output import Maintenance
 from circuit_maintenance_parser.data import NotificationData
-from circuit_maintenance_parser.parser import ICal
+from circuit_maintenance_parser.parser import ICal, EmailDateParser
 from circuit_maintenance_parser.errors import ProcessorError, ProviderError
 from circuit_maintenance_parser.processor import CombinedProcessor, SimpleProcessor, GenericProcessor
 
@@ -106,7 +106,7 @@ class Cogent(GenericProvider):
     """Cogent provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        SimpleProcessor(data_parsers=[HtmlParserCogent1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserCogent1]),
     ]
     _default_organizer = "[email protected]"
 
@@ -121,7 +121,7 @@ class GTT(GenericProvider):
     """GTT provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        SimpleProcessor(data_parsers=[HtmlParserGTT1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserGTT1]),
     ]
     _default_organizer = "[email protected]"
 
@@ -130,7 +130,7 @@ class Lumen(GenericProvider):
     """Lumen provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        SimpleProcessor(data_parsers=[HtmlParserLumen1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserLumen1]),
     ]
     _default_organizer = "[email protected]"
 
@@ -139,7 +139,7 @@ class Megaport(GenericProvider):
     """Megaport provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        SimpleProcessor(data_parsers=[HtmlParserMegaport1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserMegaport1]),
     ]
     _default_organizer = "[email protected]"
 
@@ -160,8 +160,8 @@ class Seaborn(GenericProvider):
     """Seaborn provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        CombinedProcessor(data_parsers=[HtmlParserSeaborn1, SubjectParserSeaborn1]),
-        CombinedProcessor(data_parsers=[HtmlParserSeaborn2, SubjectParserSeaborn2]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserSeaborn1, SubjectParserSeaborn1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserSeaborn2, SubjectParserSeaborn2]),
     ]
     _default_organizer = "[email protected]"
 
@@ -177,7 +177,7 @@ class Telstra(GenericProvider):
 
     _processors: List[GenericProcessor] = [
         SimpleProcessor(data_parsers=[ICal]),
-        SimpleProcessor(data_parsers=[HtmlParserTelstra1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserTelstra1]),
     ]
     _default_organizer = "[email protected]"
 
@@ -186,7 +186,7 @@ class Turkcell(GenericProvider):
     """Turkcell provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        SimpleProcessor(data_parsers=[HtmlParserTurkcell1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserTurkcell1]),
     ]
     _default_organizer = "[email protected]"
 
@@ -195,7 +195,7 @@ class Verizon(GenericProvider):
     """Verizon provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        SimpleProcessor(data_parsers=[HtmlParserVerizon1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserVerizon1]),
     ]
     _default_organizer = "[email protected]"
 

tests/unit/data/date/email_date_1

@@ -0,0 +1 @@
+Mon, 1 Feb 2021 09:33:34 +0000

tests/unit/data/date/email_date_1_result.json

@@ -0,0 +1,5 @@
+[
+    {
+    "stamp": 1612172014
+    }
+]

tests/unit/data/seaborn/seaborn1_result.json

@@ -12,7 +12,7 @@
         "organizer": "[email protected]",
         "provider": "seaborn",
         "sequence": 1,
-        "stamp": null,
+        "stamp": 1629131396,
         "start": 1628733600,
         "status": "RE-SCHEDULED",
         "summary": "An emergency work will be carried out to relocate fiber cable due to civil works in the zone.",

tests/unit/data/seaborn/seaborn2_result.json

@@ -12,10 +12,10 @@
         "organizer": "[email protected]",
         "provider": "seaborn",
         "sequence": 1,
-        "stamp": null,
+        "stamp": 1629131396,
         "start": 1628733600,
         "status": "RE-SCHEDULED",
         "summary": "An emergency work will be carried out to relocate fiber cable due to civil works in the zone.",
         "uid": "0"
     }
-]
+]

tests/unit/data/seaborn/seaborn3_result.json

@@ -12,7 +12,7 @@
         "organizer": "[email protected]",
         "provider": "seaborn",
         "sequence": 1,
-        "stamp": null,
+        "stamp": 1629131347,
         "start": 1628827200,
         "status": "CONFIRMED",
         "summary": "A scheduled work will be carried out in order to perform OTDR measures.",