Merge pull request #105 from pke11y/pk_colt_html

Changing Colt parsing to use Subject and remove ICal
networktocode · Nov 2, 2021 · 06eb0b2 · 06eb0b2
2 parents 120e7ef + 4e2e19b
commit 06eb0b2
Show file tree

Hide file tree

Showing 11 changed files with 191 additions and 69 deletions.
diff --git a/circuit_maintenance_parser/parsers/colt.py b/circuit_maintenance_parser/parsers/colt.py
@@ -4,66 +4,18 @@
 import re
 import csv
 import io
-import base64
 
-from icalendar import Calendar  # type: ignore
-
-from circuit_maintenance_parser.parser import Csv
-from circuit_maintenance_parser.errors import ParserError
+from dateutil import parser
 from circuit_maintenance_parser.output import Status, Impact, CircuitImpact
-from circuit_maintenance_parser.parser import ICal
+from circuit_maintenance_parser.parser import EmailSubjectParser, Csv
 
 logger = logging.getLogger(__name__)
 
 # pylint: disable=too-many-branches
 
 
-class ICalParserColt1(ICal):
-    """Colt Notifications Parser based on ICal notifications."""
-
-    def parse(self, raw: bytes):
-        """Method that returns a list of Maintenance objects."""
-        result = []
-
-        # iCalendar data sometimes comes encoded with base64
-        # TODO: add a test case
-        try:
-            gcal = Calendar.from_ical(base64.b64decode(raw))
-        except ValueError:
-            gcal = Calendar.from_ical(raw)
-
-        if not gcal:
-            raise ParserError("Not a valid iCalendar data received")
-
-        for component in gcal.walk():
-            if component.name == "VEVENT":
-                maintenance_id = ""
-                account = ""
-
-                summary_match = re.search(
-                    r"^.*?[-]\s(?P<maintenance_id>CRQ[\S]+).*?,\s*(?P<account>\d+)$", str(component.get("SUMMARY"))
-                )
-                if summary_match:
-                    maintenance_id = summary_match.group("maintenance_id")
-                    account = summary_match.group("account")
-
-                data = {
-                    "account": account,
-                    "maintenance_id": maintenance_id,
-                    "status": Status("CONFIRMED"),
-                    "start": round(component.get("DTSTART").dt.timestamp()),
-                    "end": round(component.get("DTEND").dt.timestamp()),
-                    "stamp": round(component.get("DTSTAMP").dt.timestamp()),
-                    "summary": str(component.get("SUMMARY")),
-                    "sequence": int(component.get("SEQUENCE")),
-                }
-                result.append(data)
-
-        return result
-
-
 class CsvParserColt1(Csv):
-    """Colt Notifications partial parser for circuit-ID's in CSV notifications."""
+    """Colt Notifications partial parser in CSV notifications."""
 
     @staticmethod
     def parse_csv(raw):
@@ -73,4 +25,37 @@ def parse_csv(raw):
             parsed_csv = csv.DictReader(csv_data, dialect=csv.excel_tab)
             for row in parsed_csv:
                 data["circuits"].append(CircuitImpact(impact=Impact("OUTAGE"), circuit_id=row["Circuit ID"].strip()))
+                if not data.get("account"):
+                    search = re.search(r"\d+", row["OCN"].strip())
+                    if search:
+                        data["account"] = search.group()
+        return [data]
+
+
+class SubjectParserColt1(EmailSubjectParser):
+    """Subject parser for Colt notifications."""
+
+    def parse_subject(self, subject):
+        """Parse subject.
+
+        Example:
+        - [ EXTERNAL ] MAINTENANCE ALERT: CRQ1-12345678 24/10/2021 04:00:00 GMT - 24/10/2021 11:00:00 GMT is about to START
+        - [ EXTERNAL ] MAINTENANCE ALERT: CRQ1-12345678 31/10/2021 00:00:00 GMT - 31/10/2021 07:30:00 GMT - COMPLETED
+        """
+        data = {}
+        search = re.search(
+            r"\[.+\](.+).+?(CRQ\w+-\w+).+?(\d+/\d+/\d+\s\d+:\d+:\d+\s[A-Z]+).+?(\d+/\d+/\d+\s\d+:\d+:\d+\s[A-Z]+).+?([A-Z]+)",
+            subject,
+        )
+        if search:
+            data["maintenance_id"] = search.group(2)
+            data["start"] = self.dt2ts(parser.parse(search.group(3)))
+            data["end"] = self.dt2ts(parser.parse(search.group(4)))
+            if search.group(5) == "START":
+                data["status"] = Status("IN-PROCESS")
+            elif search.group(5) == "COMPLETED":
+                data["status"] = Status("COMPLETED")
+            else:
+                data["status"] = Status("CONFIRMED")
+            data["summary"] = subject
         return [data]
diff --git a/circuit_maintenance_parser/provider.py b/circuit_maintenance_parser/provider.py
@@ -19,7 +19,7 @@
 from circuit_maintenance_parser.parsers.aquacomms import HtmlParserAquaComms1, SubjectParserAquaComms1
 from circuit_maintenance_parser.parsers.aws import SubjectParserAWS1, TextParserAWS1
 from circuit_maintenance_parser.parsers.cogent import HtmlParserCogent1
-from circuit_maintenance_parser.parsers.colt import ICalParserColt1, CsvParserColt1
+from circuit_maintenance_parser.parsers.colt import CsvParserColt1, SubjectParserColt1
 from circuit_maintenance_parser.parsers.equinix import HtmlParserEquinix, SubjectParserEquinix
 from circuit_maintenance_parser.parsers.gtt import HtmlParserGTT1
 from circuit_maintenance_parser.parsers.hgc import HtmlParserHGC1, HtmlParserHGC2, SubjectParserHGC1
@@ -193,7 +193,7 @@ class Colt(GenericProvider):
     """Cogent provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        CombinedProcessor(data_parsers=[ICalParserColt1, CsvParserColt1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, CsvParserColt1, SubjectParserColt1]),
     ]
     _default_organizer = "[email protected]"
 

diff --git a/tests/unit/data/colt/colt1 b/tests/unit/data/colt/colt1
@@ -21,7 +21,7 @@ ORGANIZER:[email protected]
 LOCATION:Colt Internal Maintenance
 PRIORITY:5
 SEQUENCE:0
-SUMMARY;LANGUAGE=en-us:Update: Colt Service Affecting Maintenance Notification - CRQ1-12345678 [06/8/2021 21:00:00 GMT - 07/8/2021 05:00:00 GMT] for ACME, 12345000
+SUMMARY;LANGUAGE=en-us:[ EXTERNAL ] Colt Service Affecting Maintenance Notification - CRQ1-12345678 [06/8/2021 22:00:00 GMT - 07/8/2021 06:00:00 GMT] for ACME,  12345000
 TRANSP:OPAQUE
 UID:MS00NDE1Mjc5NDc3NTM4ODE4MTFJbXBhY3RlZA==
 X-ALT-DESC;FMTTYPE=text/html:<html xmlns:v='urn:schemas-microsoft-com:vml'  xmlns:o='urn:schemas-microsoft-com:office:office' xmlns:w='urn:schemas-mic rosoft-com:office:word' xmlns:m='http://schemas.microsoft.com/office/2004/12/omml' xmlns='http://www.w3.org/TR/REC-html40'><head><meta name=ProgId content=Word.Document><meta name=Generator content='Microsoft Word 15'><meta name=Originator content='Microsoft Word 15'><link rel=File-List href='cid:[email protected]'> </head> <body lang=EN-US link='#0563C1' vlink='#954F72' style='tab-interval:.5in'><table width="100%" border="0" cellspacing="0" cellpadding="0"><tr><td align="center"><span style="font-size:12pt;font-family:'Calibri',sans-serif;color:#C00000;text-align:center"><strong>This is a system generated calendar invite. Accepting or Declining is meant only for the purpose of a Calendar display.  The timing of the invite is set to the user’s local system time and should not to be confused with the text displayed withi

diff --git a/tests/unit/data/colt/colt1_result.json b/tests/unit/data/colt/colt1_result.json
@@ -7,6 +7,6 @@
     "stamp": 1627653788,
     "start": 1628283600,
     "status": "CONFIRMED",
-    "summary": "Update: Colt Service Affecting Maintenance Notification - CRQ1-12345678 [06/8/2021 21:00:00 GMT - 07/8/2021 05:00:00 GMT] for ACME, 12345000"
+    "summary": "[ EXTERNAL ] Colt Service Affecting Maintenance Notification - CRQ1-12345678 [06/8/2021 22:00:00 GMT - 07/8/2021 06:00:00 GMT] for ACME,  12345000"
   }
 ]
diff --git a/tests/unit/data/colt/colt2_result.json b/tests/unit/data/colt/colt2_result.json
@@ -1,5 +1,6 @@
 [
   {
+    "account": "123456",    
     "circuits": [
       {
         "impact": "OUTAGE",

diff --git a/tests/unit/data/colt/colt3.eml b/tests/unit/data/colt/colt3.eml
@@ -1,8 +1,7 @@
 MIME-Version: 1.0
 Date: Sat, 4 Sep 2021 14:02:52 +0100
 Message-ID: <CAP1aGUfh2mCXB=[email protected]>
-Subject:  EXTERNAL ] Colt Service Affecting Maintenance Notification -
-  CRQ1-12345678 [06/8/2021 22:00:00 GMT - 07/8/2021 06:00:00 GMT] for ACME,  12345000
+Subject:  [ EXTERNAL ] Colt Service Affecting Maintenance Notification - CRQ1-12345678 [06/8/2021 22:00:00 GMT - 07/8/2021 06:00:00 GMT] for ACME,  12345000
 From: Maintenance Request <[email protected]>
 To: Maintenance Request <[email protected]>
 Content-Type: multipart/mixed; boundary="000000000000a43c2305cb2b086c"

diff --git a/tests/unit/data/colt/colt3_result.json b/tests/unit/data/colt/colt3_result.json
@@ -1,18 +1,18 @@
 [
   {
-    "account": "12345000",
-    "end": 1628312400,
+    "account": "123456",
+    "end": 1625724000,
     "maintenance_id": "CRQ1-12345678",
     "circuits": [
       {
         "impact": "OUTAGE",
         "circuit_id": "C-1234567"
       }
     ],
-    "sequence": 0,
-    "stamp": 1627653788,
-    "start": 1628283600,
+    "sequence": 1,
+    "stamp": 1630760572,
+    "start": 1623189600,
     "status": "CONFIRMED",
-    "summary": "Update: Colt Service Affecting Maintenance Notification - CRQ1-12345678 [06/8/2021 21:00:00 GMT - 07/8/2021 05:00:00 GMT] for ACME, 12345000"
+    "summary": "[ EXTERNAL ] Colt Service Affecting Maintenance Notification - CRQ1-12345678 [06/8/2021 22:00:00 GMT - 07/8/2021 06:00:00 GMT] for ACME,  12345000"
   }
 ]
diff --git a/tests/unit/data/colt/colt4.eml b/tests/unit/data/colt/colt4.eml
@@ -0,0 +1,119 @@
+MIME-Version: 1.0
+Date: Mon, 1 Nov 2021 11:51:41 +0000
+Subject: [ EXTERNAL ] MAINTENANCE ALERT: CRQ1-12345678 31/10/2021 00:00:00 GMT - 31/10/2021 07:30:00 GMT - COMPLETED
+From: Maintenance Request <[email protected]>
+To: Maintenance Request <[email protected]>
+Content-Type: multipart/mixed; boundary="000000000000e8c2b105cfb8cc38"
+
+--000000000000e8c2b105cfb8cc38
+Content-Type: multipart/alternative; boundary="000000000000e8c2ae05cfb8cc36"
+
+--000000000000e8c2ae05cfb8cc36
+Content-Type: text/plain; charset="UTF-8"
+
+[ EXTERNAL ]
+
+Dear Sir/Madam
+
+Colt confirms that a planned maintenance which your organization has
+previously been notified under reference *CRQ1-12345678 *with start
+time *31/10/2021
+00:00:00 GMT* has been completed.
+
+For more information about the activity, including a list of the affected
+service(s) please log into your Colt online account:
+https://prodidm.colt.net/
+
+Should you ascertain that you are experiencing any connection issues after
+the works, please contact our technical service desk to investigate the
+matter https://www.colt.net/support/
+
+*This mail has been automatically generated, please do not reply it.*
+
+
+Kind regards
+Colt change management
+
+[Colt Disclaimer] This email is from an entity of the Colt group of
+companies. Colt Group Holdings Limited, Colt House, 20 Great Eastern
+Street, London, EC2A 3EH, United Kingdom, registered in England and Wales,
+under company number 11530966. Corporate and contact information for our
+entities can be found at https://www.colt.net/legal/colt-group-of-companies/.
+Internet communications are not secure and Colt does not accept
+responsibility for the accurate transmission of this message. Content of
+this email or its attachments is not legally or contractually binding
+unless expressly previously agreed in writing by Colt
+
+--000000000000e8c2ae05cfb8cc36
+Content-Type: text/html; charset="UTF-8"
+Content-Transfer-Encoding: quoted-printable
+
+<div dir=3D"ltr">[ EXTERNAL ] <div><p class=3D"gmail-x_MsoNormal" style=3D"=
+margin:0cm 0cm 0pt"><span style=3D"font-family:arial,sans-serif"><span styl=
+e=3D"font-size:10pt">Dear Sir/Madam</span></span><br>=C2=A0<br><span style=
+=3D"font-family:arial,sans-serif"><span style=3D"font-size:10pt">Colt confi=
+rms that a planned maintenance which your organization has previously been =
+notified under reference=C2=A0</span></span><strong><span style=3D"font-fam=
+ily:arial,sans-serif"><span style=3D"font-size:10pt">CRQ1-12345678 </span><=
+/span></strong><span style=3D"font-family:arial,sans-serif"><span style=3D"=
+font-size:10pt">with start time=C2=A0</span></span><strong><span style=3D"f=
+ont-family:arial,sans-serif"><span style=3D"font-size:10pt">31/10/2021 00:0=
+0:00=C2=A0GMT</span></span></strong><span style=3D"font-family:arial,sans-s=
+erif"><span style=3D"font-size:10pt">=C2=A0has been completed.=C2=A0</span>=
+</span><br>=C2=A0<br><span style=3D"font-family:arial,sans-serif"><span sty=
+le=3D"font-size:10pt">For more information about the activity, including a =
+list of the affected service(s) please log into your Colt online account:=
+=C2=A0<a href=3D"https://prodidm.colt.net/" target=3D"_blank" rel=3D"noopen=
+er noreferrer">https://prodidm.colt.net/</a></span></span><br>=C2=A0<br><sp=
+an style=3D"font-family:arial,sans-serif"><span style=3D"font-size:10pt">Sh=
+ould
+ you ascertain that you are experiencing any connection issues after the
+ works, please contact our technical service desk to investigate the=20
+matter<span style=3D"color:rgb(112,48,160)">=C2=A0<a href=3D"https://www.co=
+lt.net/support/" target=3D"_blank" rel=3D"noopener noreferrer"><span style=
+=3D"color:rgb(112,48,160)">https://www.colt.net/support/</span></a></span><=
+/span></span><br>=C2=A0<br><strong><em><span style=3D"font-family:arial,san=
+s-serif"><span style=3D"font-size:10pt">This mail has been automatically ge=
+nerated, please do not reply it.</span></span></em></strong><br>=C2=A0<br>=
+=C2=A0<br><span style=3D"font-family:arial,sans-serif"><span style=3D"font-=
+size:10pt">Kind regards</span></span><br><span style=3D"font-family:arial,s=
+ans-serif"><span style=3D"font-size:10pt">Colt change management</span></sp=
+an><br>=C2=A0<span style=3D"font-family:&quot;Arial&quot;,&quot;sans-serif&=
+quot;;font-size:10pt"></span></p>[Colt
+ Disclaimer] This email is from an entity of the Colt group of=20
+companies. Colt Group Holdings Limited, Colt House, 20 Great Eastern=20
+Street, London, EC2A 3EH, United Kingdom, registered in England and=20
+Wales, under company number 11530966. Corporate and contact information=20
+for our entities can be found at=20
+<a href=3D"https://www.colt.net/legal/colt-group-of-companies/">https://www=
+.colt.net/legal/colt-group-of-companies/</a>. Internet=20
+communications are not secure and Colt does not accept responsibility=20
+for the accurate transmission of this message. Content of this email or=20
+its attachments is not legally or contractually binding unless expressly
+ previously agreed in writing by Colt </div></div>
+
+--000000000000e8c2ae05cfb8cc36--
+--000000000000e8c2b105cfb8cc38
+Content-Type: text/csv; charset="UTF-16LE"; name="colt2.csv"
+Content-Disposition: attachment; filename="colt2.csv"
+Content-Transfer-Encoding: base64
+X-Attachment-Id: f_kvglq75m0
+Content-ID: <f_kvglq75m0>
+
+//5PAEMATgAJAEwAZQBnAGEAbAAgAEMAdQBzAHQAbwBtAGUAcgAJAE8AcgBkAGUAcgAgAE4AdQBt
+AGIAZQByAAkAQwBpAHIAYwB1AGkAdAAgAEkARAAJAEMAdQBzAHQAbwBtAGUAcgAgAFIAZQBmACAA
+MQAJAEMAdQBzAHQAbwBtAGUAcgAgAFIAZQBmACAAMgAJAFMAZQByAHYAaQBjAGUACQBBACAAQwB1
+AHMAdABvAG0AZQByAAkAQQAgAEEAZABkAHIAZQBzAHMACQBBACAAUABvAHMAdABjAG8AZABlAAkA
+QQAgAFQAbwB3AG4AIABDAGkAdAB5AAkAQgAgAEMAdQBzAHQAbwBtAGUAcgAJAEIAIABBAGQAZABy
+AGUAcwBzAAkAQgAgAFAAbwBzAHQAYwBvAGQAZQAJAEIAIABUAG8AdwBuACAAQwBpAHQAeQAJAA0A
+CgBPAEMATgA6ACAAMQAyADMANAA1ADYACQBBAEMATQBFACAARQBVAFIATwBQAEUAIABTAEEACQA5
+ADgANwA2ADUANAAzADIAMQAvADEAMgAzADQANQAtADYANwA4ADkACQBDAC0AMQAyADMANAA1ADYA
+NwAJAAkAQgBlAGwAZwBpAHUAbQAgAC0AIABCAHIAdQBzAHMAZQBsAHMACQBJAFAAIABBAEMAQwBF
+AFMAUwA6ACAAMQAgAEcAQgBQAFMAOwAgAFUATgBQAFIATwBUAEUAQwBUAEUARAA7ACAATgBPACAA
+UgBFAFMASQBMAEkARQBOAEMARQA7ACAATgBPACAAQwBPAEwAVAAgAFIATwBVAFQARQBSADsAIABG
+AEwAQQBUACAAUgBBAFQARQAgAEIASQBMAEwASQBOAEcAOwAgADMAUgBEACAAUABBAFIAVABZACAA
+TABFAEEAUwBFAEQAIABMAEkATgBFADsAIABFAFQASABFAFIATgBFAFQAOwAgADEAMAAwADAAQgBB
+AFMARQAtAFQAOwAgAFIASgA0ADUACQBBAEMATQBFACAARQBVAFIATwBQAEUAIABTAEEACQBNAEEA
+SQBOACAAUwBUAFIARQBFAFQACQAxADIAMwA0AAkAQgBSAFUAUwBTAEUATABTAAkAQwBvAGwAdAAJ
+AAkACQAJAA==
+--000000000000e8c2b105cfb8cc38--
diff --git a/tests/unit/data/colt/colt4_result.json b/tests/unit/data/colt/colt4_result.json
@@ -0,0 +1,18 @@
+[
+  {
+    "account": "123456",
+    "end": 1635665400,
+    "maintenance_id": "CRQ1-12345678",
+    "circuits": [
+      {
+        "impact": "OUTAGE",
+        "circuit_id": "C-1234567"
+      }
+    ],
+    "sequence": 1,
+    "stamp": 1635767501,
+    "start": 1635638400,
+    "status": "COMPLETED",
+    "summary": "[ EXTERNAL ] MAINTENANCE ALERT: CRQ1-12345678 31/10/2021 00:00:00 GMT - 31/10/2021 07:30:00 GMT - COMPLETED"
+  }
+]
diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py
@@ -91,6 +91,11 @@
             [("email", Path(dir_path, "data", "colt", "colt3.eml")),],
             [Path(dir_path, "data", "colt", "colt3_result.json"),],
         ),
+        (
+            Colt,
+            [("email", Path(dir_path, "data", "colt", "colt4.eml")),],
+            [Path(dir_path, "data", "colt", "colt4_result.json"),],
+        ),
         # Equinix
         (
             Equinix,

diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py
@@ -11,7 +11,7 @@
 from circuit_maintenance_parser.parsers.aws import SubjectParserAWS1, TextParserAWS1
 
 # from circuit_maintenance_parser.parsers.cogent import HtmlParserCogent1
-from circuit_maintenance_parser.parsers.colt import ICalParserColt1, CsvParserColt1
+from circuit_maintenance_parser.parsers.colt import CsvParserColt1
 from circuit_maintenance_parser.parsers.equinix import HtmlParserEquinix, SubjectParserEquinix
 from circuit_maintenance_parser.parsers.gtt import HtmlParserGTT1
 from circuit_maintenance_parser.parsers.hgc import HtmlParserHGC1, HtmlParserHGC2
@@ -87,11 +87,6 @@
         #     Path(dir_path, "data", "cogent", "cogent2_result.json"),
         # ),
         # Colt
-        (
-            ICalParserColt1,
-            Path(dir_path, "data", "colt", "colt1"),
-            Path(dir_path, "data", "colt", "colt1_result.json"),
-        ),
         (
             CsvParserColt1,
             Path(dir_path, "data", "colt", "colt2.csv"),