Merge pull request #120 from networktocode/gfm-still-more-zayo

Handle yet more Zayo variants

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v2.0.7 - 2021-MM-DD
+
+### Fixed
+
+- #120 - Improve handling of Zayo notifications.
+
 ## v2.0.6 - 2021-11-30
 
 ### Added

circuit_maintenance_parser/parsers/zayo.py

@@ -1,5 +1,6 @@
 """Zayo parser."""
 import logging
+import re
 from typing import Dict
 
 import bs4  # type: ignore
@@ -22,15 +23,19 @@ class SubjectParserZayo1(EmailSubjectParser):
              END OF WINDOW NOTIFICATION***Customer Inc.***ZAYO TTN-0000123456 Planned***
              ***Customer Inc***ZAYO TTN-0001234567 Emergency MAINTENANCE NOTIFICATION***
              RESCHEDULE NOTIFICATION***Customer Inc***ZAYO TTN-0005423873 Planned***
+
+    Some degenerate examples have been seen as well:
+            [notices] CANCELLED NOTIFICATION***Customer,inc***ZAYO TTN-0005432100 Planned**
+            [notices] Rescheduled  Maintenance***ZAYO TTN-0005471719 MAINTENANCE NOTIFICATION***
     """
 
     def parse_subject(self, subject):
         """Parse subject of email message."""
         data = {}
-        tokens = subject.split("***")
+        tokens = re.split(r"\*+", subject)
         if len(tokens) == 4:
             data["account"] = tokens[1]
-            data["maintenance_id"] = tokens[2].split(" ")[1]
+        data["maintenance_id"] = tokens[-2].split(" ")[1]
         return [data]
 
 
@@ -48,7 +53,7 @@ def parse_html(self, soup):
                 text = soup.get_text()
                 if "will be commencing momentarily" in text:
                     data["status"] = Status("IN-PROCESS")
-                elif "has been completed" in text:
+                elif "has been completed" in text or "has closed" in text:
                     data["status"] = Status("COMPLETED")
 
         return [data]

circuit_maintenance_parser/provider.py

@@ -335,6 +335,11 @@ class Verizon(GenericProvider):
 class Zayo(GenericProvider):
     """Zayo provider custom class."""
 
+    _include_filter = {
+        "text/html": ["Maintenance Ticket #"],
+        "html": ["Maintenance Ticket #"],
+    }
+
     _processors: List[GenericProcessor] = [
         CombinedProcessor(data_parsers=[EmailDateParser, SubjectParserZayo1, HtmlParserZayo1]),
     ]

tests/unit/data/zayo/zayo7_html_parser_result.json

@@ -0,0 +1,15 @@
+[
+    {
+        "circuits": [
+            {
+                "circuit_id": "/IPYX/100722/   /ZYO /",
+                "impact": "OUTAGE"
+            }
+        ],
+        "end": 1637067600,
+        "maintenance_id": "TTN-0005432100",
+        "start": 1636876860,
+        "status": "COMPLETED",
+        "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages"
+    }
+]

tests/unit/data/zayo/zayo7_result.json

@@ -0,0 +1,17 @@
+[
+    {
+        "account": "Example Inc.",
+        "circuits": [
+            {
+                "circuit_id": "/IPYX/100722/   /ZYO /",
+                "impact": "OUTAGE"
+            }
+        ],
+        "end": 1637067600,
+        "maintenance_id": "TTN-0005432100",
+        "stamp": 1637068032,
+        "start": 1636876860,
+        "status": "COMPLETED",
+        "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages"
+    }
+]

tests/unit/data/zayo/zayo7_subject_parser_result.json

@@ -0,0 +1,6 @@
+[
+    {
+        "account": "Example Inc.",
+        "maintenance_id": "TTN-0005432100"
+    }
+]

tests/unit/data/zayo/zayo_bad_html.html

@@ -1 +1 @@
-aaa
+Maintenance Ticket #: aaa

tests/unit/data/zayo/zayo_missing_maintenance_id.html

@@ -12,7 +12,12 @@
 0=C2=A0</span><br style=3D"color:rgb(0,0,0)"><br style=3D"color:rgb(0,0,0)"=
 ><b style=3D"color:rgb(0,0,0)">Customer:=C2=A0</b><span style=3D"color:rgb(=
 0,0,0)">clientX=C2=A0</span><br style=3D"color:rgb(0,0,0)"><br style=3D"colo=
-r:rgb(0,0,0)"><b style=3D"color:rgb(0,0,0)">Maintenance Window=C2=A0</b><br=
+r:rgb(0,0,0)">
+<b sty=
+le=3D"color:rgb(0,0,0)">Maintenance Ticket #:=C2=A0</b><span style=3D"co=
+lor:rgb(0,0,0)"></span><br style=3D"c=
+olor:rgb(0,0,0)"><br style=3D"color:rgb(0,0,0)">
+<b style=3D"color:rgb(0,0,0)">Maintenance Window=C2=A0</b><br=
  style=3D"color:rgb(0,0,0)"><br style=3D"color:rgb(0,0,0)"><b style=3D"colo=
 r:rgb(0,0,0)">1<sup>st</sup>=C2=A0Activity Date=C2=A0</b><br style=3D"color=
 :rgb(0,0,0)"><span style=3D"color:rgb(0,0,0)">25-Sep-2020 00:01 to 25-Sep-2=

tests/unit/data/zayo/zayo_subject_1.txt

@@ -0,0 +1,2 @@
+Subject: [maint-notices] CANCELLED NOTIFICATION***Some Customer,inc***ZAYO
+ TTN-0005432100 Planned**

tests/unit/data/zayo/zayo_subject_1_result.json

@@ -0,0 +1,6 @@
+[
+    {
+        "account": "Some Customer,inc",
+        "maintenance_id": "TTN-0005432100"
+    }
+]

tests/unit/data/zayo/zayo_subject_2.txt

@@ -0,0 +1,2 @@
+Subject: [maint-notices] Rescheduled  Maintenance***ZAYO TTN-0005432100
+ MAINTENANCE NOTIFICATION***

tests/unit/data/zayo/zayo_subject_2_result.json

@@ -0,0 +1,5 @@
+[
+    {
+        "maintenance_id": "TTN-0005432100"
+    }
+]

tests/unit/test_e2e.py

@@ -471,6 +471,11 @@
             Zayo,
             [("email", Path(dir_path, "data", "zayo", "zayo6.eml")),],
             [Path(dir_path, "data", "zayo", "zayo6_result.json"),],
+        ),
+        (
+            Zayo,
+            [("email", Path(dir_path, "data", "zayo", "zayo7.eml")),],
+            [Path(dir_path, "data", "zayo", "zayo7_result.json"),],
         ),  # pylint: disable=too-many-locals
     ],
 )
@@ -596,7 +601,7 @@ def test_provider_get_maintenances(provider_class, test_data_files, result_parse
 Details:
 - Processor CombinedProcessor from Zayo failed due to: 1 validation error for Maintenance
 maintenance_id
-  field required (type=value_error.missing)
+  String is empty or 'None' (type=value_error)
 """,
         ),
         (
@@ -608,7 +613,7 @@ def test_provider_get_maintenances(provider_class, test_data_files, result_parse
 Failed creating Maintenance notification for Zayo.
 Details:
 - Processor CombinedProcessor from Zayo failed due to: HtmlParserZayo1 parser was not able to extract the expected data for each maintenance.
-  - Raw content: b'aaa'
+  - Raw content: b'Maintenance Ticket #: aaa\\n'
   - Result: [{}]
 """,
         ),

tests/unit/test_parsers.py

@@ -296,6 +296,16 @@
             Path(dir_path, "data", "verizon", "verizon3_result.json"),
         ),
         # Zayo
+        (
+            SubjectParserZayo1,
+            Path(dir_path, "data", "zayo", "zayo_subject_1.txt"),
+            Path(dir_path, "data", "zayo", "zayo_subject_1_result.json"),
+        ),
+        (
+            SubjectParserZayo1,
+            Path(dir_path, "data", "zayo", "zayo_subject_2.txt"),
+            Path(dir_path, "data", "zayo", "zayo_subject_2_result.json"),
+        ),
         (
             HtmlParserZayo1,
             Path(dir_path, "data", "zayo", "zayo1.html"),
@@ -341,6 +351,16 @@
             Path(dir_path, "data", "zayo", "zayo6.eml"),
             Path(dir_path, "data", "zayo", "zayo6_subject_parser_result.json"),
         ),
+        (
+            HtmlParserZayo1,
+            Path(dir_path, "data", "zayo", "zayo7.eml"),
+            Path(dir_path, "data", "zayo", "zayo7_html_parser_result.json"),
+        ),
+        (
+            SubjectParserZayo1,
+            Path(dir_path, "data", "zayo", "zayo7.eml"),
+            Path(dir_path, "data", "zayo", "zayo7_subject_parser_result.json"),
+        ),
         # Email Date
         (
             EmailDateParser,