Merge branch 'develop' into kc_aquacomms

CHANGELOG.md

@@ -27,6 +27,7 @@
 - #60 - Added new provider `Seaborn` using `Html` and a new parser for Email Subject: `EmailSubjectParser`
 - #61 - Added new provider `Colt` using `ICal` and `Csv`
 - #66 - Added new provider `Momentum` using `Html` and `EmailSubjectParser`
+- #73 - Added new provider `Sparkle` using `Html` and `EmailSubjectParser`. Added support for multiple maintenances with `CombinedProcessor`.
 - #75 - Added new provider `AquaComms` using `Html` and `EmailSubjectParser`
 
 ### Fixed

README.md

@@ -52,6 +52,7 @@ By default, there is a `GenericProvider` that support a `SimpleProcessor` using
 - Megaport
 - Momentum
 - Seaborn
+- Sparkle
 - Telstra
 - Turkcell
 - Verizon

circuit_maintenance_parser/__init__.py

@@ -18,6 +18,7 @@
     NTT,
     PacketFabric,
     Seaborn,
+    Sparkle,
     Telia,
     Telstra,
     Turkcell,
@@ -39,6 +40,7 @@
     NTT,
     PacketFabric,
     Seaborn,
+    Sparkle,
     Telia,
     Telstra,
     Turkcell,

circuit_maintenance_parser/parsers/sparkle.py

@@ -0,0 +1,94 @@
+"""Sparkle parser."""
+import logging
+from dateutil import parser
+
+from circuit_maintenance_parser.errors import ParserError
+from circuit_maintenance_parser.parser import CircuitImpact, Html, Impact, Status
+
+logger = logging.getLogger(__name__)
+
+
+class HtmlParserSparkle1(Html):
+    """Notifications HTML Parser 1 for Sparkle notifications.
+
+    Example:
+        <table>
+            <tbody>
+                <tr>
+                    <td><p></p>Maintenance ID</td>
+                    <td><p></p>1111 / 2222</td>
+                </tr>
+                <tr>
+                    <td><p></p>Start Date/Time (UTC) Day 1</td>
+                    <td><p></p>08/10/2021 03:00 UTC</td>
+                </tr>
+                <tr>
+                    <td><p></p>End Date/Time (UTC) Day 1</td>
+                    <td><p></p>08/10/2021 11:00 UTC</td>
+                </tr>
+                <tr>
+                    <td><p></p>Start Date/Time (UTC) Day 2</td>
+                    <td><p></p>08/11/2021 03:00 UTC</td>
+                </tr>
+                <tr>
+                    <td><p></p>End Date/Time (UTC) Day 2</td>
+                    <td><p></p>08/11/2021 11:00 UTC</td>
+                </tr>
+                ...
+            </tbody>
+        </table>
+    """
+
+    def parse_html(self, soup):
+        """Execute parsing."""
+        data = {}
+        try:
+            return self.parse_tables(soup.find_all("table"), data)
+        except Exception as exc:
+            raise ParserError from exc
+
+    def clean_string(self, string):
+        """Remove hex characters and new lines."""
+        return self.remove_hex_characters(string.replace("\n", "")).strip()
+
+    @staticmethod
+    def set_all_tickets(tickets, attribute, value):
+        """Set the same value for all notifications."""
+        for ticket in tickets:
+            ticket[attribute] = value
+
+    def parse_tables(self, tables, data_base):
+        """Parse HTML tables."""
+        data = []
+        for table in tables:
+            tr_elements = table.find_all("tr")
+            for idx, tr_element in enumerate(tr_elements):
+                td_elements = tr_element.find_all("td")
+                if "sparkle ticket number" in td_elements[0].text.lower():
+                    tickets = self.clean_string(td_elements[1].text).split("/ ")
+                    for ticket_id in tickets:
+                        ticket = data_base.copy()
+                        ticket["maintenance_id"] = ticket_id
+                        if "start date/time" in tr_elements[idx + 1].text.lower():
+                            start = self.clean_string(tr_elements[idx + 1].find_all("td")[1].text)
+                            ticket["start"] = self.dt2ts(parser.parse(start))
+                        else:
+                            raise ParserError("Unable to find start time for ticket " + ticket_id)
+                        if "end date/time" in tr_elements[idx + 2].text.lower():
+                            end = self.clean_string(tr_elements[idx + 2].find_all("td")[1].text)
+                            ticket["end"] = self.dt2ts(parser.parse(end))
+                        else:
+                            raise ParserError("Unable to find end time for ticket " + ticket_id)
+                        idx += 2
+                        data.append(ticket)
+                elif "circuits involved" in td_elements[0].text.lower():
+                    self.set_all_tickets(
+                        data,
+                        "circuits",
+                        [CircuitImpact(impact=Impact.OUTAGE, circuit_id=self.clean_line(td_elements[1].text))],
+                    )
+                elif "description of work" in td_elements[0].text.lower():
+                    self.set_all_tickets(data, "summary", self.clean_string(td_elements[1].text))
+        self.set_all_tickets(data, "status", Status.CONFIRMED)
+        self.set_all_tickets(data, "account", "Not Available")
+        return data

circuit_maintenance_parser/processor.py

@@ -116,17 +116,28 @@ class CombinedProcessor(GenericProcessor):
     combined_maintenance_data: Dict = {}
 
     def process_hook(self, maintenances_extracted_data, maintenances_data):
-        """All the parsers contribute with a subset of data that is extended."""
-        # We only expect one data object from these parsers
+        """All the parsers contribute with a subset of data that is extended.
+
+        For some notifications there can be multiple maintenances in a single file. To handle this, maintenances are store in a
+        list where each of them can be extended with the extra processors.
+        """
         if len(maintenances_extracted_data) == 1:
             self.combined_maintenance_data.update(maintenances_extracted_data[0])
         else:
-            raise ProcessorError(f"Unexpected data retrieved from parser: {maintenances_extracted_data}")
+            maintenances_data.extend(maintenances_extracted_data)
 
     def post_process_hook(self, maintenances_data):
         """After processing all the parsers, we try to combine all the data together."""
         self.extend_processor_data(self.combined_maintenance_data)
-        try:
-            maintenances_data.append(Maintenance(**self.combined_maintenance_data))
-        except ValidationError as exc:
-            raise ProcessorError("Not enough information available to create a Maintenance notification.") from exc
+        if not maintenances_data:
+            maintenances = [{}]
+        else:
+            maintenances = maintenances_data.copy()
+            maintenances_data.clear()
+
+        for maintenance in maintenances:
+            try:
+                combined_data = {**self.combined_maintenance_data, **maintenance}
+                maintenances_data.append(Maintenance(**combined_data))
+            except ValidationError as exc:
+                raise ProcessorError("Not enough information available to create a Maintenance notification.") from exc

circuit_maintenance_parser/provider.py

@@ -28,6 +28,7 @@
     SubjectParserSeaborn1,
     SubjectParserSeaborn2,
 )
+from circuit_maintenance_parser.parsers.sparkle import HtmlParserSparkle1
 from circuit_maintenance_parser.parsers.telstra import HtmlParserTelstra1
 from circuit_maintenance_parser.parsers.turkcell import HtmlParserTurkcell1
 from circuit_maintenance_parser.parsers.verizon import HtmlParserVerizon1
@@ -207,6 +208,15 @@ class Seaborn(GenericProvider):
     _default_organizer = "[email protected]"
 
 
+class Sparkle(GenericProvider):
+    """Sparkle provider custom class."""
+
+    _processors: List[GenericProcessor] = [
+        CombinedProcessor(data_parsers=[HtmlParserSparkle1, EmailDateParser]),
+    ]
+    _default_organizer = "[email protected]"
+
+
 class Telia(GenericProvider):
     """Telia provider custom class."""