Fix tutorial workflows (#5091)

* Fix broken templates in tutorial map file * Switch from `urllib3` to `requests` for tutorials
cylc · Aug 25, 2022 · 00e1a6c · 00e1a6c
1 parent 026ce5c
commit 00e1a6c
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 102 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -28,9 +28,8 @@ creating a new release entry be sure to copy & paste the span tag with the
 `actions:bind` attribute, which is used by a regex to find the text to be
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
-
 -------------------------------------------------------------------------------
-## __cylc-8.0.2 (<span actions:bind='release-date'>Released YYYY-MM-DD</span>)__
+## __cylc-8.0.2 (<span actions:bind='release-date'>Pending YYYY-MM-DD</span>)__
 
 Maintenance release.
 
@@ -39,16 +38,12 @@ Maintenance release.
 [#5067](https://github.com/cylc/cylc-flow/pull/5067) - Datastore fix for
 taskdefs removed before restart.
 
--------------------------------------------------------------------------------
-## __cylc-8.0.2 (<span actions:bind='release-date'>Pending YYYY-MM-DD</span>)__
-
-Maintenance release.
-
-### Fixes
-
 [#5066](https://github.com/cylc/cylc-flow/pull/5066) - Fix bug where
 .cylcignore only found if `cylc install` is run in source directory.
 
+[#5091](https://github.com/cylc/cylc-flow/pull/5091) - Fix problems with
+tutorial workflows.
+
 -------------------------------------------------------------------------------
 ## __cylc-8.0.1 (<span actions:bind='release-date'>Released 2022-08-16</span>)__
 

diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/get-observations b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/get-observations
@@ -39,7 +39,6 @@ import os
 import re
 
 import requests
-import urllib3
 
 
 BASE_URL = ('http://datapoint.metoffice.gov.uk/public/data/'
@@ -76,31 +75,20 @@ def get_datapoint_data(site_id, time, api_key):
     """Get weather data from the DataPoint service."""
     # The URL required to get the data.
     time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime('%Y-%m-%dT%H:%MZ')
-    url = BASE_URL.format(time=time, site_id=site_id,
-                          api_key=api_key)
-
-    # HTTP header to prevent 403 error:
-    hdr = {'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 '
-                          '(KHTML, like Gecko) Chrome/23.0.1271.64 '
-                          'Safari/537.11'),
-           'Accept': ('text/html,application/xhtml+xml,application/xml;'
-                      'q=0.9,*/*;q=0.8'),
-           'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
-           'Accept-Encoding': 'none',
-           'Accept-Language': 'en-US,en;q=0.8',
-           'Connection': 'keep-alive'}
-    http = urllib3.PoolManager()
-    req = http.request('GET', url, headers=hdr)
-    if req.status != 200:
-        raise Exception(f'{url} returned exit code {req.status}')
+    url = BASE_URL.format(time=time, site_id=site_id, api_key=api_key)
+    req = requests.get(url)
+    if req.status_code != 200:
+        raise Exception(f'{url} returned exit code {req.status_code}')
     # Get the data and parse it as JSON.
     print('Opening URL: %s' % url)
-    return json.loads(req.data)['SiteRep']['DV']['Location']
+    return req.json()['SiteRep']['DV']['Location']
 
 
 def get_archived_data(site_id, time):
     """Return archived weather data from the workflow directory."""
-    print([os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json'])
+    print(
+        [os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json']
+    )
     path = os.path.join(
         os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json')
     print('Opening File: %s' % path)
@@ -117,7 +105,7 @@ def extract_observations(data):
 
 def reference_lat_long(site_id):
     """Extract lat-long from a reference file."""
-    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
+    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR'], 'etc/met-office-sites.dat')
     info = Path(src).read_text()
     siteinfo = re.findall(f'^{int(site_id):05d}.*', info, re.MULTILINE)
     if not siteinfo:
@@ -177,7 +165,7 @@ def synop_grab(site_id, time):
 def get_nearby_site(site_id, badsites):
     """Use Pythagoras to find the next nearest site."""
     lat, lon = reference_lat_long(site_id)
-    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
+    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR'], 'etc/met-office-sites.dat')
     info = Path(src).read_text()
     dist = 100
     result = 0
@@ -195,7 +183,6 @@ def get_nearby_site(site_id, badsites):
     return int(result[0]), dist
 
 
-
 def main(site_id, api_key=None):
     cycle_point = os.environ['CYLC_TASK_CYCLE_POINT']
 
@@ -222,7 +209,6 @@ def main(site_id, api_key=None):
 
         obs = extract_observations(data)
 
-
     # Write observations.
     with open('wind.csv', 'w+') as data_file:
         data_file.write(', '.join(obs))

diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/get-rainfall b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/get-rainfall
@@ -38,7 +38,7 @@ import math
 import os
 import shutil
 
-import urllib3
+import requests
 
 try:
     from PIL import Image
@@ -126,22 +126,11 @@ def get_datapoint_radar_image(filename, time, api_key):
     time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime(
         '%Y-%m-%dT%H:%M:%SZ')
     url = URL.format(time=time, api_key=api_key)
-    # HTTP header to prevent 403 error:
-    hdr = {'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 '
-                          '(KHTML, like Gecko) Chrome/23.0.1271.64 '
-                          'Safari/537.11'),
-           'Accept': ('text/html,application/xhtml+xml,application/xml;'
-                      'q=0.9,*/*;q=0.8'),
-           'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
-           'Accept-Encoding': 'none',
-           'Accept-Language': 'en-US,en;q=0.8',
-           'Connection': 'keep-alive'}
-    http = urllib3.PoolManager()
-    req = http.request('GET', url, headers=hdr)
-    if req.status != 200:
-        raise Exception(f'{url} returned exit code {req.status}')
-    with open(filename, 'bw+') as png_file:
-        png_file.write(req.data)
+    req = requests.get(url)
+    if req.status_code != 200:
+        raise Exception(f'{url} returned exit code {req.status_code}')
+    with open(filename, 'bw') as png_file:
+        png_file.write(req.content)
 
 
 def get_archived_radar_image(filename, time):

diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/lib/template/map.html b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/lib/template/map.html
@@ -31,11 +31,7 @@
         }
 
         // Forecast data as dict {'lead_time': [2d data matrix]}.
-        var data = {
-            {
-                data
-            }
-        };
+        var data = {{ data }};
 
         // Annotate map with data.
         var lng;
@@ -47,36 +43,12 @@
             rects = L.layerGroup();
             for (let pos_y in data[time]) {
                 for (let pos_x in data[time][0]) {
-                    lng = (pos_x * {
-                        {
-                            resolution
-                        }
-                    }) + {
-                        {
-                            lng1
-                        }
-                    };
-                    lat = {
-                        {
-                            lat2
-                        }
-                    } - (pos_y * {
-                        {
-                            resolution
-                        }
-                    });
+                    lng = (pos_x * {{ resolution }}) + {{ lng1 }};
+                    lat = {{ lat2 }} - (pos_y * {{ resolution }});
                     L.rectangle(
                         [
                             [lat, lng],
-                            [lat + {
-                                {
-                                    resolution
-                                }
-                            }, lng + {
-                                {
-                                    resolution
-                                }
-                            }]
+                            [lat + {{ resolution }}, lng + {{ resolution }}]
                         ], {
                             color: get_colour(data[time][pos_y][pos_x]),
                             weight: 0,
@@ -105,24 +77,8 @@
 
         // Zoom / center map to fit domain.
         map.fitBounds([
-            [{
-                {
-                    lat1
-                }
-            }, {
-                {
-                    lng1
-                }
-            }],
-            [{
-                {
-                    lat2
-                }
-            }, {
-                {
-                    lng2
-                }
-            }]
+            [{{ lat1 }}, {{ lng1 }}],
+            [{{ lat2 }}, {{ lng2 }}]
         ]);
 
         // Add map tiles.
@@ -134,4 +90,4 @@
     </script>
 </body>
 
-</html>
+</html>
diff --git a/setup.cfg b/setup.cfg
@@ -130,7 +130,6 @@ tests =
 tutorials =
     pillow
     requests
-    urllib3
 all =
     %(empy)s
     %(graph)s