Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix tutorial workflows #5091

Merged
merged 3 commits into from
Aug 25, 2022
Merged

Fix tutorial workflows #5091

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
101f044
Fix broken templates in tutorial map file
MetRonnie Aug 18, 2022
85daea4
Switch from `urllib3` to `requests` for tutorials
MetRonnie Aug 19, 2022
75b3632
Update changelog
MetRonnie Aug 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 4 additions & 9 deletions CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ creating a new release entry be sure to copy & paste the span tag with the
`actions:bind` attribute, which is used by a regex to find the text to be
updated. Only the first match gets replaced, so it's fine to leave the old
ones in. -->

-------------------------------------------------------------------------------
## __cylc-8.0.2 (<span actions:bind='release-date'>Released YYYY-MM-DD</span>)__
## __cylc-8.0.2 (<span actions:bind='release-date'>Pending YYYY-MM-DD</span>)__

Maintenance release.

Expand All @@ -39,16 +38,12 @@ Maintenance release.
[#5067](https://github.com/cylc/cylc-flow/pull/5067) - Datastore fix for
taskdefs removed before restart.

-------------------------------------------------------------------------------
## __cylc-8.0.2 (<span actions:bind='release-date'>Pending YYYY-MM-DD</span>)__

Maintenance release.

### Fixes

[#5066](https://github.com/cylc/cylc-flow/pull/5066) - Fix bug where
.cylcignore only found if `cylc install` is run in source directory.

[#5091](https://github.com/cylc/cylc-flow/pull/5091) - Fix problems with
tutorial workflows.

-------------------------------------------------------------------------------
## __cylc-8.0.1 (<span actions:bind='release-date'>Released 2022-08-16</span>)__

Expand Down
34 changes: 10 additions & 24 deletions cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/get-observations
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ import os
import re

import requests
import urllib3


BASE_URL = ('http://datapoint.metoffice.gov.uk/public/data/'
Expand Down Expand Up @@ -76,31 +75,20 @@ def get_datapoint_data(site_id, time, api_key):
"""Get weather data from the DataPoint service."""
# The URL required to get the data.
time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime('%Y-%m-%dT%H:%MZ')
url = BASE_URL.format(time=time, site_id=site_id,
api_key=api_key)

# HTTP header to prevent 403 error:
hdr = {'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 '
'(KHTML, like Gecko) Chrome/23.0.1271.64 '
'Safari/537.11'),
'Accept': ('text/html,application/xhtml+xml,application/xml;'
'q=0.9,*/*;q=0.8'),
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}
http = urllib3.PoolManager()
req = http.request('GET', url, headers=hdr)
if req.status != 200:
raise Exception(f'{url} returned exit code {req.status}')
url = BASE_URL.format(time=time, site_id=site_id, api_key=api_key)
req = requests.get(url)
if req.status_code != 200:
raise Exception(f'{url} returned exit code {req.status_code}')
# Get the data and parse it as JSON.
print('Opening URL: %s' % url)
return json.loads(req.data)['SiteRep']['DV']['Location']
return req.json()['SiteRep']['DV']['Location']


def get_archived_data(site_id, time):
"""Return archived weather data from the workflow directory."""
print([os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json'])
print(
[os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json']
)
path = os.path.join(
os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json')
print('Opening File: %s' % path)
Expand All @@ -117,7 +105,7 @@ def extract_observations(data):

def reference_lat_long(site_id):
"""Extract lat-long from a reference file."""
src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR'], 'etc/met-office-sites.dat')
info = Path(src).read_text()
siteinfo = re.findall(f'^{int(site_id):05d}.*', info, re.MULTILINE)
if not siteinfo:
Expand Down Expand Up @@ -177,7 +165,7 @@ def synop_grab(site_id, time):
def get_nearby_site(site_id, badsites):
"""Use Pythagoras to find the next nearest site."""
lat, lon = reference_lat_long(site_id)
src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR'], 'etc/met-office-sites.dat')
info = Path(src).read_text()
dist = 100
result = 0
Expand All @@ -195,7 +183,6 @@ def get_nearby_site(site_id, badsites):
return int(result[0]), dist



def main(site_id, api_key=None):
cycle_point = os.environ['CYLC_TASK_CYCLE_POINT']

Expand All @@ -222,7 +209,6 @@ def main(site_id, api_key=None):

obs = extract_observations(data)


# Write observations.
with open('wind.csv', 'w+') as data_file:
data_file.write(', '.join(obs))
Expand Down
23 changes: 6 additions & 17 deletions cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/get-rainfall
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import math
import os
import shutil

import urllib3
import requests

try:
from PIL import Image
Expand Down Expand Up @@ -126,22 +126,11 @@ def get_datapoint_radar_image(filename, time, api_key):
time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime(
'%Y-%m-%dT%H:%M:%SZ')
url = URL.format(time=time, api_key=api_key)
# HTTP header to prevent 403 error:
hdr = {'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 '
'(KHTML, like Gecko) Chrome/23.0.1271.64 '
'Safari/537.11'),
'Accept': ('text/html,application/xhtml+xml,application/xml;'
'q=0.9,*/*;q=0.8'),
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}
http = urllib3.PoolManager()
req = http.request('GET', url, headers=hdr)
if req.status != 200:
raise Exception(f'{url} returned exit code {req.status}')
with open(filename, 'bw+') as png_file:
png_file.write(req.data)
req = requests.get(url)
if req.status_code != 200:
raise Exception(f'{url} returned exit code {req.status_code}')
with open(filename, 'bw') as png_file:
png_file.write(req.content)


def get_archived_radar_image(filename, time):
Expand Down
58 changes: 7 additions & 51 deletions cylc/flow/etc/tutorial/cylc-forecasting-workflow/lib/template/map.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,7 @@
}

// Forecast data as dict {'lead_time': [2d data matrix]}.
var data = {
{
data
}
};
var data = {{ data }};

// Annotate map with data.
var lng;
Expand All @@ -47,36 +43,12 @@
rects = L.layerGroup();
for (let pos_y in data[time]) {
for (let pos_x in data[time][0]) {
lng = (pos_x * {
{
resolution
}
}) + {
{
lng1
}
};
lat = {
{
lat2
}
} - (pos_y * {
{
resolution
}
});
lng = (pos_x * {{ resolution }}) + {{ lng1 }};
lat = {{ lat2 }} - (pos_y * {{ resolution }});
L.rectangle(
[
[lat, lng],
[lat + {
{
resolution
}
}, lng + {
{
resolution
}
}]
[lat + {{ resolution }}, lng + {{ resolution }}]
], {
color: get_colour(data[time][pos_y][pos_x]),
weight: 0,
Expand Down Expand Up @@ -105,24 +77,8 @@

// Zoom / center map to fit domain.
map.fitBounds([
[{
{
lat1
}
}, {
{
lng1
}
}],
[{
{
lat2
}
}, {
{
lng2
}
}]
[{{ lat1 }}, {{ lng1 }}],
[{{ lat2 }}, {{ lng2 }}]
]);

// Add map tiles.
Expand All @@ -134,4 +90,4 @@
</script>
</body>

</html>
</html>
1 change: 0 additions & 1 deletion setup.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,6 @@ tests =
tutorials =
pillow
requests
urllib3
all =
%(empy)s
%(graph)s
Expand Down