Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate CLI functionality to plugin layer subset 2 #2003

Merged
merged 46 commits into from
Jul 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
a84a709
Standardise move logic from CLI to plugin
cpelley Nov 13, 2023
1f935d3
Revert to minimal changes
cpelley Apr 29, 2024
5d56669
Merge branch 'master' into using_plugin_framework
cpelley Apr 29, 2024
3647b12
Reverted further changes
cpelley Apr 29, 2024
0103bb5
MAINT: Changes based on feedback
cpelley May 3, 2024
6453967
MAINT: Flake8 changes
cpelley May 3, 2024
85a169c
CLI categorical
cpelley May 17, 2024
ce00db6
cloud-condensation-level CLI
cpelley May 17, 2024
b2c006f
cloud_top_temperature CLI
cpelley May 17, 2024
ac9c8a4
Fixed categorical
cpelley May 17, 2024
2d09920
Fixed cloud_condensation_level
cpelley May 17, 2024
4b5138f
combine CLI
cpelley May 17, 2024
fbe391e
new copy_attributes
cpelley May 17, 2024
b5819d9
Fluffed out some tests
cpelley May 20, 2024
738a71e
formatting
cpelley May 20, 2024
a3a0a73
Merge branch 'using_plugin_framework' into MIGRATE_CLI_FUNCTIONALITY_EPP
cpelley May 20, 2024
562ee52
standardisemetadata
cpelley May 20, 2024
c9b6e3a
MAINT: Changes based on feedback
cpelley May 21, 2024
dbb7a2a
MAINT: Remainder feedback from review
cpelley May 24, 2024
41574cc
Merge remote-tracking branch 'origin/master' into MIGRATE_CLI_FUNCTIO…
cpelley May 24, 2024
380493c
Updated license header to align with master
cpelley May 24, 2024
7826fe3
MAINT: generalised as_cube and as_cubes
cpelley May 29, 2024
a989fed
MAINT: More generalised as_cube still
cpelley May 29, 2024
2164259
MAINT: changes based on review
cpelley Jun 4, 2024
a1185e7
Rename metaplugin class name
cpelley Jun 6, 2024
0a1a855
Corrected type hint
cpelley Jun 7, 2024
bfe6406
improver/cli/extract.py
cpelley May 29, 2024
2733fda
improver/cli/freezing_rain.py
cpelley May 29, 2024
f627dd6
improver/cli/hail_fraction.py
cpelley May 29, 2024
596d2dd
improver/cli/hail_size.py
cpelley May 29, 2024
c79c4c7
improver/cli/interpolate_using_difference.py
cpelley May 29, 2024
357d6fc
improver/cli/lightning_from_cape_and_precip.py
cpelley May 29, 2024
77ce9fa
improver/cli/max_in_height.py
cpelley May 29, 2024
fded8c6
improver/cli/nbhood.py
cpelley May 29, 2024
c5791ff
improver/cli/phase_change_level.py
cpelley May 29, 2024
d705d93
improver/cli/phase_probability.py
cpelley May 29, 2024
2507bec
improver/cli/snow_splitter.py
cpelley May 29, 2024
70a3392
improver/cli/vertical_updraught.py
cpelley May 29, 2024
471fbd6
improver/cli/wet_bulb_freezing_level.py
cpelley May 29, 2024
8f1b5c3
improver/cli/wet_bulb_temperature_integral.py
cpelley May 29, 2024
4d652f9
improver/cli/wet_bulb_temperature.py
cpelley May 29, 2024
1f56e8d
Subset of tests
cpelley Jun 11, 2024
5c9c797
MAINT: changes based on feedback
cpelley Jul 5, 2024
584f4d5
Merge remote-tracking branch 'origin/master' into MIGRATE_CLI_FUNCTIO…
cpelley Jul 5, 2024
31500cb
Merge remote-tracking branch 'origin/master' into MIGRATE_CLI_FUNCTIO…
cpelley Jul 9, 2024
26e791f
MAINT: chages based on feedback
cpelley Jul 11, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions improver/blending/weighted_blend.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from improver.blending.utilities import find_blend_dim_coord, store_record_run_as_coord
from improver.metadata.constants import FLOAT_DTYPE, PERC_COORD
from improver.metadata.forecast_times import rebadge_forecasts_as_latest_cycle
from improver.utilities.complex_conversion import complex_to_deg, deg_to_complex
from improver.utilities.cube_manipulation import (
MergeCubes,
collapsed,
Expand All @@ -29,7 +30,6 @@
get_dim_coord_names,
sort_coord_in_cube,
)
from improver.wind_calculations.wind_direction import WindDirection


class MergeCubesForWeightedBlending(BasePlugin):
Expand Down Expand Up @@ -631,7 +631,7 @@ def weighted_mean(self, cube: Cube, weights: Optional[Cube]) -> Cube:

# If units are degrees, convert degrees to complex numbers.
if cube.units == "degrees":
cube.data = WindDirection.deg_to_complex(cube.data)
cube.data = deg_to_complex(cube.data)

weights_array = self.get_weights_array(cube, weights)

Expand All @@ -658,7 +658,7 @@ def weighted_mean(self, cube: Cube, weights: Optional[Cube]) -> Cube:

# If units are degrees, convert complex numbers back to degrees.
if cube.units == "degrees":
result.data = WindDirection.complex_to_deg(result.data)
result.data = complex_to_deg(result.data)

return result

Expand Down
11 changes: 3 additions & 8 deletions improver/cli/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,8 @@ def process(
A single cube matching the input constraints or None. If no
sub-cube is found within the cube that matches the constraints.
"""
from improver.utilities.cube_extraction import extract_subcube
from improver.utilities.cube_extraction import ExtractSubCube

result = extract_subcube(cube, constraints, units)

if result is None and ignore_failure:
return cube
if result is None:
msg = "Constraint(s) could not be matched in input cube"
raise ValueError(msg)
plugin = ExtractSubCube(constraints, units=units, ignore_failure=ignore_failure)
result = plugin.process(cube)
return result
4 changes: 1 addition & 3 deletions improver/cli/freezing_rain.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ def process(*cubes: cli.inputcube, model_id_attr: str = None):
A cube of freezing rain rate or accumulation probabilities.

"""
from iris.cube import CubeList

from improver.precipitation_type.freezing_rain import FreezingRain

return FreezingRain(model_id_attr=model_id_attr)(CubeList(cubes))
return FreezingRain(model_id_attr=model_id_attr)(*cubes)
30 changes: 1 addition & 29 deletions improver/cli/hail_fraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,34 +27,6 @@ def process(*cubes: cli.inputcubelist, model_id_attr: str = None):
A single cube containing the hail fraction.

"""
from iris.cube import CubeList

from improver.precipitation_type.hail_fraction import HailFraction
from improver.utilities.flatten import flatten

(
vertical_updraught,
hail_size,
cloud_condensation_level,
convective_cloud_top,
hail_melting_level,
altitude,
) = CubeList(flatten(cubes)).extract(
[
"maximum_vertical_updraught",
"diameter_of_hail_stones",
"air_temperature_at_condensation_level",
"air_temperature_at_convective_cloud_top",
"altitude_of_rain_from_hail_falling_level",
"surface_altitude",
]
)

return HailFraction(model_id_attr=model_id_attr)(
vertical_updraught,
hail_size,
cloud_condensation_level,
convective_cloud_top,
hail_melting_level,
altitude,
)
return HailFraction(model_id_attr=model_id_attr)(*cubes)
20 changes: 1 addition & 19 deletions improver/cli/hail_size.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,24 +33,6 @@ def process(*cubes: cli.inputcubelist, model_id_attr: str = None):
iris.cube.Cube:
Cube of diameter_of_hail (m).
"""

from iris.cube import CubeList

from improver.psychrometric_calculations.hail_size import HailSize
from improver.utilities.flatten import flatten

cubes = flatten(cubes)
(temperature, ccl_pressure, ccl_temperature, wet_bulb_zero, orography,) = CubeList(
cubes
).extract(
[
"air_temperature",
"air_pressure_at_condensation_level",
"air_temperature_at_condensation_level",
"wet_bulb_freezing_level_altitude",
"surface_altitude",
]
)
return HailSize(model_id_attr=model_id_attr)(
ccl_temperature, ccl_pressure, temperature, wet_bulb_zero, orography,
)
return HailSize(model_id_attr=model_id_attr)(*cubes)
4 changes: 2 additions & 2 deletions improver/cli/interpolate_using_difference.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def process(
"""
from improver.utilities.interpolation import InterpolateUsingDifference

result = InterpolateUsingDifference()(
cube, reference_cube, limit=limit, limit_as_maximum=limit_as_maximum
result = InterpolateUsingDifference(limit_as_maximum=limit_as_maximum)(
cube, reference_cube=reference_cube, limit=limit,
)
return result
6 changes: 1 addition & 5 deletions improver/cli/lightning_from_cape_and_precip.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,6 @@ def process(
iris.cube.Cube:
Cube of probabilities of lightning relative to a zero rate thresholds
"""
from iris.cube import CubeList

from improver.lightning import LightningFromCapePrecip

result = LightningFromCapePrecip()(CubeList(cubes), model_id_attr=model_id_attr)

return result
return LightningFromCapePrecip(model_id_attr=model_id_attr)(*cubes)
60 changes: 13 additions & 47 deletions improver/cli/nbhood.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,51 +96,17 @@ def process(
RuntimeError:
If degree_as_complex is used with neighbourhood_shape='circular'.
"""
from improver.nbhood import radius_by_lead_time
from improver.nbhood.nbhood import (
GeneratePercentilesFromANeighbourhood,
NeighbourhoodProcessing,
)
from improver.utilities.pad_spatial import remove_cube_halo
from improver.wind_calculations.wind_direction import WindDirection

if neighbourhood_output == "percentiles":
if weighted_mode:
raise RuntimeError(
"weighted_mode cannot be used with" 'neighbourhood_output="percentiles"'
)
if degrees_as_complex:
raise RuntimeError("Cannot generate percentiles from complex numbers")

if neighbourhood_shape == "circular":
if degrees_as_complex:
raise RuntimeError(
"Cannot process complex numbers with circular neighbourhoods"
)

if degrees_as_complex:
# convert cube data into complex numbers
cube.data = WindDirection.deg_to_complex(cube.data)
from improver.nbhood.nbhood import MetaNeighbourhood

radius_or_radii, lead_times = radius_by_lead_time(radii, lead_times)

if neighbourhood_output == "probabilities":
result = NeighbourhoodProcessing(
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
weighted_mode=weighted_mode,
sum_only=area_sum,
re_mask=True,
)(cube, mask_cube=mask)
elif neighbourhood_output == "percentiles":
result = GeneratePercentilesFromANeighbourhood(
radius_or_radii, lead_times=lead_times, percentiles=percentiles,
)(cube)

if degrees_as_complex:
# convert neighbourhooded cube back to degrees
result.data = WindDirection.complex_to_deg(result.data)
if halo_radius is not None:
result = remove_cube_halo(result, halo_radius)
return result
plugin = MetaNeighbourhood(
neighbourhood_output=neighbourhood_output,
neighbourhood_shape=neighbourhood_shape,
radii=radii,
lead_times=lead_times,
degrees_as_complex=degrees_as_complex,
weighted_mode=weighted_mode,
area_sum=area_sum,
percentiles=percentiles,
halo_radius=halo_radius,
)
return plugin(cube, mask=mask)
2 changes: 1 addition & 1 deletion improver/cli/phase_change_level.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,5 +72,5 @@ def process(
horizontal_interpolation=horizontal_interpolation,
model_id_attr=model_id_attr,
)
result = plugin(cubes)
result = plugin(*cubes)
return result
4 changes: 1 addition & 3 deletions improver/cli/phase_probability.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,8 @@ def process(*cubes: cli.inputcube):
The name of the orography cube must be "surface_altitude".
The name of the site ancillary most be "grid_neighbours".
"""
from iris.cube import CubeList

from improver.psychrometric_calculations.precip_phase_probability import (
PrecipPhaseProbability,
)

return PrecipPhaseProbability()(CubeList(cubes))
return PrecipPhaseProbability()(*cubes)
4 changes: 1 addition & 3 deletions improver/cli/snow_splitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ def process(*cubes: cli.inputcube, output_is_rain: bool):
on precipitation cube)

"""
from iris.cube import CubeList

from improver.precipitation_type.snow_splitter import SnowSplitter

return SnowSplitter(output_is_rain=output_is_rain)(CubeList(cubes))
return SnowSplitter(output_is_rain=output_is_rain)(*cubes)
2 changes: 1 addition & 1 deletion improver/cli/vertical_updraught.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,4 @@ def process(*cubes: cli.inputcube, model_id_attr: str = None):
"""
from improver.wind_calculations.vertical_updraught import VerticalUpdraught

return VerticalUpdraught(model_id_attr=model_id_attr)(cubes)
return VerticalUpdraught(model_id_attr=model_id_attr)(*cubes)
11 changes: 4 additions & 7 deletions improver/cli/wet_bulb_freezing_level.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,8 @@ def process(wet_bulb_temperature: cli.inputcube):
Cube of wet-bulb freezing level.

"""
from improver.utilities.cube_extraction import ExtractLevel
from improver.psychrometric_calculations.wet_bulb_temperature import (
MetaWetBulbFreezingLevel,
)

wet_bulb_freezing_level = ExtractLevel(
positive_correlation=False, value_of_level=273.15
)(wet_bulb_temperature)
wet_bulb_freezing_level.rename("wet_bulb_freezing_level_altitude")

return wet_bulb_freezing_level
return MetaWetBulbFreezingLevel()(wet_bulb_temperature)
2 changes: 1 addition & 1 deletion improver/cli/wet_bulb_temperature.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,4 @@ def process(

return WetBulbTemperature(
precision=convergence_condition, model_id_attr=model_id_attr
)(cubes)
)(*cubes)
22 changes: 16 additions & 6 deletions improver/lightning.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# See LICENSE in the root of the repository for full licensing details.
"""Module containing lightning classes."""
from datetime import timedelta
from typing import Tuple
from typing import Tuple, Union

import iris
import numpy as np
Expand All @@ -18,6 +18,7 @@
generate_mandatory_attributes,
)
from improver.threshold import LatitudeDependentThreshold
from improver.utilities.common_input_handle import as_cubelist
from improver.utilities.cube_checker import spatial_coords_match
from improver.utilities.rescale import rescale
from improver.utilities.spatial import create_vicinity_coord
Expand All @@ -44,6 +45,17 @@ class LightningFromCapePrecip(PostProcessingPlugin):

"""

def __init__(self, model_id_attr: str = None) -> None:
"""
Initialise the plugin with the model_id_attr.

Args:
model_id_attr:
The name of the dataset attribute to be used to identify the source
model when blending data from different models.
"""
self._model_id_attr = model_id_attr

@staticmethod
def _get_inputs(cubes: CubeList) -> Tuple[Cube, Cube]:
"""
Expand Down Expand Up @@ -96,17 +108,14 @@ def _get_inputs(cubes: CubeList) -> Tuple[Cube, Cube]:
raise ValueError("Supplied cubes do not have the same spatial coordinates")
return cape, precip

def process(self, cubes: CubeList, model_id_attr: str = None) -> Cube:
def process(self, *cubes: Union[Cube, CubeList]) -> Cube:
"""
From the supplied CAPE and precipitation-rate cubes, calculate a probability
of lightning cube.

Args:
cubes:
Cubes of CAPE and Precipitation rate.
model_id_attr:
The name of the dataset attribute to be used to identify the source
model when blending data from different models.

Returns:
Cube of lightning data
Expand All @@ -115,6 +124,7 @@ def process(self, cubes: CubeList, model_id_attr: str = None) -> Cube:
ValueError:
If one of the cubes is not found or doesn't match the other
"""
cubes = as_cubelist(*cubes)
cape, precip = self._get_inputs(cubes)

cape_true = LatitudeDependentThreshold(
Expand All @@ -137,7 +147,7 @@ def process(self, cubes: CubeList, model_id_attr: str = None) -> Cube:
template_cube=precip,
data=data.astype(FLOAT_DTYPE),
mandatory_attributes=generate_mandatory_attributes(
cubes, model_id_attr=model_id_attr
cubes, model_id_attr=self._model_id_attr
),
)

Expand Down
Loading
Loading