Thresholding masked data (#1905)

* Adds filling masked values prior to thresholding and associated tests

* formatting

---------

Co-authored-by: Marcus Spelman <[email protected]>

improver/cli/threshold.py

@@ -47,6 +47,7 @@ def process(
     fuzzy_factor: float = None,
     collapse_coord: str = None,
     vicinity: cli.comma_separated_list = None,
+    fill_masked: float = None,
 ):
     """Module to apply thresholding to a parameter dataset.
 
@@ -102,6 +103,9 @@ def process(
             Binary land-sea mask data. True for land-points, False for sea.
             Restricts in-vicinity processing to only include points of a
             like mask value.
+        fill_masked (float):
+            If provided all masked points in cube will be replaced with the
+            provided value before thresholding.
 
     Returns:
         iris.cube.Cube:
@@ -122,7 +126,6 @@ def process(
         )
     if threshold_config and fuzzy_factor:
         raise ValueError("--threshold-config cannot be used for fuzzy thresholding")
-
     if threshold_config:
         thresholds = []
         fuzzy_bounds = []
@@ -160,11 +163,15 @@ def process(
     elif collapse_coord is not None:
         raise ValueError("Cannot collapse over non-realization coordinate")
 
+    if fill_masked is not None:
+        fill_masked = float(fill_masked)
+
     return BasicThreshold(
         thresholds,
         fuzzy_factor=fuzzy_factor,
         fuzzy_bounds=fuzzy_bounds,
         threshold_units=threshold_units,
         comparison_operator=comparison_operator,
         each_threshold_func=each_threshold_func_list,
+        fill_masked=fill_masked,
     )(cube)

improver/threshold.py

@@ -69,6 +69,7 @@ def __init__(
         threshold_units: Optional[str] = None,
         comparison_operator: str = ">",
         each_threshold_func: Union[Callable, List[Callable]] = (),
+        fill_masked: Optional[float] = None,
     ) -> None:
         """
         Set up for processing an in-or-out of threshold field, including the
@@ -124,6 +125,9 @@ def __init__(
             each_threshold_func:
                 Callable or sequence of callables to apply after thresholding.
                 Eg vicinity processing or collapse over ensemble realizations.
+            fill_masked:
+                If provided all masked points in cube will be replaced with the
+                provided value.
 
         Raises:
             ValueError: If using a fuzzy factor with a threshold of 0.0.
@@ -171,6 +175,8 @@ def __init__(
             each_threshold_func = (each_threshold_func,)
         self.each_threshold_func = each_threshold_func
 
+        self.fill_masked = fill_masked
+
     def _generate_fuzzy_bounds(
         self, fuzzy_factor_loc: float
     ) -> List[Tuple[float, float]]:
@@ -300,6 +306,9 @@ def process(self, input_cube: Cube) -> Cube:
         if np.isnan(input_cube.data).any():
             raise ValueError("Error: NaN detected in input cube data")
 
+        if self.fill_masked is not None:
+            input_cube.data = np.ma.filled(input_cube.data, self.fill_masked)
+
         self.threshold_coord_name = input_cube.name()
 
         thresholded_cubes = iris.cube.CubeList()

improver_tests/acceptance/SHA256SUMS

@@ -627,6 +627,7 @@ d00745d7911caf4968ec9befa9a1dc71fd29744bbe12b02649c4693bd5b0aecf  ./threshold/fu
 678c1daa00ebeb9b072d43f18050ab2565179e335cba35b1cf19620be81d275b  ./threshold/json/threshold_config.json
 c2f8c6157532c30c02c00f45a71c01b58f0307b385f58034d3b158442d2eed8c  ./threshold/masked_collapse/input.nc
 d9041ae3a8f47e4b337f332c7f6ffe0208f17a2c11c77a197e7db0e01c9c56f4  ./threshold/masked_collapse/kgo.nc
+f3bad618cf481fb82975367a30a6eb174c2bfb74e4d26b9c0c39382ff3f3c993  ./threshold/masked_collapse/kgo_mask_filled.nc
 4a9edf8649156e2b11dc253c36bb8f1d0537fde8682e77b7d0395211692944e7  ./threshold/multiple_thresholds/kgo.nc
 eb3fdc9400401ec47d95961553aed452abcbd91891d0fbca106b3a05131adaa9  ./threshold/threshold_units/kgo.nc
 ba2a059b7dfc94a7ff1561f07ac006a7d5d8bea2399c3352fab430327af1cfb2  ./threshold/threshold_units_fuzzy_factor/kgo.nc

improver_tests/acceptance/test_threshold.py

@@ -120,11 +120,14 @@ def test_collapse_realization(tmp_path):
     acc.compare(output_path, kgo_path)
 
 
-def test_collapse_realization_masked_data(tmp_path):
+@pytest.mark.parametrize(
+    "extra_arg,kgo", (([], "kgo.nc"), (["--fill-masked", "inf"], "kgo_mask_filled.nc"))
+)
+def test_collapse_realization_masked_data(tmp_path, extra_arg, kgo):
     """Test thresholding and collapsing realizations where the data being
     thresholded is masked."""
     kgo_dir = acc.kgo_root() / "threshold/masked_collapse"
-    kgo_path = kgo_dir / "kgo.nc"
+    kgo_path = kgo_dir / kgo
     input_path = kgo_dir / "input.nc"
     output_path = tmp_path / "output.nc"
     args = [
@@ -136,6 +139,7 @@ def test_collapse_realization_masked_data(tmp_path):
         "--collapse-coord",
         "realization",
     ]
+    args += extra_arg
     run_cli(args)
     acc.compare(output_path, kgo_path)
 

improver_tests/threshold/test_BasicThreshold.py

@@ -202,6 +202,14 @@ def test_masked_array(self):
         self.assertArrayAlmostEqual(result.data.data, expected_result_array)
         self.assertArrayEqual(result.data.mask, self.masked_cube.data.mask)
 
+    def test_fill_masked(self):
+        """Test plugin when masked points are replaced with fill value"""
+        plugin = Threshold(0.6, fill_masked=np.inf)
+        result = plugin(self.masked_cube)
+        expected_result = np.zeros_like(self.masked_cube.data)
+        expected_result[0][0] = 1.0
+        self.assertArrayEqual(result.data, expected_result)
+
     def test_threshold_fuzzy(self):
         """Test when a point is in the fuzzy threshold area."""
         plugin = Threshold(0.6, fuzzy_factor=self.fuzzy_factor)