From 25cdc1098508cbccc841bb08293697d8cc0530b4 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Wed, 23 Oct 2024 10:08:17 -0400 Subject: [PATCH] Add to varinfo config, re-enable MissingReprojectedDataError, modify docstrings --- swath_projector/earthdata_varinfo_config.json | 25 +++++++++++++++++-- swath_projector/nc_merge.py | 3 ++- swath_projector/utilities.py | 3 ++- tests/unit/test_nc_merge.py | 3 +-- tests/unit/test_utilities.py | 12 ++++----- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/swath_projector/earthdata_varinfo_config.json b/swath_projector/earthdata_varinfo_config.json index a7b2dab..25069d4 100644 --- a/swath_projector/earthdata_varinfo_config.json +++ b/swath_projector/earthdata_varinfo_config.json @@ -9,6 +9,27 @@ "VNP10": "VIIRS", "TEMPO_O3TOT_L2": "TEMPO" }, + "ExcludedScienceVariables": [ + { + "Applicability": { + "Mission": "TEMPO", + "ShortNamePath": "TEMPO_O3TOT_L2" + }, + "VariablePattern": [ + "/support_data/a_priori_layer_o3", + "/support_data/cal_adjustment", + "/support_data/dNdR", + "/support_data/layer_efficiency", + "/support_data/lut_wavelength", + "/support_data/N_value", + "/support_data/N_value_residual", + "/support_data/ozone_sensitivity_ratio", + "/support_data/step_1_N_value_residual", + "/support_data/step_2_N_value_residual", + "/support_data/temp_sensitivity_ratio" + ] + } + ], "MetadataOverrides": [ { "Applicability": { @@ -28,7 +49,7 @@ "Applicability": { "Mission": "TEMPO", "ShortNamePath": "TEMPO_O3TOT_L2", - "VariablePattern": "/product/.*|/support_data/.*" + "VariablePattern": "^/product/.*|^/support_data/.*" }, "Attributes": [ { @@ -36,7 +57,7 @@ "Value": "/geolocation/latitude, /geolocation/longitude" } ], - "_Description": "TEMPO variables only have simple, single names for coordinates, but need full paths." + "_Description": "TEMPO_O3TOT_L2 variables only contain basenames for coordinates, which are found in sibling hierarchical groups. This rule fully qualifies the paths to these coordinates." } ] } diff --git a/swath_projector/nc_merge.py b/swath_projector/nc_merge.py index dc8f5c6..b9fc615 100644 --- a/swath_projector/nc_merge.py +++ b/swath_projector/nc_merge.py @@ -13,6 +13,7 @@ from netCDF4 import Dataset, Variable from varinfo import VarInfoFromNetCDF4 +from swath_projector.exceptions import MissingReprojectedDataError from swath_projector.utilities import get_variable_file_path, variable_in_dataset # Values needed for history_json attribute @@ -94,7 +95,7 @@ def create_output( else: logger.error(f'Cannot find "{dataset_file}".') - # QuickFix (DAS-2216) Ignore missing reprojections + raise MissingReprojectedDataError(variable_name) def set_output_attributes( diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index c513fdf..df67795 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -31,7 +31,8 @@ def get_variable_values( As the variable data are returned as a `numpy.ma.MaskedArray`, the will return no data in the filled pixels. To ensure that the data are correctly handled, the fill value is applied to masked pixels using the - `filled` method. + `filled` method. The variable values are transposed if the `along-track` + dimension size is less than the `across-track` dimension size. """ # TODO: Remove in favour of apply2D or process_subdimension. diff --git a/tests/unit/test_nc_merge.py b/tests/unit/test_nc_merge.py index dace742..e5c5421 100644 --- a/tests/unit/test_nc_merge.py +++ b/tests/unit/test_nc_merge.py @@ -2,7 +2,7 @@ import logging import os from datetime import datetime -from unittest import TestCase, skip +from unittest import TestCase from unittest.mock import Mock, patch from netCDF4 import Dataset @@ -186,7 +186,6 @@ def test_same_data_type(self): output_data_type = out_dataset[test_variable].datatype self.assertEqual(input_data_type, output_data_type, 'Should be equal') - @skip("Test disabled for DAS-2216 quick fix that ignores missing reprojections") def test_missing_file_raises_error(self): """If a science variable should be included in the output, but there is no associated output file, an exception should be raised. diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 1d752b4..b31e00f 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -366,7 +366,7 @@ def test_make_array_two_dimensional(self): class TestTransposeIfXdimLessThanYdim(TestCase): def test_wider_than_tall(self): - # Test case where x dim < y dim and should transpose + """Test case where x dim < y dim and should transpose.""" input_array = np.ma.array([[1, 2, 3], [4, 5, 6]]) expected_output = np.ma.array([[1, 4], [2, 5], [3, 6]]) result = transpose_if_xdim_less_than_ydim(input_array) @@ -374,33 +374,33 @@ def test_wider_than_tall(self): self.assertEqual(result.shape, (3, 2)) def test_taller_than_wide(self): - # Test case where x < y and should not transpose. + """Test case where x < y and should not transpose.""" input_array = np.ma.array([[1, 2], [3, 4], [5, 6]]) result = transpose_if_xdim_less_than_ydim(input_array) np.testing.assert_array_equal(result, input_array) self.assertEqual(result.shape, (3, 2)) def test_square_array(self): - # test where y dim == x dim and should not transpose. + """Test case where y dim == x dim and should not transpose.""" input_array = np.ma.array([[1, 2], [3, 4]]) result = transpose_if_xdim_less_than_ydim(input_array) np.testing.assert_array_equal(result, input_array) self.assertEqual(result.shape, (2, 2)) def test_1d_array(self): - # Test case with a 1D array + """Test case with a 1D array""" input_array = np.ma.array([1, 2, 3]) with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'): transpose_if_xdim_less_than_ydim(input_array) def test_3d_array(self): - # Test case with a 3D array + """Test case with a 3D array""" input_array = np.ma.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'): transpose_if_xdim_less_than_ydim(input_array) def test_masked_array(self): - # Test case with a masked array + """Test case with a masked array""" input_array = np.ma.array( [[1, 2, 3], [4, 5, 6]], mask=[[True, False, False], [False, True, False]] )