Feature/pysurfex experiment (#44)

* Update pyproject.toml

* Namelist blocks as input

* Only replace if string

* Black

* Remove print

pyproject.toml

@@ -46,7 +46,7 @@
     python = "^3.8"
     dateutils = "^0.6.12"
     f90nml = "^1.4.3"
-    humanize = "^3.14.0"
+    humanize = ">=3.14.0"
     numexpr = "^2.8.4"
     numpy = "^1.20.1"
     pandas = "^1.3.0"

pysurfex/binary_input.py

@@ -49,7 +49,6 @@ def __init__(self, data):
     def archive_files(self):
         """Archive files."""
         for output_file, target in self.data.items():
-
             logging.info("%s -> %s", output_file, target)
             command = "mv"
             if isinstance(target, dict):
@@ -95,7 +94,6 @@ def __init__(self, data):
     def prepare_input(self):
         """Prepare input."""
         for target, input_file in self.data.items():
-
             logging.info("%s -> %s", target, input_file)
             logging.debug(os.path.realpath(target))
             command = None
@@ -358,8 +356,9 @@ def substitute(self, key, val, macros=None, micro="@", check_parsing=False):
         pkey = key
         pval = val
         for spath_key, spath_val in self.platform.system_file_paths.items():
-            pkey = pkey.replace(f"{micro}{spath_key}{micro}", spath_val)
-            pval = pval.replace(f"{micro}{spath_key}{micro}", spath_val)
+            if isinstance(spath_val, str):
+                pkey = pkey.replace(f"{micro}{spath_key}{micro}", spath_val)
+                pval = pval.replace(f"{micro}{spath_key}{micro}", spath_val)
         if macros is not None:
             for macro_key, macro_val in macros.items():
                 pkey = pkey.replace(f"{micro}{macro_key}{micro}", macro_val)

pysurfex/cli.py

@@ -139,7 +139,6 @@ def run_first_guess_for_oi(**kwargs):
     cache = Cache(3600)
     f_g = None
     for var in variables:
-
         inputfile = kwargs.get("inputfile")
         fileformat = kwargs.get("inputformat")
         logging.debug("inputfile: %s", inputfile)
@@ -289,6 +288,16 @@ def run_masterodb(**kwargs):
     archive = kwargs["archive"]
     only_archive = kwargs["only_archive"]
     print_namelist = kwargs["print_namelist"]
+    try:
+        consistency = kwargs["no_consistency"]
+        if consistency:
+            consistency = False
+    except KeyError:
+        consistency = True
+    try:
+        assemble = kwargs["assemble_file"]
+    except KeyError:
+        assemble = None
 
     check_existence = True
     if "tolerate_missing" in kwargs:
@@ -346,7 +355,12 @@ def run_masterodb(**kwargs):
     if os.path.isfile(namelist_path):
         with open(namelist_path, mode="r", encoding="utf-8") as file_handler:
             nam_defs = yaml.safe_load(file_handler)
-        nam_gen = NamelistGenerator(mode, config, nam_defs)
+        if assemble is not None:
+            with open(assemble, mode="r", encoding="utf8") as fh:
+                assemble = json.load(fh)
+        nam_gen = NamelistGenerator(
+            mode, config, nam_defs, assemble=assemble, consistency=consistency
+        )
         my_settings = nam_gen.nml
         if input_binary_data is None:
             raise RuntimeError("input_binary_data not set")
@@ -402,7 +416,6 @@ def run_masterodb(**kwargs):
         if output is not None:
             exists = os.path.exists(output)
         if not exists or force:
-
             if binary is None:
                 my_batch = None
 
@@ -532,6 +545,17 @@ def run_surfex_binary(mode, **kwargs):
     if "forc_zs" in kwargs:
         forc_zs = kwargs["forc_zs"]
 
+    try:
+        consistency = kwargs["no_consistency"]
+        if consistency:
+            consistency = False
+    except KeyError:
+        consistency = True
+    try:
+        assemble = kwargs["assemble_file"]
+    except KeyError:
+        assemble = None
+
     if mode == "pgd":
         pgd = True
         need_pgd = False
@@ -593,7 +617,12 @@ def run_surfex_binary(mode, **kwargs):
         if os.path.isfile(namelist_path):
             with open(namelist_path, mode="r", encoding="utf-8") as file_handler:
                 nam_defs = yaml.safe_load(file_handler)
-            nam_gen = NamelistGenerator(mode, config, nam_defs)
+            if assemble is not None:
+                with open(assemble, mode="r", encoding="utf8") as fh:
+                    assemble = json.load(fh)
+            nam_gen = NamelistGenerator(
+                mode, config, nam_defs, assemble=assemble, consistency=consistency
+            )
 
             my_settings = nam_gen.nml
             if mode == "pgd":

pysurfex/cmd_parsing.py

@@ -852,6 +852,16 @@ def parse_args_masterodb(argv):
         nargs="?",
         help="Full path of MASTERODB binary",
     )
+    parser.add_argument(
+        "--no-consistency", dest="no_consistency", action="store_true", required=False
+    )
+    parser.add_argument(
+        "--assemble",
+        type=str,
+        required=False,
+        help="Path to file containing list of namelist blocks",
+        default=None,
+    )
 
     if len(argv) == 0:
         parser.print_help()
@@ -986,6 +996,16 @@ def parse_args_surfex_binary(argv, mode):
         help="JSON file with archive output",
     )
     parser.add_argument("binary", type=str, help="Command to run")
+    parser.add_argument(
+        "--no-consistency", dest="no_consistency", action="store_true", required=False
+    )
+    parser.add_argument(
+        "--assemble",
+        type=str,
+        required=False,
+        help="Path to file containing list of namelist blocks",
+        default=None,
+    )
 
     if len(argv) == 0:
         parser.print_help()
@@ -2213,7 +2233,6 @@ def variable_parser(needles, argv, parser):
 
     kwargs = {}
     for needle in needles:
-
         argv = argv_string[start_indices[needle] : end_indices[needle]].split()
         opt = parser.parse_args(argv)
         vargs = {}

pysurfex/file.py

@@ -1323,7 +1323,6 @@ def read(self, variable, times):
         tstep = 0
         col = 0
         for line in self.file.read().splitlines():
-
             words = line.split()
             if len(words) > 0:
                 for i, word in enumerate(words):

pysurfex/geo.py

@@ -653,7 +653,6 @@ def __init__(self, from_json, recreate=False):
                 and "ncols"
                 and "nrows" in domain_dict["nam_ign"]
             ):
-
                 self.clambert = domain_dict["nam_ign"]["clambert"]
                 npoints = domain_dict["nam_ign"]["npoints"]
                 self.x_x = domain_dict["nam_ign"]["xx"]
@@ -811,7 +810,6 @@ def ign_mask(pxall, pyall, xxx, yyy, recreate):
 
         for i, pxall_val in enumerate(pxall):
             for pyall_val in pyall:
-
                 count = count + 1
                 for k, xval in enumerate(xxx):
                     if xval == pxall_val and yyy[k] == pyall_val:
@@ -933,7 +931,6 @@ def set_domain(settings, domain, hm_mode=False):
     if isinstance(settings, dict):
         if domain in settings:
             if hm_mode:
-
                 ezone = 11
                 if "EZONE" in settings[domain]:
                     ezone = settings[domain]["EZONE"]

pysurfex/input_methods.py

@@ -33,7 +33,6 @@ def get_datasources(obs_time, settings):
     """
     datasources = []
     for obs_set in settings:
-
         kwargs = {}
         kwargs.update({"label": obs_set})
 

pysurfex/namelist.py

@@ -59,7 +59,7 @@ def __init__(self, program, config, definitions, assemble=None, consistency=True
             self.assemble = self.namelist_blocks()
         else:
             self.assemble = assemble
-        logging.debug(self.assemble)
+        logging.info("Namelist blocks: %s", self.assemble)
         nlres = self.assemble_namelist()
         self.nml = f90nml.Namelist(nlres)
         if consistency:
@@ -118,7 +118,6 @@ def namelist_blocks(self):
 
         # Program specific settings
         if self.program == "pgd":
-
             input_blocks += ["pgd", "pgd_cover", "pgd_zs"]
             eco_sg = self.config.get_setting("SURFEX#COVER#SG")
             if eco_sg:
@@ -376,7 +375,6 @@ def concistency(self, nml):
 
         # Program specific settings
         if self.program == "pgd":
-
             problems = self.check_nml_setting(
                 problems,
                 nml,
@@ -406,7 +404,6 @@ def concistency(self, nml):
                 self.config.get_setting("SURFEX#TILES#TOWN"),
             )
             if self.config.get_setting("SURFEX#TOWN#LTOWN_TO_ROCK"):
-
                 if self.config.get_setting("SURFEX#TILES#TOWN") != "NONE":
                     logging.warning(
                         "WARNING: TOWN is not NONE and you want LTOWN_TO_ROCK. "

pysurfex/namelist_legacy.py

@@ -1443,7 +1443,6 @@ def set_pgd_namelist(self, merged_dict):
             self.config.get_setting("SURFEX#TILES#TOWN"),
         )
         if self.config.get_setting("SURFEX#TOWN#LTOWN_TO_ROCK"):
-
             if self.config.get_setting("SURFEX#TILES#TOWN") != "NONE":
                 logging.warning(
                     "WARNING: TOWN is not NONE and you want LTOWN_TO_ROCK. "
@@ -1908,7 +1907,6 @@ def prepare_offline_perturbation(self, merged_dict):
         merged_dict["NAM_VAR"].update({"NIVAR": 0})
         merged_dict["NAM_IO_VARASSIM"].update({"LPRT": False})
         if self.config.get_setting("SURFEX#ASSIM#SCHEMES#ISBA") == "EKF":
-
             merged_dict = self.merge_json_namelist_file(
                 merged_dict, self.input_path + "/offline_assim_pert.json"
             )
@@ -1941,7 +1939,6 @@ def prepare_offline_perturbation(self, merged_dict):
             merged_dict = self.sub(merged_dict, "NAM_VAR", "NVAR", nvar)
 
         if self.config.get_setting("SURFEX#ASSIM#SCHEMES#ISBA") == "ENKF":
-
             merged_dict = self.merge_json_namelist_file(
                 merged_dict, self.input_path + "/offline_assim_pert.json"
             )

pysurfex/netcdf.py

@@ -848,7 +848,6 @@ def oi2soda(dtg, t2m=None, rh2m=None, s_d=None, s_m=None, output=None):
     """
 
     def check_input_to_soda_dimensions(my_nx, my_ny, nx1, ny1):
-
         if my_nx < 0:
             my_nx = nx1
         if my_ny < 0:

pysurfex/platform_deps.py

@@ -288,7 +288,6 @@ def parse_setting(
         """
         # Check on arguments
         if isinstance(setting, str):
-
             if basedtg is not None:
                 if isinstance(basedtg, str):
                     basedtg = as_datetime(basedtg)

pysurfex/pseudoobs.py

@@ -171,7 +171,6 @@ def snow_pseudo_obs_cryoclim(
     providers = []
     logging.debug("p_fg_snow_depth.shape[0]=%s", p_fg_snow_depth.shape[0])
     for i in range(0, p_fg_snow_depth.shape[0]):
-
         p_snow_fg = p_fg_snow_depth[i]
         logging.debug("%s %s %s %s", i, p_snow_fg, res_lons[i], res_lats[i])
         if not np.isnan(p_snow_fg):
@@ -319,7 +318,6 @@ def sm_obs_sentinel(
     lafs = []
     providers = []
     for i in range(0, p_fg_sm.shape[0]):
-
         p_sm_fg = p_fg_sm[i]
         if not np.isnan(p_sm_fg):
             # Check if in grid

pysurfex/run.py

@@ -293,7 +293,6 @@ def __init__(
         if self.prepfile.input_file is not None and os.path.abspath(
             self.prepfile.filename
         ) != os.path.abspath(self.prepfile.input_file):
-
             logging.info("Input PREP file is: %s", self.prepfile.input_file)
             remove_existing_file(self.prepfile.input_file, self.prepfile.filename)
             os.symlink(self.prepfile.input_file, self.prepfile.filename)

pysurfex/titan.py

@@ -1052,7 +1052,6 @@ def __init__(self, blacklist):
         blacklist_stid = {}
         if "lons" in blacklist and "lats" in blacklist:
             for i in range(0, len(blacklist["lons"])):
-
                 if len(blacklist["lons"]) != len(blacklist["lats"]):
                     raise RuntimeError(
                         "Blacklist must have the same length for both lons and lats"
@@ -1092,7 +1091,6 @@ def test(self, dataset, mask, code=100):
         flags = dataset.flags
         for i, lon_val in enumerate(dataset.lons):
             if i in mask:
-
                 lon = Observation.format_lon(lon_val)
                 lat = Observation.format_lat(dataset.lats[i])
                 stid = dataset.stids[i]
@@ -1342,7 +1340,6 @@ def define_quality_control(test_list, settings, an_time, domain_geo=None, blackl
             tests.append(Buddy(**kwargs))
 
         elif qct.lower() == "climatology":
-
             if test_options is not None:
                 opts = ["minval", "maxval", "offset"]
                 for opt in opts:
@@ -1351,7 +1348,6 @@ def define_quality_control(test_list, settings, an_time, domain_geo=None, blackl
             tests.append(Climatology(an_time, **kwargs))
 
         elif qct.lower() == "sct":
-
             if test_options is not None:
                 opts = [
                     "num_min",
@@ -1687,7 +1683,6 @@ def perform_tests(self):
             mask = []
             findex = 0
             for obs_set in self.datasources:
-
                 if obs_set.label == "":
                     raise RuntimeError(
                         "Observations set for quality control are "
@@ -2003,7 +1998,6 @@ def merge_json_qc_data_sets(an_time, filenames, qc_flag=None, skip_flags=None):
     index_pos = {}
     data = {}
     for filename in filenames:
-
         if os.path.exists(filename):
             with open(filename, mode="r", encoding="utf-8") as file_handler:
                 data1 = json.load(file_handler)

tests/smoke/test_cli_fg_titan_oi_obsmon.py

@@ -583,7 +583,6 @@ def _qc_gridpp_obsmon(
 
 
 def obsmon_test(var, qc_fname, first_guess_file, analysis_file, db_file):
-
     translation = {
         "t2m": "air_temperature_2m",
         "rh2m": "relative_humidity_2m",

tests/smoke/test_cli_misc.py

@@ -65,7 +65,6 @@ def test_cli_set_geo_from_obs_set(obsset_fname, tmp_path_factory):
 
 
 def test_cryoclim_pseudoobs(tmp_path_factory, data_cryoclim_nc_file, firstguess4gridpp):
-
     out_fname = f"{tmp_path_factory.getbasetemp().as_posix()}/output_cryoclim.json"
     argv = [
         "-step",
@@ -86,7 +85,6 @@ def test_cryoclim_pseudoobs(tmp_path_factory, data_cryoclim_nc_file, firstguess4
 def test_cryoclim_pseudoobs_iv(
     tmp_path_factory, data_cryoclim_nc_file, firstguess4gridpp
 ):
-
     out_fname = f"{tmp_path_factory.getbasetemp().as_posix()}/output_cryoclim2.json"
     argv = [
         "-step",
@@ -109,7 +107,6 @@ def test_cryoclim_pseudoobs_iv(
 def test_cryoclim_pseudoobs_iv_slope_glacier_mask_netcdf(
     tmp_path_factory, data_cryoclim_nc_file, firstguess4gridpp, data_surfex_pgd_nc_file
 ):
-
     out_fname = f"{tmp_path_factory.getbasetemp().as_posix()}/output_cryoclim4.json"
     argv = [
         "-step",
@@ -204,7 +201,6 @@ def data_sentinel_nc_file(tmp_path_factory):
 
 
 def test_sentinel(tmp_path_factory, data_sentinel_nc_file, firstguess4gridpp):
-
     out_fname = f"{tmp_path_factory.getbasetemp().as_posix()}/output_sentinel.json"
     argv = [
         "-step",

tests/smoke/test_cli_run_binary.py

@@ -165,7 +165,6 @@ def test_run_prep(
     tmp_path_factory,
     input_binary_data_file,
 ):
-
     # PREP
 
     pgd = tmp_path_factory.getbasetemp() / "PGD_input.nc"
@@ -209,7 +208,6 @@ def test_run_prep_old(
     conf_proj_2x3_file,
     tmp_path_factory,
 ):
-
     # PREP
     wrk = tmp_path_factory.getbasetemp() / "nam_old"
     wrk.mkdir(exist_ok=True)