Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Removes tags.yml from module and subworkflow template #2995

Merged
merged 6 commits into from
Jun 21, 2024
Merged

Removes tags.yml from module and subworkflow template #2995

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
33179ea
Removes tags.yml from module and subworkflow template
adamrtalbot May 22, 2024
4cb6fd1
mypy fixes
adamrtalbot May 22, 2024
8eb4f41
re-add checks for pytest dirs existing
adamrtalbot May 22, 2024
95976e5
remove test_tags_yml from pytest test
mirpedrol May 22, 2024
03e78f9
Merge branch 'dev' into remove_tags.yml
mirpedrol May 23, 2024
34eaf36
Merge branch 'dev' into remove_tags.yml
adamrtalbot Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .prettierignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ slackreport.json
docs/api/_build
testing
nf_core/module-template/meta.yml
nf_core/module-template/tests/tags.yml
nf_core/subworkflow-template/tests/tags.yml
nf_core/pipeline-template/nextflow_schema.json
# don't run on things handled by ruff
*.py
Expand Down
46 changes: 34 additions & 12 deletions nf_core/components/create.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ def create(self):
├── meta.yml
├── environment.yml
└── tests
├── main.nf.test
└── tags.yml
└── main.nf.test
```

The function will attempt to automatically find a Bioconda package called <component>
Expand All @@ -102,8 +101,7 @@ def create(self):
├── main.nf
├── meta.yml
└── tests
├── main.nf.test
└── tags.yml
└── main.nf.test
```

"""
Expand Down Expand Up @@ -228,7 +226,13 @@ def _get_bioconda_tool(self):
log.info(f"Could not find a Docker/Singularity container ({e})")

def _get_module_structure_components(self):
process_label_defaults = ["process_single", "process_low", "process_medium", "process_high", "process_long"]
process_label_defaults = [
"process_single",
"process_low",
"process_medium",
"process_high",
"process_long",
]
if self.process_label is None:
log.info(
"Provide an appropriate resource label for the process, taken from the "
Expand All @@ -252,7 +256,8 @@ def _get_module_structure_components(self):
)
while self.has_meta is None:
self.has_meta = rich.prompt.Confirm.ask(
"[violet]Will the module require a meta map of sample information?", default=True
"[violet]Will the module require a meta map of sample information?",
default=True,
)

def _render_template(self):
Expand All @@ -262,7 +267,8 @@ def _render_template(self):
object_attrs = vars(self)
# Run jinja2 for each file in the template folder
env = jinja2.Environment(
loader=jinja2.PackageLoader("nf_core", f"{self.component_type[:-1]}-template"), keep_trailing_newline=True
loader=jinja2.PackageLoader("nf_core", f"{self.component_type[:-1]}-template"),
keep_trailing_newline=True,
)
for template_fn, dest_fn in self.file_paths.items():
log.debug(f"Rendering template file: '{template_fn}'")
Expand Down Expand Up @@ -372,7 +378,13 @@ def _get_component_dirs(self):

if self.component_type == "modules":
# If a subtool, check if there is a module called the base tool name already
parent_tool_main_nf = Path(self.directory, self.component_type, self.org, self.component, "main.nf")
parent_tool_main_nf = Path(
self.directory,
self.component_type,
self.org,
self.component,
"main.nf",
)
if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest:
raise UserWarning(
f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'"
Expand All @@ -393,7 +405,6 @@ def _get_component_dirs(self):
file_paths["meta.yml"] = component_dir / "meta.yml"
if self.component_type == "modules":
file_paths["environment.yml"] = component_dir / "environment.yml"
file_paths["tests/tags.yml"] = component_dir / "tests" / "tags.yml"
file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test"

return file_paths
Expand Down Expand Up @@ -428,11 +439,15 @@ def _copy_old_files(self, component_old_path):
shutil.copyfile(component_old_path / "meta.yml", self.file_paths["meta.yml"])
if self.component_type == "modules":
log.debug("Copying original environment.yml file")
shutil.copyfile(component_old_path / "environment.yml", self.file_paths["environment.yml"])
shutil.copyfile(
component_old_path / "environment.yml",
self.file_paths["environment.yml"],
)
if (component_old_path / "templates").is_dir():
log.debug("Copying original templates directory")
shutil.copytree(
component_old_path / "templates", self.file_paths["environment.yml"].parent / "templates"
component_old_path / "templates",
self.file_paths["environment.yml"].parent / "templates",
)
# Create a nextflow.config file if it contains information other than publishDir
pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir)
Expand All @@ -447,7 +462,14 @@ def _copy_old_files(self, component_old_path):
if len(config_lines) > 11:
log.debug("Copying nextflow.config file from pytest tests")
with open(
Path(self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config"),
Path(
self.directory,
self.component_type,
self.org,
self.component_dir,
"tests",
"nextflow.config",
),
"w+",
) as ofh:
ofh.write(config_lines)
Expand Down
10 changes: 8 additions & 2 deletions nf_core/components/nfcore_component.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,14 @@ class NFCoreComponent:
"""

def __init__(
self, component_name, repo_url, component_dir, repo_type, base_dir, component_type, remote_component=True
self,
component_name,
repo_url,
component_dir,
repo_type,
base_dir,
component_type,
remote_component=True,
):
"""
Initialize the object
Expand Down Expand Up @@ -59,7 +66,6 @@ def __init__(
self.org = repo_dir
self.nftest_testdir = Path(self.component_dir, "tests")
self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test")
self.tags_yml = Path(self.nftest_testdir, "tags.yml")

if self.repo_type == "pipeline":
patch_fn = f"{self.component_name.replace('/', '-')}.diff"
Expand Down
2 changes: 0 additions & 2 deletions nf_core/module-template/tests/tags.yml
View file
Open in desktop

This file was deleted.

103 changes: 63 additions & 40 deletions nf_core/modules/lint/module_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def module_tests(_, module: NFCoreComponent):
Lint the tests of a module in ``nf-core/modules``

It verifies that the test directory exists
and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``.
and contains a ``main.nf.test`` and a ``main.nf.test.snap``

"""
repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1]
Expand All @@ -29,19 +29,43 @@ def module_tests(_, module: NFCoreComponent):
module.passed.append(("test_dir_exists", "nf-test test directory exists", module.nftest_testdir))
else:
if is_pytest:
module.warned.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir))
module.warned.append(
(
"test_dir_exists",
"nf-test directory is missing",
module.nftest_testdir,
)
)
else:
module.failed.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir))
module.failed.append(
(
"test_dir_exists",
"nf-test directory is missing",
module.nftest_testdir,
)
)
return

# Lint the test main.nf file
if module.nftest_main_nf.is_file():
module.passed.append(("test_main_nf_exists", "test `main.nf.test` exists", module.nftest_main_nf))
else:
if is_pytest:
module.warned.append(("test_main_nf_exists", "test `main.nf.test` does not exist", module.nftest_main_nf))
module.warned.append(
(
"test_main_nf_exists",
"test `main.nf.test` does not exist",
module.nftest_main_nf,
)
)
else:
module.failed.append(("test_main_nf_exists", "test `main.nf.test` does not exist", module.nftest_main_nf))
module.failed.append(
(
"test_main_nf_exists",
"test `main.nf.test` does not exist",
module.nftest_main_nf,
)
)

if module.nftest_main_nf.is_file():
# Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test
Expand All @@ -50,7 +74,11 @@ def module_tests(_, module: NFCoreComponent):
snap_file = module.nftest_testdir / "main.nf.test.snap"
if snap_file.is_file():
module.passed.append(
("test_snapshot_exists", "snapshot file `main.nf.test.snap` exists", snap_file)
(
"test_snapshot_exists",
"snapshot file `main.nf.test.snap` exists",
snap_file,
)
)
# Validate no empty files
with open(snap_file) as snap_fh:
Expand Down Expand Up @@ -133,7 +161,11 @@ def module_tests(_, module: NFCoreComponent):
)
else:
module.failed.append(
("test_snapshot_exists", "snapshot file `main.nf.test.snap` does not exist", snap_file)
(
"test_snapshot_exists",
"snapshot file `main.nf.test.snap` does not exist",
snap_file,
)
)
# Verify that tags are correct.
main_nf_tags = module._get_main_nf_tags(module.nftest_main_nf)
Expand All @@ -148,7 +180,13 @@ def module_tests(_, module: NFCoreComponent):
if tag not in main_nf_tags:
missing_tags.append(tag)
if len(missing_tags) == 0:
module.passed.append(("test_main_tags", "Tags adhere to guidelines", module.nftest_main_nf))
module.passed.append(
(
"test_main_tags",
"Tags adhere to guidelines",
module.nftest_main_nf,
)
)
else:
module.failed.append(
(
Expand All @@ -174,41 +212,20 @@ def module_tests(_, module: NFCoreComponent):
)
else:
module.passed.append(
("test_pytest_yml", "module with nf-test not in pytest_modules.yml", pytest_yml_path)
)
except FileNotFoundError:
module.warned.append(("test_pytest_yml", "Could not open pytest_modules.yml file", pytest_yml_path))

if module.tags_yml.is_file():
# Check that tags.yml exists and it has the correct entry
module.passed.append(("test_tags_yml_exists", "file `tags.yml` exists", module.tags_yml))
with open(module.tags_yml) as fh:
tags_yml = yaml.safe_load(fh)
if module.component_name in tags_yml.keys():
module.passed.append(("test_tags_yml", "correct entry in tags.yml", module.tags_yml))
if f"modules/{module.org}/{module.component_name}/**" in tags_yml[module.component_name]:
module.passed.append(("test_tags_yml", "correct path in tags.yml", module.tags_yml))
else:
module.failed.append(
(
"test_tags_yml",
f"incorrect path in tags.yml, expected `modules/{module.org}/{module.component_name}/**`, got `{tags_yml[module.component_name][0]}`",
module.tags_yml,
"test_pytest_yml",
"module with nf-test not in pytest_modules.yml",
pytest_yml_path,
)
)
else:
module.failed.append(
(
"test_tags_yml",
f"incorrect key in tags.yml, should be `{module.component_name}`, got `{list(tags_yml.keys())[0]}`.",
module.tags_yml,
)
except FileNotFoundError:
module.warned.append(
(
"test_pytest_yml",
"Could not open pytest_modules.yml file",
pytest_yml_path,
)
else:
if is_pytest:
module.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml))
else:
module.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml))
)

# Check that the old test directory does not exist
if not is_pytest:
Expand All @@ -222,4 +239,10 @@ def module_tests(_, module: NFCoreComponent):
)
)
else:
module.passed.append(("test_old_test_dir", "Old pytests don't exist for this module", old_test_dir))
module.passed.append(
(
"test_old_test_dir",
"Old pytests don't exist for this module",
old_test_dir,
)
)
2 changes: 0 additions & 2 deletions nf_core/subworkflow-template/tests/tags.yml
View file
Open in desktop

This file was deleted.

Loading
Loading