From 27ca57396b034044549ac369e1de1c98b32e8fec Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Fri, 12 Apr 2024 14:15:35 -0400 Subject: [PATCH 01/23] Draft for Gentoo configuration --- neurodocker/templates/gentoo.yaml | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 neurodocker/templates/gentoo.yaml diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml new file mode 100644 index 00000000..966f781d --- /dev/null +++ b/neurodocker/templates/gentoo.yaml @@ -0,0 +1,37 @@ +--- +# Instructions to add NeuroDebian repositories. + +name: gentoo +url: https://www.gentoo.org/ +binaries: + urls: + australia: http://neuro.debian.net/lists/{{ self.os_codename }}.au.{{ self.full_or_libre }} + china-tsinghua: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj1.{{ self.full_or_libre }} + china-scitech: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj2.{{ self.full_or_libre }} + china-zhejiang: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-zj.{{ self.full_or_libre }} + germany-munich: http://neuro.debian.net/lists/{{ self.os_codename }}.de-m.{{ self.full_or_libre }} + germany-magdeburg: http://neuro.debian.net/lists/{{ self.os_codename }}.de-md.{{ self.full_or_libre }} + greece: http://neuro.debian.net/lists/{{ self.os_codename }}.gr.{{ self.full_or_libre }} + japan: http://neuro.debian.net/lists/{{ self.os_codename }}.jp.{{ self.full_or_libre }} + usa-ca: http://neuro.debian.net/lists/{{ self.os_codename }}.us-ca.{{ self.full_or_libre }} + usa-nh: http://neuro.debian.net/lists/{{ self.os_codename }}.us-nh.{{ self.full_or_libre }} + usa-tn: http://neuro.debian.net/lists/{{ self.os_codename }}.us-tn.{{ self.full_or_libre }} + arguments: + required: + - version + - os_codename + optional: + full_or_libre: full + dependencies: + apt: + - ca-certificates + - curl + - dirmngr + - gnupg2 + instructions: | + {{ self.install_dependencies() }} + curl -fsSL {{ self.urls[self.version] }} \ + > /etc/apt/sources.list.d/neurodebian.sources.list + curl -sSL https://dl.dropbox.com/s/zxs209o955q6vkg/neurodebian.gpg | apt-key add - + (apt-key adv --refresh-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 || true) + apt-get -qq update From 304d24eabdfaeafd530030783de17c0e02d9d8fd Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Fri, 12 Apr 2024 14:20:05 -0400 Subject: [PATCH 02/23] Use HTTPS --- neurodocker/templates/gentoo.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 966f781d..4d1486c3 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -5,17 +5,17 @@ name: gentoo url: https://www.gentoo.org/ binaries: urls: - australia: http://neuro.debian.net/lists/{{ self.os_codename }}.au.{{ self.full_or_libre }} - china-tsinghua: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj1.{{ self.full_or_libre }} - china-scitech: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj2.{{ self.full_or_libre }} - china-zhejiang: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-zj.{{ self.full_or_libre }} - germany-munich: http://neuro.debian.net/lists/{{ self.os_codename }}.de-m.{{ self.full_or_libre }} - germany-magdeburg: http://neuro.debian.net/lists/{{ self.os_codename }}.de-md.{{ self.full_or_libre }} - greece: http://neuro.debian.net/lists/{{ self.os_codename }}.gr.{{ self.full_or_libre }} - japan: http://neuro.debian.net/lists/{{ self.os_codename }}.jp.{{ self.full_or_libre }} - usa-ca: http://neuro.debian.net/lists/{{ self.os_codename }}.us-ca.{{ self.full_or_libre }} - usa-nh: http://neuro.debian.net/lists/{{ self.os_codename }}.us-nh.{{ self.full_or_libre }} - usa-tn: http://neuro.debian.net/lists/{{ self.os_codename }}.us-tn.{{ self.full_or_libre }} + australia: https://neuro.debian.net/lists/{{ self.os_codename }}.au.{{ self.full_or_libre }} + china-tsinghua: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj1.{{ self.full_or_libre }} + china-scitech: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj2.{{ self.full_or_libre }} + china-zhejiang: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-zj.{{ self.full_or_libre }} + germany-munich: https://neuro.debian.net/lists/{{ self.os_codename }}.de-m.{{ self.full_or_libre }} + germany-magdeburg: https://neuro.debian.net/lists/{{ self.os_codename }}.de-md.{{ self.full_or_libre }} + greece: https://neuro.debian.net/lists/{{ self.os_codename }}.gr.{{ self.full_or_libre }} + japan: https://neuro.debian.net/lists/{{ self.os_codename }}.jp.{{ self.full_or_libre }} + usa-ca: https://neuro.debian.net/lists/{{ self.os_codename }}.us-ca.{{ self.full_or_libre }} + usa-nh: https://neuro.debian.net/lists/{{ self.os_codename }}.us-nh.{{ self.full_or_libre }} + usa-tn: https://neuro.debian.net/lists/{{ self.os_codename }}.us-tn.{{ self.full_or_libre }} arguments: required: - version From 32cd2106639c9af3858afafb9aba540bafcc33ca Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Thu, 25 Apr 2024 17:38:04 -0400 Subject: [PATCH 03/23] Adding test template to develop gentoo support --- .../cli/tests/test_build_images_with_cli.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index f5444535..23d50473 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -113,3 +113,23 @@ def test_json_roundtrip(cmd: str, inputs: str, tmp_path: Path): stdout, _ = run_fn(img, args=["env"]) assert "CAT=FOO" in stdout assert "DOG=BAR" in stdout + +def test_gentoo_image(tmp_path: Path): + # also add singularity like in the test above + + cmd = "neurodocker generate docker" + + _TemplateRegistry._reset() + runner = CliRunner() + result = runner.invoke( + generate, + [ + cmd, + "--pkg-manager apt", + "--base-image neurodebian:bullseye", + "--ants version=2.4.3", + "--user nonroot" + ], + ) + assert result.exit_code == 0, result.output + (tmp_path / "specs.json").write_text(result.output) From 91df3067dddb7d488ff9d0e92ce0679345a1cb6b Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Thu, 25 Apr 2024 17:47:15 -0400 Subject: [PATCH 04/23] Added containerfile contents --- neurodocker/templates/gentoo.yaml | 72 ++++++++++++++++++------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 4d1486c3..3c29b6d0 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -4,34 +4,46 @@ name: gentoo url: https://www.gentoo.org/ binaries: - urls: - australia: https://neuro.debian.net/lists/{{ self.os_codename }}.au.{{ self.full_or_libre }} - china-tsinghua: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj1.{{ self.full_or_libre }} - china-scitech: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj2.{{ self.full_or_libre }} - china-zhejiang: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-zj.{{ self.full_or_libre }} - germany-munich: https://neuro.debian.net/lists/{{ self.os_codename }}.de-m.{{ self.full_or_libre }} - germany-magdeburg: https://neuro.debian.net/lists/{{ self.os_codename }}.de-md.{{ self.full_or_libre }} - greece: https://neuro.debian.net/lists/{{ self.os_codename }}.gr.{{ self.full_or_libre }} - japan: https://neuro.debian.net/lists/{{ self.os_codename }}.jp.{{ self.full_or_libre }} - usa-ca: https://neuro.debian.net/lists/{{ self.os_codename }}.us-ca.{{ self.full_or_libre }} - usa-nh: https://neuro.debian.net/lists/{{ self.os_codename }}.us-nh.{{ self.full_or_libre }} - usa-tn: https://neuro.debian.net/lists/{{ self.os_codename }}.us-tn.{{ self.full_or_libre }} - arguments: - required: - - version - - os_codename - optional: - full_or_libre: full - dependencies: - apt: - - ca-certificates - - curl - - dirmngr - - gnupg2 instructions: | - {{ self.install_dependencies() }} - curl -fsSL {{ self.urls[self.version] }} \ - > /etc/apt/sources.list.d/neurodebian.sources.list - curl -sSL https://dl.dropbox.com/s/zxs209o955q6vkg/neurodebian.gpg | apt-key add - - (apt-key adv --refresh-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 || true) - apt-get -qq update + FROM docker.io/gentoo/portage:20240324 as portage + FROM docker.io/gentoo/stage3:20240318 + # copy the entire portage volume in + COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo + RUN emerge -v --noreplace dev-vcs/git + RUN emerge -v1u portage + # Pinned commits for the dependency tree state + ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 + ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448 + ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" + # This will be bound, and contents available outside of container + RUN mkdir /outputs + COPY .gentoo/portage/ /etc/portage/ + # Moving gentoo repo from default rsync to git + RUN rm /var/db/repos/gentoo -rf + # Cloning manually to prevent vdb update, pinning state via git + # Allegedly it's better to chain everything in one command, something with container layers 🤔 + RUN \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \ + mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \ + git remote add origin ${REPO_URL} && \ + git fetch --filter="blob:none" origin $gentoo_hash && \ + git reset --hard $gentoo_hash && rm .git -rf && popd && \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \ + mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \ + git remote add origin ${REPO_URL} && \ + git fetch --filter="blob:none" origin $science_hash && \ + git reset --hard $science_hash && rm .git -rf && popd + # Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively + # Christian from the future: Maybe we want the option to re-sync if we're debugging it interactively... + #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "s/sync-type *= *git/sync-type =/g" + #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-uri/d" + #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-git-verify-commit-signature/d" + # Make sure all CPU flags supported by the hardware are whitelisted + # This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg. + # Removing it is safe, software will just not take full advantage of processor capabilities. + #RUN emerge cpuid2cpuflags + #RUN echo "*/* $(cpuid2cpuflags)" > /etc/portage/package.use/00cpu-flags + ### Emerge cool stuff here + ### Autounmask-continue enables all features on dependencies which the top level packages require + ### By default this needs user confirmation which would interrupt the build. + RUN emerge --autounmask-continue afni fsl From 3428ab9c8a7fd0c46fbf7a8dcd575e3b0c75b9d6 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Thu, 25 Apr 2024 18:20:52 -0400 Subject: [PATCH 05/23] Corrected command --- neurodocker/cli/tests/test_build_images_with_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index 23d50473..5ad32914 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -117,7 +117,7 @@ def test_json_roundtrip(cmd: str, inputs: str, tmp_path: Path): def test_gentoo_image(tmp_path: Path): # also add singularity like in the test above - cmd = "neurodocker generate docker" + cmd = "docker" _TemplateRegistry._reset() runner = CliRunner() From 7f72cf7031bb9484d691f7e6323cdcf1b0d3e086 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Thu, 25 Apr 2024 18:56:57 -0400 Subject: [PATCH 06/23] Trying to add package manager support --- neurodocker/reproenv/types.py | 4 ++-- neurodocker/templates/afni.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/neurodocker/reproenv/types.py b/neurodocker/reproenv/types.py index 31f61e6d..17f3c0ab 100644 --- a/neurodocker/reproenv/types.py +++ b/neurodocker/reproenv/types.py @@ -18,8 +18,8 @@ allowed_installation_methods = {"binaries", "source"} installation_methods_type = Literal["binaries", "source"] -allowed_pkg_managers = {"apt", "yum"} -pkg_managers_type = Literal["apt", "yum"] +allowed_pkg_managers = {"apt", "portage", "yum"} +pkg_managers_type = Literal["apt", "portage", "yum"] # Cross-reference the dictionary types below with the JSON schemas. diff --git a/neurodocker/templates/afni.yaml b/neurodocker/templates/afni.yaml index 61247abe..2418b7d2 100644 --- a/neurodocker/templates/afni.yaml +++ b/neurodocker/templates/afni.yaml @@ -72,6 +72,8 @@ binaries: - which - unzip - ncurses-compat-libs + portage: + - sci-biology/ants debs: - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb - http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb From 01fd95aac196c8c38caead72e460f531e4bced9a Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 30 Apr 2024 13:47:24 -0400 Subject: [PATCH 07/23] ignore Yarik's venvs/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4cf7ecc9..028772ae 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,4 @@ cython_debug/ # Pycharm .idea/ +venvs From a5df85fd1d8966c1fdef2960fac6f06a1f1bcf7c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 30 Apr 2024 13:50:55 -0400 Subject: [PATCH 08/23] We are not yet fully ready for gentoo in recipes --- neurodocker/templates/afni.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/afni.yaml b/neurodocker/templates/afni.yaml index 2418b7d2..288eabbe 100644 --- a/neurodocker/templates/afni.yaml +++ b/neurodocker/templates/afni.yaml @@ -72,8 +72,8 @@ binaries: - which - unzip - ncurses-compat-libs - portage: - - sci-biology/ants + # portage: + # - sci-biology/ants debs: - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb - http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb From 39b79fc25bb5f5cb624daa6a3fa8febb79de13dd Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 30 Apr 2024 13:51:56 -0400 Subject: [PATCH 09/23] Provide base skeleton for gentoo to pass test_cli tests --- neurodocker/cli/tests/test_cli.py | 8 +++---- neurodocker/reproenv/renderers.py | 21 +++++++++++++++++++ neurodocker/reproenv/schemas/renderer.json | 2 ++ neurodocker/reproenv/types.py | 1 + .../templates/gentoo-portage/make.conf | 21 +++++++++++++++++++ .../package.accept_keywords/gen | 2 ++ .../gentoo-portage/package.mask/bugs | 2 ++ .../gentoo-portage/repos.conf/gentoo | 6 ++++++ .../gentoo-portage/repos.conf/science | 6 ++++++ neurodocker/templates/gentoo.yaml | 8 +++++-- 10 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 neurodocker/templates/gentoo-portage/make.conf create mode 100644 neurodocker/templates/gentoo-portage/package.accept_keywords/gen create mode 100644 neurodocker/templates/gentoo-portage/package.mask/bugs create mode 100644 neurodocker/templates/gentoo-portage/repos.conf/gentoo create mode 100644 neurodocker/templates/gentoo-portage/repos.conf/science diff --git a/neurodocker/cli/tests/test_cli.py b/neurodocker/cli/tests/test_cli.py index ba54339e..fe3b0894 100644 --- a/neurodocker/cli/tests/test_cli.py +++ b/neurodocker/cli/tests/test_cli.py @@ -19,7 +19,7 @@ def test_fail_on_empty_args(cmd: str): @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_fail_on_no_base(cmd: str, pkg_manager: str): runner = CliRunner() result = runner.invoke(generate, [cmd, "--pkg-manager", pkg_manager]) @@ -34,7 +34,7 @@ def test_fail_on_no_pkg_manager(cmd: str): @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_minimal_args(cmd: str, pkg_manager: str): runner = CliRunner() result = runner.invoke( @@ -107,7 +107,7 @@ def test_optioneatall_type_issue_498(): @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_all_args(cmd: str, pkg_manager: str): runner = CliRunner() result = runner.invoke( @@ -156,7 +156,7 @@ def test_all_args(cmd: str, pkg_manager: str): # is what registers the templates. Using the `docker` function # (`reproenv generate docker`) directly does not fire `generate`. @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_render_registered(cmd: str, pkg_manager: str): template_path = Path(__file__).parent runner = CliRunner(env={"REPROENV_TEMPLATE_PATH": str(template_path)}) diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 2df1a4c5..89a58679 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -753,6 +753,8 @@ def _indent_run_instruction(string: str, indent=4) -> str: def _install(pkgs: list[str], pkg_manager: str, opts: str = None) -> str: if pkg_manager == "apt": return _apt_install(pkgs, opts) + elif pkg_manager == "portage": + return _portage_install(pkgs, opts) elif pkg_manager == "yum": return _yum_install(pkgs, opts) # TODO: add debs here? @@ -802,6 +804,25 @@ def install_one(url: str): return s +def _portage_install(pkgs: list[str], opts: str = None, sort=True) -> str: + """Return command to install packages with `portage` (Gentoo). + + `opts` are options passed to `emerge`. + Default is "--autounmask-continue". + """ + pkgs = sorted(pkgs) if sort else pkgs + opts = "--autounmask-continue" if opts is None else opts + + s = """\ +emerge {opts} \\ + {pkgs} +rm -rf /var/tmp/portage/* +""".format( + opts=opts, pkgs=" \\\n ".join(pkgs) + ) + return s.strip() + + def _yum_install(pkgs: list[str], opts: str = None, sort=True) -> str: """Return command to install packages with `yum` (CentOS, Fedora). diff --git a/neurodocker/reproenv/schemas/renderer.json b/neurodocker/reproenv/schemas/renderer.json index 169fa1f9..6597d7ae 100644 --- a/neurodocker/reproenv/schemas/renderer.json +++ b/neurodocker/reproenv/schemas/renderer.json @@ -11,10 +11,12 @@ "type": "string", "enum": [ "apt", + "portage", "yum" ], "examples": [ "apt", + "portage", "yum" ] }, diff --git a/neurodocker/reproenv/types.py b/neurodocker/reproenv/types.py index 17f3c0ab..6cab0f83 100644 --- a/neurodocker/reproenv/types.py +++ b/neurodocker/reproenv/types.py @@ -32,6 +32,7 @@ class _InstallationDependenciesType(TypedDict, total=False): apt: list[str] debs: list[str] + portage: list[str] yum: list[str] diff --git a/neurodocker/templates/gentoo-portage/make.conf b/neurodocker/templates/gentoo-portage/make.conf new file mode 100644 index 00000000..5d2eded2 --- /dev/null +++ b/neurodocker/templates/gentoo-portage/make.conf @@ -0,0 +1,21 @@ +### This file contains system-wide build variables, including Gentoo variables such as USE, which enable/disable optional package features. + +COMMON_FLAGS="-O2 -pipe -march=native" +# Comment the following out on systems with less than 8 threads +MAKEOPTS="--jobs 8 --load-average 9" +CFLAGS="${COMMON_FLAGS}" +CXXFLAGS="${COMMON_FLAGS}" +FCFLAGS="${COMMON_FLAGS}" +FFLAGS="${COMMON_FLAGS}" + +# NOTE: This stage was built with the bindist Use flag enabled + +# This sets the language of build output to English. +# Please keep this setting intact when reporting bugs. +LC_MESSAGES=C + +USE="${USE} science" +ACCEPT_LICENSE="*" + +# Needed in the container environment +#FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" diff --git a/neurodocker/templates/gentoo-portage/package.accept_keywords/gen b/neurodocker/templates/gentoo-portage/package.accept_keywords/gen new file mode 100644 index 00000000..467373bd --- /dev/null +++ b/neurodocker/templates/gentoo-portage/package.accept_keywords/gen @@ -0,0 +1,2 @@ +### This is needed because ::science packages are generally not marked as stable +*/* ~amd64 diff --git a/neurodocker/templates/gentoo-portage/package.mask/bugs b/neurodocker/templates/gentoo-portage/package.mask/bugs new file mode 100644 index 00000000..1e593ebc --- /dev/null +++ b/neurodocker/templates/gentoo-portage/package.mask/bugs @@ -0,0 +1,2 @@ +### This is empty, thankfully. +### If we find bugs in some version of some package we can blacklist the package, version, or feature that causes it here. diff --git a/neurodocker/templates/gentoo-portage/repos.conf/gentoo b/neurodocker/templates/gentoo-portage/repos.conf/gentoo new file mode 100644 index 00000000..5d53ee40 --- /dev/null +++ b/neurodocker/templates/gentoo-portage/repos.conf/gentoo @@ -0,0 +1,6 @@ +[gentoo] +location = /var/db/repos/gentoo +# We sync manually, but we need sync-uri to be written down somewhere to do so +sync-type = git +sync-uri = https://anongit.gentoo.org/git/repo/gentoo.git +sync-git-verify-commit-signature = yes diff --git a/neurodocker/templates/gentoo-portage/repos.conf/science b/neurodocker/templates/gentoo-portage/repos.conf/science new file mode 100644 index 00000000..4a03df2a --- /dev/null +++ b/neurodocker/templates/gentoo-portage/repos.conf/science @@ -0,0 +1,6 @@ +[science] +location = /var/db/repos/science +# We sync manually, but we need sync-uri to be written down somewhere to do so +sync-type = git +sync-uri = https://anongit.gentoo.org/git/proj/sci.git +priority = 7777 diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 3c29b6d0..f72ed953 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -4,6 +4,11 @@ name: gentoo url: https://www.gentoo.org/ binaries: + # ATM template jsonschema demands having urls and instructions. + # In the future we might use urls to point to git repositories + # which are currently under gentoo-portage/ config files. + urls: + something: not-used instructions: | FROM docker.io/gentoo/portage:20240324 as portage FROM docker.io/gentoo/stage3:20240318 @@ -17,7 +22,7 @@ binaries: ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" # This will be bound, and contents available outside of container RUN mkdir /outputs - COPY .gentoo/portage/ /etc/portage/ + COPY gentoo-portage/ /etc/portage/ # Moving gentoo repo from default rsync to git RUN rm /var/db/repos/gentoo -rf # Cloning manually to prevent vdb update, pinning state via git @@ -46,4 +51,3 @@ binaries: ### Emerge cool stuff here ### Autounmask-continue enables all features on dependencies which the top level packages require ### By default this needs user confirmation which would interrupt the build. - RUN emerge --autounmask-continue afni fsl From 6bca6f5c5e80351927969465007e2e3a9d54890a Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 30 Apr 2024 14:07:09 -0400 Subject: [PATCH 10/23] Add some demo options for instructions -- the problem is that it is for the "RUN" not for the FROM --- neurodocker/templates/gentoo.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index f72ed953..035fe8a0 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -9,9 +9,13 @@ binaries: # which are currently under gentoo-portage/ config files. urls: something: not-used + arguments: + optional: + portage_image_version: "20240324" + stage3_image_version: "20240318" instructions: | - FROM docker.io/gentoo/portage:20240324 as portage - FROM docker.io/gentoo/stage3:20240318 + FROM docker.io/gentoo/portage:{{ self.portage_image_version }} as portage + FROM docker.io/gentoo/stage3:{{ self.stage3_image_version }} # copy the entire portage volume in COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo RUN emerge -v --noreplace dev-vcs/git From 9b0d75fb69f06f2582e42d314e4c95eaf88c1e29 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 1 May 2024 14:31:24 -0400 Subject: [PATCH 11/23] image versions not handled in this template, reusing parameters for hash --- neurodocker/templates/gentoo.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 035fe8a0..2edc70f6 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -11,18 +11,15 @@ binaries: something: not-used arguments: optional: - portage_image_version: "20240324" - stage3_image_version: "20240318" + gentoo_hash: "2d25617a1d085316761b06c17a93ec972f172fc6" + science_hash: "73916dd3680ffd92e5bd3d32b262e5d78c86a448" instructions: | - FROM docker.io/gentoo/portage:{{ self.portage_image_version }} as portage - FROM docker.io/gentoo/stage3:{{ self.stage3_image_version }} - # copy the entire portage volume in COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo RUN emerge -v --noreplace dev-vcs/git RUN emerge -v1u portage # Pinned commits for the dependency tree state - ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 - ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448 + ARG gentoo_hash={{ self.gentoo_hash }} + ARG science_hash={{ self.science_hash }} ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" # This will be bound, and contents available outside of container RUN mkdir /outputs From c8cb92290b179adfa9bcc9561140803925a238e8 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 1 May 2024 14:44:42 -0400 Subject: [PATCH 12/23] Embedding Gentoo image resource identifiers into the logic --- neurodocker/reproenv/renderers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 89a58679..e621e7ee 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -518,10 +518,14 @@ def entrypoint(self, args: list[str]) -> DockerRenderer: @_log_instruction def from_(self, base_image: str, as_: str = None) -> DockerRenderer: """Add a Dockerfile `FROM` instruction.""" - if as_ is None: - s = "FROM " + base_image + if base_image == "gentoo": + s = f"FROM docker.io/gentoo/portage:20240324 as portage \n" + s += f"FROM docker.io/gentoo/stage3:20240318" else: - s = f"FROM {base_image} AS {as_}" + if as_ is None: + s = "FROM " + base_image + else: + s = f"FROM {base_image} AS {as_}" self._parts.append(s) return self From 456c479a69fd1a1e49866ff7a230cad690479818 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 24 May 2024 12:30:35 -0400 Subject: [PATCH 13/23] Making gentoo template/invocation work (more TODOs) We did not figure out good way to do all of that like reacting to specific date. Might need some dedicated handling or option like ndfreeze. Also to figure out how to make it work without specifyin any optional argument --- neurodocker/reproenv/renderers.py | 11 +++- neurodocker/templates/gentoo.yaml | 87 +++++++++++++++++-------------- 2 files changed, 58 insertions(+), 40 deletions(-) diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index e621e7ee..6869fcf1 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -519,8 +519,15 @@ def entrypoint(self, args: list[str]) -> DockerRenderer: def from_(self, base_image: str, as_: str = None) -> DockerRenderer: """Add a Dockerfile `FROM` instruction.""" if base_image == "gentoo": - s = f"FROM docker.io/gentoo/portage:20240324 as portage \n" - s += f"FROM docker.io/gentoo/stage3:20240318" + # TODO (if we can, likely not): make neurodocker argument "portage_date" for which we + # figure out the corresponding stage3 date and hashes. + s = "FROM docker.io/gentoo/portage:20240324 as portage\n" + s += "FROM docker.io/gentoo/stage3:20240318\n" + s += "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\n" + # TODO: figure out hashes for the date + #s += "ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\n" + #s += "ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\n" + s += "ARG FEATURES=\"-ipc-sandbox -network-sandbox -pid-sandbox\"\n" else: if as_ is None: s = "FROM " + base_image diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 2edc70f6..de4900ba 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -13,42 +13,53 @@ binaries: optional: gentoo_hash: "2d25617a1d085316761b06c17a93ec972f172fc6" science_hash: "73916dd3680ffd92e5bd3d32b262e5d78c86a448" + # Below echo commands will be indented by _indent_run_instruction which + # would cause trailing spaces to be added to the files. But it should be ok + # for us, thus we are not bothering to workaround via inlining. instructions: | - COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo - RUN emerge -v --noreplace dev-vcs/git - RUN emerge -v1u portage - # Pinned commits for the dependency tree state - ARG gentoo_hash={{ self.gentoo_hash }} - ARG science_hash={{ self.science_hash }} - ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" - # This will be bound, and contents available outside of container - RUN mkdir /outputs - COPY gentoo-portage/ /etc/portage/ - # Moving gentoo repo from default rsync to git - RUN rm /var/db/repos/gentoo -rf - # Cloning manually to prevent vdb update, pinning state via git - # Allegedly it's better to chain everything in one command, something with container layers 🤔 - RUN \ - REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \ - mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \ - git remote add origin ${REPO_URL} && \ - git fetch --filter="blob:none" origin $gentoo_hash && \ - git reset --hard $gentoo_hash && rm .git -rf && popd && \ - REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \ - mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \ - git remote add origin ${REPO_URL} && \ - git fetch --filter="blob:none" origin $science_hash && \ - git reset --hard $science_hash && rm .git -rf && popd - # Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively - # Christian from the future: Maybe we want the option to re-sync if we're debugging it interactively... - #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "s/sync-type *= *git/sync-type =/g" - #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-uri/d" - #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-git-verify-commit-signature/d" - # Make sure all CPU flags supported by the hardware are whitelisted - # This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg. - # Removing it is safe, software will just not take full advantage of processor capabilities. - #RUN emerge cpuid2cpuflags - #RUN echo "*/* $(cpuid2cpuflags)" > /etc/portage/package.use/00cpu-flags - ### Emerge cool stuff here - ### Autounmask-continue enables all features on dependencies which the top level packages require - ### By default this needs user confirmation which would interrupt the build. + mkdir -p /etc/portage/; \ + echo -e "\ + \nCOMMON_FLAGS=\"-O2 -pipe -march=native\" \ + \nMAKEOPTS=\"--jobs 8 --load-average 9\" \ + \nCFLAGS=\"\${COMMON_FLAGS}\" \ + \nCXXFLAGS=\"\${COMMON_FLAGS}\" \ + \nFCFLAGS=\"\${COMMON_FLAGS}\" \ + \nFFLAGS=\"\${COMMON_FLAGS}\" \ + \nLC_MESSAGES=C \ + \nUSE=\"\${USE} science\" \ + \nACCEPT_LICENSE=\"*\" \ + " > "/etc/portage/make.conf"; \ + mkdir -p "/etc/portage/package.accept_keywords"; \ + echo -e "*/* ~amd64" > "/etc/portage/package.accept_keywords/gen" ; \ + mkdir -p "/etc/portage/package.mask"; \ + touch "/etc/portage/package.mask/bugs"; \ + mkdir -p "/etc/portage/repos.conf" ; \ + echo -e "[gentoo] \ + \nlocation = /var/db/repos/gentoo \ + \nsync-type = git \ + \nsync-uri = https://anongit.gentoo.org/git/repo/gentoo.git \ + \nsync-git-verify-commit-signature = yes" > "/etc/portage/repos.conf/gentoo"; \ + echo -e "[science] \ + \nlocation = /var/db/repos/science \ + \nsync-type = git \ + \nsync-uri = https://anongit.gentoo.org/git/proj/sci.git \ + \npriority = 7777" > "/etc/portage/repos.conf/science"; \ + emerge -v --noreplace dev-vcs/git \ + && emerge -v1u portage \ + && mkdir /outputs \ + && rm /var/db/repos/gentoo -rf \ + && git config --global init.defaultBranch master \ + && \ + set -x && export GIT_TRACE=1 && \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \ + git clone --depth 1 ${REPO_URL} /var/db/repos/gentoo && \ + cd /var/db/repos/gentoo && \ + git fetch --depth 1 origin {{ self.gentoo_hash }} && \ + git reset --hard {{ self.gentoo_hash }} && \ + rm .git -rf && \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \ + git clone --depth 1 ${REPO_URL} /var/db/repos/science && \ + cd /var/db/repos/science && \ + git fetch --depth 1 origin {{ self.science_hash }} && \ + git reset --hard {{ self.science_hash }} && \ + rm .git -rf From 30ab193cfdcc3bda4fd19fafd830a03f72209436 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Tue, 28 May 2024 14:10:52 -0400 Subject: [PATCH 14/23] Added new hashes --- neurodocker/templates/gentoo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index de4900ba..22269275 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -11,8 +11,8 @@ binaries: something: not-used arguments: optional: - gentoo_hash: "2d25617a1d085316761b06c17a93ec972f172fc6" - science_hash: "73916dd3680ffd92e5bd3d32b262e5d78c86a448" + gentoo_hash: "47e776639abff6a0c61e3b04d7224cb4dfc4854b" + science_hash: "0075b0aa4f9993e0bc54e9d627329332555d16e0" # Below echo commands will be indented by _indent_run_instruction which # would cause trailing spaces to be added to the files. But it should be ok # for us, thus we are not bothering to workaround via inlining. From e7f1720b7f4784a5b26daa9a269db6e32bbade73 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Wed, 29 May 2024 13:53:09 -0400 Subject: [PATCH 15/23] Updated hashes and dates after test, provide AFNI with version fix as well --- neurodocker/reproenv/renderers.py | 4 ++-- neurodocker/templates/gentoo.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 6869fcf1..c32edefd 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -521,8 +521,8 @@ def from_(self, base_image: str, as_: str = None) -> DockerRenderer: if base_image == "gentoo": # TODO (if we can, likely not): make neurodocker argument "portage_date" for which we # figure out the corresponding stage3 date and hashes. - s = "FROM docker.io/gentoo/portage:20240324 as portage\n" - s += "FROM docker.io/gentoo/stage3:20240318\n" + s = "FROM docker.io/gentoo/portage:20240529 as portage\n" + s += "FROM docker.io/gentoo/stage3:20240527\n" s += "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\n" # TODO: figure out hashes for the date #s += "ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\n" diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 22269275..6d8a95b5 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -11,8 +11,8 @@ binaries: something: not-used arguments: optional: - gentoo_hash: "47e776639abff6a0c61e3b04d7224cb4dfc4854b" - science_hash: "0075b0aa4f9993e0bc54e9d627329332555d16e0" + gentoo_hash: "0e9370b45a589867220384ca6c63bc6bcaec3f74" + science_hash: "6d9b772f10ed51682604e5c1cb6f9c278a5c6c72" # Below echo commands will be indented by _indent_run_instruction which # would cause trailing spaces to be added to the files. But it should be ok # for us, thus we are not bothering to workaround via inlining. From 758951bde08016f94ec3027cab32f9b25ce2bdaf Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Fri, 7 Jun 2024 13:23:19 -0400 Subject: [PATCH 16/23] updated hashes --- neurodocker/templates/gentoo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 6d8a95b5..5b32fbf9 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -12,7 +12,7 @@ binaries: arguments: optional: gentoo_hash: "0e9370b45a589867220384ca6c63bc6bcaec3f74" - science_hash: "6d9b772f10ed51682604e5c1cb6f9c278a5c6c72" + science_hash: "5307342730267714f7019d62f77b2d9bf7624d8c" # Below echo commands will be indented by _indent_run_instruction which # would cause trailing spaces to be added to the files. But it should be ok # for us, thus we are not bothering to workaround via inlining. From 8b5df1dea048ab0f05cf3571775a65a0c8237618 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 11 Jun 2024 18:19:29 -0400 Subject: [PATCH 17/23] Rudimentary fix to blunt stab at a test + added TODOs --- .../cli/tests/test_build_images_with_cli.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index 5ad32914..32177f5e 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -26,7 +26,13 @@ ], ) @pytest.mark.parametrize( - ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "centos:7")] + ["pkg_manager", "base_image"], + [ + ("apt", "debian:buster-slim"), + ("yum", "centos:7"), + # TODO: make it work -- seems has difficulty installing jq, may be curl isn't there or alike + # ("portage", "gentoo"), + ], ) def test_build_image_from_registered( tmp_path: Path, cmd: str, pkg_manager: str, base_image: str @@ -114,8 +120,9 @@ def test_json_roundtrip(cmd: str, inputs: str, tmp_path: Path): assert "CAT=FOO" in stdout assert "DOG=BAR" in stdout + def test_gentoo_image(tmp_path: Path): - # also add singularity like in the test above + # TODO: also add singularity like in the test above cmd = "docker" @@ -125,11 +132,13 @@ def test_gentoo_image(tmp_path: Path): generate, [ cmd, - "--pkg-manager apt", - "--base-image neurodebian:bullseye", - "--ants version=2.4.3", - "--user nonroot" + "--pkg-manager", "portage", + "--base-image", "gentoo", + # TODO: this flips it out for some reason in the test, works on CLI + # "--gentoo", "gentoo_hash=0e9370b45a589867220384ca6c63bc6bcaec3f74", + "--install", "afni", # TODO: replace with some tiny / quick to install pkg ], ) assert result.exit_code == 0, result.output (tmp_path / "specs.json").write_text(result.output) + # TODO: add more testing here of the result that it is usable From 722e555991c61635f6ae26b6c20c130dffcea760 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Mon, 29 Jul 2024 09:55:26 -0400 Subject: [PATCH 18/23] Removed config tracking in ancillary files --- .../templates/gentoo-portage/make.conf | 21 ------------------- .../package.accept_keywords/gen | 2 -- .../gentoo-portage/package.mask/bugs | 2 -- .../gentoo-portage/repos.conf/gentoo | 6 ------ .../gentoo-portage/repos.conf/science | 6 ------ 5 files changed, 37 deletions(-) delete mode 100644 neurodocker/templates/gentoo-portage/make.conf delete mode 100644 neurodocker/templates/gentoo-portage/package.accept_keywords/gen delete mode 100644 neurodocker/templates/gentoo-portage/package.mask/bugs delete mode 100644 neurodocker/templates/gentoo-portage/repos.conf/gentoo delete mode 100644 neurodocker/templates/gentoo-portage/repos.conf/science diff --git a/neurodocker/templates/gentoo-portage/make.conf b/neurodocker/templates/gentoo-portage/make.conf deleted file mode 100644 index 5d2eded2..00000000 --- a/neurodocker/templates/gentoo-portage/make.conf +++ /dev/null @@ -1,21 +0,0 @@ -### This file contains system-wide build variables, including Gentoo variables such as USE, which enable/disable optional package features. - -COMMON_FLAGS="-O2 -pipe -march=native" -# Comment the following out on systems with less than 8 threads -MAKEOPTS="--jobs 8 --load-average 9" -CFLAGS="${COMMON_FLAGS}" -CXXFLAGS="${COMMON_FLAGS}" -FCFLAGS="${COMMON_FLAGS}" -FFLAGS="${COMMON_FLAGS}" - -# NOTE: This stage was built with the bindist Use flag enabled - -# This sets the language of build output to English. -# Please keep this setting intact when reporting bugs. -LC_MESSAGES=C - -USE="${USE} science" -ACCEPT_LICENSE="*" - -# Needed in the container environment -#FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" diff --git a/neurodocker/templates/gentoo-portage/package.accept_keywords/gen b/neurodocker/templates/gentoo-portage/package.accept_keywords/gen deleted file mode 100644 index 467373bd..00000000 --- a/neurodocker/templates/gentoo-portage/package.accept_keywords/gen +++ /dev/null @@ -1,2 +0,0 @@ -### This is needed because ::science packages are generally not marked as stable -*/* ~amd64 diff --git a/neurodocker/templates/gentoo-portage/package.mask/bugs b/neurodocker/templates/gentoo-portage/package.mask/bugs deleted file mode 100644 index 1e593ebc..00000000 --- a/neurodocker/templates/gentoo-portage/package.mask/bugs +++ /dev/null @@ -1,2 +0,0 @@ -### This is empty, thankfully. -### If we find bugs in some version of some package we can blacklist the package, version, or feature that causes it here. diff --git a/neurodocker/templates/gentoo-portage/repos.conf/gentoo b/neurodocker/templates/gentoo-portage/repos.conf/gentoo deleted file mode 100644 index 5d53ee40..00000000 --- a/neurodocker/templates/gentoo-portage/repos.conf/gentoo +++ /dev/null @@ -1,6 +0,0 @@ -[gentoo] -location = /var/db/repos/gentoo -# We sync manually, but we need sync-uri to be written down somewhere to do so -sync-type = git -sync-uri = https://anongit.gentoo.org/git/repo/gentoo.git -sync-git-verify-commit-signature = yes diff --git a/neurodocker/templates/gentoo-portage/repos.conf/science b/neurodocker/templates/gentoo-portage/repos.conf/science deleted file mode 100644 index 4a03df2a..00000000 --- a/neurodocker/templates/gentoo-portage/repos.conf/science +++ /dev/null @@ -1,6 +0,0 @@ -[science] -location = /var/db/repos/science -# We sync manually, but we need sync-uri to be written down somewhere to do so -sync-type = git -sync-uri = https://anongit.gentoo.org/git/proj/sci.git -priority = 7777 From 5420c0404455b006e28153405fa0ec1b2c067316 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Tue, 30 Jul 2024 11:55:54 -0400 Subject: [PATCH 19/23] Updated hashes --- neurodocker/templates/gentoo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 5b32fbf9..0ae27c1c 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -11,8 +11,8 @@ binaries: something: not-used arguments: optional: - gentoo_hash: "0e9370b45a589867220384ca6c63bc6bcaec3f74" - science_hash: "5307342730267714f7019d62f77b2d9bf7624d8c" + gentoo_hash: "282c4b518562e6c194761d9ce0882f30672219c5" + science_hash: "e4579f5851734f245b02cc1e98f3b69d9df67e09" # Below echo commands will be indented by _indent_run_instruction which # would cause trailing spaces to be added to the files. But it should be ok # for us, thus we are not bothering to workaround via inlining. From 54c98f66becdd52c9ca7da44c0f59deb8155b7aa Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Tue, 30 Jul 2024 11:56:05 -0400 Subject: [PATCH 20/23] better package for testing --- neurodocker/cli/tests/test_build_images_with_cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index 32177f5e..e1d7b198 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -134,9 +134,7 @@ def test_gentoo_image(tmp_path: Path): cmd, "--pkg-manager", "portage", "--base-image", "gentoo", - # TODO: this flips it out for some reason in the test, works on CLI - # "--gentoo", "gentoo_hash=0e9370b45a589867220384ca6c63bc6bcaec3f74", - "--install", "afni", # TODO: replace with some tiny / quick to install pkg + "--install", " app-text/asciidoc", # TODO: replace with some tiny / quick to install pkg ], ) assert result.exit_code == 0, result.output From 5fd6303d871d23d73f1ae7a168b877ef8601c675 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Tue, 30 Jul 2024 12:12:51 -0400 Subject: [PATCH 21/23] Even smaller package for testing --- neurodocker/cli/tests/test_build_images_with_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index e1d7b198..d8f7abdb 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -134,7 +134,7 @@ def test_gentoo_image(tmp_path: Path): cmd, "--pkg-manager", "portage", "--base-image", "gentoo", - "--install", " app-text/asciidoc", # TODO: replace with some tiny / quick to install pkg + "--install", "app-misc/mime-types", # TODO: replace with some tiny / quick to install pkg ], ) assert result.exit_code == 0, result.output From cc0d86e9a4b9ec02c3f66e69fef0d7a12941b192 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Fri, 2 Aug 2024 00:27:53 -0400 Subject: [PATCH 22/23] flake8 fixes --- neurodocker/cli/tests/test_build_images_with_cli.py | 5 +++-- neurodocker/reproenv/renderers.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index d8f7abdb..f9fe473f 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -30,7 +30,8 @@ [ ("apt", "debian:buster-slim"), ("yum", "centos:7"), - # TODO: make it work -- seems has difficulty installing jq, may be curl isn't there or alike + # TODO: make it work -- seems has difficulty installing jq, + # may be curl isn't there or alike # ("portage", "gentoo"), ], ) @@ -134,7 +135,7 @@ def test_gentoo_image(tmp_path: Path): cmd, "--pkg-manager", "portage", "--base-image", "gentoo", - "--install", "app-misc/mime-types", # TODO: replace with some tiny / quick to install pkg + "--install", "app-misc/mime-types", ], ) assert result.exit_code == 0, result.output diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index c32edefd..4cec9ceb 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -519,14 +519,15 @@ def entrypoint(self, args: list[str]) -> DockerRenderer: def from_(self, base_image: str, as_: str = None) -> DockerRenderer: """Add a Dockerfile `FROM` instruction.""" if base_image == "gentoo": - # TODO (if we can, likely not): make neurodocker argument "portage_date" for which we + # TODO (if we can, likely not): + # make neurodocker argument "portage_date" for which we # figure out the corresponding stage3 date and hashes. s = "FROM docker.io/gentoo/portage:20240529 as portage\n" s += "FROM docker.io/gentoo/stage3:20240527\n" s += "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\n" # TODO: figure out hashes for the date - #s += "ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\n" - #s += "ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\n" + # s += "ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\n" + # s += "ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\n" s += "ARG FEATURES=\"-ipc-sandbox -network-sandbox -pid-sandbox\"\n" else: if as_ is None: From 6d76eb58d239f35e31d84ec094e50d67d8d157fc Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 15 Aug 2024 13:10:04 +0200 Subject: [PATCH 23/23] run precommit --- neurodocker/cli/tests/test_build_images_with_cli.py | 11 +++++++---- neurodocker/reproenv/renderers.py | 2 +- neurodocker/templates/gentoo.yaml | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index f9fe473f..71ac83fe 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -33,7 +33,7 @@ # TODO: make it work -- seems has difficulty installing jq, # may be curl isn't there or alike # ("portage", "gentoo"), - ], + ], ) def test_build_image_from_registered( tmp_path: Path, cmd: str, pkg_manager: str, base_image: str @@ -133,9 +133,12 @@ def test_gentoo_image(tmp_path: Path): generate, [ cmd, - "--pkg-manager", "portage", - "--base-image", "gentoo", - "--install", "app-misc/mime-types", + "--pkg-manager", + "portage", + "--base-image", + "gentoo", + "--install", + "app-misc/mime-types", ], ) assert result.exit_code == 0, result.output diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 4cec9ceb..0d4429fe 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -528,7 +528,7 @@ def from_(self, base_image: str, as_: str = None) -> DockerRenderer: # TODO: figure out hashes for the date # s += "ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\n" # s += "ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\n" - s += "ARG FEATURES=\"-ipc-sandbox -network-sandbox -pid-sandbox\"\n" + s += 'ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox"\n' else: if as_ is None: s = "FROM " + base_image diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml index 0ae27c1c..62a7426c 100644 --- a/neurodocker/templates/gentoo.yaml +++ b/neurodocker/templates/gentoo.yaml @@ -11,8 +11,8 @@ binaries: something: not-used arguments: optional: - gentoo_hash: "282c4b518562e6c194761d9ce0882f30672219c5" - science_hash: "e4579f5851734f245b02cc1e98f3b69d9df67e09" + gentoo_hash: 282c4b518562e6c194761d9ce0882f30672219c5 + science_hash: e4579f5851734f245b02cc1e98f3b69d9df67e09 # Below echo commands will be indented by _indent_run_instruction which # would cause trailing spaces to be added to the files. But it should be ok # for us, thus we are not bothering to workaround via inlining.