Merge pull request opendatahub-io#193 from tumido/change_python_image

feat: use python image with pre-installed kfp package
MichaelClifford · Nov 20, 2024 · f3b78c0 · f3b78c0
2 parents 3eecce2 + a0da880
commit f3b78c0
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 38 deletions.
diff --git a/pipeline.yaml b/pipeline.yaml
@@ -721,9 +721,7 @@ deploymentSpec:
         - -c
         - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\
-          \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"'  &&\
-          \  python3 -m pip install --quiet --no-warn-script-location 'huggingface_hub'\
+          \ python3 -m pip install --quiet --no-warn-script-location 'huggingface_hub'\
           \ && \"$0\" \"$@\"\n"
         - sh
         - -ec
@@ -739,7 +737,7 @@ deploymentSpec:
           \ *\n\ndef huggingface_importer_op(repo_name: str, model_path: str = \"\
           /model\"):\n    from huggingface_hub import snapshot_download\n\n    snapshot_download(repo_id=repo_name,\
           \ cache_dir=\"/tmp\", local_dir=model_path)\n\n"
-        image: registry.access.redhat.com/ubi9/python-311:latest
+        image: quay.io/modh/odh-generic-data-science-notebook:v3-2024b-20241111
     exec-knowledge-processed-data-to-artifact-op:
       container:
         args:
@@ -791,13 +789,6 @@ deploymentSpec:
         - list_models_in_directory_op
         command:
         - sh
-        - -c
-        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
-          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\
-          \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
-          $0\" \"$@\"\n"
-        - sh
         - -ec
         - 'program_path=$(mktemp -d)
 
@@ -811,7 +802,7 @@ deploymentSpec:
           \ *\n\ndef list_models_in_directory_op(models_folder: str) -> List[str]:\n\
           \    import os\n\n    models = os.listdir(models_folder)\n    return models\n\
           \n"
-        image: registry.access.redhat.com/ubi9/python-311:latest
+        image: quay.io/modh/odh-generic-data-science-notebook:v3-2024b-20241111
     exec-pvc-to-model-op:
       container:
         args:
@@ -837,13 +828,6 @@ deploymentSpec:
         - pytorchjob_manifest_op
         command:
         - sh
-        - -c
-        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
-          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\
-          \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
-          $0\" \"$@\"\n"
-        - sh
         - -ec
         - 'program_path=$(mktemp -d)
 
@@ -984,7 +968,7 @@ deploymentSpec:
           \   - name: output\n                      persistentVolumeClaim:\n     \
           \                   claimName: {output_pvc_name}\n        \"\"\"\n    )\n\
           \n    return Outputs(manifest, name)\n\n"
-        image: registry.access.redhat.com/ubi9/python-311:latest
+        image: quay.io/modh/odh-generic-data-science-notebook:v3-2024b-20241111
     exec-pytorchjob-manifest-op-2:
       container:
         args:
@@ -994,13 +978,6 @@ deploymentSpec:
         - pytorchjob_manifest_op
         command:
         - sh
-        - -c
-        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
-          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\
-          \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
-          $0\" \"$@\"\n"
-        - sh
         - -ec
         - 'program_path=$(mktemp -d)
 
@@ -1141,7 +1118,7 @@ deploymentSpec:
           \   - name: output\n                      persistentVolumeClaim:\n     \
           \                   claimName: {output_pvc_name}\n        \"\"\"\n    )\n\
           \n    return Outputs(manifest, name)\n\n"
-        image: registry.access.redhat.com/ubi9/python-311:latest
+        image: quay.io/modh/odh-generic-data-science-notebook:v3-2024b-20241111
     exec-run-final-eval-op:
       container:
         args:

diff --git a/training/components.py b/training/components.py
@@ -120,7 +120,7 @@ def knowledge_processed_data_to_artifact_op(
     )
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def pytorchjob_manifest_op(
     model_pvc_name: str,
     input_pvc_name: str,

diff --git a/training/faked/components.py b/training/faked/components.py
@@ -8,7 +8,7 @@
 from utils.consts import PYTHON_IMAGE, TOOLBOX_IMAGE
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def pytorchjob_manifest_op(
     model_pvc_name: str,
     input_pvc_name: str,
@@ -19,7 +19,7 @@ def pytorchjob_manifest_op(
     return Outputs("", "")
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def data_processing_op(
     model_path: str = "/model",
     sdg_path: str = "/data/sdg",

diff --git a/utils/components.py b/utils/components.py
@@ -51,7 +51,7 @@ def pvc_to_model_op(model: dsl.Output[dsl.Model], pvc_path: str):
     )
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def list_models_in_directory_op(models_folder: str) -> List[str]:
     import os
 
@@ -61,6 +61,7 @@ def list_models_in_directory_op(models_folder: str) -> List[str]:
 
 @dsl.component(
     base_image=PYTHON_IMAGE,
+    install_kfp_package=False,
     packages_to_install=["huggingface_hub"],
 )
 def huggingface_importer_op(repo_name: str, model_path: str = "/model"):

diff --git a/utils/consts.py b/utils/consts.py
@@ -1,4 +1,4 @@
-PYTHON_IMAGE = "registry.access.redhat.com/ubi9/python-311:latest"
+PYTHON_IMAGE = "quay.io/modh/odh-generic-data-science-notebook:v3-2024b-20241111"
 TOOLBOX_IMAGE = "registry.access.redhat.com/ubi9/toolbox"
 OC_IMAGE = "registry.redhat.io/openshift4/ose-cli"
 RHELAI_IMAGE = "quay.io/redhat-et/ilab:1.2"
diff --git a/utils/faked/components.py b/utils/faked/components.py
@@ -5,26 +5,26 @@
 from ..consts import PYTHON_IMAGE
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def kubectl_apply_op(manifest: str):
     return
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def kubectl_wait_for_op(condition: str, kind: str, name: str):
     return
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def huggingface_importer_op(repo_name: str, model_path: str = "/model"):
     return
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def pvc_to_mt_bench_op(mt_bench_output: dsl.Output[dsl.Artifact], pvc_path: str):
     return
 
 
-@dsl.component(base_image=PYTHON_IMAGE)
+@dsl.component(base_image=PYTHON_IMAGE, install_kfp_package=False)
 def pvc_to_model_op(model: dsl.Output[dsl.Model], pvc_path: str):
     return