VirtualEnv can be created from setup.py files

agentos/cli.py

@@ -12,6 +12,7 @@
 
 from agentos import ArgumentSet, Component
 from agentos.registry import Registry
+from agentos.repo import Repo
 from agentos.run import Run
 from agentos.virtual_env import VirtualEnv
 
@@ -265,7 +266,35 @@ def publish(
 @agentos_cmd.command()
 @_option_assume_yes
 def clear_env_cache(assume_yes):
+    """
+    This command clears all virtual environments that have been cached by
+    AgentOS in your local file system.  All the virtual environments can be
+    automatically recreated when re-running a Component ``requirements_path``
+    specified.
+    """
+    VirtualEnv.clear_env_cache(assume_yes=assume_yes)
+
+
+@agentos_cmd.command()
+@_option_assume_yes
+def clear_repo_cache(assume_yes):
+    """
+    This command clears all git repos that have been cached by AgentOS on your
+    local file system.  These repos will be recreated if as you run Components
+    that require them.
+    """
+    Repo.clear_repo_cache(assume_yes=assume_yes)
+
+
+@agentos_cmd.command()
+@_option_assume_yes
+def clear_cache(assume_yes):
+    """
+    This command clears all virtual environments AND git repos that have been
+    cached by AgentOS on your local file system.
+    """
     VirtualEnv.clear_env_cache(assume_yes=assume_yes)
+    Repo.clear_repo_cache(assume_yes=assume_yes)
 
 
 # Copied from https://github.com/mlflow/mlflow/blob/3958cdf9664ade34ebcf5960bee215c80efae992/mlflow/cli.py#L188 # noqa: E501

agentos/component.py

@@ -440,10 +440,11 @@ def _build_virtual_env(self) -> VirtualEnv:
         for c in self.dependency_list(include_parents=True):
             if c.requirements_path is None:
                 continue
-            full_req_path = self.repo.get_local_file_path(
-                c.identifier.version, c.requirements_path
-            ).absolute()
-            req_paths.add(full_req_path)
+            for req_path in c.requirements_path.split(";"):
+                full_req_path = self.repo.get_local_file_path(
+                    c.identifier.version, req_path
+                ).absolute()
+                req_paths.add(full_req_path)
         return VirtualEnv.from_requirements_paths(req_paths)
 
     def _handle_repo_spec(self, repos):

agentos/repo.py

@@ -19,7 +19,7 @@
 from agentos.identifiers import ComponentIdentifier, RepoIdentifier
 from agentos.registry import InMemoryRegistry, Registry
 from agentos.specs import NestedRepoSpec, RepoSpec, RepoSpecKeys, flatten_spec
-from agentos.utils import AOS_GLOBAL_REPOS_DIR
+from agentos.utils import AOS_GLOBAL_REPOS_DIR, _clear_cache_path
 
 logger = logging.getLogger(__name__)
 
@@ -51,13 +51,13 @@ def __eq__(self, other: object) -> bool:
     @staticmethod
     def from_spec(spec: NestedRepoSpec, base_dir: str = None) -> "Repo":
         flat_spec = flatten_spec(spec)
-        if flat_spec["type"] == RepoType.LOCAL.value:
+        if flat_spec[RepoSpecKeys.TYPE] == RepoType.LOCAL.value:
             return LocalRepo.from_spec(spec, base_dir)
-        if flat_spec["type"] == RepoType.GITHUB.value:
+        if flat_spec[RepoSpecKeys.TYPE] == RepoType.GITHUB.value:
             return GitHubRepo.from_spec(spec)
         raise PythonComponentSystemException(
-            f"Unknown repo type '{flat_spec['type']} in "
-            f"repo '{flat_spec['identifier']}'"
+            f"Unknown repo type '{flat_spec[RepoSpecKeys.TYPE]} in "
+            f"repo '{flat_spec[RepoSpecKeys.IDENTIFIER]}'"
         )
 
     @classmethod
@@ -75,6 +75,17 @@ def from_github(
         url = f"https://github.com/{github_account}/{repo_name}"
         return GitHubRepo(identifier, url)
 
+    def clear_repo_cache(
+        repo_cache_path: Path = None, assume_yes: bool = False
+    ) -> None:
+        """
+        Completely removes all the repos that have been created or checked
+        out in the ``repo_cache_path``. Pass True to ``assume_yes`` to run
+        non-interactively.
+        """
+        repo_cache_path = repo_cache_path or AOS_GLOBAL_REPOS_DIR
+        _clear_cache_path(repo_cache_path, assume_yes)
+
     @abc.abstractmethod
     def to_spec(self, flatten: bool = False) -> RepoSpec:
         return NotImplementedError  # type: ignore

agentos/utils.py

@@ -1,4 +1,5 @@
 import pprint
+import shutil
 from pathlib import Path
 from typing import Dict, Optional
 
@@ -60,6 +61,25 @@ def parse_github_web_ui_url(
     return project_name, repo_name, branch_name, repo_path
 
 
+def _clear_cache_path(cache_path: Path, assume_yes: bool):
+    cache_path = Path(cache_path).absolute()
+    if not cache_path.exists():
+        print(f"Cache path {cache_path} does not exist.  Aborting...")
+        return
+    answer = None
+    if assume_yes:
+        answer = "y"
+    else:
+        answer = input(
+            f"This will remove everything under {cache_path}. Continue? [Y/N] "
+        )
+    if assume_yes or answer.lower() in ["y", "yes"]:
+        shutil.rmtree(cache_path)
+        print("Cache cleared...")
+        return
+    print("Aborting...")
+
+
 def generate_dummy_dev_registry(
     version_string: str = "for_tests_dummy_dev_registry",
 ) -> Dict:

agentos/virtual_env.py

@@ -1,6 +1,5 @@
 import hashlib
 import os
-import shutil
 import subprocess
 import sys
 import sysconfig
@@ -10,7 +9,7 @@
 
 import yaml
 
-from agentos.utils import AOS_GLOBAL_REQS_DIR
+from agentos.utils import AOS_GLOBAL_REQS_DIR, _clear_cache_path
 
 
 class VirtualEnv:
@@ -73,19 +72,7 @@ def clear_env_cache(
         for Components.  Pass True to ``assume_yes`` to run non-interactively.
         """
         env_cache_path = env_cache_path or AOS_GLOBAL_REQS_DIR
-        answer = None
-        if assume_yes:
-            answer = "y"
-        else:
-            answer = input(
-                f"This will remove everything under {env_cache_path}.  "
-                "Continue? [Y/N] "
-            )
-        if assume_yes or answer.lower() in ["y", "yes"]:
-            shutil.rmtree(env_cache_path)
-            print("Cache cleared...")
-            return
-        print("Aborting...")
+        _clear_cache_path(env_cache_path, assume_yes)
 
     def _save_default_env_info(self):
         self._default_sys_path = [p for p in sys.path]
@@ -103,25 +90,25 @@ def activate(self) -> None:
         """
         assert self.venv_path.exists(), f"{self.venv_path} does not exist!"
         self._save_default_env_info()
-        self._set_venv_sys_path()
-        self._set_venv_sys_attributes()
-        self._set_venv_sys_environment()
+        self._exec_activate_this_script()
         self._venv_is_active = True
         print(f"VirtualEnv: Running in Python venv at {self.venv_path}")
 
-    def _set_venv_sys_path(self):
-        sys_path_copy = [p for p in sys.path]
-        self._clear_sys_path()
-        for p in sys_path_copy:
-            if p.startswith(sys.base_prefix):
-                sys.path.append(p)
+    def _exec_activate_this_script(self):
+        """
+        To see how to manually activate a virtual environment in a running
+        interpreter, look at the ``activate_this.py` script automatically
+        created in the bin directory of a virtual environment.
+
+        For example, if your virtual environment is found at ``/foo/bar/venv``,
+        then you can find the script at ``/foo/bar/venv/bin/activate_this.py``.
+        """
+        scripts_path = self.venv_path / "bin"
         if sys.platform in ["win32", "win64"]:
-            sys.path.insert(0, str(self.venv_path / "Scripts"))
-            sys.path.append(str(self.venv_path / "Lib" / "site-packages"))
-        else:
-            sys.path.insert(0, str(self.venv_path / "bin"))
-            versioned_lib_path = self.venv_path / "lib" / self._py_version
-            sys.path.append(str(versioned_lib_path / "site-packages"))
+            scripts_path = self.venv_path / "Scripts"
+        activate_script_path = scripts_path / "activate_this.py"
+        with open(activate_script_path) as file_in:
+            exec(file_in.read(), {"__file__": activate_script_path})
 
     def _clear_sys_path(self):
         while len(sys.path) > 0:
@@ -197,13 +184,19 @@ def install_requirements_file(
         python_path = self.venv_path / "bin" / "python"
         if sys.platform in ["win32", "win64"]:
             python_path = self.venv_path / "Scripts" / "python.exe"
+
+        install_flag = "-r"
+        install_path = str(req_path)
+        if req_path.name == "setup.py":
+            install_flag = "-e"
+            install_path = str(req_path.parent)
         cmd = [
             str(python_path),
             "-m",
             "pip",
             "install",
-            "-r",
-            str(req_path),
+            install_flag,
+            install_path,
         ]
         embedded_pip_flags = self._get_embedded_pip_flags(req_path)
         for flag, value in embedded_pip_flags.items():

example_agents/rllib_agent/requirements.txt

@@ -1,11 +1,13 @@
-# When updating here, also update conda_env.yaml
-requests==2.21.0  # Needed till https://github.com/ray-project/ray/issues/8373 fixed.
-
 # See https://pytorch.org/get-started/locally/ for the command to install
 # the CPU-only torch for each platform.
 # Pass the following option to pip on linux to get the CPU only torch
-# -f https://download.pytorch.org/whl/torch_stable.html
-torch==1.9.1+cpu; sys_platform == 'linux'
-torch==1.9.1; sys_platform == 'windows'
-torch==1.9.1; sys_platform == 'darwin'
-ray[rllib]==1.7.0
+# -f https://download.pytorch.org/whl/cpu/torch_stable.html
+torch==1.10.2+cpu; sys_platform == 'linux'
+torch==1.10.2; sys_platform == 'windows'
+torch==1.10.2; sys_platform == 'darwin'
+ray[rllib]==1.10.0
+gym==0.21.0
+
+# agentos_pip_cmdline:
+#   linux:
+#       "-f": "https://download.pytorch.org/whl/cpu/torch_stable.html"

example_agents/sb3_agent/requirements.txt

@@ -1,6 +1,6 @@
 # agentos_pip_cmdline:
 #   linux:
-#       "-f": "https://download.pytorch.org/whl/torch_stable.html"
+#       "-f": "https://download.pytorch.org/whl/cpu/torch_stable.html"
 
 stable_baselines3==1.4.0
 dm-env==1.5

install_requirements.py

@@ -11,28 +11,27 @@
 ACME_R2D2_REQS_PATH = EXAMPLE_AGENT_PATH / "acme_r2d2" / "requirements.txt"
 WEB_REQS_PATH = REPO_ROOT / "web" / "requirements.txt"
 
+PYTORCH_CPU_URL = "https://download.pytorch.org/whl/cpu/torch_stable.html"
+
 
 def install_with_pip(pip):  # install with given pip
-    subprocess.run([pip, "install", "-r", DEV_REQS_PATH])
+    _run([pip, "install", "-r", DEV_REQS_PATH])
     if sys.platform == "linux":
         # Get CPU-only version of torch in case CUDA is not proper configured
-        subprocess.run(
-            [
-                pip,
-                "install",
-                "-r",
-                RLLIB_REQS_PATH,
-                "-f",
-                "https://download.pytorch.org/whl/torch_stable.html",
-            ]
-        )
-        subprocess.run([pip, "install", "-r", ACME_DQN_REQS_PATH])
-        subprocess.run([pip, "install", "-r", ACME_R2D2_REQS_PATH])
+        _run([pip, "install", "-r", RLLIB_REQS_PATH, "-f", PYTORCH_CPU_URL])
+        _run([pip, "install", "-r", SB3_REQS_PATH, "-f", PYTORCH_CPU_URL])
+        _run([pip, "install", "-r", ACME_DQN_REQS_PATH])
+        _run([pip, "install", "-r", ACME_R2D2_REQS_PATH])
     else:
-        subprocess.run([pip, "install", "-r", RLLIB_REQS_PATH])
-    subprocess.run([pip, "install", "-r", SB3_REQS_PATH])
-    subprocess.run([pip, "install", "-r", WEB_REQS_PATH])
-    subprocess.run([pip, "install", "-e", REPO_ROOT])
+        _run([pip, "install", "-r", RLLIB_REQS_PATH])
+        _run([pip, "install", "-r", SB3_REQS_PATH])
+    _run([pip, "install", "-r", WEB_REQS_PATH])
+    _run([pip, "install", "-e", REPO_ROOT])
+
+
+def _run(cmd):
+    print(f"\n==========\nRUNNING:\n\t{cmd }\n==========\n")
+    subprocess.run(cmd)
 
 
 def install_requirements():

setup.py

@@ -22,9 +22,6 @@
     packages=find_packages(),
     install_requires=[
         "click>=7.0",
-        "gym==0.17.1",
-        "numpy>=1.18.5",
-        "dm-env>=1.5",
         "pyyaml>=5.4.1",
         "mlflow>=1.20.2",
         "dulwich==0.20.28",

tests/test_agents/setup_py_agent/agent.py

@@ -0,0 +1,11 @@
+import arrow  # noqa: F401
+
+
+# A basic agent.
+class BasicAgent:
+    DEFAULT_ENTRY_POINT = "evaluate"
+
+    def evaluate(self):
+        import bottle  # noqa: F401
+
+        print("Successful venv test")

tests/test_agents/setup_py_agent/setup.py

@@ -0,0 +1,19 @@
+from setuptools import setup
+
+setup(
+    name="setup_py_agent",
+    description=(
+        "An agent with requirements in setup.py for AOS testing purposes"
+    ),
+    version="1.0.0",
+    install_requires=[
+        "arrow",
+        "bottle",
+    ],
+    license="Apache License 2.0",
+    python_requires=">=3.5",
+    url="https://agentos.org",
+    project_urls={
+        "Source Code": "https://github.com/agentos-project/agentos",
+    },
+)