From 27aeca295f14b922fe70d073945a8d0824d91bd6 Mon Sep 17 00:00:00 2001 From: Nick Jalbert Date: Sat, 5 Mar 2022 20:30:34 +0600 Subject: [PATCH 1/4] Add isort as part of code formatting --- agentos/__init__.py | 15 ++++----- agentos/agent_run.py | 8 +++-- agentos/argument_set.py | 6 ++-- agentos/cli.py | 11 ++++--- agentos/component.py | 23 +++++++------ agentos/component_run.py | 6 ++-- agentos/core.py | 3 +- agentos/identifiers.py | 1 + agentos/registry.py | 33 +++++++++---------- agentos/repo.py | 18 +++++----- agentos/run.py | 5 +-- agentos/run_command.py | 9 ++--- agentos/specs.py | 3 +- agentos/utils.py | 2 +- agentos/virtual_env.py | 11 ++++--- documentation/conf.py | 1 - example_agents/acme_dqn/agent.py | 3 +- example_agents/acme_dqn/environment.py | 7 ++-- example_agents/acme_dqn/network.py | 6 ++-- example_agents/acme_r2d2/agent.py | 1 + example_agents/acme_r2d2/dataset.py | 9 ++--- example_agents/acme_r2d2/network.py | 2 +- example_agents/acme_r2d2/policy.py | 6 ++-- example_agents/acme_r2d2/trainer.py | 13 ++++---- example_agents/chatbot/env.py | 9 ++--- example_agents/chatbot/env_utils.py | 5 +-- example_agents/chatbot/main.py | 6 ++-- example_agents/evolutionary_agent/agent.py | 6 ++-- .../free_energy_tutorial/main.py | 4 ++- example_agents/rl_agents/dqn_agent.py | 8 +++-- .../rl_agents/random_nn_policy_agent.py | 5 +-- example_agents/rl_agents/reinforce_agent.py | 6 ++-- example_agents/rllib_agent/main.py | 2 +- example_agents/sb3_agent/agent.py | 1 + example_agents/sb3_agent/environment.py | 7 ++-- example_agents/sb3_agent/sb3_run.py | 8 +++-- scripts/build_docs.py | 4 +-- scripts/format_code.py | 27 ++++++++++----- scripts/lint_code.py | 6 ++-- scripts/shared.py | 3 +- setup.py | 3 +- tests/example_agents/test_acme_dqn.py | 5 ++- tests/example_agents/test_acme_r2d2.py | 5 ++- tests/example_agents/test_chatbot.py | 2 +- tests/example_agents/test_gh_sb3.py | 3 +- tests/example_agents/test_rllib.py | 3 +- tests/example_agents/test_sb3.py | 3 +- tests/test_component.py | 11 ++++--- tests/test_core.py | 8 ++--- tests/test_registry.py | 9 ++--- tests/test_repo.py | 4 +-- tests/test_specs.py | 4 +-- tests/test_venv_management.py | 7 ++-- tests/utils.py | 3 +- web/aos_web/settings.py | 5 +-- web/leaderboard/urls.py | 1 - web/leaderboard/views.py | 10 ++---- web/registry/admin.py | 5 +-- .../management/commands/import_registry.py | 5 ++- web/registry/migrations/0001_initial.py | 2 +- web/registry/models.py | 4 ++- web/registry/serializers.py | 3 +- web/registry/tests/test_integration.py | 19 +++++------ web/registry/tests/test_models.py | 7 ++-- web/registry/tests/test_registry.py | 4 +-- web/registry/urls.py | 10 +++--- web/registry/views.py | 16 ++++----- 67 files changed, 251 insertions(+), 219 deletions(-) diff --git a/agentos/__init__.py b/agentos/__init__.py index fe621b34..37bab2e6 100644 --- a/agentos/__init__.py +++ b/agentos/__init__.py @@ -1,6 +1,9 @@ """The ``agentos`` module provides an API for building learning agents.""" -from agentos.version import VERSION as __version__ # noqa: F401 +from agentos.agent_run import AgentRun +from agentos.argument_set import ArgumentSet +from agentos.component import Component +from agentos.component_run import ComponentRun, active_component_run from agentos.core import ( Agent, Dataset, @@ -10,23 +13,19 @@ Runnable, Trainer, ) - from agentos.registry import Registry -from agentos.component import Component -from agentos.component_run import ComponentRun, active_component_run -from agentos.repo import Repo, LocalRepo, GitHubRepo -from agentos.argument_set import ArgumentSet +from agentos.repo import GitHubRepo, LocalRepo, Repo from agentos.run import Run from agentos.run_command import RunCommand from agentos.specs import ( + ArgumentSetSpec, ComponentSpec, RepoSpec, - ArgumentSetSpec, RunSpec, flatten_spec, unflatten_spec, ) -from agentos.agent_run import AgentRun +from agentos.version import VERSION as __version__ # noqa: F401 __all__ = [ "Agent", diff --git a/agentos/agent_run.py b/agentos/agent_run.py index 647a726e..2eca921b 100644 --- a/agentos/agent_run.py +++ b/agentos/agent_run.py @@ -1,9 +1,11 @@ import statistics -from typing import Optional from collections import namedtuple -from agentos.run import Run -from mlflow.utils.mlflow_tags import MLFLOW_PARENT_RUN_ID, MLFLOW_RUN_NAME +from typing import Optional + from mlflow.entities import RunStatus +from mlflow.utils.mlflow_tags import MLFLOW_PARENT_RUN_ID, MLFLOW_RUN_NAME + +from agentos.run import Run _EPISODE_KEY = "episode_count" _STEP_KEY = "step_count" diff --git a/agentos/argument_set.py b/agentos/argument_set.py index 50cde528..10acb863 100644 --- a/agentos/argument_set.py +++ b/agentos/argument_set.py @@ -1,8 +1,10 @@ import copy -import yaml import json from hashlib import sha1 -from typing import TypeVar, Mapping, Dict +from typing import Dict, Mapping, TypeVar + +import yaml + from agentos.specs import ArgumentSetSpec # Use Python generics (https://mypy.readthedocs.io/en/stable/generics.html) diff --git a/agentos/cli.py b/agentos/cli.py index 23fc3e41..12cb0fbe 100644 --- a/agentos/cli.py +++ b/agentos/cli.py @@ -4,14 +4,15 @@ """ import os import sys -import yaml -import click from datetime import datetime from pathlib import Path -from agentos import Component -from agentos import ArgumentSet -from agentos.run import Run + +import click +import yaml + +from agentos import ArgumentSet, Component from agentos.registry import Registry +from agentos.run import Run from agentos.virtual_env import VirtualEnv diff --git a/agentos/component.py b/agentos/component.py index 47866690..7504052f 100644 --- a/agentos/component.py +++ b/agentos/component.py @@ -1,26 +1,25 @@ +import importlib +import logging import sys import uuid -import logging -import importlib from hashlib import sha1 from pathlib import Path +from typing import Any, Dict, Sequence, Type, TypeVar, Union + from dill.source import getsource as dill_getsource -from typing import Union, TypeVar, Dict, Type, Any, Sequence from rich import print as rich_print from rich.tree import Tree -from agentos.run_command import RunCommand + +from agentos.argument_set import ArgumentSet from agentos.component_run import ComponentRun +from agentos.exceptions import RegistryException from agentos.identifiers import ComponentIdentifier +from agentos.registry import InMemoryRegistry, Registry +from agentos.repo import GitHubRepo, LocalRepo, Repo, RepoType +from agentos.run_command import RunCommand from agentos.specs import ComponentSpec, ComponentSpecKeys, unflatten_spec -from agentos.registry import ( - Registry, - InMemoryRegistry, -) -from agentos.exceptions import RegistryException -from agentos.repo import RepoType, Repo, LocalRepo, GitHubRepo -from agentos.argument_set import ArgumentSet -from agentos.virtual_env import VirtualEnv, NoOpVirtualEnv from agentos.utils import parse_github_web_ui_url +from agentos.virtual_env import NoOpVirtualEnv, VirtualEnv logger = logging.getLogger(__name__) diff --git a/agentos/component_run.py b/agentos/component_run.py index c4f1b715..2d3d54d0 100644 --- a/agentos/component_run.py +++ b/agentos/component_run.py @@ -1,7 +1,9 @@ import tempfile -from mlflow.utils.mlflow_tags import MLFLOW_RUN_NAME from pathlib import Path -from typing import Any, Optional, Mapping +from typing import Any, Mapping, Optional + +from mlflow.utils.mlflow_tags import MLFLOW_RUN_NAME + from agentos.exceptions import PythonComponentSystemException from agentos.identifiers import RunIdentifier from agentos.registry import Registry diff --git a/agentos/core.py b/agentos/core.py index fa177a24..720c3383 100644 --- a/agentos/core.py +++ b/agentos/core.py @@ -2,7 +2,8 @@ import time from collections import namedtuple from threading import Thread -from typing import Sequence, Optional +from typing import Optional, Sequence + from agentos.agent_run import AgentRun diff --git a/agentos/identifiers.py b/agentos/identifiers.py index ff6f6636..4af9f6a6 100644 --- a/agentos/identifiers.py +++ b/agentos/identifiers.py @@ -5,6 +5,7 @@ class ComponentIdentifier(str): allows referring to Components both as [name] and [name]==[version] (e.g., in registries or the command line). """ + def __new__( cls, identifier_or_name: str, diff --git a/agentos/registry.py b/agentos/registry.py index f88df857..7db05d04 100644 --- a/agentos/registry.py +++ b/agentos/registry.py @@ -1,40 +1,42 @@ import abc +import json import logging import os -import yaml -import json import pprint import shutil import tarfile import tempfile -import requests from pathlib import Path -from typing import Dict, Sequence, Union, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Optional, Sequence, Union + +import requests +import yaml from dotenv import load_dotenv + from agentos.identifiers import ( ComponentIdentifier, - RunIdentifier, RepoIdentifier, RunCommandIdentifier, + RunIdentifier, ) from agentos.specs import ( - flatten_spec, - unflatten_spec, - is_flat, - json_encode_flat_spec_field, - RepoSpec, ComponentSpec, NestedComponentSpec, - RunSpec, + RepoSpec, RunCommandSpec, + RunSpec, + flatten_spec, + is_flat, + json_encode_flat_spec_field, + unflatten_spec, ) logger = logging.getLogger(__name__) if TYPE_CHECKING: from agentos.component import Component - from agentos.run import Run from agentos.repo import Repo + from agentos.run import Run # add USE_LOCAL_SERVER=True to .env to talk to local server load_dotenv() @@ -194,8 +196,7 @@ def get_component_spec( return {} if len(components) > 1: versions = [ - ComponentIdentifier(c_id).version - for c_id in components.keys() + ComponentIdentifier(c_id).version for c_id in components.keys() ] version_str = "\n - ".join(versions) raise LookupError( @@ -233,9 +234,7 @@ def get_specs_transitively_by_id( repo_spec = self.get_repo_spec(repo_id, flatten=flatten) repo_specs[repo_id] = repo_spec for d_id in inner_spec.get("dependencies", {}).values(): - component_identifiers.append( - ComponentIdentifier(d_id) - ) + component_identifiers.append(ComponentIdentifier(d_id)) return list(component_specs.values()), list(repo_specs.values()) def has_component_by_id(self, identifier: ComponentIdentifier) -> bool: diff --git a/agentos/repo.py b/agentos/repo.py index 6442cc51..ab2a583a 100644 --- a/agentos/repo.py +++ b/agentos/repo.py @@ -1,25 +1,25 @@ -import os -import sys import abc import logging +import os +import sys import uuid from enum import Enum -from typing import TypeVar, Dict, Tuple, Union from pathlib import Path +from typing import Dict, Tuple, TypeVar, Union + from dulwich import porcelain -from dulwich.repo import Repo as PorcelainRepo -from dulwich.objectspec import parse_ref -from dulwich.objectspec import parse_commit from dulwich.errors import NotGitRepository +from dulwich.objectspec import parse_commit, parse_ref +from dulwich.repo import Repo as PorcelainRepo from agentos.exceptions import ( BadGitStateException, PythonComponentSystemException, ) -from agentos.utils import AOS_GLOBAL_REPOS_DIR from agentos.identifiers import ComponentIdentifier, RepoIdentifier -from agentos.specs import RepoSpec, NestedRepoSpec, RepoSpecKeys, flatten_spec -from agentos.registry import Registry, InMemoryRegistry +from agentos.registry import InMemoryRegistry, Registry +from agentos.specs import NestedRepoSpec, RepoSpec, RepoSpecKeys, flatten_spec +from agentos.utils import AOS_GLOBAL_REPOS_DIR logger = logging.getLogger(__name__) diff --git a/agentos/run.py b/agentos/run.py index a00004ac..6a69f4ad 100644 --- a/agentos/run.py +++ b/agentos/run.py @@ -3,14 +3,15 @@ from pathlib import Path from typing import Sequence from urllib.parse import urlparse + from mlflow.entities import RunStatus from mlflow.exceptions import MlflowException from mlflow.tracking import MlflowClient from mlflow.tracking.context import registry as context_registry + from agentos.identifiers import RunIdentifier from agentos.registry import Registry -from agentos.specs import RunSpec -from agentos.specs import unflatten_spec +from agentos.specs import RunSpec, unflatten_spec class Run: diff --git a/agentos/run_command.py b/agentos/run_command.py index 79c28eaa..90ddf392 100644 --- a/agentos/run_command.py +++ b/agentos/run_command.py @@ -1,14 +1,15 @@ from hashlib import sha1 from typing import TYPE_CHECKING + +from agentos.identifiers import RunCommandIdentifier, RunIdentifier from agentos.registry import Registry -from agentos.specs import RunCommandSpec, RunCommandSpecKeys, unflatten_spec -from agentos.identifiers import RunIdentifier, RunCommandIdentifier from agentos.run import Run +from agentos.specs import RunCommandSpec, RunCommandSpecKeys, unflatten_spec # Avoids circular imports if TYPE_CHECKING: - from agentos.component import Component from agentos.argument_set import ArgumentSet + from agentos.component import Component class RunCommand: @@ -115,8 +116,8 @@ def from_spec( spec_identifier = key inner_spec = value component_id = inner_spec[RunCommandSpecKeys.COMPONENT_ID] - from agentos.component import Component from agentos.argument_set import ArgumentSet + from agentos.component import Component component = Component.from_registry(registry, component_id) arg_set = ArgumentSet.from_spec( diff --git a/agentos/specs.py b/agentos/specs.py index e17a6da2..1e7b9b6c 100644 --- a/agentos/specs.py +++ b/agentos/specs.py @@ -19,7 +19,8 @@ import copy import json -from typing import Mapping, Union, Any +from typing import Any, Mapping, Union + from agentos.identifiers import ComponentIdentifier FlatSpec = Mapping[str, str] diff --git a/agentos/utils.py b/agentos/utils.py index bd7aee5c..41c51728 100644 --- a/agentos/utils.py +++ b/agentos/utils.py @@ -1,8 +1,8 @@ import pprint -import yaml from pathlib import Path from typing import Dict, Optional +import yaml AOS_GLOBAL_CONFIG_DIR = Path.home() / ".agentos" AOS_GLOBAL_CACHE_DIR = AOS_GLOBAL_CONFIG_DIR / "cache" diff --git a/agentos/virtual_env.py b/agentos/virtual_env.py index 72b6d6a0..4d105584 100644 --- a/agentos/virtual_env.py +++ b/agentos/virtual_env.py @@ -1,13 +1,14 @@ +import hashlib import os -import sys -import yaml import shutil -import hashlib -import sysconfig import subprocess +import sys +import sysconfig +from contextlib import contextmanager from pathlib import Path from typing import Sequence -from contextlib import contextmanager + +import yaml from agentos.utils import AOS_GLOBAL_REQS_DIR diff --git a/documentation/conf.py b/documentation/conf.py index 43954de2..6b3a39e5 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -13,7 +13,6 @@ import os from importlib.machinery import SourceFileLoader - version = ( SourceFileLoader( "agentos.version", os.path.join("..", "agentos", "version.py") diff --git a/example_agents/acme_dqn/agent.py b/example_agents/acme_dqn/agent.py index 7092f9b8..7baa8657 100644 --- a/example_agents/acme_dqn/agent.py +++ b/example_agents/acme_dqn/agent.py @@ -1,6 +1,7 @@ import acme -from acme.agents.tf import dqn import numpy as np +from acme.agents.tf import dqn + from agentos import active_component_run diff --git a/example_agents/acme_dqn/environment.py b/example_agents/acme_dqn/environment.py index 4daaa968..431b79f2 100644 --- a/example_agents/acme_dqn/environment.py +++ b/example_agents/acme_dqn/environment.py @@ -1,11 +1,10 @@ # Thin wrapper around cartpole from OpenAI's Gym toolkit # This env models a cart with a pole balancing on top of it -import agentos import gym import numpy as np -from dm_env import specs -from dm_env import TimeStep -from dm_env import StepType +from dm_env import StepType, TimeStep, specs + +import agentos class CartPole(agentos.Environment): diff --git a/example_agents/acme_dqn/network.py b/example_agents/acme_dqn/network.py index 87d401d5..bf7b9f85 100644 --- a/example_agents/acme_dqn/network.py +++ b/example_agents/acme_dqn/network.py @@ -1,8 +1,10 @@ import shutil -import sonnet as snt import tempfile -import tensorflow as tf from pathlib import Path + +import sonnet as snt +import tensorflow as tf + from agentos.run import Run diff --git a/example_agents/acme_r2d2/agent.py b/example_agents/acme_r2d2/agent.py index 5414e443..229918d5 100644 --- a/example_agents/acme_r2d2/agent.py +++ b/example_agents/acme_r2d2/agent.py @@ -1,4 +1,5 @@ import acme + from agentos import active_component_run diff --git a/example_agents/acme_r2d2/dataset.py b/example_agents/acme_r2d2/dataset.py index 14779e92..69b99835 100644 --- a/example_agents/acme_r2d2/dataset.py +++ b/example_agents/acme_r2d2/dataset.py @@ -1,10 +1,11 @@ # TODO - update reqs -import agentos -from acme import datasets +import reverb import tensorflow as tf -from acme.tf import utils as tf2_utils +from acme import datasets from acme.adders import reverb as adders -import reverb +from acme.tf import utils as tf2_utils + +import agentos class ReverbDataset(agentos.Dataset): diff --git a/example_agents/acme_r2d2/network.py b/example_agents/acme_r2d2/network.py index 2ae86b2f..0af99b85 100644 --- a/example_agents/acme_r2d2/network.py +++ b/example_agents/acme_r2d2/network.py @@ -1,5 +1,5 @@ -from acme.tf import networks import sonnet as snt +from acme.tf import networks class R2D2Network: diff --git a/example_agents/acme_r2d2/policy.py b/example_agents/acme_r2d2/policy.py index 23ab7caa..3ba93030 100644 --- a/example_agents/acme_r2d2/policy.py +++ b/example_agents/acme_r2d2/policy.py @@ -1,8 +1,8 @@ -from acme.agents.tf import actors -from acme.tf import utils as tf2_utils -import trfl import numpy as np import sonnet as snt +import trfl +from acme.agents.tf import actors +from acme.tf import utils as tf2_utils class R2D2Policy: diff --git a/example_agents/acme_r2d2/trainer.py b/example_agents/acme_r2d2/trainer.py index 687af1ee..e018275c 100644 --- a/example_agents/acme_r2d2/trainer.py +++ b/example_agents/acme_r2d2/trainer.py @@ -2,14 +2,15 @@ # See acme/agents/tf/r2d2/learning.py for original code source import copy import functools -import agentos -import tree -from attrdict import AttrDict -from acme.tf import utils as tf2_utils -from acme.tf import losses -from acme.tf import networks + import sonnet as snt import tensorflow as tf +import tree +from acme.tf import losses, networks +from acme.tf import utils as tf2_utils +from attrdict import AttrDict + +import agentos class R2D2Trainer(agentos.Trainer): diff --git a/example_agents/chatbot/env.py b/example_agents/chatbot/env.py index caaffdf4..df35b29b 100644 --- a/example_agents/chatbot/env.py +++ b/example_agents/chatbot/env.py @@ -1,9 +1,10 @@ -from collections import deque -from gym import Space -import numpy as np -from secrets import token_bytes import string import threading +from collections import deque +from secrets import token_bytes + +import numpy as np +from gym import Space class TextSpace(Space): diff --git a/example_agents/chatbot/env_utils.py b/example_agents/chatbot/env_utils.py index 108d3ce8..f0439485 100644 --- a/example_agents/chatbot/env_utils.py +++ b/example_agents/chatbot/env_utils.py @@ -1,7 +1,8 @@ -import numpy as np -from sys import stdin import threading import time +from sys import stdin + +import numpy as np class RandomSpeaker(threading.Thread): diff --git a/example_agents/chatbot/main.py b/example_agents/chatbot/main.py index 63d26537..06012a97 100644 --- a/example_agents/chatbot/main.py +++ b/example_agents/chatbot/main.py @@ -1,8 +1,10 @@ -import agentos from collections import deque + +from numpy import random as np_random + +import agentos from example_agents.chatbot.env import MultiChatEnv from example_agents.chatbot.env_utils import CommandLineClient -from numpy import random as np_random class ChatBot(agentos.Runnable): diff --git a/example_agents/evolutionary_agent/agent.py b/example_agents/evolutionary_agent/agent.py index 8734d84e..1c92b740 100644 --- a/example_agents/evolutionary_agent/agent.py +++ b/example_agents/evolutionary_agent/agent.py @@ -5,13 +5,15 @@ $ pip install -r requirements.txt $ agentos run agent.py gym.envs.classic_control.CartPoleEnv """ -import agentos import copy + import gym -from gym.envs.classic_control import CartPoleEnv import numpy as np +from gym.envs.classic_control import CartPoleEnv from tensorflow import keras +import agentos + class TFPolicy(agentos.Policy): def __init__(self, tf_model): diff --git a/example_agents/predictive_coding/free_energy_tutorial/main.py b/example_agents/predictive_coding/free_energy_tutorial/main.py index 0451919d..d695db49 100644 --- a/example_agents/predictive_coding/free_energy_tutorial/main.py +++ b/example_agents/predictive_coding/free_energy_tutorial/main.py @@ -2,13 +2,15 @@ Based on https://sciencedirect.com/science/article/pii/S0022249615000759 by Rafal Bogacz. """ -import agentos from collections import defaultdict from decimal import Decimal + import gym import matplotlib.pyplot as plt import numpy as np +import agentos + np.random.seed(47) env_stats = defaultdict(list) diff --git a/example_agents/rl_agents/dqn_agent.py b/example_agents/rl_agents/dqn_agent.py index 4a8dafd0..77d89433 100644 --- a/example_agents/rl_agents/dqn_agent.py +++ b/example_agents/rl_agents/dqn_agent.py @@ -1,9 +1,11 @@ -import agentos import random -import tensorflow as tf -import tensorflow.keras as keras + import numpy as np import pandas as pd +import tensorflow as tf +import tensorflow.keras as keras + +import agentos class EpsilonGreedyTFPolicy: diff --git a/example_agents/rl_agents/random_nn_policy_agent.py b/example_agents/rl_agents/random_nn_policy_agent.py index 1f78033d..8d1bb7af 100644 --- a/example_agents/rl_agents/random_nn_policy_agent.py +++ b/example_agents/rl_agents/random_nn_policy_agent.py @@ -4,9 +4,10 @@ TensorFlow NN policy for each step but doesn't do any learning. """ -import agentos -from tensorflow import keras import numpy as np +from tensorflow import keras + +import agentos class SingleLayerTFPolicy(agentos.Policy): diff --git a/example_agents/rl_agents/reinforce_agent.py b/example_agents/rl_agents/reinforce_agent.py index 09d6d75b..aabb8076 100644 --- a/example_agents/rl_agents/reinforce_agent.py +++ b/example_agents/rl_agents/reinforce_agent.py @@ -13,11 +13,12 @@ TODO: Add max_steps_per_iter to agent init. """ -import agentos import numpy as np import tensorflow as tf -from tensorflow import keras import tensorflow_probability as tfp +from tensorflow import keras + +import agentos class TwoLayerTFPolicy(agentos.Policy): @@ -123,6 +124,7 @@ def rollout_step(policy, obs): if __name__ == "__main__": import argparse + from gym.envs.classic_control import CartPoleEnv parser = argparse.ArgumentParser( diff --git a/example_agents/rllib_agent/main.py b/example_agents/rllib_agent/main.py index b7abeace..8cb403f4 100644 --- a/example_agents/rllib_agent/main.py +++ b/example_agents/rllib_agent/main.py @@ -1,8 +1,8 @@ import ray import ray.rllib.agents.registry as rllib_registry -from ray.tune.registry import register_env as rllib_reg_env from ray.rllib.agents.trainer import COMMON_CONFIG from ray.rllib.utils import merge_dicts +from ray.tune.registry import register_env as rllib_reg_env class RLlibAgent: diff --git a/example_agents/sb3_agent/agent.py b/example_agents/sb3_agent/agent.py index c5229d0a..fdd7312c 100644 --- a/example_agents/sb3_agent/agent.py +++ b/example_agents/sb3_agent/agent.py @@ -1,5 +1,6 @@ from stable_baselines3 import PPO from stable_baselines3.common.evaluation import evaluate_policy + from agentos import active_component_run diff --git a/example_agents/sb3_agent/environment.py b/example_agents/sb3_agent/environment.py index 5bdae0a4..25815774 100644 --- a/example_agents/sb3_agent/environment.py +++ b/example_agents/sb3_agent/environment.py @@ -1,9 +1,10 @@ # Thin wrapper around cartpole from OpenAI's Gym toolkit # This env models a cart with a pole balancing on top of it -import agentos -from gym.envs.classic_control.cartpole import CartPoleEnv -from dm_env import specs import numpy as np +from dm_env import specs +from gym.envs.classic_control.cartpole import CartPoleEnv + +import agentos class CartPole(CartPoleEnv, agentos.Environment): diff --git a/example_agents/sb3_agent/sb3_run.py b/example_agents/sb3_agent/sb3_run.py index 1452c326..5cec05db 100644 --- a/example_agents/sb3_agent/sb3_run.py +++ b/example_agents/sb3_agent/sb3_run.py @@ -1,11 +1,13 @@ -import tempfile import shutil +import tempfile from pathlib import Path +from typing import Optional + from stable_baselines3 import PPO -from stable_baselines3.common.policies import BasePolicy from stable_baselines3.common.callbacks import BaseCallback +from stable_baselines3.common.policies import BasePolicy + from agentos.agent_run import AgentRun -from typing import Optional class EvaluateCallback: diff --git a/scripts/build_docs.py b/scripts/build_docs.py index 5dd130a7..72521703 100644 --- a/scripts/build_docs.py +++ b/scripts/build_docs.py @@ -9,9 +9,9 @@ import os from subprocess import Popen +from shared import docs_build_dir, docs_dir + import agentos -from shared import docs_dir -from shared import docs_build_dir parser = argparse.ArgumentParser( description="Build the AgentOS docs. Any arguments that are provided " diff --git a/scripts/format_code.py b/scripts/format_code.py index 3810a4e7..5a15a27f 100644 --- a/scripts/format_code.py +++ b/scripts/format_code.py @@ -13,12 +13,9 @@ import os import sys from pathlib import Path -from subprocess import run -from subprocess import PIPE -from subprocess import STDOUT +from subprocess import PIPE, STDOUT, run -from shared import root_dir -from shared import traverse_tracked_files +from shared import root_dir, traverse_tracked_files returncode = 0 @@ -26,14 +23,22 @@ "agentos/templates/agent.py", ] +CHECK_ARG = "--check" + def format_file(path): - global returncode extension = os.path.splitext(path)[1] if extension != ".py": return - cmd = ["black", "--line-length=79", path] - if len(sys.argv) > 1 and sys.argv[1] == "--check": + black_cmd = ["black", "--line-length=79", path] + _run_command(path, black_cmd) + isort_cmd = ["isort", "-m" "VERTICAL_HANGING_INDENT", "--tc", path] + _run_command(path, isort_cmd) + + +def _run_command(path, cmd): + global returncode + if CHECK_ARG in sys.argv: cmd.append("--check") result = run(cmd, stdout=PIPE, stderr=STDOUT) returncode = returncode | result.returncode @@ -44,8 +49,12 @@ def format_file(path): print() -if len(sys.argv) > 1: +check_count = 1 if CHECK_ARG in sys.argv else 0 + +if len(sys.argv) - check_count > 1: for arg in sys.argv[1:]: + if arg == "--check": + continue path = Path(arg).absolute() format_file(path) else: diff --git a/scripts/lint_code.py b/scripts/lint_code.py index a6f82920..3b83568e 100644 --- a/scripts/lint_code.py +++ b/scripts/lint_code.py @@ -9,11 +9,9 @@ import os import sys from pathlib import Path -from subprocess import run -from subprocess import PIPE +from subprocess import PIPE, run -from shared import root_dir -from shared import traverse_tracked_files +from shared import root_dir, traverse_tracked_files returncode = 0 diff --git a/scripts/shared.py b/scripts/shared.py index 57ede374..18c94f70 100644 --- a/scripts/shared.py +++ b/scripts/shared.py @@ -7,8 +7,7 @@ """ import os -from subprocess import run -from subprocess import DEVNULL +from subprocess import DEVNULL, run scripts_dir = os.path.dirname(os.path.abspath(__file__)) root_dir = os.path.normpath(os.path.join(scripts_dir, os.pardir)) diff --git a/setup.py b/setup.py index 105dc634..419bfce9 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ import os from importlib.machinery import SourceFileLoader -from setuptools import setup, find_packages + +from setuptools import find_packages, setup version = ( SourceFileLoader("agentos.version", os.path.join("agentos", "version.py")) diff --git a/tests/example_agents/test_acme_dqn.py b/tests/example_agents/test_acme_dqn.py index 8be9b79b..e418a205 100644 --- a/tests/example_agents/test_acme_dqn.py +++ b/tests/example_agents/test_acme_dqn.py @@ -1,8 +1,7 @@ import pytest -from tests.utils import run_test_command -from tests.utils import is_linux -from tests.utils import ACME_DQN_AGENT_DIR + from agentos.cli import run +from tests.utils import ACME_DQN_AGENT_DIR, is_linux, run_test_command test_args = ["agent", "--use-outer-env"] test_kwargs = { diff --git a/tests/example_agents/test_acme_r2d2.py b/tests/example_agents/test_acme_r2d2.py index 13654e1d..1ee0bd68 100644 --- a/tests/example_agents/test_acme_r2d2.py +++ b/tests/example_agents/test_acme_r2d2.py @@ -1,8 +1,7 @@ import pytest -from tests.utils import run_test_command -from tests.utils import is_linux -from tests.utils import ACME_R2D2_AGENT_DIR + from agentos.cli import run +from tests.utils import ACME_R2D2_AGENT_DIR, is_linux, run_test_command test_args = ["agent", "--use-outer-env"] test_kwargs = { diff --git a/tests/example_agents/test_chatbot.py b/tests/example_agents/test_chatbot.py index 8fb497ba..e45225a0 100644 --- a/tests/example_agents/test_chatbot.py +++ b/tests/example_agents/test_chatbot.py @@ -1,5 +1,5 @@ -from example_agents.chatbot.main import ChatBot from example_agents.chatbot.env import MultiChatEnv +from example_agents.chatbot.main import ChatBot def test_chatbot(capsys): diff --git a/tests/example_agents/test_gh_sb3.py b/tests/example_agents/test_gh_sb3.py index 7523e173..a046a7d5 100644 --- a/tests/example_agents/test_gh_sb3.py +++ b/tests/example_agents/test_gh_sb3.py @@ -1,6 +1,5 @@ -from tests.utils import run_test_command -from tests.utils import GH_SB3_AGENT_DIR from agentos.cli import run +from tests.utils import GH_SB3_AGENT_DIR, run_test_command test_args = ["agent", "--use-outer-env"] test_kwargs = {"--registry-file": str(GH_SB3_AGENT_DIR / "components.yaml")} diff --git a/tests/example_agents/test_rllib.py b/tests/example_agents/test_rllib.py index 300e1289..07300491 100644 --- a/tests/example_agents/test_rllib.py +++ b/tests/example_agents/test_rllib.py @@ -1,6 +1,5 @@ -from tests.utils import run_test_command -from tests.utils import RLLIB_AGENT_DIR from agentos.cli import run +from tests.utils import RLLIB_AGENT_DIR, run_test_command test_args = ["agent", "--use-outer-env"] test_kwargs = {"--registry-file": str(RLLIB_AGENT_DIR / "components.yaml")} diff --git a/tests/example_agents/test_sb3.py b/tests/example_agents/test_sb3.py index ccadf233..7b12767b 100644 --- a/tests/example_agents/test_sb3.py +++ b/tests/example_agents/test_sb3.py @@ -1,6 +1,5 @@ -from tests.utils import run_test_command -from tests.utils import SB3_AGENT_DIR from agentos.cli import run +from tests.utils import SB3_AGENT_DIR, run_test_command test_args = ["sb3_agent", "--use-outer-env"] test_kwargs = {"--registry-file": str(SB3_AGENT_DIR / "components.yaml")} diff --git a/tests/test_component.py b/tests/test_component.py index 1a7e2baf..9c3464ed 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -1,13 +1,14 @@ """Test suite for AgentOS Component.""" +from unittest.mock import DEFAULT, patch + import pytest -from unittest.mock import patch -from unittest.mock import DEFAULT +from utils import run_in_dir, run_test_command + +from agentos.cli import init from agentos.component import Component from agentos.component_run import ComponentRun -from agentos.virtual_env import auto_revert_venv from agentos.run_command import RunCommand -from utils import run_test_command, run_in_dir -from agentos.cli import init +from agentos.virtual_env import auto_revert_venv # We define these classes at the module global level so that diff --git a/tests/test_core.py b/tests/test_core.py index d227e337..a99ca2fd 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,10 +4,10 @@ """ import os from pathlib import Path -from utils import run_test_command -from utils import run_in_dir -from utils import SB3_AGENT_DIR -from agentos.cli import init, run, freeze, status + +from utils import SB3_AGENT_DIR, run_in_dir, run_test_command + +from agentos.cli import freeze, init, run, status def test_cli_init(tmpdir): diff --git a/tests/test_registry.py b/tests/test_registry.py index e9e8fcb4..5be01111 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,10 +1,11 @@ """Test suite for AgentOS Registry.""" import pytest -from tests.utils import is_linux, RANDOM_AGENT_DIR, CHATBOT_AGENT_DIR -from agentos.registry import Registry + +from agentos import ArgumentSet from agentos.component import Component +from agentos.registry import Registry from agentos.utils import generate_dummy_dev_registry -from agentos import ArgumentSet +from tests.utils import CHATBOT_AGENT_DIR, RANDOM_AGENT_DIR, is_linux @pytest.mark.skipif(not is_linux(), reason="Acme only available on posix") @@ -103,8 +104,8 @@ def test_registry_from_dict(): def test_registry_from_file(): - from agentos.exceptions import RegistryException from agentos.argument_set import ArgumentSet + from agentos.exceptions import RegistryException r = Registry.from_yaml(RANDOM_AGENT_DIR / "components.yaml") random_local_ag = Component.from_registry(r, "agent") diff --git a/tests/test_repo.py b/tests/test_repo.py index b85ed16d..851852f4 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,9 +1,9 @@ -from agentos.repo import Repo, LocalRepo from agentos.component import Component +from agentos.repo import LocalRepo, Repo from tests.utils import ( + TESTING_BRANCH_NAME, TESTING_GITHUB_ACCOUNT, TESTING_GITHUB_REPO, - TESTING_BRANCH_NAME, ) diff --git a/tests/test_specs.py b/tests/test_specs.py index db87e1ae..e6922d0e 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -1,6 +1,6 @@ -from agentos.specs import flatten_spec, unflatten_spec from agentos.registry import Registry -from tests.utils import RANDOM_AGENT_DIR, GH_SB3_AGENT_DIR +from agentos.specs import flatten_spec, unflatten_spec +from tests.utils import GH_SB3_AGENT_DIR, RANDOM_AGENT_DIR def test_flatten_spec(): diff --git a/tests/test_venv_management.py b/tests/test_venv_management.py index a121d737..1388c3c8 100644 --- a/tests/test_venv_management.py +++ b/tests/test_venv_management.py @@ -1,10 +1,11 @@ import sys -import pytest from pathlib import Path -from tests.utils import run_test_command -from tests.utils import TEST_VENV_AGENT_DIR + +import pytest + from agentos.cli import run from agentos.virtual_env import VirtualEnv +from tests.utils import TEST_VENV_AGENT_DIR, run_test_command def _clean_up_sys_modules(): diff --git a/tests/utils.py b/tests/utils.py index 11e54cd0..252600c9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,9 @@ import os import sys +from contextlib import contextmanager from pathlib import Path + from click.testing import CliRunner -from contextlib import contextmanager ROOT_DIR = Path(__file__).parent.parent EXAMPLE_AGENT_DIR = ROOT_DIR / "example_agents" diff --git a/web/aos_web/settings.py b/web/aos_web/settings.py index de9f70b7..4f4b9a61 100644 --- a/web/aos_web/settings.py +++ b/web/aos_web/settings.py @@ -10,10 +10,11 @@ https://docs.djangoproject.com/en/3.2/ref/settings/ """ -from pathlib import Path -import dj_database_url import os import sys +from pathlib import Path + +import dj_database_url IS_TEST = "test" in sys.argv diff --git a/web/leaderboard/urls.py b/web/leaderboard/urls.py index 6049bc1c..dbed05fa 100644 --- a/web/leaderboard/urls.py +++ b/web/leaderboard/urls.py @@ -2,7 +2,6 @@ from . import views - urlpatterns = [ path("empty_database", views.empty_database, name="empty-database"), path("", views.index, name="index"), diff --git a/web/leaderboard/views.py b/web/leaderboard/views.py index b83906ae..5add110d 100644 --- a/web/leaderboard/views.py +++ b/web/leaderboard/views.py @@ -1,12 +1,8 @@ -from django.urls import reverse from django.conf import settings +from django.http import HttpResponseBadRequest, HttpResponseRedirect from django.shortcuts import render -from django.http import HttpResponseRedirect -from django.http import HttpResponseBadRequest -from registry.models import ComponentDependency -from registry.models import Component -from registry.models import Repo -from registry.models import Run +from django.urls import reverse +from registry.models import Component, ComponentDependency, Repo, Run def index(request): diff --git a/web/registry/admin.py b/web/registry/admin.py index 3ace3050..a8f08c9a 100644 --- a/web/registry/admin.py +++ b/web/registry/admin.py @@ -1,9 +1,6 @@ from django.contrib import admin -from .models import Component -from .models import ComponentDependency -from .models import Repo -from .models import Run +from .models import Component, ComponentDependency, Repo, Run admin.site.register(Component) admin.site.register(ComponentDependency) diff --git a/web/registry/management/commands/import_registry.py b/web/registry/management/commands/import_registry.py index 6d25970f..1a5018a2 100644 --- a/web/registry/management/commands/import_registry.py +++ b/web/registry/management/commands/import_registry.py @@ -1,8 +1,7 @@ -from django.core.management.base import BaseCommand -from registry.models import Component -from registry.models import ComponentRelease import requests import yaml +from django.core.management.base import BaseCommand +from registry.models import Component, ComponentRelease # {'2048': # { diff --git a/web/registry/migrations/0001_initial.py b/web/registry/migrations/0001_initial.py index b10d1e31..0e97267a 100644 --- a/web/registry/migrations/0001_initial.py +++ b/web/registry/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.11 on 2022-02-14 18:12 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/web/registry/models.py b/web/registry/models.py index 80559756..1b7cb9ee 100644 --- a/web/registry/models.py +++ b/web/registry/models.py @@ -1,6 +1,8 @@ +from typing import Dict, List + from django.db import models from rest_framework.exceptions import ValidationError -from typing import Dict, List + from agentos.identifiers import ComponentIdentifier diff --git a/web/registry/serializers.py b/web/registry/serializers.py index 855f0171..056b45a7 100644 --- a/web/registry/serializers.py +++ b/web/registry/serializers.py @@ -1,6 +1,7 @@ from rest_framework import serializers from rest_framework.reverse import reverse -from .models import Repo, ComponentDependency, Component, RunCommand, Run + +from .models import Component, ComponentDependency, Repo, Run, RunCommand def _get_link_from_id(serializer, view_name, obj_id): diff --git a/web/registry/tests/test_integration.py b/web/registry/tests/test_integration.py index f03d8de5..b3832204 100644 --- a/web/registry/tests/test_integration.py +++ b/web/registry/tests/test_integration.py @@ -1,16 +1,15 @@ from django.test import LiveServerTestCase -from agentos.registry import WebRegistry -from registry.models import ( - Repo as RepoModel, - Component as ComponentModel, - RunCommand as RunCommandModel, - Run as RunModel, -) -from agentos.component_run import ComponentRun -from agentos.run_command import RunCommand +from registry.models import Component as ComponentModel +from registry.models import Repo as RepoModel +from registry.models import Run as RunModel +from registry.models import RunCommand as RunCommandModel + from agentos.component import Component +from agentos.component_run import ComponentRun +from agentos.registry import WebRegistry from agentos.repo import Repo -from tests.utils import TESTING_GITHUB_REPO_URL, TESTING_BRANCH_NAME +from agentos.run_command import RunCommand +from tests.utils import TESTING_BRANCH_NAME, TESTING_GITHUB_REPO_URL agentos_repo_spec = { "AgentOSRepo": { diff --git a/web/registry/tests/test_models.py b/web/registry/tests/test_models.py index cf554d01..d1730eb5 100644 --- a/web/registry/tests/test_models.py +++ b/web/registry/tests/test_models.py @@ -1,8 +1,9 @@ -from registry.models import Repo, ComponentDependency, Component -from registry.serializers import ComponentSerializer -from django.test import LiveServerTestCase import logging +from django.test import LiveServerTestCase +from registry.models import Component, ComponentDependency, Repo +from registry.serializers import ComponentSerializer + logger = logging.getLogger(__name__) diff --git a/web/registry/tests/test_registry.py b/web/registry/tests/test_registry.py index 430de255..897edcd9 100644 --- a/web/registry/tests/test_registry.py +++ b/web/registry/tests/test_registry.py @@ -1,9 +1,9 @@ # import yaml # import json from pathlib import Path + from django.core.files.base import File -from django.test import TestCase -from django.test import Client +from django.test import Client, TestCase from django.urls import reverse from registry.models import Component, Repo, Run, RunCommand diff --git a/web/registry/urls.py b/web/registry/urls.py index f1efc2ad..5c9baca5 100644 --- a/web/registry/urls.py +++ b/web/registry/urls.py @@ -1,13 +1,13 @@ -from django.urls import path, include +from django.urls import include, path +from rest_framework.routers import DefaultRouter + from .views import ( AgentRunViewSet, - RunViewSet, - RunCommandViewSet, ComponentViewSet, RepoViewSet, + RunCommandViewSet, + RunViewSet, ) -from rest_framework.routers import DefaultRouter - router = DefaultRouter() router.register(r"agentruns", AgentRunViewSet, basename="agentrun") diff --git a/web/registry/views.py b/web/registry/views.py index 980589d5..4efeaa18 100644 --- a/web/registry/views.py +++ b/web/registry/views.py @@ -1,23 +1,23 @@ -from django.db import transaction +import json +from django.db import transaction # from django.urls import reverse from django.http import HttpResponse from rest_framework import viewsets +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError from rest_framework.permissions import AllowAny from rest_framework.request import Request from rest_framework.response import Response -from rest_framework.decorators import action -from rest_framework.exceptions import ValidationError -import json -from .models import Repo, Component, ComponentDependency, RunCommand, Run + +from .models import Component, ComponentDependency, Repo, Run, RunCommand from .serializers import ( - RunSerializer, - RepoSerializer, ComponentSerializer, + RepoSerializer, RunCommandSerializer, + RunSerializer, ) - # TODO: This used to be a member of ComponentViewSet that also called a static # method of the Component model ingest_registry_dict(), but it should be # refactored to be its own stand-alone view, since ingest_registry view From 6f62d4163360dfe5e6e62978fd98a766ed3d9490 Mon Sep 17 00:00:00 2001 From: Nick Jalbert Date: Sat, 5 Mar 2022 20:49:58 +0600 Subject: [PATCH 2/4] Add isort to dev-requirements.txtt --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index e92c9522..cadc5ffd 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -13,6 +13,7 @@ sphinx_rtd_theme==1.0.0 click>=7.0 # when updating, also update in ../setup.py # Tests requirements +isort==5.10.1 black==21.12b0 cloudpickle==1.3.0 # gym 0.17.1 in setup.py requires cloudpickle<1.4.0,>=1.2.0 flake8==4.0.1 From 18abb8768995b0ea28d674eb7fe4af5e8d13c894 Mon Sep 17 00:00:00 2001 From: Nick Jalbert Date: Sat, 5 Mar 2022 20:56:17 +0600 Subject: [PATCH 3/4] remove commented out import --- web/registry/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/web/registry/views.py b/web/registry/views.py index 4efeaa18..c0802cf9 100644 --- a/web/registry/views.py +++ b/web/registry/views.py @@ -1,7 +1,6 @@ import json from django.db import transaction -# from django.urls import reverse from django.http import HttpResponse from rest_framework import viewsets from rest_framework.decorators import action From c523c9230c28d97396f243d6ab990e519d270b5e Mon Sep 17 00:00:00 2001 From: Nick Jalbert Date: Sat, 5 Mar 2022 21:09:52 +0600 Subject: [PATCH 4/4] Update usage docstring in format and lint script --- scripts/format_code.py | 3 +++ scripts/lint_code.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/scripts/format_code.py b/scripts/format_code.py index 5a15a27f..ce8880f6 100644 --- a/scripts/format_code.py +++ b/scripts/format_code.py @@ -6,6 +6,9 @@ # Format all code $ python scripts/format_code.py + # Format specific files + $ python scripts/format_code.py [path/to/file1] [path/to/file2] + # Print files that will be formatted, but don't actually format $ python scripts/format_code.py --check """ diff --git a/scripts/lint_code.py b/scripts/lint_code.py index 3b83568e..6115a46d 100644 --- a/scripts/lint_code.py +++ b/scripts/lint_code.py @@ -3,7 +3,11 @@ To use:: + # Lint all code $ python scripts/lint_code.py + + # Lint specific files + $ python scripts/lint_code.py [path/to/file1] [path/to/file2] """ import os