Refactor challenges to use cycle count instead of time (#4222)

Co-authored-by: Richard Beales <[email protected]>

.github/workflows/ci.yml

@@ -45,6 +45,10 @@ jobs:
         run: isort . --check
         if: success() || failure()
 
+      - name: Check mypy formatting
+        run: mypy
+        if: success() || failure()
+
   test:
     permissions:
       # Gives the action the necessary permissions for publishing new

.pre-commit-config.yaml

@@ -21,6 +21,10 @@ repos:
     hooks:
       - id: black
         language_version: python3.10
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.3.0'
+    hooks:
+      - id: mypy
 
   - repo: local
     hooks:

mypy.ini

@@ -0,0 +1,10 @@
+[mypy]
+follow_imports = skip
+check_untyped_defs = True
+disallow_untyped_defs = True
+files = tests/integration/challenges/**/*.py
+
+[mypy-requests.*]
+ignore_missing_imports = True
+[mypy-yaml.*]
+ignore_missing_imports = True

requirements.txt

@@ -36,7 +36,7 @@ gitpython==3.1.31
 auto-gpt-plugin-template
 mkdocs
 pymdown-extensions
-
+mypy
 
 # OpenAI and Generic plugins import
 openapi-python-client==0.13.4

tests/integration/agent_factory.py

@@ -12,7 +12,7 @@
 def agent_test_config(config: Config):
     was_continuous_mode = config.continuous_mode
     was_temperature = config.temperature
-    config.set_continuous_mode(True)
+    config.set_continuous_mode(False)
     config.set_temperature(0)
     yield config
     config.set_continuous_mode(was_continuous_mode)

tests/integration/challenges/basic_abilities/test_browse_website.py

@@ -0,0 +1,23 @@
+import pytest
+from pytest_mock import MockerFixture
+
+from autogpt.agent import Agent
+from autogpt.commands.file_operations import read_file
+from tests.integration.challenges.utils import run_interaction_loop
+from tests.utils import requires_api_key
+
+CYCLE_COUNT = 2
+
+
+@requires_api_key("OPENAI_API_KEY")
+@pytest.mark.vcr
+def test_browse_website(
+    browser_agent: Agent,
+    patched_api_requestor: MockerFixture,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    file_path = browser_agent.workspace.get_path("browse_website.txt")
+    run_interaction_loop(monkeypatch, browser_agent, CYCLE_COUNT)
+
+    content = read_file(file_path)
+    assert "£25.89" in content, f"Expected £25.89, got {content}"

tests/integration/challenges/basic_abilities/test_write_file.py

@@ -0,0 +1,23 @@
+import pytest
+from pytest_mock import MockerFixture
+
+from autogpt.agent import Agent
+from autogpt.commands.file_operations import read_file
+from tests.integration.challenges.utils import run_interaction_loop
+from tests.utils import requires_api_key
+
+CYCLE_COUNT = 3
+
+
+@requires_api_key("OPENAI_API_KEY")
+@pytest.mark.vcr
+def test_write_file(
+    writer_agent: Agent,
+    patched_api_requestor: MockerFixture,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    file_path = str(writer_agent.workspace.get_path("hello_world.txt"))
+    run_interaction_loop(monkeypatch, writer_agent, CYCLE_COUNT)
+
+    content = read_file(file_path)
+    assert content == "Hello World", f"Expected 'Hello World', got {content}"

tests/integration/challenges/conftest.py

@@ -1,17 +1,21 @@
 import pytest
+from _pytest.config import Config
+from _pytest.config.argparsing import Parser
+from _pytest.fixtures import FixtureRequest
+from _pytest.monkeypatch import MonkeyPatch
 
 
-def pytest_addoption(parser):
+def pytest_addoption(parser: Parser) -> None:
     parser.addoption(
         "--level", action="store", default=None, type=int, help="Specify test level"
     )
 
 
-def pytest_configure(config):
+def pytest_configure(config: Config) -> None:
     config.option.level = config.getoption("--level")
 
 
 @pytest.fixture
-def user_selected_level(request) -> int:
+def user_selected_level(request: FixtureRequest) -> int:
     ## used for challenges in the goal oriented tests
     return request.config.option.level

tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py

@@ -1,45 +1,30 @@
-import contextlib
-from functools import wraps
-from typing import Generator
-
 import pytest
+from pytest_mock import MockerFixture
 
 from autogpt.commands.file_operations import read_file, write_to_file
-from tests.integration.agent_utils import run_interaction_loop
-from tests.integration.challenges.utils import run_multiple_times
+from tests.integration.challenges.utils import run_interaction_loop, run_multiple_times
 from tests.utils import requires_api_key
 
-
-def input_generator(input_sequence: list) -> Generator[str, None, None]:
-    """
-    Creates a generator that yields input strings from the given sequence.
-
-    :param input_sequence: A list of input strings.
-    :return: A generator that yields input strings.
-    """
-    yield from input_sequence
+CYCLE_COUNT = 3
+from autogpt.agent import Agent
 
 
-# @pytest.skip("Nobody beat this challenge yet")
 @pytest.mark.skip("This challenge hasn't been beaten yet.")
 @pytest.mark.vcr
 @requires_api_key("OPENAI_API_KEY")
 @run_multiple_times(3)
 def test_information_retrieval_challenge_a(
-    get_company_revenue_agent, monkeypatch, patched_api_requestor
+    get_company_revenue_agent: Agent,
+    monkeypatch: pytest.MonkeyPatch,
+    patched_api_requestor: MockerFixture,
 ) -> None:
     """
     Test the challenge_a function in a given agent by mocking user inputs and checking the output file content.
 
     :param get_company_revenue_agent: The agent to test.
     :param monkeypatch: pytest's monkeypatch utility for modifying builtins.
     """
-    input_sequence = ["s", "s", "s", "s", "s", "EXIT"]
-    gen = input_generator(input_sequence)
-    monkeypatch.setattr("builtins.input", lambda _: next(gen))
-
-    with contextlib.suppress(SystemExit):
-        run_interaction_loop(get_company_revenue_agent, None)
+    run_interaction_loop(monkeypatch, get_company_revenue_agent, CYCLE_COUNT)
 
     file_path = str(get_company_revenue_agent.workspace.get_path("output.txt"))
     content = read_file(file_path)

tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py

@@ -1,43 +1,29 @@
-import contextlib
-from typing import Generator
-
 import pytest
 import yaml
 
+from autogpt.agent import Agent
 from autogpt.commands.file_operations import read_file
-from tests.integration.agent_utils import run_interaction_loop
-from tests.integration.challenges.utils import run_multiple_times
+from tests.integration.challenges.utils import run_interaction_loop, run_multiple_times
 from tests.utils import requires_api_key
 
-
-def input_generator(input_sequence: list) -> Generator[str, None, None]:
-    """
-    Creates a generator that yields input strings from the given sequence.
-
-    :param input_sequence: A list of input strings.
-    :return: A generator that yields input strings.
-    """
-    yield from input_sequence
+CYCLE_COUNT = 6
 
 
 @pytest.mark.skip("This challenge hasn't been beaten yet.")
 @pytest.mark.vcr
 @requires_api_key("OPENAI_API_KEY")
 @run_multiple_times(3)
-def test_information_retrieval_challenge_a(kubernetes_agent, monkeypatch) -> None:
+def test_kubernetes_template_challenge_a(
+    kubernetes_agent: Agent, monkeypatch: pytest.MonkeyPatch
+) -> None:
     """
     Test the challenge_a function in a given agent by mocking user inputs
     and checking the output file content.
 
     :param get_company_revenue_agent: The agent to test.
     :param monkeypatch: pytest's monkeypatch utility for modifying builtins.
     """
-    input_sequence = ["s", "s", "s", "s", "s", "EXIT"]
-    gen = input_generator(input_sequence)
-    monkeypatch.setattr("builtins.input", lambda _: next(gen))
-
-    with contextlib.suppress(SystemExit):
-        run_interaction_loop(kubernetes_agent, None)
+    run_interaction_loop(monkeypatch, kubernetes_agent, CYCLE_COUNT)
 
     file_path = str(kubernetes_agent.workspace.get_path("kube.yaml"))
     content = read_file(file_path)