diff --git a/apps/rendering/benchmark/minilight/src/__init__.py b/apps/rendering/benchmark/minilight/src/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/codecov.yml b/codecov.yml index 5cbd182b89..e02fb41ba1 100644 --- a/codecov.yml +++ b/codecov.yml @@ -14,9 +14,9 @@ coverage: target: auto changes: false ignore: - - "apps/core/benchmark/minilight/*" - "apps/rendering/resources/taskcollector/*" - "golem/database/schemas" + - "golem/envs/docker/benchmark/minilight/*" - "golem/testutils.py" - "golem/tools/pyuic.py" - "save/*" diff --git a/golem/environments/environment.py b/golem/environments/environment.py index 66b3ee81a6..bd0194daed 100644 --- a/golem/environments/environment.py +++ b/golem/environments/environment.py @@ -3,10 +3,9 @@ from os import path -from apps.rendering.benchmark.minilight.src.minilight import make_perf_test - from golem.core.common import get_golem_path from golem.environments.minperformancemultiplier import MinPerformanceMultiplier +from golem.envs.docker.benchmark.minilight import make_perf_test from golem.model import Performance @@ -113,9 +112,7 @@ def get_min_accepted_performance(cls) -> float: def run_default_benchmark(cls, save=False): logger = logging.getLogger('golem.task.benchmarkmanager') logger.info('Running benchmark for %s', cls.get_id()) - test_file = path.join(get_golem_path(), 'apps', 'rendering', - 'benchmark', 'minilight', 'cornellbox.ml.txt') - performance = make_perf_test(test_file) + performance = make_perf_test() logger.info('%s performance is %.2f', cls.get_id(), performance) if save: Performance.update_or_create(cls.get_id(), performance) diff --git a/golem/envs/__init__.py b/golem/envs/__init__.py index 0902283095..445acb701f 100644 --- a/golem/envs/__init__.py +++ b/golem/envs/__init__.py @@ -281,6 +281,12 @@ def start(self) -> Deferred: """ Start the computation. Assumes current status is 'PREPARED'. """ raise NotImplementedError + @abstractmethod + def wait_until_stopped(self) -> Deferred: + """ Can be called after calling `start` to wait until the runtime has + stopped """ + raise NotImplementedError + @abstractmethod def stop(self) -> Deferred: """ Interrupt the computation. Assumes current status is 'RUNNING'. """ @@ -450,6 +456,11 @@ def clean_up(self) -> Deferred: 'ERROR'. """ raise NotImplementedError + @abstractmethod + def run_benchmark(self) -> Deferred: + """ Get the general performace score for this environment. """ + raise NotImplementedError + @classmethod @abstractmethod def metadata(cls) -> EnvMetadata: diff --git a/golem/envs/docker/benchmark/Dockerfile b/golem/envs/docker/benchmark/Dockerfile new file mode 100644 index 0000000000..ce24f2bc82 --- /dev/null +++ b/golem/envs/docker/benchmark/Dockerfile @@ -0,0 +1,8 @@ +FROM golemfactory/base:1.5 + +MAINTAINER Golem Tech + +COPY minilight /golem/minilight +COPY entrypoint.py /golem/ + +ENTRYPOINT ["python3", "/golem/entrypoint.py"] diff --git a/golem/envs/docker/benchmark/entrypoint.py b/golem/envs/docker/benchmark/entrypoint.py new file mode 100644 index 0000000000..db92e7b8dd --- /dev/null +++ b/golem/envs/docker/benchmark/entrypoint.py @@ -0,0 +1,6 @@ +from minilight import make_perf_test + + +if __name__ == '__main__': + score = make_perf_test() + print(score) diff --git a/golem/envs/docker/benchmark/minilight/__init__.py b/golem/envs/docker/benchmark/minilight/__init__.py new file mode 100644 index 0000000000..1958db68e5 --- /dev/null +++ b/golem/envs/docker/benchmark/minilight/__init__.py @@ -0,0 +1,9 @@ +from pathlib import Path + +from .src.minilight import make_perf_test as make_perf_test_impl + +TESTFILE = Path(__file__).parent / 'cornellbox.ml.txt' + + +def make_perf_test() -> float: + return make_perf_test_impl(str(TESTFILE)) diff --git a/apps/rendering/benchmark/minilight/cornellbox.ml.txt b/golem/envs/docker/benchmark/minilight/cornellbox.ml.txt similarity index 100% rename from apps/rendering/benchmark/minilight/cornellbox.ml.txt rename to golem/envs/docker/benchmark/minilight/cornellbox.ml.txt diff --git a/apps/rendering/benchmark/minilight/__init__.py b/golem/envs/docker/benchmark/minilight/src/__init__.py similarity index 100% rename from apps/rendering/benchmark/minilight/__init__.py rename to golem/envs/docker/benchmark/minilight/src/__init__.py diff --git a/apps/rendering/benchmark/minilight/src/camera.py b/golem/envs/docker/benchmark/minilight/src/camera.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/camera.py rename to golem/envs/docker/benchmark/minilight/src/camera.py diff --git a/apps/rendering/benchmark/minilight/src/image.py b/golem/envs/docker/benchmark/minilight/src/image.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/image.py rename to golem/envs/docker/benchmark/minilight/src/image.py diff --git a/apps/rendering/benchmark/minilight/src/img.py b/golem/envs/docker/benchmark/minilight/src/img.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/img.py rename to golem/envs/docker/benchmark/minilight/src/img.py diff --git a/apps/rendering/benchmark/minilight/src/maxilight.py b/golem/envs/docker/benchmark/minilight/src/maxilight.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/maxilight.py rename to golem/envs/docker/benchmark/minilight/src/maxilight.py diff --git a/apps/rendering/benchmark/minilight/src/minilight.py b/golem/envs/docker/benchmark/minilight/src/minilight.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/minilight.py rename to golem/envs/docker/benchmark/minilight/src/minilight.py diff --git a/apps/rendering/benchmark/minilight/src/randommini.py b/golem/envs/docker/benchmark/minilight/src/randommini.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/randommini.py rename to golem/envs/docker/benchmark/minilight/src/randommini.py diff --git a/apps/rendering/benchmark/minilight/src/raytracer.py b/golem/envs/docker/benchmark/minilight/src/raytracer.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/raytracer.py rename to golem/envs/docker/benchmark/minilight/src/raytracer.py diff --git a/apps/rendering/benchmark/minilight/src/rendertask.py b/golem/envs/docker/benchmark/minilight/src/rendertask.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/rendertask.py rename to golem/envs/docker/benchmark/minilight/src/rendertask.py diff --git a/apps/rendering/benchmark/minilight/src/rendertaskcreator.py b/golem/envs/docker/benchmark/minilight/src/rendertaskcreator.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/rendertaskcreator.py rename to golem/envs/docker/benchmark/minilight/src/rendertaskcreator.py diff --git a/apps/rendering/benchmark/minilight/src/renderworker.py b/golem/envs/docker/benchmark/minilight/src/renderworker.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/renderworker.py rename to golem/envs/docker/benchmark/minilight/src/renderworker.py diff --git a/apps/rendering/benchmark/minilight/src/scene.py b/golem/envs/docker/benchmark/minilight/src/scene.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/scene.py rename to golem/envs/docker/benchmark/minilight/src/scene.py diff --git a/apps/rendering/benchmark/minilight/src/spatialindex.py b/golem/envs/docker/benchmark/minilight/src/spatialindex.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/spatialindex.py rename to golem/envs/docker/benchmark/minilight/src/spatialindex.py diff --git a/apps/rendering/benchmark/minilight/src/surfacepoint.py b/golem/envs/docker/benchmark/minilight/src/surfacepoint.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/surfacepoint.py rename to golem/envs/docker/benchmark/minilight/src/surfacepoint.py diff --git a/apps/rendering/benchmark/minilight/src/task_data_0.py b/golem/envs/docker/benchmark/minilight/src/task_data_0.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/task_data_0.py rename to golem/envs/docker/benchmark/minilight/src/task_data_0.py diff --git a/apps/rendering/benchmark/minilight/src/taskablelight.py b/golem/envs/docker/benchmark/minilight/src/taskablelight.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/taskablelight.py rename to golem/envs/docker/benchmark/minilight/src/taskablelight.py diff --git a/apps/rendering/benchmark/minilight/src/taskablerenderer.py b/golem/envs/docker/benchmark/minilight/src/taskablerenderer.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/taskablerenderer.py rename to golem/envs/docker/benchmark/minilight/src/taskablerenderer.py diff --git a/apps/rendering/benchmark/minilight/src/triangle.py b/golem/envs/docker/benchmark/minilight/src/triangle.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/triangle.py rename to golem/envs/docker/benchmark/minilight/src/triangle.py diff --git a/apps/rendering/benchmark/minilight/src/vector3f.py b/golem/envs/docker/benchmark/minilight/src/vector3f.py similarity index 100% rename from apps/rendering/benchmark/minilight/src/vector3f.py rename to golem/envs/docker/benchmark/minilight/src/vector3f.py diff --git a/golem/envs/docker/cpu.py b/golem/envs/docker/cpu.py index dbc638308a..45ccd8cb34 100644 --- a/golem/envs/docker/cpu.py +++ b/golem/envs/docker/cpu.py @@ -1,4 +1,5 @@ import logging +import os from pathlib import Path from socket import socket, SocketIO, SHUT_WR from threading import Thread, Lock @@ -7,7 +8,7 @@ NamedTuple, Tuple, Iterator, Union, Iterable from docker.errors import APIError -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.internet.threads import deferToThread from urllib3.contrib.pyopenssl import WrappedSocket @@ -285,6 +286,12 @@ def _spawn_status_update_thread(_): f"Starting container '{self._container_id}' failed.")) return deferred_start + def wait_until_stopped(self) -> Deferred: + def _wait_until_stopped(): + while self.status() == RuntimeStatus.RUNNING: + sleep(1) + return deferToThread(_wait_until_stopped) + def stop(self) -> Deferred: with self._status_lock: self._assert_status(self._status, RuntimeStatus.RUNNING) @@ -404,7 +411,7 @@ class DockerCPUEnvironment(Environment): MIN_MEMORY_MB: ClassVar[int] = 1024 MIN_CPU_COUNT: ClassVar[int] = 1 - SHARED_DIR_PATH: ClassVar[str] = '/golem' + SHARED_DIR_PATH: ClassVar[str] = '/golem/work' NETWORK_MODE: ClassVar[str] = 'none' DNS_SERVERS: ClassVar[List[str]] = [] @@ -431,6 +438,8 @@ class DockerCPUEnvironment(Environment): 'sys_tty_config' ] + BENCHMARK_IMAGE = 'igorgolem/cpu_benchmark:1.0' + @classmethod def supported(cls) -> EnvSupportStatus: logger.info('Checking environment support status...') @@ -505,6 +514,28 @@ def _clean_up(): return deferToThread(_clean_up) + @inlineCallbacks + def run_benchmark(self) -> Deferred: + image, tag = self.BENCHMARK_IMAGE.split(':') + yield self.install_prerequisites(DockerPrerequisites( + image=image, + tag=tag, + )) + payload = DockerPayload( + image=image, + tag=tag, + user=None if is_windows() else str(os.getuid()), + env={}, + ) + runtime = self.runtime(payload) + yield runtime.prepare() + yield runtime.start() + yield runtime.wait_until_stopped() + # Benchmark is supposed to output a single line containing a float value + stdout = list(runtime.stdout('utf-8')) + yield runtime.clean_up() + return float(stdout[0]) + @classmethod def metadata(cls) -> EnvMetadata: return EnvMetadata( diff --git a/scripts/pyinstaller/hooks/hook-golem.py b/scripts/pyinstaller/hooks/hook-golem.py index 3743af22a2..cb7f08de6f 100644 --- a/scripts/pyinstaller/hooks/hook-golem.py +++ b/scripts/pyinstaller/hooks/hook-golem.py @@ -11,8 +11,6 @@ ('apps/*.ini', 'apps/'), ('apps/core/resources/images/*', 'apps/core/resources/images/'), - ('apps/rendering/benchmark/minilight/cornellbox.ml.txt', - 'apps/rendering/benchmark/minilight/'), ('apps/blender/resources/images/*.Dockerfile', 'apps/blender/resources/images/'), ('apps/blender/resources/images/entrypoints/scripts/render_tools/templates/' @@ -29,6 +27,8 @@ ('golem/RELEASE-VERSION', 'golem/'), ('golem/TERMS.html', 'golem/'), ('golem/database/schemas/*.py', 'golem/database/schemas/'), + ('golem/envs/docker/benchmark/minilight/cornellbox.ml.txt', + 'golem/envs/docker/benchmark/minilight/'), ('golem/network/concent/resources/ssl/certs/*.crt', 'golem/network/concent/resources/ssl/certs/'), ('scripts/docker/create-share.ps1', 'scripts/docker/'), diff --git a/setup.py b/setup.py index 6eb4a2e2b0..0b3c5a4728 100755 --- a/setup.py +++ b/setup.py @@ -80,8 +80,9 @@ path.normpath('apps/registered_test.ini'), path.normpath('apps/images.ini') ]), - (path.normpath('../../golem/apps/rendering/benchmark/minilight'), [ - path.normpath('apps/rendering/benchmark/minilight/cornellbox.ml.txt'), + (path.normpath('../../golem/golem/envs/docker/benchmark/minilight'), [ + path.normpath( + 'golem/envs/docker/benchmark/minilight/cornellbox.ml.txt'), ]), (path.normpath( '../../golem/apps/blender/resources/images/entrypoints/scripts/' diff --git a/tests/golem/envs/docker/cpu/test_env.py b/tests/golem/envs/docker/cpu/test_env.py index 4b10100735..e36a976e0b 100644 --- a/tests/golem/envs/docker/cpu/test_env.py +++ b/tests/golem/envs/docker/cpu/test_env.py @@ -1,6 +1,5 @@ from logging import Logger from pathlib import Path -from subprocess import SubprocessError from unittest.mock import patch as _patch, Mock, MagicMock, ANY from twisted.trial.unittest import TestCase diff --git a/tests/golem/envs/docker/cpu/test_integration.py b/tests/golem/envs/docker/cpu/test_integration.py index c9dc55723b..91467065cf 100644 --- a/tests/golem/envs/docker/cpu/test_integration.py +++ b/tests/golem/envs/docker/cpu/test_integration.py @@ -72,8 +72,7 @@ def _clean_up_runtime(): self.assertEqual(test_input, test_output) # Wait for exit and delete container - while runtime.status() == RuntimeStatus.RUNNING: - time.sleep(1) + yield runtime.wait_until_stopped() self.assertEqual(runtime.status(), RuntimeStatus.STOPPED) yield runtime.clean_up() self.assertEqual(runtime.status(), RuntimeStatus.TORN_DOWN) @@ -81,3 +80,15 @@ def _clean_up_runtime(): # Clean up the environment yield env.clean_up() self.assertEqual(env.status(), EnvStatus.DISABLED) + + @inlineCallbacks + def test_benchmark(self): + config = DockerCPUConfig(work_dir=Path(tempfile.gettempdir())) + env = DockerCPUEnvironment(config) + yield env.prepare() + + Whitelist.add(env.BENCHMARK_IMAGE.split('/')[0]) + score = yield env.run_benchmark() + self.assertGreater(score, 0) + + yield env.clean_up()