From 7ad329c06c76aed147d04eb709a686d0f7226460 Mon Sep 17 00:00:00 2001
From: Igor Adamski <igor@golem.network>
Date: Mon, 27 May 2019 16:16:45 +0200
Subject: [PATCH 1/2] Add run_benchmark to environments

---
 .../benchmark/minilight/src/__init__.py       |  0
 codecov.yml                                   |  2 +-
 golem/environments/environment.py             |  7 ++--
 golem/envs/__init__.py                        | 11 ++++++
 golem/envs/docker/benchmark/Dockerfile        |  8 +++++
 golem/envs/docker/benchmark/entrypoint.py     |  6 ++++
 .../docker/benchmark/minilight/__init__.py    |  9 +++++
 .../benchmark/minilight/cornellbox.ml.txt     |  0
 .../benchmark/minilight/src}/__init__.py      |  0
 .../docker}/benchmark/minilight/src/camera.py |  0
 .../docker}/benchmark/minilight/src/image.py  |  0
 .../docker}/benchmark/minilight/src/img.py    |  0
 .../benchmark/minilight/src/maxilight.py      |  0
 .../benchmark/minilight/src/minilight.py      |  0
 .../benchmark/minilight/src/randommini.py     |  0
 .../benchmark/minilight/src/raytracer.py      |  0
 .../benchmark/minilight/src/rendertask.py     |  0
 .../minilight/src/rendertaskcreator.py        |  0
 .../benchmark/minilight/src/renderworker.py   |  0
 .../docker}/benchmark/minilight/src/scene.py  |  0
 .../benchmark/minilight/src/spatialindex.py   |  0
 .../benchmark/minilight/src/surfacepoint.py   |  0
 .../benchmark/minilight/src/task_data_0.py    |  0
 .../benchmark/minilight/src/taskablelight.py  |  0
 .../minilight/src/taskablerenderer.py         |  0
 .../benchmark/minilight/src/triangle.py       |  0
 .../benchmark/minilight/src/vector3f.py       |  0
 golem/envs/docker/cpu.py                      | 35 +++++++++++++++++--
 scripts/pyinstaller/hooks/hook-golem.py       |  4 +--
 setup.py                                      |  5 +--
 tests/golem/envs/docker/cpu/test_env.py       |  1 -
 .../golem/envs/docker/cpu/test_integration.py | 15 ++++++--
 32 files changed, 88 insertions(+), 15 deletions(-)
 delete mode 100644 apps/rendering/benchmark/minilight/src/__init__.py
 create mode 100644 golem/envs/docker/benchmark/Dockerfile
 create mode 100644 golem/envs/docker/benchmark/entrypoint.py
 create mode 100644 golem/envs/docker/benchmark/minilight/__init__.py
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/cornellbox.ml.txt (100%)
 rename {apps/rendering/benchmark/minilight => golem/envs/docker/benchmark/minilight/src}/__init__.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/camera.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/image.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/img.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/maxilight.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/minilight.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/randommini.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/raytracer.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/rendertask.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/rendertaskcreator.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/renderworker.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/scene.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/spatialindex.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/surfacepoint.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/task_data_0.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/taskablelight.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/taskablerenderer.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/triangle.py (100%)
 rename {apps/rendering => golem/envs/docker}/benchmark/minilight/src/vector3f.py (100%)

diff --git a/apps/rendering/benchmark/minilight/src/__init__.py b/apps/rendering/benchmark/minilight/src/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/codecov.yml b/codecov.yml
index 5cbd182b89..e02fb41ba1 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -14,9 +14,9 @@ coverage:
         target: auto
     changes: false
   ignore:
-    - "apps/core/benchmark/minilight/*"
     - "apps/rendering/resources/taskcollector/*"
     - "golem/database/schemas"
+    - "golem/envs/docker/benchmark/minilight/*"
     - "golem/testutils.py"
     - "golem/tools/pyuic.py"
     - "save/*"
diff --git a/golem/environments/environment.py b/golem/environments/environment.py
index 66b3ee81a6..bd0194daed 100644
--- a/golem/environments/environment.py
+++ b/golem/environments/environment.py
@@ -3,10 +3,9 @@
 
 from os import path
 
-from apps.rendering.benchmark.minilight.src.minilight import make_perf_test
-
 from golem.core.common import get_golem_path
 from golem.environments.minperformancemultiplier import MinPerformanceMultiplier
+from golem.envs.docker.benchmark.minilight import make_perf_test
 from golem.model import Performance
 
 
@@ -113,9 +112,7 @@ def get_min_accepted_performance(cls) -> float:
     def run_default_benchmark(cls, save=False):
         logger = logging.getLogger('golem.task.benchmarkmanager')
         logger.info('Running benchmark for %s', cls.get_id())
-        test_file = path.join(get_golem_path(), 'apps', 'rendering',
-                              'benchmark', 'minilight', 'cornellbox.ml.txt')
-        performance = make_perf_test(test_file)
+        performance = make_perf_test()
         logger.info('%s performance is %.2f', cls.get_id(), performance)
         if save:
             Performance.update_or_create(cls.get_id(), performance)
diff --git a/golem/envs/__init__.py b/golem/envs/__init__.py
index 0902283095..445acb701f 100644
--- a/golem/envs/__init__.py
+++ b/golem/envs/__init__.py
@@ -281,6 +281,12 @@ def start(self) -> Deferred:
         """ Start the computation. Assumes current status is 'PREPARED'. """
         raise NotImplementedError
 
+    @abstractmethod
+    def wait_until_stopped(self) -> Deferred:
+        """ Can be called after calling `start` to wait until the runtime has
+            stopped """
+        raise NotImplementedError
+
     @abstractmethod
     def stop(self) -> Deferred:
         """ Interrupt the computation. Assumes current status is 'RUNNING'. """
@@ -450,6 +456,11 @@ def clean_up(self) -> Deferred:
             'ERROR'. """
         raise NotImplementedError
 
+    @abstractmethod
+    def run_benchmark(self) -> Deferred:
+        """ Get the general performace score for this environment. """
+        raise NotImplementedError
+
     @classmethod
     @abstractmethod
     def metadata(cls) -> EnvMetadata:
diff --git a/golem/envs/docker/benchmark/Dockerfile b/golem/envs/docker/benchmark/Dockerfile
new file mode 100644
index 0000000000..ce24f2bc82
--- /dev/null
+++ b/golem/envs/docker/benchmark/Dockerfile
@@ -0,0 +1,8 @@
+FROM golemfactory/base:1.5
+
+MAINTAINER Golem Tech <tech@golem.network>
+
+COPY minilight /golem/minilight
+COPY entrypoint.py /golem/
+
+ENTRYPOINT ["python3", "/golem/entrypoint.py"]
diff --git a/golem/envs/docker/benchmark/entrypoint.py b/golem/envs/docker/benchmark/entrypoint.py
new file mode 100644
index 0000000000..db92e7b8dd
--- /dev/null
+++ b/golem/envs/docker/benchmark/entrypoint.py
@@ -0,0 +1,6 @@
+from minilight import make_perf_test
+
+
+if __name__ == '__main__':
+    score = make_perf_test()
+    print(score)
diff --git a/golem/envs/docker/benchmark/minilight/__init__.py b/golem/envs/docker/benchmark/minilight/__init__.py
new file mode 100644
index 0000000000..1958db68e5
--- /dev/null
+++ b/golem/envs/docker/benchmark/minilight/__init__.py
@@ -0,0 +1,9 @@
+from pathlib import Path
+
+from .src.minilight import make_perf_test as make_perf_test_impl
+
+TESTFILE = Path(__file__).parent / 'cornellbox.ml.txt'
+
+
+def make_perf_test() -> float:
+    return make_perf_test_impl(str(TESTFILE))
diff --git a/apps/rendering/benchmark/minilight/cornellbox.ml.txt b/golem/envs/docker/benchmark/minilight/cornellbox.ml.txt
similarity index 100%
rename from apps/rendering/benchmark/minilight/cornellbox.ml.txt
rename to golem/envs/docker/benchmark/minilight/cornellbox.ml.txt
diff --git a/apps/rendering/benchmark/minilight/__init__.py b/golem/envs/docker/benchmark/minilight/src/__init__.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/__init__.py
rename to golem/envs/docker/benchmark/minilight/src/__init__.py
diff --git a/apps/rendering/benchmark/minilight/src/camera.py b/golem/envs/docker/benchmark/minilight/src/camera.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/camera.py
rename to golem/envs/docker/benchmark/minilight/src/camera.py
diff --git a/apps/rendering/benchmark/minilight/src/image.py b/golem/envs/docker/benchmark/minilight/src/image.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/image.py
rename to golem/envs/docker/benchmark/minilight/src/image.py
diff --git a/apps/rendering/benchmark/minilight/src/img.py b/golem/envs/docker/benchmark/minilight/src/img.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/img.py
rename to golem/envs/docker/benchmark/minilight/src/img.py
diff --git a/apps/rendering/benchmark/minilight/src/maxilight.py b/golem/envs/docker/benchmark/minilight/src/maxilight.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/maxilight.py
rename to golem/envs/docker/benchmark/minilight/src/maxilight.py
diff --git a/apps/rendering/benchmark/minilight/src/minilight.py b/golem/envs/docker/benchmark/minilight/src/minilight.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/minilight.py
rename to golem/envs/docker/benchmark/minilight/src/minilight.py
diff --git a/apps/rendering/benchmark/minilight/src/randommini.py b/golem/envs/docker/benchmark/minilight/src/randommini.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/randommini.py
rename to golem/envs/docker/benchmark/minilight/src/randommini.py
diff --git a/apps/rendering/benchmark/minilight/src/raytracer.py b/golem/envs/docker/benchmark/minilight/src/raytracer.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/raytracer.py
rename to golem/envs/docker/benchmark/minilight/src/raytracer.py
diff --git a/apps/rendering/benchmark/minilight/src/rendertask.py b/golem/envs/docker/benchmark/minilight/src/rendertask.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/rendertask.py
rename to golem/envs/docker/benchmark/minilight/src/rendertask.py
diff --git a/apps/rendering/benchmark/minilight/src/rendertaskcreator.py b/golem/envs/docker/benchmark/minilight/src/rendertaskcreator.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/rendertaskcreator.py
rename to golem/envs/docker/benchmark/minilight/src/rendertaskcreator.py
diff --git a/apps/rendering/benchmark/minilight/src/renderworker.py b/golem/envs/docker/benchmark/minilight/src/renderworker.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/renderworker.py
rename to golem/envs/docker/benchmark/minilight/src/renderworker.py
diff --git a/apps/rendering/benchmark/minilight/src/scene.py b/golem/envs/docker/benchmark/minilight/src/scene.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/scene.py
rename to golem/envs/docker/benchmark/minilight/src/scene.py
diff --git a/apps/rendering/benchmark/minilight/src/spatialindex.py b/golem/envs/docker/benchmark/minilight/src/spatialindex.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/spatialindex.py
rename to golem/envs/docker/benchmark/minilight/src/spatialindex.py
diff --git a/apps/rendering/benchmark/minilight/src/surfacepoint.py b/golem/envs/docker/benchmark/minilight/src/surfacepoint.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/surfacepoint.py
rename to golem/envs/docker/benchmark/minilight/src/surfacepoint.py
diff --git a/apps/rendering/benchmark/minilight/src/task_data_0.py b/golem/envs/docker/benchmark/minilight/src/task_data_0.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/task_data_0.py
rename to golem/envs/docker/benchmark/minilight/src/task_data_0.py
diff --git a/apps/rendering/benchmark/minilight/src/taskablelight.py b/golem/envs/docker/benchmark/minilight/src/taskablelight.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/taskablelight.py
rename to golem/envs/docker/benchmark/minilight/src/taskablelight.py
diff --git a/apps/rendering/benchmark/minilight/src/taskablerenderer.py b/golem/envs/docker/benchmark/minilight/src/taskablerenderer.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/taskablerenderer.py
rename to golem/envs/docker/benchmark/minilight/src/taskablerenderer.py
diff --git a/apps/rendering/benchmark/minilight/src/triangle.py b/golem/envs/docker/benchmark/minilight/src/triangle.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/triangle.py
rename to golem/envs/docker/benchmark/minilight/src/triangle.py
diff --git a/apps/rendering/benchmark/minilight/src/vector3f.py b/golem/envs/docker/benchmark/minilight/src/vector3f.py
similarity index 100%
rename from apps/rendering/benchmark/minilight/src/vector3f.py
rename to golem/envs/docker/benchmark/minilight/src/vector3f.py
diff --git a/golem/envs/docker/cpu.py b/golem/envs/docker/cpu.py
index dbc638308a..16a3b1e9d1 100644
--- a/golem/envs/docker/cpu.py
+++ b/golem/envs/docker/cpu.py
@@ -1,4 +1,5 @@
 import logging
+import os
 from pathlib import Path
 from socket import socket, SocketIO, SHUT_WR
 from threading import Thread, Lock
@@ -7,7 +8,7 @@
     NamedTuple, Tuple, Iterator, Union, Iterable
 
 from docker.errors import APIError
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.internet.threads import deferToThread
 from urllib3.contrib.pyopenssl import WrappedSocket
 
@@ -285,6 +286,12 @@ def _spawn_status_update_thread(_):
             f"Starting container '{self._container_id}' failed."))
         return deferred_start
 
+    def wait_until_stopped(self) -> Deferred:
+        def _wait_until_stopped():
+            while self.status() == RuntimeStatus.RUNNING:
+                sleep(1)
+        return deferToThread(_wait_until_stopped)
+
     def stop(self) -> Deferred:
         with self._status_lock:
             self._assert_status(self._status, RuntimeStatus.RUNNING)
@@ -404,7 +411,7 @@ class DockerCPUEnvironment(Environment):
     MIN_MEMORY_MB: ClassVar[int] = 1024
     MIN_CPU_COUNT: ClassVar[int] = 1
 
-    SHARED_DIR_PATH: ClassVar[str] = '/golem'
+    SHARED_DIR_PATH: ClassVar[str] = '/golem/work'
 
     NETWORK_MODE: ClassVar[str] = 'none'
     DNS_SERVERS: ClassVar[List[str]] = []
@@ -431,6 +438,8 @@ class DockerCPUEnvironment(Environment):
         'sys_tty_config'
     ]
 
+    BENCHMARK_IMAGE = 'igorgolem/cpu_benchmark:1.0'
+
     @classmethod
     def supported(cls) -> EnvSupportStatus:
         logger.info('Checking environment support status...')
@@ -505,6 +514,28 @@ def _clean_up():
 
         return deferToThread(_clean_up)
 
+    @inlineCallbacks
+    def run_benchmark(self) -> Deferred:
+        image, tag = self.BENCHMARK_IMAGE.split(':')
+        yield self.install_prerequisites(DockerPrerequisites(
+            image=image,
+            tag=tag,
+        ))
+        payload = DockerPayload(
+            image=image,
+            tag=tag,
+            user=None if is_windows() else str(os.getuid()),
+            env={},
+        )
+        runtime = self.runtime(payload)
+        yield runtime.prepare()
+        yield runtime.start()
+        yield runtime.wait_until_stopped()
+        # Benchmark is supposed to output a single line containing a float value
+        score = float(list(runtime.stdout('utf-8'))[0])
+        yield runtime.clean_up()
+        return score
+
     @classmethod
     def metadata(cls) -> EnvMetadata:
         return EnvMetadata(
diff --git a/scripts/pyinstaller/hooks/hook-golem.py b/scripts/pyinstaller/hooks/hook-golem.py
index 3743af22a2..cb7f08de6f 100644
--- a/scripts/pyinstaller/hooks/hook-golem.py
+++ b/scripts/pyinstaller/hooks/hook-golem.py
@@ -11,8 +11,6 @@
     ('apps/*.ini', 'apps/'),
     ('apps/core/resources/images/*',
      'apps/core/resources/images/'),
-    ('apps/rendering/benchmark/minilight/cornellbox.ml.txt',
-     'apps/rendering/benchmark/minilight/'),
     ('apps/blender/resources/images/*.Dockerfile',
      'apps/blender/resources/images/'),
     ('apps/blender/resources/images/entrypoints/scripts/render_tools/templates/'
@@ -29,6 +27,8 @@
     ('golem/RELEASE-VERSION', 'golem/'),
     ('golem/TERMS.html', 'golem/'),
     ('golem/database/schemas/*.py', 'golem/database/schemas/'),
+    ('golem/envs/docker/benchmark/minilight/cornellbox.ml.txt',
+     'golem/envs/docker/benchmark/minilight/'),
     ('golem/network/concent/resources/ssl/certs/*.crt',
      'golem/network/concent/resources/ssl/certs/'),
     ('scripts/docker/create-share.ps1', 'scripts/docker/'),
diff --git a/setup.py b/setup.py
index 6eb4a2e2b0..0b3c5a4728 100755
--- a/setup.py
+++ b/setup.py
@@ -80,8 +80,9 @@
             path.normpath('apps/registered_test.ini'),
             path.normpath('apps/images.ini')
         ]),
-        (path.normpath('../../golem/apps/rendering/benchmark/minilight'), [
-            path.normpath('apps/rendering/benchmark/minilight/cornellbox.ml.txt'),
+        (path.normpath('../../golem/golem/envs/docker/benchmark/minilight'), [
+            path.normpath(
+                'golem/envs/docker/benchmark/minilight/cornellbox.ml.txt'),
         ]),
         (path.normpath(
             '../../golem/apps/blender/resources/images/entrypoints/scripts/'
diff --git a/tests/golem/envs/docker/cpu/test_env.py b/tests/golem/envs/docker/cpu/test_env.py
index 4b10100735..e36a976e0b 100644
--- a/tests/golem/envs/docker/cpu/test_env.py
+++ b/tests/golem/envs/docker/cpu/test_env.py
@@ -1,6 +1,5 @@
 from logging import Logger
 from pathlib import Path
-from subprocess import SubprocessError
 from unittest.mock import patch as _patch, Mock, MagicMock, ANY
 
 from twisted.trial.unittest import TestCase
diff --git a/tests/golem/envs/docker/cpu/test_integration.py b/tests/golem/envs/docker/cpu/test_integration.py
index c9dc55723b..91467065cf 100644
--- a/tests/golem/envs/docker/cpu/test_integration.py
+++ b/tests/golem/envs/docker/cpu/test_integration.py
@@ -72,8 +72,7 @@ def _clean_up_runtime():
         self.assertEqual(test_input, test_output)
 
         # Wait for exit and delete container
-        while runtime.status() == RuntimeStatus.RUNNING:
-            time.sleep(1)
+        yield runtime.wait_until_stopped()
         self.assertEqual(runtime.status(), RuntimeStatus.STOPPED)
         yield runtime.clean_up()
         self.assertEqual(runtime.status(), RuntimeStatus.TORN_DOWN)
@@ -81,3 +80,15 @@ def _clean_up_runtime():
         # Clean up the environment
         yield env.clean_up()
         self.assertEqual(env.status(), EnvStatus.DISABLED)
+
+    @inlineCallbacks
+    def test_benchmark(self):
+        config = DockerCPUConfig(work_dir=Path(tempfile.gettempdir()))
+        env = DockerCPUEnvironment(config)
+        yield env.prepare()
+
+        Whitelist.add(env.BENCHMARK_IMAGE.split('/')[0])
+        score = yield env.run_benchmark()
+        self.assertGreater(score, 0)
+
+        yield env.clean_up()

From 4e801abc6e3f1aa556bd8e46d35fbb85b9041cb5 Mon Sep 17 00:00:00 2001
From: Krigpl <3904252+Krigpl@users.noreply.github.com>
Date: Mon, 24 Jun 2019 12:38:46 +0200
Subject: [PATCH 2/2] Apply suggestions from code review

Co-Authored-By: Adam Wierzbicki <awierzbicki@golem.network>
---
 golem/envs/docker/cpu.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/golem/envs/docker/cpu.py b/golem/envs/docker/cpu.py
index 16a3b1e9d1..45ccd8cb34 100644
--- a/golem/envs/docker/cpu.py
+++ b/golem/envs/docker/cpu.py
@@ -532,9 +532,9 @@ def run_benchmark(self) -> Deferred:
         yield runtime.start()
         yield runtime.wait_until_stopped()
         # Benchmark is supposed to output a single line containing a float value
-        score = float(list(runtime.stdout('utf-8'))[0])
+        stdout = list(runtime.stdout('utf-8'))
         yield runtime.clean_up()
-        return score
+        return float(stdout[0])
 
     @classmethod
     def metadata(cls) -> EnvMetadata: