diff --git a/.github/workflows/run_unit_tests.yml b/.github/workflows/run_unit_tests.yml index e61bbf60..a5f084bc 100644 --- a/.github/workflows/run_unit_tests.yml +++ b/.github/workflows/run_unit_tests.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [windows-latest, ubuntu-22.04] - python-version: [3.8, 3.9, "3.10", 3.11] + python-version: [3.8, 3.9, "3.10", 3.11, 3.12] steps: - uses: actions/checkout@v4 with: @@ -39,7 +39,7 @@ jobs: PYTHONUTF8: 1 run: | python -m pip install --upgrade pip - pip install git+https://github.com/spine-tools/Spine-Database-API.git#egg=spinedb_api + pip install -r requirements.txt pip install .[dev] - name: List packages run: diff --git a/.gitignore b/.gitignore index c6057896..f02a67f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,13 @@ .idea __pycache__ -build -dist -spine_engine.egg-info +build/ +dist/ +spine_engine.egg-info/ .coverage htmlcov/ -spine_engine/server/certs +spine_engine/server/certs/ spine_engine/server/received_projects spine_engine/version.py -tests/server/test_project -tests/server/util/output +tests/server/test_project/ +tests/server/util/output/ docs/source/autoapi/ diff --git a/README.md b/README.md index be8b5c79..03dfd52d 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,6 @@ To install Spine Engine into an existing Python environment, run $ pip install spine_engine -### Dependencies - -Spine Engine installation will install [dagster](https://dagster.readthedocs.io/en/master/index.html). - - 
diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index e5ec38da..00000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -black == 24.4.2 -pre-commit == 2.0.1 -pylint >= 2.3.0 --r "docs/requirements.txt" diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index a4a65339..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -sphinx >= 1.7.5 -sphinx_rtd_theme >= 0.4.0 -recommonmark >=0.5.0 -sphinx-autoapi >=1.1.0 -jinja2 < 3.0 # Dagster 0.12.8 requires Jinja2<3.0 -markupsafe < 2.1 # Jinja2<3.0 tries to import soft_unicode, which has been removed in markupsafe 2.1 diff --git a/docs/source/autoapi/index.rst b/docs/source/autoapi/index.rst deleted file mode 100644 index 968dd4bf..00000000 --- a/docs/source/autoapi/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -API Reference -============= - -This page contains auto-generated API reference documentation [#f1]_. - -.. toctree:: - :titlesonly: - - /autoapi/spine_engine/index - -.. [#f1] Created with `sphinx-autoapi `_ \ No newline at end of file diff --git a/docs/source/autoapi/spine_engine/config/index.rst b/docs/source/autoapi/spine_engine/config/index.rst deleted file mode 100644 index 2182a788..00000000 --- a/docs/source/autoapi/spine_engine/config/index.rst +++ /dev/null @@ -1,105 +0,0 @@ -:py:mod:`spine_engine.config` -============================= - -.. py:module:: spine_engine.config - -.. autoapi-nested-parse:: - - Application constants. - - :author: P. Savolainen (VTT) - :date: 2.1.2018 - - - -Module Contents ---------------- - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.config._executable - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.config._on_windows - spine_engine.config.GAMS_EXECUTABLE - spine_engine.config.GAMSIDE_EXECUTABLE - spine_engine.config.JULIA_EXECUTABLE - spine_engine.config.PYTHON_EXECUTABLE - spine_engine.config._frozen - spine_engine.config._path_to_executable - spine_engine.config.APPLICATION_PATH - spine_engine.config.EMBEDDED_PYTHON - spine_engine.config.TOOL_OUTPUT_DIR - spine_engine.config.GIMLET_WORK_DIR_NAME - - -.. py:data:: _on_windows - - - - -.. py:function:: _executable(name) - - Appends a .exe extension to `name` on Windows platform. - - -.. py:data:: GAMS_EXECUTABLE - - - - -.. py:data:: GAMSIDE_EXECUTABLE - - - - -.. py:data:: JULIA_EXECUTABLE - - - - -.. py:data:: PYTHON_EXECUTABLE - - - - -.. py:data:: _frozen - - - - -.. py:data:: _path_to_executable - - - - -.. py:data:: APPLICATION_PATH - - - - -.. py:data:: EMBEDDED_PYTHON - - - - -.. py:data:: TOOL_OUTPUT_DIR - :annotation: = output - - - -.. py:data:: GIMLET_WORK_DIR_NAME - :annotation: = work - - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_manager/index.rst deleted file mode 100644 index 16e61193..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_manager/index.rst +++ /dev/null @@ -1,164 +0,0 @@ -:py:mod:`spine_engine.execution_managers.conda_kernel_spec_manager` -=================================================================== - -.. py:module:: spine_engine.execution_managers.conda_kernel_spec_manager - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.conda_kernel_spec_manager.CondaKernelSpecManager - - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.conda_kernel_spec_manager.CACHE_TIMEOUT - spine_engine.execution_managers.conda_kernel_spec_manager.RUNNER_COMMAND - - -.. py:data:: CACHE_TIMEOUT - :annotation: = 60 - - - -.. py:data:: RUNNER_COMMAND - :annotation: = ['python', '-m', 'spine_engine.execution_managers.conda_kernel_spec_runner'] - - - -.. py:class:: CondaKernelSpecManager(**kwargs) - - Bases: :py:obj:`jupyter_client.kernelspec.KernelSpecManager` - - A custom KernelSpecManager able to search for conda environments and - create kernelspecs for them. - - Create a configurable given a config config. - - :param config: If this is empty, default values are used. If config is a - :class:`Config` instance, it will be used to configure the - instance. - :type config: Config - :param parent: The parent Configurable instance of this object. - :type parent: Configurable instance, optional - - .. rubric:: Notes - - Subclasses of Configurable must call the :meth:`__init__` method of - :class:`Configurable` *before* doing anything else and using - :func:`super`:: - - class MyConfigurable(Configurable): - def __init__(self, config=None): - super(MyConfigurable, self).__init__(config=config) - # Then any other code you need to finish initialization. - - This ensures that instances will be configured properly. - - .. py:attribute:: conda_only - - - - - .. py:attribute:: env_filter - - - - - .. py:attribute:: kernelspec_path - - - - - .. py:attribute:: name_format - - - - - .. py:method:: _validate_kernelspec_path(self, proposal) - - - .. py:method:: clean_kernel_name(kname) - :staticmethod: - - Replaces invalid characters in the Jupyter kernelname, with - a bit of effort to preserve readability. - - - .. py:method:: _conda_info(self) - :property: - - Get and parse the whole conda information output - - Caches the information for CACHE_TIMEOUT seconds, as this is - relatively expensive. - - - .. py:method:: _all_envs(self) - - Find all of the environments we should be checking. We skip - environments in the conda-bld directory as well as environments - that match our env_filter regex. Returns a dict with canonical - environment names as keys, and full paths as values. - - - .. py:method:: _all_specs(self) - - Find the all kernel specs in all environments. - - Returns a dict with unique env names as keys, and the kernel.json - content as values, modified so that they can be run properly in - their native environments. - - Caches the information for CACHE_TIMEOUT seconds, as this is - relatively expensive. - - - .. py:method:: _conda_kspecs(self) - :property: - - Get (or refresh) the cache of conda kernels - - - - .. py:method:: find_kernel_specs(self) - - Returns a dict mapping kernel names to resource directories. - - The update process also adds the resource dir for the conda - environments. - - - .. py:method:: get_kernel_spec(self, kernel_name) - - Returns a :class:`KernelSpec` instance for the given kernel_name. - - Additionally, conda kernelspecs are generated on the fly - accordingly with the detected environments. - - - .. py:method:: get_all_specs(self) - - Returns a dict mapping kernel names to dictionaries with two - entries: "resource_dir" and "spec". This was added to fill out - the full public interface to KernelManagerSpec. - - - .. py:method:: remove_kernel_spec(self, name) - - Remove a kernel spec directory by name. - - Returns the path that was deleted. - - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_runner/index.rst b/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_runner/index.rst deleted file mode 100644 index fc06cdad..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_runner/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -:py:mod:`spine_engine.execution_managers.conda_kernel_spec_runner` -================================================================== - -.. py:module:: spine_engine.execution_managers.conda_kernel_spec_runner - - -Module Contents ---------------- - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.conda_kernel_spec_runner.exec_in_env - - - -.. py:function:: exec_in_env(conda_prefix, env_path, *command) - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/execution_manager_base/index.rst b/docs/source/autoapi/spine_engine/execution_managers/execution_manager_base/index.rst deleted file mode 100644 index 9c94c0f2..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/execution_manager_base/index.rst +++ /dev/null @@ -1,52 +0,0 @@ -:py:mod:`spine_engine.execution_managers.execution_manager_base` -================================================================ - -.. py:module:: spine_engine.execution_managers.execution_manager_base - -.. autoapi-nested-parse:: - - Contains the ExecutionManagerBase class. - - :authors: M. Marin (KTH) - :date: 12.10.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase - - - - -.. py:class:: ExecutionManagerBase(logger) - - Base class for all tool instance execution managers. - - Class constructor. - - :param logger: a logger instance - :type logger: LoggerInterface - - .. py:method:: run_until_complete(self) - :abstractmethod: - - Runs until completion. - - :returns: return code - :rtype: int - - - .. py:method:: stop_execution(self) - :abstractmethod: - - Stops execution gracefully. - - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/index.rst b/docs/source/autoapi/spine_engine/execution_managers/index.rst deleted file mode 100644 index 93fe245d..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/index.rst +++ /dev/null @@ -1,21 +0,0 @@ -:py:mod:`spine_engine.execution_managers` -========================================= - -.. py:module:: spine_engine.execution_managers - - -Submodules ----------- -.. toctree:: - :titlesonly: - :maxdepth: 1 - - conda_kernel_spec_manager/index.rst - conda_kernel_spec_runner/index.rst - execution_manager_base/index.rst - kernel_execution_manager/index.rst - persistent_execution_manager/index.rst - process_execution_manager/index.rst - spine_repl/index.rst - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/kernel_execution_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/kernel_execution_manager/index.rst deleted file mode 100644 index 0bcf0c9a..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/kernel_execution_manager/index.rst +++ /dev/null @@ -1,165 +0,0 @@ -:py:mod:`spine_engine.execution_managers.kernel_execution_manager` -================================================================== - -.. py:module:: spine_engine.execution_managers.kernel_execution_manager - -.. autoapi-nested-parse:: - - Contains the KernelExecutionManager class and subclasses, and some convenience functions. - - :authors: M. Marin (KTH) - :date: 12.10.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.kernel_execution_manager._KernelManagerFactory - spine_engine.execution_managers.kernel_execution_manager.KernelExecutionManager - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.kernel_execution_manager.get_kernel_manager - spine_engine.execution_managers.kernel_execution_manager.pop_kernel_manager - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.kernel_execution_manager._kernel_manager_factory - - -.. py:class:: _KernelManagerFactory - - .. py:attribute:: _kernel_managers - - - Maps tuples (kernel name, group id) to associated KernelManager. - - .. py:attribute:: _key_by_connection_file - - - Maps connection file string to tuple (kernel_name, group_id). Mostly for fast lookup in ``restart_kernel()`` - - .. py:method:: _make_kernel_manager(self, kernel_name, group_id) - - Creates a new kernel manager for given kernel and group id if none exists, and returns it. - - :param kernel_name: the kernel - :type kernel_name: str - :param group_id: item group that will execute using this kernel - :type group_id: str - - :returns: KernelManager - - - .. py:method:: new_kernel_manager(self, kernel_name, group_id, logger, extra_switches=None, environment='', **kwargs) - - Creates a new kernel manager for given kernel and group id if none exists. - Starts the kernel if not started, and returns it. - - :param kernel_name: the kernel - :type kernel_name: str - :param group_id: item group that will execute using this kernel - :type group_id: str - :param logger: for logging - :type logger: LoggerInterface - :param extra_switches: List of additional switches to julia or python. - These come before the 'programfile'. - :type extra_switches: list, optional - :param environment: "conda" to launch a Conda kernel spec. "" for a regular kernel spec - :type environment: str - :param `**kwargs`: optional. Keyword arguments passed to ``KernelManager.start_kernel()`` - - :returns: KernelManager - - - .. py:method:: get_kernel_manager(self, connection_file) - - Returns a kernel manager for given connection file if any. - - :param connection_file: path of connection file - :type connection_file: str - - :returns: KernelManager or None - - - .. py:method:: pop_kernel_manager(self, connection_file) - - Returns a kernel manager for given connection file if any. - It also removes it from cache. - - :param connection_file: path of connection file - :type connection_file: str - - :returns: KernelManager or None - - - -.. py:data:: _kernel_manager_factory - - - - -.. py:function:: get_kernel_manager(connection_file) - - -.. py:function:: pop_kernel_manager(connection_file) - - -.. py:class:: KernelExecutionManager(logger, kernel_name, *commands, group_id=None, workdir=None, startup_timeout=60, extra_switches=None, environment='', **kwargs) - - Bases: :py:obj:`spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase` - - Base class for all tool instance execution managers. - - :param logger: - :type logger: LoggerInterface - :param kernel_name: the kernel - :type kernel_name: str - :param \*commands: Commands to execute in the kernel - :param group_id: item group that will execute using this kernel - :type group_id: str, optional - :param workdir: item group that will execute using this kernel - :type workdir: str, optional - :param startup_timeout: How much to wait for the kernel, used in ``KernelClient.wait_for_ready()`` - :type startup_timeout: int, optional - :param extra_switches: List of additional switches to launch julia. - These come before the 'programfile'. - :type extra_switches: list, optional - :param environment: "conda" to launch a Conda kernel spec. "" for a regular kernel spec. - :type environment: str - :param \*\*kwargs: Keyword arguments passed to ``KernelManager.start_kernel()`` - :type \*\*kwargs: optional - - .. py:method:: run_until_complete(self) - - Runs until completion. - - :returns: return code - :rtype: int - - - .. py:method:: _do_run(self) - - - .. py:method:: stop_execution(self) - - Stops execution gracefully. - - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/persistent_execution_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/persistent_execution_manager/index.rst deleted file mode 100644 index c54dadf4..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/persistent_execution_manager/index.rst +++ /dev/null @@ -1,528 +0,0 @@ -:py:mod:`spine_engine.execution_managers.persistent_execution_manager` -====================================================================== - -.. py:module:: spine_engine.execution_managers.persistent_execution_manager - -.. autoapi-nested-parse:: - - Contains PersistentManagerBase, PersistentExecutionManagerBase classes and subclasses, - as well as some convenience functions. - - :authors: M. Marin (KTH) - :date: 12.10.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.persistent_execution_manager.PersistentManagerBase - spine_engine.execution_managers.persistent_execution_manager.JuliaPersistentManager - spine_engine.execution_managers.persistent_execution_manager.PythonPersistentManager - spine_engine.execution_managers.persistent_execution_manager._PersistentManagerFactory - spine_engine.execution_managers.persistent_execution_manager.PersistentExecutionManagerBase - spine_engine.execution_managers.persistent_execution_manager.JuliaPersistentExecutionManager - spine_engine.execution_managers.persistent_execution_manager.PythonPersistentExecutionManager - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.persistent_execution_manager.restart_persistent - spine_engine.execution_managers.persistent_execution_manager.interrupt_persistent - spine_engine.execution_managers.persistent_execution_manager.kill_persistent_processes - spine_engine.execution_managers.persistent_execution_manager.issue_persistent_command - spine_engine.execution_managers.persistent_execution_manager.get_persistent_completions - spine_engine.execution_managers.persistent_execution_manager.get_persistent_history_item - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.persistent_execution_manager._persistent_manager_factory - - -.. py:class:: PersistentManagerBase(args, cwd=None) - - - :param args: the arguments to launch the persistent process - :type args: list - :param cwd: the directory where to start the process - :type cwd: str, optional - - .. py:method:: language(self) - :property: - - Returns the underlying language for UI customization in toolbox. - - :returns: str - - - .. py:method:: _init_args(self) - :abstractmethod: - - Returns init args for Popen. - - Subclasses must reimplement to include appropriate switches to ensure the - process is interactive, or to load modules for internal communication. - - :returns: list - - - .. py:method:: _sentinel_command(host, port) - :staticmethod: - :abstractmethod: - - Returns a command in the underlying language, that sends a sentinel to a socket listening on given - host/port. - - Used to synchronize with the persistent process. - - :param host: - :type host: str - :param port: - :type port: int - - :returns: str - - - .. py:method:: _start_persistent(self) - - Starts the persistent process. - - - .. py:method:: _log_stdout(self) - - Puts stdout from the process into the queue (it will be consumed by issue_command()). - - - .. py:method:: _log_stderr(self) - - Puts stderr from the process into the queue (it will be consumed by issue_command()). - - - .. py:method:: _is_complete(self, cmd) - - - .. py:method:: issue_command(self, cmd, add_history=False) - - Issues cmd to the persistent process and returns an iterator of stdout and stderr messages. - - Each message is a dictionary with two keys: - - - "type": either "stdout" or "stderr" - - "data": the actual message string. - - :param cmd: - :type cmd: str - - :returns: generator - - - .. py:method:: _make_complete_command(self) - - - .. py:method:: _issue_command_and_wait_for_idle(self, cmd, add_history) - - Issues command and wait for idle. - - :param cmd: Command to pass to the persistent process - :type cmd: str - :param add_history: Whether or not to add the command to history - :type add_history: bool - - - .. py:method:: _issue_command(self, cmd, is_complete=True, add_history=False) - - Writes command to the process's stdin and flushes. - - :param cmd: Command to pass to the persistent process - :type cmd: str - :param is_complete: Whether or not the command is complete - :type is_complete: bool - :param add_history: Whether or not to add the command to history - :type add_history: bool - - - .. py:method:: _wait(self) - - Waits for the persistent process to become idle. - - This is implemented by writing the sentinel to stdin and waiting for the _SENTINEL to come out of stdout. - - :returns: bool - - - .. py:method:: _listen_and_enqueue(host, port, queue) - :staticmethod: - - Listens on the server and enqueues all data received. - - - .. py:method:: _communicate(self, request, arg, receive=True) - - Sends a request to the persistent process with the given argument. - - :param request: One of the supported requests - :type request: str - :param arg: Request argument - :param receive: If True (the default) also receives the response and returns it. - :type receive: bool, optional - - :returns: response, or None if the ``receive`` argument is False - :rtype: str or NoneType - - - .. py:method:: get_completions(self, text) - - Returns a list of autocompletion options for given text. - - :param text: Text to complete - :type text: str - - :returns: List of options - :rtype: list(str) - - - .. py:method:: get_history_item(self, index) - - Returns the history item given by index. - - :param index: Index to retrieve, one-based, 1 means most recent - :type index: int - - :returns: str - - - .. py:method:: restart_persistent(self) - - Restarts the persistent process. - - - .. py:method:: interrupt_persistent(self) - - Interrupts the persistent process. - - - .. py:method:: _do_interrupt_persistent(self) - - - .. py:method:: is_persistent_alive(self) - - Whether or not the persistent is still alive and ready to receive commands. - - :returns: bool - - - .. py:method:: kill_process(self) - - - -.. py:class:: JuliaPersistentManager(args, cwd=None) - - Bases: :py:obj:`PersistentManagerBase` - - - :param args: the arguments to launch the persistent process - :type args: list - :param cwd: the directory where to start the process - :type cwd: str, optional - - .. py:method:: language(self) - :property: - - See base class. - - - .. py:method:: _init_args(self) - - See base class. - - - .. py:method:: _sentinel_command(host, port) - :staticmethod: - - Returns a command in the underlying language, that sends a sentinel to a socket listening on given - host/port. - - Used to synchronize with the persistent process. - - :param host: - :type host: str - :param port: - :type port: int - - :returns: str - - - -.. py:class:: PythonPersistentManager(args, cwd=None) - - Bases: :py:obj:`PersistentManagerBase` - - - :param args: the arguments to launch the persistent process - :type args: list - :param cwd: the directory where to start the process - :type cwd: str, optional - - .. py:method:: language(self) - :property: - - See base class. - - - .. py:method:: _init_args(self) - - See base class. - - - .. py:method:: _sentinel_command(host, port) - :staticmethod: - - Returns a command in the underlying language, that sends a sentinel to a socket listening on given - host/port. - - Used to synchronize with the persistent process. - - :param host: - :type host: str - :param port: - :type port: int - - :returns: str - - - -.. py:class:: _PersistentManagerFactory - - .. py:attribute:: _persistent_managers - - - Maps tuples (process args) to associated PersistentManagerBase. - - .. py:method:: new_persistent_manager(self, constructor, logger, args, group_id, cwd=None) - - Creates a new persistent for given args and group id if none exists. - - :param constructor: the persistent manager constructor - :type constructor: function - :param logger: - :type logger: LoggerInterface - :param args: the arguments to launch the persistent process - :type args: list - :param group_id: item group that will execute using this persistent - :type group_id: str - :param cwd: directory where to start the persistent - :type cwd: str, optional - - :returns: persistent manager - :rtype: PersistentManagerBase - - - .. py:method:: restart_persistent(self, key) - - Restart a persistent process. - - :param key: persistent identifier - :type key: tuple - - - .. py:method:: interrupt_persistent(self, key) - - Interrupts a persistent process. - - :param key: persistent identifier - :type key: tuple - - - .. py:method:: issue_persistent_command(self, key, cmd) - - Issues a command to a persistent process. - - :param key: persistent identifier - :type key: tuple - :param cmd: command to issue - :type cmd: str - - :returns: stdio and stderr messages (dictionaries with two keys: type, and data) - :rtype: generator - - - .. py:method:: get_persistent_completions(self, key, text) - - Returns a list of completion options. - - :param key: persistent identifier - :type key: tuple - :param text: text to complete - :type text: str - - :returns: options that match given text - :rtype: list of str - - - .. py:method:: get_persistent_history_item(self, key, index) - - Issues a command to a persistent process. - - :param key: persistent identifier - :type key: tuple - :param index: index of the history item, most recen first - :type index: int - - :returns: history item or empty string if none - :rtype: str - - - .. py:method:: kill_manager_processes(self) - - Kills persistent managers' Popen instances. - - - -.. py:data:: _persistent_manager_factory - - - - -.. py:function:: restart_persistent(key) - - See _PersistentManagerFactory. - - -.. py:function:: interrupt_persistent(key) - - See _PersistentManagerFactory. - - -.. py:function:: kill_persistent_processes() - - Kills all persistent processes. - - On Windows systems the work directories are reserved by the ``Popen`` objects owned by the persistent managers. - They need to be killed to before the directories can be modified, e.g. deleted or renamed. - - -.. py:function:: issue_persistent_command(key, cmd) - - See _PersistentManagerFactory. - - -.. py:function:: get_persistent_completions(key, text) - - See _PersistentManagerFactory. - - -.. py:function:: get_persistent_history_item(key, index) - - See _PersistentManagerFactory. - - -.. py:class:: PersistentExecutionManagerBase(logger, args, commands, alias, group_id=None, workdir=None) - - Bases: :py:obj:`spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase` - - Base class for managing execution of commands on a persistent process. - - Class constructor. - - :param logger: a logger instance - :type logger: LoggerInterface - :param args: List of args to start the persistent process - :type args: list - :param commands: List of commands to execute in the persistent process - :type commands: list - :param group_id: item group that will execute using this kernel - :type group_id: str, optional - :param workdir: item group that will execute using this kernel - :type workdir: str, optional - - .. py:method:: alias(self) - :property: - - - .. py:method:: persistent_manager_factory() - :staticmethod: - :abstractmethod: - - Returns a function to create a persistent manager for this execution - (a subclass of PersistentManagerBase) - - :returns: function - - - .. py:method:: run_until_complete(self) - - See base class. - - - .. py:method:: stop_execution(self) - - See base class. - - - -.. py:class:: JuliaPersistentExecutionManager(logger, args, commands, alias, group_id=None, workdir=None) - - Bases: :py:obj:`PersistentExecutionManagerBase` - - Manages execution of commands on a Julia persistent process. - - Class constructor. - - :param logger: a logger instance - :type logger: LoggerInterface - :param args: List of args to start the persistent process - :type args: list - :param commands: List of commands to execute in the persistent process - :type commands: list - :param group_id: item group that will execute using this kernel - :type group_id: str, optional - :param workdir: item group that will execute using this kernel - :type workdir: str, optional - - .. py:method:: persistent_manager_factory() - :staticmethod: - - See base class. - - - -.. py:class:: PythonPersistentExecutionManager(logger, args, commands, alias, group_id=None, workdir=None) - - Bases: :py:obj:`PersistentExecutionManagerBase` - - Manages execution of commands on a Python persistent process. - - Class constructor. - - :param logger: a logger instance - :type logger: LoggerInterface - :param args: List of args to start the persistent process - :type args: list - :param commands: List of commands to execute in the persistent process - :type commands: list - :param group_id: item group that will execute using this kernel - :type group_id: str, optional - :param workdir: item group that will execute using this kernel - :type workdir: str, optional - - .. py:method:: persistent_manager_factory() - :staticmethod: - - See base class. - - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/process_execution_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/process_execution_manager/index.rst deleted file mode 100644 index 2a115542..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/process_execution_manager/index.rst +++ /dev/null @@ -1,62 +0,0 @@ -:py:mod:`spine_engine.execution_managers.process_execution_manager` -=================================================================== - -.. py:module:: spine_engine.execution_managers.process_execution_manager - -.. autoapi-nested-parse:: - - Contains the ProcessExecutionManager class. - - :authors: M. Marin (KTH) - :date: 12.10.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.process_execution_manager.ProcessExecutionManager - - - - -.. py:class:: ProcessExecutionManager(logger, program, *args, workdir=None) - - Bases: :py:obj:`spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase` - - Base class for all tool instance execution managers. - - Class constructor. - - :param logger: a logger instance - :type logger: LoggerInterface - :param program: Path to program to run in the subprocess (e.g. julia.exe) - :type program: str - :param args: List of argument for the program (e.g. path to script file) - :type args: list - - .. py:method:: run_until_complete(self) - - Runs until completion. - - :returns: return code - :rtype: int - - - .. py:method:: stop_execution(self) - - Stops execution gracefully. - - - .. py:method:: _log_stdout(self, stdout) - - - .. py:method:: _log_stderr(self, stderr) - - - diff --git a/docs/source/autoapi/spine_engine/execution_managers/spine_repl/index.rst b/docs/source/autoapi/spine_engine/execution_managers/spine_repl/index.rst deleted file mode 100644 index 038e1f94..00000000 --- a/docs/source/autoapi/spine_engine/execution_managers/spine_repl/index.rst +++ /dev/null @@ -1,104 +0,0 @@ -:py:mod:`spine_engine.execution_managers.spine_repl` -==================================================== - -.. py:module:: spine_engine.execution_managers.spine_repl - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.spine_repl.SpineDBServer - spine_engine.execution_managers.spine_repl._RequestHandler - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.spine_repl.completions - spine_engine.execution_managers.spine_repl.add_history - spine_engine.execution_managers.spine_repl.history_item - spine_engine.execution_managers.spine_repl.is_complete - spine_engine.execution_managers.spine_repl.start_server - spine_engine.execution_managers.spine_repl.send_sentinel - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.execution_managers.spine_repl.readline - - -.. py:data:: readline - - - - -.. py:class:: SpineDBServer(server_address, RequestHandlerClass, bind_and_activate=True) - - Bases: :py:obj:`socketserver.ThreadingMixIn`, :py:obj:`socketserver.TCPServer` - - Mix-in class to handle each request in a new thread. - - Constructor. May be extended, do not override. - - .. py:attribute:: allow_reuse_address - :annotation: = True - - - - -.. py:class:: _RequestHandler(request, client_address, server) - - Bases: :py:obj:`socketserver.BaseRequestHandler` - - Base class for request handler classes. - - This class is instantiated for each request to be handled. The - constructor sets the instance variables request, client_address - and server, and then calls the handle() method. To implement a - specific service, all you need to do is to derive a class which - defines a handle() method. - - The handle() method can find the request as self.request, the - client address as self.client_address, and the server (in case it - needs access to per-server information) as self.server. Since a - separate instance is created for each request, the handle() method - can define other arbitrary instance variables. - - - .. py:method:: handle(self) - - - -.. py:function:: completions(text) - - -.. py:function:: add_history(line) - - -.. py:function:: history_item(index) - - -.. py:function:: is_complete(cmd) - - -.. py:function:: start_server(address) - - :param address: Server address - :type address: tuple(str,int) - - -.. py:function:: send_sentinel(host, port) - - diff --git a/docs/source/autoapi/spine_engine/index.rst b/docs/source/autoapi/spine_engine/index.rst deleted file mode 100644 index dae87444..00000000 --- a/docs/source/autoapi/spine_engine/index.rst +++ /dev/null @@ -1,394 +0,0 @@ -:py:mod:`spine_engine` -====================== - -.. py:module:: spine_engine - - -Subpackages ------------ -.. toctree:: - :titlesonly: - :maxdepth: 3 - - execution_managers/index.rst - multithread_executor/index.rst - project_item/index.rst - utils/index.rst - - -Submodules ----------- -.. toctree:: - :titlesonly: - :maxdepth: 1 - - config/index.rst - load_project_items/index.rst - project_item_loader/index.rst - spine_engine/index.rst - spine_engine_server/index.rst - version/index.rst - - -Package Contents ----------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.ExecutionDirection - spine_engine.SpineEngine - spine_engine.SpineEngineState - spine_engine.ItemExecutionFinishState - - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.__version__ - - -.. py:class:: ExecutionDirection - - Bases: :py:obj:`enum.Enum` - - Generic enumeration. - - Derive from this class to define new enumerations. - - .. py:attribute:: FORWARD - - - - - .. py:attribute:: BACKWARD - - - - - .. py:method:: __str__(self) - - Return str(self). - - - -.. py:class:: SpineEngine(items=None, specifications=None, connections=None, items_module_name='spine_items', settings=None, project_dir=None, execution_permits=None, node_successors=None, debug=False) - - An engine for executing a Spine Toolbox DAG-workflow. - - :param items: List of executable item dicts. - :type items: list(dict) - :param specifications: A mapping from item type to list of specification dicts. - :type specifications: dict(str,list(dict)) - :param connections: List of connection dicts - :type connections: list of dict - :param items_module_name: name of the Python module that contains project items - :type items_module_name: str - :param settings: Toolbox execution settings. - :type settings: dict - :param project_dir: Path to project directory. - :type project_dir: str - :param execution_permits: A mapping from item name to a boolean value, False indicating that - the item is not executed, only its resources are collected. - :type execution_permits: dict(str,bool) - :param node_successors: A mapping from item name to list of successor item names, dictating the dependencies. - :type node_successors: dict(str,list(str)) - :param debug: Whether debug mode is active or not. - :type debug: bool - - .. py:method:: _make_item_specifications(self, specifications, project_item_loader, items_module_name) - - Instantiates item specifications. - - :param specifications: A mapping from item type to list of specification dicts. - :type specifications: dict - :param project_item_loader: loader instance - :type project_item_loader: ProjectItemLoader - :param items_module_name: name of the Python module that contains the project items - :type items_module_name: str - - :returns: mapping from item type to a dict that maps specification names to specification instances - :rtype: dict - - - .. py:method:: _make_item(self, item_name, direction) - - Recreates item from project item dictionary. Note that all items are created twice. - One for the backward pipeline, the other one for the forward pipeline. - - - .. py:method:: get_event(self) - - Returns the next event in the stream. Calling this after receiving the event of type "dag_exec_finished" - will raise StopIterationError. - - - .. py:method:: state(self) - - - .. py:method:: _get_event_stream(self) - - Returns an iterator of tuples (event_type, event_data). - - TODO: Describe the events in depth. - - - .. py:method:: answer_prompt(self, item_name, accepted) - - - .. py:method:: run(self) - - Runs this engine. - - - .. py:method:: _process_event(self, event) - - Processes events from a pipeline. - - :param event: an event - :type event: DagsterEvent - - - .. py:method:: stop(self) - - Stops the engine. - - - .. py:method:: _stop_item(self, item) - - - .. py:method:: _make_pipeline(self) - - Returns a PipelineDefinition for executing this engine. - - :returns: PipelineDefinition - - - .. py:method:: _make_backward_solid_def(self, item_name) - - Returns a SolidDefinition for executing the given item in the backward sweep. - - :param item_name: The project item that gets executed by the solid. - :type item_name: str - - - .. py:method:: _make_forward_solid_def(self, item_name) - - Returns a SolidDefinition for executing the given item. - - :param item_name: - :type item_name: str - - :returns: SolidDefinition - - - .. py:method:: _execute_item(self, context, item_name, forward_resource_stacks, backward_resources) - - Executes the given item using the given forward resource stacks and backward resources. - Returns list of output resource stacks. - - Called by ``_make_forward_solid_def.compute_fn``. - - For each element yielded by ``_filtered_resources_iterator``, spawns a thread that runs ``_execute_item_filtered``. - - :param context: - :param item_name: - :type item_name: str - :param forward_resource_stacks: - :type forward_resource_stacks: list(tuple(ProjectItemResource)) - :param backward_resources: - :type backward_resources: list(ProjectItemResource) - - :returns: list(tuple(ProjectItemResource)) - - - .. py:method:: _execute_item_filtered(self, item, filtered_forward_resources, filtered_backward_resources, output_resources_list, success) - - Executes the given item using the given filtered resources. Target for threads in ``_execute_item``. - - :param item: - :type item: ExecutableItemBase - :param filtered_forward_resources: - :type filtered_forward_resources: list(ProjectItemResource) - :param filtered_backward_resources: - :type filtered_backward_resources: list(ProjectItemResource) - :param output_resources_list: A list of lists, to append the - output resources generated by the item. - :type output_resources_list: list(list(ProjectItemResource)) - :param success: A list of one element, to write the outcome of the execution. - :type success: list - - - .. py:method:: _filtered_resources_iterator(self, item_name, forward_resource_stacks, backward_resources, timestamp) - - Yields tuples of (filtered forward resources, filtered backward resources, filter id). - - Each tuple corresponds to a unique filter combination. Combinations are obtained by applying the cross-product - over forward resource stacks as yielded by ``_forward_resource_stacks_iterator``. - - :param item_name: - :type item_name: str - :param forward_resource_stacks: - :type forward_resource_stacks: list(tuple(ProjectItemResource)) - :param backward_resources: - :type backward_resources: list(ProjectItemResource) - :param timestamp: timestamp for the execution filter - :type timestamp: str - - :returns: forward resources, backward resources, filter id - :rtype: Iterator(tuple(list,list,str)) - - - .. py:method:: _expand_resource_stack(self, item_name, resource_stack) - - Expands a resource stack if possible. - - If the stack has more than one resource, returns the unaltered stack. - - Otherwise, if the stack has only one resource but there are no filters defined for that resource, - again, returns the unaltered stack. - - Otherwise, returns an expanded stack of as many resources as filter stacks defined for the only one resource. - Each resource in the expanded stack is a clone of the original, with one of the filter stacks - applied to the URL. - - :param item_name: resource receiving item's name - :type item_name: str - :param resource_stack: - :type resource_stack: tuple(ProjectItemResource) - - :returns: tuple(ProjectItemResource) - - - .. py:method:: _filter_stacks(self, item_name, resource_label) - - Computes filter stacks. - - Stacks are computed as the cross-product of all individual filters defined for a resource. - - :param item_name: item's name - :type item_name: str - :param resource_label: resource's label - :type resource_label: str - - :returns: filter stacks - :rtype: list of list - - - .. py:method:: _convert_forward_resources(self, item_name, resources) - - Converts resources as they're being forwarded to given item. - The conversion is dictated by the connection the resources traverse in order to reach the item. - - :param item_name: receiving item's name - :type item_name: str - :param resources: resources to convert - :type resources: list of ProjectItemResource - - :returns: converted resources - :rtype: list of ProjectItemResource - - - .. py:method:: _make_dependencies(self) - - Returns a dictionary of dependencies according to the given dictionaries of injectors. - - :returns: a dictionary to pass to the PipelineDefinition constructor as dependencies - :rtype: dict - - - -.. py:class:: SpineEngineState - - Bases: :py:obj:`enum.Enum` - - Generic enumeration. - - Derive from this class to define new enumerations. - - .. py:attribute:: SLEEPING - :annotation: = 1 - - - - .. py:attribute:: RUNNING - :annotation: = 2 - - - - .. py:attribute:: USER_STOPPED - :annotation: = 3 - - - - .. py:attribute:: FAILED - :annotation: = 4 - - - - .. py:attribute:: COMPLETED - :annotation: = 5 - - - - .. py:method:: __str__(self) - - Return str(self). - - - -.. py:class:: ItemExecutionFinishState - - Bases: :py:obj:`enum.Enum` - - Generic enumeration. - - Derive from this class to define new enumerations. - - .. py:attribute:: SUCCESS - :annotation: = 1 - - - - .. py:attribute:: FAILURE - :annotation: = 2 - - - - .. py:attribute:: SKIPPED - :annotation: = 3 - - - - .. py:attribute:: EXCLUDED - :annotation: = 4 - - - - .. py:attribute:: STOPPED - :annotation: = 5 - - - - .. py:attribute:: NEVER_FINISHED - :annotation: = 6 - - - - .. py:method:: __str__(self) - - Return str(self). - - - -.. py:data:: __version__ - :annotation: = 0.10.1 - - - diff --git a/docs/source/autoapi/spine_engine/load_project_items/index.rst b/docs/source/autoapi/spine_engine/load_project_items/index.rst deleted file mode 100644 index fa761fff..00000000 --- a/docs/source/autoapi/spine_engine/load_project_items/index.rst +++ /dev/null @@ -1,50 +0,0 @@ -:py:mod:`spine_engine.load_project_items` -========================================= - -.. py:module:: spine_engine.load_project_items - -.. autoapi-nested-parse:: - - Functions to load project item modules. - - :author: A. Soininen (VTT) - :date: 29.4.2020 - - - -Module Contents ---------------- - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.load_project_items.load_item_specification_factories - spine_engine.load_project_items.load_executable_item_classes - - - -.. py:function:: load_item_specification_factories(items_package_name) - - Loads the project item specification factories in given project item package. - - :param items_package_name: name of the package that contains the project items - :type items_package_name: str - - :returns: a map from item type to specification factory - :rtype: dict - - -.. py:function:: load_executable_item_classes(items_package_name) - - Loads the project item executable classes included in given project item package. - - :param items_package_name: name of the package that contains the project items - :type items_package_name: str - - :returns: a map from item type to the executable item class - :rtype: dict - - diff --git a/docs/source/autoapi/spine_engine/multithread_executor/executor/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/executor/index.rst deleted file mode 100644 index d3bf4471..00000000 --- a/docs/source/autoapi/spine_engine/multithread_executor/executor/index.rst +++ /dev/null @@ -1,52 +0,0 @@ -:py:mod:`spine_engine.multithread_executor.executor` -==================================================== - -.. py:module:: spine_engine.multithread_executor.executor - -.. autoapi-nested-parse:: - - Module contains multithread_executor. - - :author: M. Marin (KTH) - :date: 1.11.2020 - - - -Module Contents ---------------- - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.multithread_executor.executor.multithread_executor - - - -.. py:function:: multithread_executor(init_context) - - A custom multithread executor. - - This simple multithread executor borrows almost all the code from dagster's builtin multiprocess executor, - but takes a twist to use threading instead of multiprocessing. - To select the multithread executor, include a fragment - such as the following in your config: - - .. code-block:: yaml - - execution: - multithread: - config: - max_concurrent: 4 - - The ``max_concurrent`` arg is optional and tells the execution engine how many threads may run - concurrently. By default, or if you set ``max_concurrent`` to be 0, this is 100 - (in attendance of a better method). - - Execution priority can be configured using the ``dagster/priority`` tag via solid metadata, - where the higher the number the higher the priority. 0 is the default and both positive - and negative numbers can be used. - - diff --git a/docs/source/autoapi/spine_engine/multithread_executor/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/index.rst deleted file mode 100644 index 3a98581c..00000000 --- a/docs/source/autoapi/spine_engine/multithread_executor/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -:py:mod:`spine_engine.multithread_executor` -=========================================== - -.. py:module:: spine_engine.multithread_executor - - -Submodules ----------- -.. toctree:: - :titlesonly: - :maxdepth: 1 - - executor/index.rst - multithread/index.rst - thread_executor/index.rst - - diff --git a/docs/source/autoapi/spine_engine/multithread_executor/multithread/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/multithread/index.rst deleted file mode 100644 index 95c09324..00000000 --- a/docs/source/autoapi/spine_engine/multithread_executor/multithread/index.rst +++ /dev/null @@ -1,82 +0,0 @@ -:py:mod:`spine_engine.multithread_executor.multithread` -======================================================= - -.. py:module:: spine_engine.multithread_executor.multithread - -.. autoapi-nested-parse:: - - Module contains MultithreadExecutor. - - :author: M. Marin (KTH) - :date: 1.11.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.multithread_executor.multithread.MultithreadExecutor - - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.multithread_executor.multithread.DELEGATE_MARKER - - -.. py:data:: DELEGATE_MARKER - :annotation: = multithread_threat_init - - - -.. py:exception:: DagsterThreadError(*args, **kwargs) - - Bases: :py:obj:`dagster.core.errors.DagsterError` - - An exception has occurred in one or more of the threads dagster manages. - This error forwards the message and stack trace for all of the collected errors. - - Initialize self. See help(type(self)) for accurate signature. - - -.. py:class:: MultithreadExecutor(retries, max_concurrent=None) - - Bases: :py:obj:`dagster.core.executor.base.Executor` - - Helper class that provides a standard way to create an ABC using - inheritance. - - .. py:method:: retries(self) - :property: - - The Retries state / policy for this instance of the Executor. Executors should allow this to be - controlled via configuration if possible. - - Returns: Retries - - - .. py:method:: execute(self, pipeline_context, execution_plan) - - For the given context and execution plan, orchestrate a series of sub plan executions in a way that satisfies the whole plan being executed. - - :param pipeline_context: The pipeline execution context. - :type pipeline_context: SystemPipelineExecutionContext - :param execution_plan: The plan to execute. - :type execution_plan: ExecutionPlan - - :returns: A stream of dagster events. - - - .. py:method:: execute_step_in_thread(self, step_key, step_context, errors) - - - diff --git a/docs/source/autoapi/spine_engine/multithread_executor/thread_executor/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/thread_executor/index.rst deleted file mode 100644 index 90d94b0f..00000000 --- a/docs/source/autoapi/spine_engine/multithread_executor/thread_executor/index.rst +++ /dev/null @@ -1,133 +0,0 @@ -:py:mod:`spine_engine.multithread_executor.thread_executor` -=========================================================== - -.. py:module:: spine_engine.multithread_executor.thread_executor - -.. autoapi-nested-parse:: - - Facilities for running arbitrary commands in child processes. - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.multithread_executor.thread_executor.ThreadEvent - spine_engine.multithread_executor.thread_executor.ThreadStartEvent - spine_engine.multithread_executor.thread_executor.ThreadDoneEvent - spine_engine.multithread_executor.thread_executor.ThreadSystemErrorEvent - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.multithread_executor.thread_executor._execute_step_in_thread - spine_engine.multithread_executor.thread_executor._poll_for_event - spine_engine.multithread_executor.thread_executor.execute_thread_step - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.multithread_executor.thread_executor.TICK - spine_engine.multithread_executor.thread_executor.THREAD_DEAD_AND_QUEUE_EMPTY - - -.. py:class:: ThreadEvent - - -.. py:class:: ThreadStartEvent - - Bases: :py:obj:`namedtuple`\ (\ :py:obj:`'ThreadStartEvent'`\ , :py:obj:`'tid'`\ ), :py:obj:`ThreadEvent` - - docstring - - Initialize self. See help(type(self)) for accurate signature. - - -.. py:class:: ThreadDoneEvent - - Bases: :py:obj:`namedtuple`\ (\ :py:obj:`'ThreadDoneEvent'`\ , :py:obj:`'tid'`\ ), :py:obj:`ThreadEvent` - - docstring - - Initialize self. See help(type(self)) for accurate signature. - - -.. py:class:: ThreadSystemErrorEvent - - Bases: :py:obj:`namedtuple`\ (\ :py:obj:`'ThreadSystemErrorEvent'`\ , :py:obj:`'tid error_info'`\ ), :py:obj:`ThreadEvent` - - docstring - - Initialize self. See help(type(self)) for accurate signature. - - -.. py:exception:: ThreadCrashException - - Bases: :py:obj:`Exception` - - Thrown when the thread crashes. - - Initialize self. See help(type(self)) for accurate signature. - - -.. py:function:: _execute_step_in_thread(event_queue, step_context, retries) - - Wraps the execution of a step. - - Handles errors and communicates across a queue with the parent process. - - -.. py:data:: TICK - - - The minimum interval at which to check for child process liveness -- default 20ms. - -.. py:data:: THREAD_DEAD_AND_QUEUE_EMPTY - :annotation: = THREAD_DEAD_AND_QUEUE_EMPTY - - Sentinel value. - -.. py:function:: _poll_for_event(thread, event_queue) - - -.. py:function:: execute_thread_step(step_context, retries) - - Execute a step in a new thread. - - This function starts a new thread whose execution target is the given step context wrapped by - _execute_step_in_thread; polls the queue for events yielded by the thread - until it dies and the queue is empty. - - This function yields a complex set of objects to enable having multiple thread - executions in flight: - - * None - nothing has happened, yielded to enable cooperative multitasking other iterators - * ThreadEvent - Family of objects that communicates state changes in the thread - * KeyboardInterrupt - Yielded in the case that an interrupt was recieved while - polling the thread. Yielded instead of raised to allow forwarding of the - interrupt to the thread and completion of the iterator for this thread and - any others that may be executing - * The actual values yielded by the thread execution - - :param step_context: The step context to execute in the child process. - :type step_context: SystemStepExecutionContext - :param retries: - :type retries: Retries - - Warning: if the thread is in an infinite loop, this will - also infinitely loop. - - diff --git a/docs/source/autoapi/spine_engine/project_item/connection/index.rst b/docs/source/autoapi/spine_engine/project_item/connection/index.rst deleted file mode 100644 index 51198c8c..00000000 --- a/docs/source/autoapi/spine_engine/project_item/connection/index.rst +++ /dev/null @@ -1,165 +0,0 @@ -:py:mod:`spine_engine.project_item.connection` -============================================== - -.. py:module:: spine_engine.project_item.connection - -.. autoapi-nested-parse:: - - Provides the :class:`Connection` class. - - :authors: A. Soininen (VTT) - :date: 12.2.2021 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item.connection.Connection - - - - -.. py:class:: Connection(source_name, source_position, destination_name, destination_position, resource_filters=None, options=None) - - Represents a connection between two project items. - - :param source_name: source project item's name - :type source_name: str - :param source_position: source anchor's position - :type source_position: str - :param destination_name: destination project item's name - :type destination_name: str - :param destination_position: destination anchor's position - :type destination_position: str - :param resource_filters: mapping from resource labels and filter types to - database ids and activity flags - :type resource_filters: dict, optional - :param options: any options, at the moment only has "use_datapackage" - :type options: dict, optional - - .. py:method:: __eq__(self, other) - - Return self==value. - - - .. py:method:: name(self) - :property: - - - .. py:method:: destination_position(self) - :property: - - Anchor's position on destination item. - - - .. py:method:: source_position(self) - :property: - - Anchor's position on source item. - - - .. py:method:: database_resources(self) - :property: - - Connection's database resources - - - .. py:method:: has_filters(self) - - Return True if connection has filters. - - :returns: True if connection has filters, False otherwise - :rtype: bool - - - .. py:method:: resource_filters(self) - :property: - - Connection's resource filters. - - - .. py:method:: use_datapackage(self) - :property: - - - .. py:method:: id_to_name(self, id_, filter_type) - - Map from scenario/tool database id to name - - - .. py:method:: receive_resources_from_source(self, resources) - - Receives resources from source item. - - :param resources: source item's resources - :type resources: Iterable of ProjectItemResource - - - .. py:method:: replace_resource_from_source(self, old, new) - - Replaces an existing resource. - - :param old: old resource - :type old: ProjectItemResource - :param new: new resource - :type new: ProjectItemResource - - - .. py:method:: fetch_database_items(self) - - Reads filter information from database. - - - .. py:method:: set_online(self, resource, filter_type, online) - - Sets the given filters online or offline. - - :param resource: Resource label - :type resource: str - :param filter_type: Either SCENARIO_FILTER_TYPE or TOOL_FILTER_TYPE, for now. - :type filter_type: str - :param online: mapping from scenario/tool id to online flag - :type online: dict - - - .. py:method:: convert_resources(self, resources) - - Called when advertising resources through this connection *in the FORWARD direction*. - Takes the initial list of resources advertised by the source item and returns a new list, - which is the one finally advertised. - - At the moment it only packs CSVs into datapackage (and again, it's only used in the FORWARD direction). - - :param resources: Resources to convert - :type resources: list of ProjectItemResource - - :returns: list of ProjectItemResource - - - .. py:method:: to_dict(self) - - Returns a dictionary representation of this Connection. - - :returns: serialized Connection - :rtype: dict - - - .. py:method:: from_dict(connection_dict) - :staticmethod: - - Restores a connection from dictionary. - - :param connection_dict: connection dictionary - :type connection_dict: dict - - :returns: restored connection - :rtype: Connection - - - diff --git a/docs/source/autoapi/spine_engine/project_item/executable_item_base/index.rst b/docs/source/autoapi/spine_engine/project_item/executable_item_base/index.rst deleted file mode 100644 index a68d6839..00000000 --- a/docs/source/autoapi/spine_engine/project_item/executable_item_base/index.rst +++ /dev/null @@ -1,186 +0,0 @@ -:py:mod:`spine_engine.project_item.executable_item_base` -======================================================== - -.. py:module:: spine_engine.project_item.executable_item_base - -.. autoapi-nested-parse:: - - Contains ExecutableItem, a project item's counterpart in execution as well as support utilities. - - :authors: A. Soininen (VTT) - :date: 30.3.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item.executable_item_base.ExecutableItemBase - - - - -.. py:class:: ExecutableItemBase(name, project_dir, logger) - - The part of a project item that is executed by the Spine Engine. - - :param name: item's name - :type name: str - :param project_dir: absolute path to project directory - :type project_dir: str - :param logger: a logger - :type logger: LoggerInterface - - .. py:method:: name(self) - :property: - - Project item's name. - - - .. py:method:: group_id(self) - :property: - - Returns the id for group-execution. - Items in the same group share a kernel, and also reuse the same kernel from past executions. - By default each item is its own group, so it executes in isolation. - NOTE: At the moment this is only used by Tool, but could be used by other items in the future? - - :returns: item's id within an execution group - :rtype: str - - - .. py:method:: filter_id(self) - :property: - - - .. py:method:: ready_to_execute(self, settings) - - Validates the internal state of this project item before execution. - - Subclasses can implement this method to do the appropriate work. - - :param settings: Application settings - :type settings: AppSettings - - :returns: True if project item is ready for execution, False otherwise - :rtype: bool - - - .. py:method:: execute(self, forward_resources, backward_resources) - - Executes this item using the given resources and returns a boolean indicating the outcome. - - Subclasses can implement this method to do the appropriate work. - - :param forward_resources: a list of ProjectItemResources from predecessors (forward) - :type forward_resources: list - :param backward_resources: a list of ProjectItemResources from successors (backward) - :type backward_resources: list - - :returns: State depending on operation success - :rtype: ItemExecutionFinishState - - - .. py:method:: exclude_execution(self, forward_resources, backward_resources) - - Excludes execution of this item. - - This method is called when the item is not selected (i.e EXCLUDED) for execution. - Only lightweight bookkeeping or processing should be done in this case, e.g. - forward input resources. - - Subclasses can implement this method to the appropriate work. - - :param forward_resources: a list of ProjectItemResources from predecessors (forward) - :type forward_resources: list - :param backward_resources: a list of ProjectItemResources from successors (backward) - :type backward_resources: list - - - .. py:method:: finish_execution(self, state) - - Does any work needed after execution given the execution success status. - - :param state: Item execution finish state - :type state: ItemExecutionFinishState - - - .. py:method:: item_type() - :staticmethod: - :abstractmethod: - - Returns the item's type identifier string. - - - .. py:method:: output_resources(self, direction) - - Returns output resources in the given direction. - - Subclasses need to implement _output_resources_backward and/or _output_resources_forward - if they want to provide resources in any direction. - - :param direction: Direction where output resources are passed - :type direction: ExecutionDirection - - :returns: a list of ProjectItemResources - :rtype: list - - - .. py:method:: stop_execution(self) - - Stops executing this item. - - - .. py:method:: _output_resources_forward(self) - - Returns output resources for forward execution. - - The default implementation returns an empty list. - - :returns: a list of ProjectItemResources - :rtype: list - - - .. py:method:: _output_resources_backward(self) - - Returns output resources for backward execution. - - The default implementation returns an empty list. - - :returns: a list of ProjectItemResources - :rtype: list - - - .. py:method:: from_dict(cls, item_dict, name, project_dir, app_settings, specifications, logger) - :classmethod: - :abstractmethod: - - Deserializes an executable item from item dictionary. - - :param item_dict: serialized project item - :type item_dict: dict - :param name: item's name - :type name: str - :param project_dir: absolute path to the project directory - :type project_dir: str - :param app_settings: Toolbox settings - :type app_settings: QSettings - :param specifications: mapping from item type to specification name to :class:`ProjectItemSpecification` - :type specifications: dict - :param logger: a logger - :type logger: LoggingInterface - - :returns: deserialized executable item - :rtype: ExecutableItemBase - - - .. py:method:: _get_specification(name, item_type, specification_name, specifications, logger) - :staticmethod: - - - diff --git a/docs/source/autoapi/spine_engine/project_item/index.rst b/docs/source/autoapi/spine_engine/project_item/index.rst deleted file mode 100644 index 20ec7975..00000000 --- a/docs/source/autoapi/spine_engine/project_item/index.rst +++ /dev/null @@ -1,28 +0,0 @@ -:py:mod:`spine_engine.project_item` -=================================== - -.. py:module:: spine_engine.project_item - -.. autoapi-nested-parse:: - - This subpackage contains base classes for project items. - - :authors: M. Marin (KTH) - :date: 8.10.2020 - - - -Submodules ----------- -.. toctree:: - :titlesonly: - :maxdepth: 1 - - connection/index.rst - executable_item_base/index.rst - project_item_info/index.rst - project_item_resource/index.rst - project_item_specification/index.rst - project_item_specification_factory/index.rst - - diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_info/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_info/index.rst deleted file mode 100644 index 35e27fa3..00000000 --- a/docs/source/autoapi/spine_engine/project_item/project_item_info/index.rst +++ /dev/null @@ -1,50 +0,0 @@ -:py:mod:`spine_engine.project_item.project_item_info` -===================================================== - -.. py:module:: spine_engine.project_item.project_item_info - -.. autoapi-nested-parse:: - - Provides the ProjectItemInfo class. - - :authors: A. Soininen (VTT) - :date: 29.4.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item.project_item_info.ProjectItemInfo - - - - -.. py:class:: ProjectItemInfo - - .. py:method:: item_category() - :staticmethod: - :abstractmethod: - - Returns the item category string, e.g., "Tools". - - :returns: item's category - :rtype: str - - - .. py:method:: item_type() - :staticmethod: - :abstractmethod: - - Returns the item type string, e.g., "Gimlet". - - :returns: item's type - :rtype: str - - - diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_resource/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_resource/index.rst deleted file mode 100644 index 4338f57f..00000000 --- a/docs/source/autoapi/spine_engine/project_item/project_item_resource/index.rst +++ /dev/null @@ -1,198 +0,0 @@ -:py:mod:`spine_engine.project_item.project_item_resource` -========================================================= - -.. py:module:: spine_engine.project_item.project_item_resource - -.. autoapi-nested-parse:: - - Provides the ProjectItemResource class. - - :authors: M. Marin (KTH) - :date: 29.4.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item.project_item_resource.ProjectItemResource - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item.project_item_resource.database_resource - spine_engine.project_item.project_item_resource.file_resource - spine_engine.project_item.project_item_resource.transient_file_resource - spine_engine.project_item.project_item_resource.file_resource_in_pack - spine_engine.project_item.project_item_resource.extract_packs - - - -.. py:class:: ProjectItemResource(provider_name, type_, label, url=None, metadata=None) - - Class to hold a resource made available by a project item and that may be consumed by another project item. - - .. attribute:: provider_name - - name of resource provider - - :type: str - - .. attribute:: type_ - - resource's type - - :type: str - - .. attribute:: label - - an identifier string - - :type: str - - .. attribute:: metadata - - resource's metadata - - :type: dict - - :param provider_name: The name of the item that provides the resource - :type provider_name: str - :param type_: The resource type, currently available types: - - - "file": url points to the file's path - - "file_pack": resource is part of a pack; url points to the file's path - - "database": url is the databases url - :type type_: str - :param label: A label that identifies the resource. - :type label: str - :param url: The url of the resource. - :type url: str, optional - :param metadata: Additional metadata providing extra information about the resource. - Currently available keys: - - - filter_stack (str): resource's filter stack - - filter_id (str): filter id - :type metadata: dict - - .. py:method:: clone(self, additional_metadata=None) - - Clones a resource and optionally updates the clone's metadata. - - :param additional_metadata: metadata to add to the clone - :type additional_metadata: dict - - :returns: cloned resource - :rtype: ProjectItemResource - - - .. py:method:: __eq__(self, other) - - Return self==value. - - - .. py:method:: __hash__(self) - - Return hash(self). - - - .. py:method:: __repr__(self) - - Return repr(self). - - - .. py:method:: url(self) - :property: - - Resource URL. - - - .. py:method:: path(self) - :property: - - Returns the resource path in the local syntax, as obtained from parsing the url. - - - .. py:method:: scheme(self) - :property: - - Returns the resource scheme, as obtained from parsing the url. - - - .. py:method:: hasfilepath(self) - :property: - - - .. py:method:: arg(self) - :property: - - - -.. py:function:: database_resource(provider_name, url, label=None) - - Constructs a database resource. - - :param provider_name: resource provider's name - :type provider_name: str - :param url: database URL - :type url: str - :param label: resource label - :type label: str, optional - - -.. py:function:: file_resource(provider_name, file_path, label=None) - - Constructs a file resource. - - :param provider_name: resource provider's name - :type provider_name: str - :param file_path: path to file - :type file_path: str - :param label: resource label - :type label: str, optional - - -.. py:function:: transient_file_resource(provider_name, label, file_path=None) - - Constructs a transient file resource. - - :param provider_name: resource provider's name - :type provider_name: str - :param label: resource label - :type label: str - :param file_path: file path if the file exists - :type file_path: str, optional - - -.. py:function:: file_resource_in_pack(provider_name, label, file_path=None) - - Constructs a file resource that is part of a resource pack. - - :param provider_name: resource provider's name - :type provider_name: str - :param label: resource label - :type label: str - :param file_path: file path if the file exists - :type file_path: str, optional - - -.. py:function:: extract_packs(resources) - - Extracts file packs from resources. - - :param resources: resources to process - :type resources: Iterable of ProjectItemResource - - :returns: list of non-pack resources and dictionary of packs keyed by label - :rtype: tuple - - diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_specification/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_specification/index.rst deleted file mode 100644 index a7d1626a..00000000 --- a/docs/source/autoapi/spine_engine/project_item/project_item_specification/index.rst +++ /dev/null @@ -1,101 +0,0 @@ -:py:mod:`spine_engine.project_item.project_item_specification` -============================================================== - -.. py:module:: spine_engine.project_item.project_item_specification - -.. autoapi-nested-parse:: - - Contains project item specification class. - - :authors: M. Marin (KTH) - :date: 7.5.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item.project_item_specification.ProjectItemSpecification - - - - -.. py:class:: ProjectItemSpecification(name, description=None, item_type='', item_category='') - - Class to hold a project item specification. - - .. attribute:: item_type - - type of the project item the specification is compatible with - - :type: str - - .. attribute:: definition_file_path - - specification's JSON file path - - :type: str - - :param name: specification name - :type name: str - :param description: description - :type description: str - :param item_type: Project item type - :type item_type: str - :param item_category: Project item category - :type item_category: str - - .. py:method:: set_name(self, name) - - Set object name and short name. - Note: Check conflicts (e.g. name already exists) - before calling this method. - - :param name: New (long) name for this object - :type name: str - - - .. py:method:: set_description(self, description) - - Set object description. - - :param description: Object description - :type description: str - - - .. py:method:: save(self) - :abstractmethod: - - Writes the specification to the path given by ``self.definition_file_path`` - - :returns: True if the operation was successful, False otherwise - :rtype: bool - - - .. py:method:: to_dict(self) - :abstractmethod: - - Returns a dict for the specification. - - :returns: specification dict - :rtype: dict - - - .. py:method:: is_equivalent(self, other) - :abstractmethod: - - Returns True if two specifications are essentially the same. - - :param other: specification to compare to - :type other: DataTransformerSpecification - - :returns: True if the specifications are equivalent, False otherwise - :rtype: bool - - - diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_specification_factory/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_specification_factory/index.rst deleted file mode 100644 index d423ff65..00000000 --- a/docs/source/autoapi/spine_engine/project_item/project_item_specification_factory/index.rst +++ /dev/null @@ -1,56 +0,0 @@ -:py:mod:`spine_engine.project_item.project_item_specification_factory` -====================================================================== - -.. py:module:: spine_engine.project_item.project_item_specification_factory - -.. autoapi-nested-parse:: - - Contains project item specification factory. - - :authors: A. Soininen (VTT) - :date: 6.5.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item.project_item_specification_factory.ProjectItemSpecificationFactory - - - - -.. py:class:: ProjectItemSpecificationFactory - - A factory to make project item specifications. - - .. py:method:: item_type() - :staticmethod: - :abstractmethod: - - Returns the project item's type. - - - .. py:method:: make_specification(definition, app_settings, logger) - :staticmethod: - :abstractmethod: - - Makes a project item specification. - - :param definition: specification's definition dictionary - :type definition: dict - :param app_settings: Toolbox settings - :type app_settings: QSettings - :param logger: a logger - :type logger: LoggerInterface - - :returns: a specification built from the given definition - :rtype: ProjectItemSpecification - - - diff --git a/docs/source/autoapi/spine_engine/project_item_loader/index.rst b/docs/source/autoapi/spine_engine/project_item_loader/index.rst deleted file mode 100644 index e30b78ed..00000000 --- a/docs/source/autoapi/spine_engine/project_item_loader/index.rst +++ /dev/null @@ -1,74 +0,0 @@ -:py:mod:`spine_engine.project_item_loader` -========================================== - -.. py:module:: spine_engine.project_item_loader - -.. autoapi-nested-parse:: - - Contains :class:`ProjectItemLoader`. - - :author: A. Soininen (VTT) - :date: 11.2.2021 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.project_item_loader.ProjectItemLoader - - - - -.. py:class:: ProjectItemLoader - - A singleton class for loading project items from multiple processes simultaneously. - - .. py:attribute:: _specification_factories - - - - - .. py:attribute:: _executable_item_classes - - - - - .. py:attribute:: _specification_factories_lock - - - - - .. py:attribute:: _executable_item_classes_lock - - - - - .. py:method:: load_item_specification_factories(self, items_module_name) - - Loads the project item specification factories in the standard Toolbox package. - - :param items_module_name: name of the Python module that contains the project items - :type items_module_name: str - - :returns: a map from item type to specification factory - :rtype: dict - - - .. py:method:: load_executable_item_classes(self, items_module_name) - - Loads the project item executable classes included in the standard Toolbox package. - - :param items_module_name: name of the Python module that contains the project items - :type items_module_name: str - - :returns: a map from item type to the executable item class - :rtype: dict - - - diff --git a/docs/source/autoapi/spine_engine/spine_engine/index.rst b/docs/source/autoapi/spine_engine/spine_engine/index.rst deleted file mode 100644 index 9bad5745..00000000 --- a/docs/source/autoapi/spine_engine/spine_engine/index.rst +++ /dev/null @@ -1,395 +0,0 @@ -:py:mod:`spine_engine.spine_engine` -=================================== - -.. py:module:: spine_engine.spine_engine - -.. autoapi-nested-parse:: - - Contains the SpineEngine class for running Spine Toolbox DAGs. - - :authors: M. Marin (KTH) - :date: 20.11.2019 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.spine_engine.ExecutionDirection - spine_engine.spine_engine.SpineEngineState - spine_engine.spine_engine.ItemExecutionFinishState - spine_engine.spine_engine.SpineEngine - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.spine_engine._shorten - spine_engine.spine_engine._make_filter_id - spine_engine.spine_engine._filter_names_from_stack - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.spine_engine.ED - - -.. py:class:: ExecutionDirection - - Bases: :py:obj:`enum.Enum` - - Generic enumeration. - - Derive from this class to define new enumerations. - - .. py:attribute:: FORWARD - - - - - .. py:attribute:: BACKWARD - - - - - .. py:method:: __str__(self) - - Return str(self). - - - -.. py:data:: ED - - - - -.. py:class:: SpineEngineState - - Bases: :py:obj:`enum.Enum` - - Generic enumeration. - - Derive from this class to define new enumerations. - - .. py:attribute:: SLEEPING - :annotation: = 1 - - - - .. py:attribute:: RUNNING - :annotation: = 2 - - - - .. py:attribute:: USER_STOPPED - :annotation: = 3 - - - - .. py:attribute:: FAILED - :annotation: = 4 - - - - .. py:attribute:: COMPLETED - :annotation: = 5 - - - - .. py:method:: __str__(self) - - Return str(self). - - - -.. py:class:: ItemExecutionFinishState - - Bases: :py:obj:`enum.Enum` - - Generic enumeration. - - Derive from this class to define new enumerations. - - .. py:attribute:: SUCCESS - :annotation: = 1 - - - - .. py:attribute:: FAILURE - :annotation: = 2 - - - - .. py:attribute:: SKIPPED - :annotation: = 3 - - - - .. py:attribute:: EXCLUDED - :annotation: = 4 - - - - .. py:attribute:: STOPPED - :annotation: = 5 - - - - .. py:attribute:: NEVER_FINISHED - :annotation: = 6 - - - - .. py:method:: __str__(self) - - Return str(self). - - - -.. py:class:: SpineEngine(items=None, specifications=None, connections=None, items_module_name='spine_items', settings=None, project_dir=None, execution_permits=None, node_successors=None, debug=False) - - An engine for executing a Spine Toolbox DAG-workflow. - - :param items: List of executable item dicts. - :type items: list(dict) - :param specifications: A mapping from item type to list of specification dicts. - :type specifications: dict(str,list(dict)) - :param connections: List of connection dicts - :type connections: list of dict - :param items_module_name: name of the Python module that contains project items - :type items_module_name: str - :param settings: Toolbox execution settings. - :type settings: dict - :param project_dir: Path to project directory. - :type project_dir: str - :param execution_permits: A mapping from item name to a boolean value, False indicating that - the item is not executed, only its resources are collected. - :type execution_permits: dict(str,bool) - :param node_successors: A mapping from item name to list of successor item names, dictating the dependencies. - :type node_successors: dict(str,list(str)) - :param debug: Whether debug mode is active or not. - :type debug: bool - - .. py:method:: _make_item_specifications(self, specifications, project_item_loader, items_module_name) - - Instantiates item specifications. - - :param specifications: A mapping from item type to list of specification dicts. - :type specifications: dict - :param project_item_loader: loader instance - :type project_item_loader: ProjectItemLoader - :param items_module_name: name of the Python module that contains the project items - :type items_module_name: str - - :returns: mapping from item type to a dict that maps specification names to specification instances - :rtype: dict - - - .. py:method:: _make_item(self, item_name, direction) - - Recreates item from project item dictionary. Note that all items are created twice. - One for the backward pipeline, the other one for the forward pipeline. - - - .. py:method:: get_event(self) - - Returns the next event in the stream. Calling this after receiving the event of type "dag_exec_finished" - will raise StopIterationError. - - - .. py:method:: state(self) - - - .. py:method:: _get_event_stream(self) - - Returns an iterator of tuples (event_type, event_data). - - TODO: Describe the events in depth. - - - .. py:method:: answer_prompt(self, item_name, accepted) - - - .. py:method:: run(self) - - Runs this engine. - - - .. py:method:: _process_event(self, event) - - Processes events from a pipeline. - - :param event: an event - :type event: DagsterEvent - - - .. py:method:: stop(self) - - Stops the engine. - - - .. py:method:: _stop_item(self, item) - - - .. py:method:: _make_pipeline(self) - - Returns a PipelineDefinition for executing this engine. - - :returns: PipelineDefinition - - - .. py:method:: _make_backward_solid_def(self, item_name) - - Returns a SolidDefinition for executing the given item in the backward sweep. - - :param item_name: The project item that gets executed by the solid. - :type item_name: str - - - .. py:method:: _make_forward_solid_def(self, item_name) - - Returns a SolidDefinition for executing the given item. - - :param item_name: - :type item_name: str - - :returns: SolidDefinition - - - .. py:method:: _execute_item(self, context, item_name, forward_resource_stacks, backward_resources) - - Executes the given item using the given forward resource stacks and backward resources. - Returns list of output resource stacks. - - Called by ``_make_forward_solid_def.compute_fn``. - - For each element yielded by ``_filtered_resources_iterator``, spawns a thread that runs ``_execute_item_filtered``. - - :param context: - :param item_name: - :type item_name: str - :param forward_resource_stacks: - :type forward_resource_stacks: list(tuple(ProjectItemResource)) - :param backward_resources: - :type backward_resources: list(ProjectItemResource) - - :returns: list(tuple(ProjectItemResource)) - - - .. py:method:: _execute_item_filtered(self, item, filtered_forward_resources, filtered_backward_resources, output_resources_list, success) - - Executes the given item using the given filtered resources. Target for threads in ``_execute_item``. - - :param item: - :type item: ExecutableItemBase - :param filtered_forward_resources: - :type filtered_forward_resources: list(ProjectItemResource) - :param filtered_backward_resources: - :type filtered_backward_resources: list(ProjectItemResource) - :param output_resources_list: A list of lists, to append the - output resources generated by the item. - :type output_resources_list: list(list(ProjectItemResource)) - :param success: A list of one element, to write the outcome of the execution. - :type success: list - - - .. py:method:: _filtered_resources_iterator(self, item_name, forward_resource_stacks, backward_resources, timestamp) - - Yields tuples of (filtered forward resources, filtered backward resources, filter id). - - Each tuple corresponds to a unique filter combination. Combinations are obtained by applying the cross-product - over forward resource stacks as yielded by ``_forward_resource_stacks_iterator``. - - :param item_name: - :type item_name: str - :param forward_resource_stacks: - :type forward_resource_stacks: list(tuple(ProjectItemResource)) - :param backward_resources: - :type backward_resources: list(ProjectItemResource) - :param timestamp: timestamp for the execution filter - :type timestamp: str - - :returns: forward resources, backward resources, filter id - :rtype: Iterator(tuple(list,list,str)) - - - .. py:method:: _expand_resource_stack(self, item_name, resource_stack) - - Expands a resource stack if possible. - - If the stack has more than one resource, returns the unaltered stack. - - Otherwise, if the stack has only one resource but there are no filters defined for that resource, - again, returns the unaltered stack. - - Otherwise, returns an expanded stack of as many resources as filter stacks defined for the only one resource. - Each resource in the expanded stack is a clone of the original, with one of the filter stacks - applied to the URL. - - :param item_name: resource receiving item's name - :type item_name: str - :param resource_stack: - :type resource_stack: tuple(ProjectItemResource) - - :returns: tuple(ProjectItemResource) - - - .. py:method:: _filter_stacks(self, item_name, resource_label) - - Computes filter stacks. - - Stacks are computed as the cross-product of all individual filters defined for a resource. - - :param item_name: item's name - :type item_name: str - :param resource_label: resource's label - :type resource_label: str - - :returns: filter stacks - :rtype: list of list - - - .. py:method:: _convert_forward_resources(self, item_name, resources) - - Converts resources as they're being forwarded to given item. - The conversion is dictated by the connection the resources traverse in order to reach the item. - - :param item_name: receiving item's name - :type item_name: str - :param resources: resources to convert - :type resources: list of ProjectItemResource - - :returns: converted resources - :rtype: list of ProjectItemResource - - - .. py:method:: _make_dependencies(self) - - Returns a dictionary of dependencies according to the given dictionaries of injectors. - - :returns: a dictionary to pass to the PipelineDefinition constructor as dependencies - :rtype: dict - - - -.. py:function:: _shorten(resource) - - -.. py:function:: _make_filter_id(resource_filter_stack) - - -.. py:function:: _filter_names_from_stack(stack) - - diff --git a/docs/source/autoapi/spine_engine/spine_engine_server/index.rst b/docs/source/autoapi/spine_engine/spine_engine_server/index.rst deleted file mode 100644 index b2a04d99..00000000 --- a/docs/source/autoapi/spine_engine/spine_engine_server/index.rst +++ /dev/null @@ -1,143 +0,0 @@ -:py:mod:`spine_engine.spine_engine_server` -========================================== - -.. py:module:: spine_engine.spine_engine_server - -.. autoapi-nested-parse:: - - Contains the SpineEngineServer class. - - :authors: M. Marin (KTH) - :date: 7.11.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.spine_engine_server.EngineRequestHandler - spine_engine.spine_engine_server.SpineEngineServer - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.spine_engine_server.start_spine_engine_server - spine_engine.spine_engine_server._shutdown_servers - - - -Attributes -~~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.spine_engine_server._servers - - -.. py:class:: EngineRequestHandler(request, client_address, server) - - Bases: :py:obj:`socketserver.BaseRequestHandler` - - The request handler class for our server. - - .. py:attribute:: _engines - - - - - .. py:attribute:: _ENCODING - :annotation: = ascii - - - - .. py:method:: _run_engine(self, data) - - Creates and engine and runs it. - - :param data: data to be passed as keyword arguments to SpineEngine() - :type data: dict - - :returns: engine id, for further calls - :rtype: str - - - .. py:method:: _get_engine_event(self, engine_id) - - Gets the next event in the engine's execution stream. - - :param engine_id: the engine id, must have been returned by run. - :type engine_id: str - - :returns: two element tuple: event type identifier string, and event data dictionary - :rtype: tuple(str,dict) - - - .. py:method:: _stop_engine(self, engine_id) - - Stops the engine. - - :param engine_id: the engine id, must have been returned by run. - :type engine_id: str - - - .. py:method:: _restart_kernel(self, connection_file) - - Restarts the jupyter kernel associated to given connection file. - - :param connection_file: path of connection file - :type connection_file: str - - - .. py:method:: _shutdown_kernel(self, connection_file) - - Shuts down the jupyter kernel associated to given connection file. - - :param connection_file: path of connection file - :type connection_file: str - - - .. py:method:: handle(self) - - - .. py:method:: _recvall(self) - - Receives and returns all data in the request. - - :returns: str - - - -.. py:class:: SpineEngineServer(server_address, RequestHandlerClass, bind_and_activate=True) - - Bases: :py:obj:`socketserver.ThreadingMixIn`, :py:obj:`socketserver.TCPServer` - - Mix-in class to handle each request in a new thread. - - Constructor. May be extended, do not override. - - .. py:attribute:: allow_reuse_address - :annotation: = True - - - - -.. py:data:: _servers - :annotation: = [] - - - -.. py:function:: start_spine_engine_server(host, port) - - -.. py:function:: _shutdown_servers() - - diff --git a/docs/source/autoapi/spine_engine/utils/command_line_arguments/index.rst b/docs/source/autoapi/spine_engine/utils/command_line_arguments/index.rst deleted file mode 100644 index 091c0a81..00000000 --- a/docs/source/autoapi/spine_engine/utils/command_line_arguments/index.rst +++ /dev/null @@ -1,42 +0,0 @@ -:py:mod:`spine_engine.utils.command_line_arguments` -=================================================== - -.. py:module:: spine_engine.utils.command_line_arguments - -.. autoapi-nested-parse:: - - Split command line arguments. - - :authors: P. Savolainen (VTT) - :date: 10.1.2018 - - - -Module Contents ---------------- - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.utils.command_line_arguments.split_cmdline_args - - - -.. py:function:: split_cmdline_args(arg_string) - - Splits a string of command line arguments into a list of tokens. - - Things in single ('') and double ("") quotes are kept as single tokens - while the quotes themselves are stripped away. - Thus, `--file="a long quoted 'file' name.txt` becomes ["--file=a long quoted 'file' name.txt"] - - :param arg_string: command line arguments as a string - :type arg_string: str - - :returns: a list of tokens - :rtype: list - - diff --git a/docs/source/autoapi/spine_engine/utils/helpers/index.rst b/docs/source/autoapi/spine_engine/utils/helpers/index.rst deleted file mode 100644 index a4b807e9..00000000 --- a/docs/source/autoapi/spine_engine/utils/helpers/index.rst +++ /dev/null @@ -1,167 +0,0 @@ -:py:mod:`spine_engine.utils.helpers` -==================================== - -.. py:module:: spine_engine.utils.helpers - -.. autoapi-nested-parse:: - - Helpers functions and classes. - - :authors: M. Marin (KTH) - :date: 20.11.2019 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.utils.helpers.Singleton - spine_engine.utils.helpers.AppSettings - - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.utils.helpers.shorten - spine_engine.utils.helpers.create_log_file_timestamp - spine_engine.utils.helpers.create_timestamp - spine_engine.utils.helpers.resolve_conda_executable - spine_engine.utils.helpers.resolve_python_interpreter - spine_engine.utils.helpers.resolve_julia_executable - spine_engine.utils.helpers.resolve_gams_executable - spine_engine.utils.helpers.resolve_executable_from_path - spine_engine.utils.helpers.inverted - spine_engine.utils.helpers.get_julia_command - spine_engine.utils.helpers.get_julia_env - - - -.. py:class:: Singleton - - Bases: :py:obj:`type` - - .. py:attribute:: _instances - - - - - .. py:method:: __call__(cls, *args, **kwargs) - - Call self as a function. - - - -.. py:class:: AppSettings(settings) - - A QSettings replacement. - - Init. - - :param settings: - :type settings: dict - - .. py:method:: value(self, key, defaultValue='') - - - -.. py:function:: shorten(name) - - Returns the 'short name' version of given name. - - -.. py:function:: create_log_file_timestamp() - - Creates a new timestamp string that is used as Data Store and Importer error log file. - - :returns: Timestamp string or empty string if failed. - - -.. py:function:: create_timestamp() - - -.. py:function:: resolve_conda_executable(conda_path) - - If given conda_path is an empty str, returns current Conda - executable from CONDA_EXE env variable if the app was started - on Conda, otherwise returns an empty string. - - -.. py:function:: resolve_python_interpreter(python_path) - - If given python_path is empty, returns the - full path to Python interpreter depending on user's - settings and whether the app is frozen or not. - - -.. py:function:: resolve_julia_executable(julia_path) - - if given julia_path is empty, tries to find the path to Julia - in user's PATH env variable. If Julia is not found in PATH, - returns an empty string. - - Note: In the long run, we should decide whether this is something we want to do - because adding julia-x.x./bin/ dir to the PATH is not recommended because this - also exposes some .dlls to other programs on user's (windows) system. I.e. it - may break other programs, and this is why the Julia installer does not - add (and does not even offer the chance to add) Julia to PATH. - - -.. py:function:: resolve_gams_executable(gams_path) - - if given gams_path is empty, tries to find the path to Gams - in user's PATH env variable. If Gams is not found in PATH, - returns an empty string. - - -.. py:function:: resolve_executable_from_path(executable_name) - - Returns full path to executable name in user's - PATH env variable. If not found, returns an empty string. - - Basically equivalent to 'where' and 'which' commands in - cmd.exe and bash respectively. - - :param executable_name: Executable filename to find (e.g. python.exe, julia.exe) - :type executable_name: str - - :returns: Full path or empty string - :rtype: str - - -.. py:function:: inverted(input_) - - Inverts a dictionary of list values. - - :param input_: - :type input_: dict - - :returns: keys are list items, and values are keys listing that item from the input dictionary - :rtype: dict - - -.. py:function:: get_julia_command(settings) - - :param settings: - :type settings: QSettings, AppSettings - - :returns: e.g. ["path/to/julia", "--project=path/to/project/"] - :rtype: list - - -.. py:function:: get_julia_env(settings) - - :param settings: - :type settings: QSettings, AppSettings - - :returns: (julia_exe, julia_project), or None if none found - :rtype: tuple, NoneType - - diff --git a/docs/source/autoapi/spine_engine/utils/index.rst b/docs/source/autoapi/spine_engine/utils/index.rst deleted file mode 100644 index 1128cce4..00000000 --- a/docs/source/autoapi/spine_engine/utils/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -:py:mod:`spine_engine.utils` -============================ - -.. py:module:: spine_engine.utils - - -Submodules ----------- -.. toctree:: - :titlesonly: - :maxdepth: 1 - - command_line_arguments/index.rst - helpers/index.rst - queue_logger/index.rst - returning_process/index.rst - serialization/index.rst - - diff --git a/docs/source/autoapi/spine_engine/utils/queue_logger/index.rst b/docs/source/autoapi/spine_engine/utils/queue_logger/index.rst deleted file mode 100644 index 2278fa0c..00000000 --- a/docs/source/autoapi/spine_engine/utils/queue_logger/index.rst +++ /dev/null @@ -1,69 +0,0 @@ -:py:mod:`spine_engine.utils.queue_logger` -========================================= - -.. py:module:: spine_engine.utils.queue_logger - -.. autoapi-nested-parse:: - - The QueueLogger class. - - :authors: M. Marin (KTH) - :date: 3.11.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.utils.queue_logger._Message - spine_engine.utils.queue_logger._Prompt - spine_engine.utils.queue_logger._ExecutionMessage - spine_engine.utils.queue_logger.QueueLogger - - - - -.. py:class:: _Message(queue, event_type, msg_type, item_name) - - .. py:method:: filter_id(self) - :property: - - - .. py:method:: emit(self, msg_text) - - - -.. py:class:: _Prompt(queue, item_name, prompt_queue) - - .. py:method:: filter_id(self) - :property: - - - .. py:method:: emit(self, prompt) - - - -.. py:class:: _ExecutionMessage(queue, event_type, item_name) - - .. py:method:: filter_id(self) - :property: - - - .. py:method:: emit(self, msg) - - - -.. py:class:: QueueLogger(queue, item_name, prompt_queue) - - A :class:`LoggerInterface` compliant logger that puts messages into a Queue. - - - .. py:method:: set_filter_id(self, filter_id) - - - diff --git a/docs/source/autoapi/spine_engine/utils/returning_process/index.rst b/docs/source/autoapi/spine_engine/utils/returning_process/index.rst deleted file mode 100644 index e27006a8..00000000 --- a/docs/source/autoapi/spine_engine/utils/returning_process/index.rst +++ /dev/null @@ -1,54 +0,0 @@ -:py:mod:`spine_engine.utils.returning_process` -============================================== - -.. py:module:: spine_engine.utils.returning_process - -.. autoapi-nested-parse:: - - The ReturningProcess class. - - :authors: M. Marin (KTH) - :date: 3.11.2020 - - - -Module Contents ---------------- - -Classes -~~~~~~~ - -.. autoapisummary:: - - spine_engine.utils.returning_process.ReturningProcess - - - - -.. py:class:: ReturningProcess(*args, **kwargs) - - Bases: :py:obj:`multiprocessing.Process` - - Process objects represent activity that is run in a separate process - - The class is analogous to `threading.Thread` - - .. py:method:: run_until_complete(self) - - Starts the process and joins it after it has finished. - - :returns: Return value of the process where the first element is a status flag - :rtype: tuple - - - .. py:method:: run(self) - - Method to be run in sub-process; can be overridden in sub-class - - - .. py:method:: terminate(self) - - Terminate process; sends SIGTERM signal or uses TerminateProcess() - - - diff --git a/docs/source/autoapi/spine_engine/utils/serialization/index.rst b/docs/source/autoapi/spine_engine/utils/serialization/index.rst deleted file mode 100644 index c6081fb9..00000000 --- a/docs/source/autoapi/spine_engine/utils/serialization/index.rst +++ /dev/null @@ -1,82 +0,0 @@ -:py:mod:`spine_engine.utils.serialization` -========================================== - -.. py:module:: spine_engine.utils.serialization - -.. autoapi-nested-parse:: - - Functions to (de)serialize stuff. - - :authors: P. Savolainen (VTT) - :date: 10.1.2018 - - - -Module Contents ---------------- - - -Functions -~~~~~~~~~ - -.. autoapisummary:: - - spine_engine.utils.serialization.path_in_dir - spine_engine.utils.serialization.serialize_path - spine_engine.utils.serialization.serialize_url - spine_engine.utils.serialization.deserialize_path - spine_engine.utils.serialization.deserialize_remote_path - - - -.. py:function:: path_in_dir(path, directory) - - Returns True if the given path is in the given directory. - - -.. py:function:: serialize_path(path, project_dir) - - Returns a dict representation of the given path. - - If path is in project_dir, converts the path to relative. - - :param path: path to serialize - :type path: str - :param project_dir: path to the project directory - :type project_dir: str - - :returns: Dictionary representing the given path - :rtype: dict - - -.. py:function:: serialize_url(url, project_dir) - - Return a dict representation of the given URL. - - If the URL is a file that is in project dir, the URL is converted to a relative path. - - :param url: a URL to serialize - :type url: str - :param project_dir: path to the project directory - :type project_dir: str - - :returns: Dictionary representing the URL - :rtype: dict - - -.. py:function:: deserialize_path(serialized, project_dir) - - Returns a deserialized path or URL. - - :param serialized: a serialized path or URL - :type serialized: dict - :param project_dir: path to the project directory - :type project_dir: str - - :returns: Path or URL as string - :rtype: str - - -.. py:function:: deserialize_remote_path(serialized, base_path) - - diff --git a/docs/source/autoapi/spine_engine/version/index.rst b/docs/source/autoapi/spine_engine/version/index.rst deleted file mode 100644 index 53d7485e..00000000 --- a/docs/source/autoapi/spine_engine/version/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -:py:mod:`spine_engine.version` -============================== - -.. py:module:: spine_engine.version - - -Module Contents ---------------- - -.. py:data:: __version__ - :annotation: = 0.10.1 - - - diff --git a/docs/source/conf.py b/docs/source/conf.py index 8211b5ba..524c373c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -67,7 +67,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/pyproject.toml b/pyproject.toml index 89c66b8f..a60bceed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,31 +7,33 @@ description = "A package to run Spine workflows." keywords = ["energy system modelling", "workflow", "optimisation", "database"] readme = {file = "README.md", content-type = "text/markdown"} classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", - "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", + "Operating System :: OS Independent", ] -requires-python = ">=3.8.1, <3.12" +requires-python = ">=3.8.1" dependencies = [ - # dagster versions lower that 1.5.7 do not support pendulum >= 3.0.0 - "pendulum < 3.0.0", - # https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates - "protobuf<3.21.0", "networkx>2.5.1", "datapackage>=1.15.2, <1.16", "jupyter_client>=6.0", "spinedb_api>=0.31.0", "pyzmq >=21.0", - # dagster 0.12.8 requires Jinja2<3.0, which tries to import - # soft_unicode, which has been removed in markupsafe 2.1 - "markupsafe < 2.1", ] [project.urls] Repository = "https://github.com/spine-tools/spine-engine" [project.optional-dependencies] -dev = ["coverage[toml]"] +dev = [ + "ipykernel", + "coverage[toml]", + "sphinx", + "recommonmark", + "sphinx_rtd_theme", + "sphinx-autoapi", + "pylint", + "black == 21.12b0" +] [build-system] requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2", "wheel", "build"] @@ -49,10 +51,10 @@ spine_engine = ["execution_managers/spine_repl.jl"] [tool.setuptools.packages.find] exclude = [ - "bin*", - "docs*", - "fig*", - "tests*", + "bin*", + "docs*", + "fig*", + "tests*", ] [tool.coverage.run] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8fc365f0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +-e git+https://github.com/spine-tools/Spine-Database-API.git@0.8-dev#egg=spinedb_api +-e . diff --git a/tests/execution_managers/test_kernel_execution_manager.py b/tests/execution_managers/test_kernel_execution_manager.py index 0d79cb44..8226d850 100644 --- a/tests/execution_managers/test_kernel_execution_manager.py +++ b/tests/execution_managers/test_kernel_execution_manager.py @@ -52,8 +52,8 @@ def test_kernel_execution_manager(self): self.assertEqual(0, retval) self.assertTrue(exec_mngr._kernel_manager.is_alive()) connection_file = exec_mngr._kernel_manager.connection_file - exec_mngr = self.release_exec_mngr_resources(exec_mngr) - exec_mngr = None + self.release_exec_mngr_resources(exec_mngr) + del exec_mngr self.assertEqual(1, _kernel_manager_factory.n_kernel_managers()) _kernel_manager_factory.shutdown_kernel_manager(connection_file) self.assertEqual(0, _kernel_manager_factory.n_kernel_managers())