diff --git a/.github/workflows/run_unit_tests.yml b/.github/workflows/run_unit_tests.yml
index e61bbf60..589ba26b 100644
--- a/.github/workflows/run_unit_tests.yml
+++ b/.github/workflows/run_unit_tests.yml
@@ -13,7 +13,7 @@ jobs:
strategy:
matrix:
os: [windows-latest, ubuntu-22.04]
- python-version: [3.8, 3.9, "3.10", 3.11]
+ python-version: [3.8, 3.9, "3.10", 3.11, 3.12]
steps:
- uses: actions/checkout@v4
with:
@@ -27,6 +27,8 @@ jobs:
cache: pip
cache-dependency-path: |
pyproject.toml
+ requirements.txt
+ dev-requirements.txt
- name: Display Python version
run:
python -c "import sys; print(sys.version)"
@@ -39,8 +41,8 @@ jobs:
PYTHONUTF8: 1
run: |
python -m pip install --upgrade pip
- pip install git+https://github.com/spine-tools/Spine-Database-API.git#egg=spinedb_api
- pip install .[dev]
+ pip install -r requirements.txt
+ pip install -r dev-requirements.txt
- name: List packages
run:
pip list
diff --git a/.gitignore b/.gitignore
index c6057896..f02a67f3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,13 +1,13 @@
.idea
__pycache__
-build
-dist
-spine_engine.egg-info
+build/
+dist/
+spine_engine.egg-info/
.coverage
htmlcov/
-spine_engine/server/certs
+spine_engine/server/certs/
spine_engine/server/received_projects
spine_engine/version.py
-tests/server/test_project
-tests/server/util/output
+tests/server/test_project/
+tests/server/util/output/
docs/source/autoapi/
diff --git a/README.md b/README.md
index be8b5c79..eca92cef 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
# Spine Engine
-[![Python](https://img.shields.io/badge/python-3.8%20|%203.9%20|%203.10%20|%203.11-blue.svg)](https://www.python.org/downloads/release/python-379/)
+[![Python](https://img.shields.io/badge/python-3.8%20|%203.9%20|%203.10%20|%203.11|%203.12-blue.svg)](https://www.python.org/downloads/release/python-379/)
[![Unit tests](https://github.com/spine-tools/spine-engine/workflows/Unit%20tests/badge.svg)](https://github.com/spine-tools/spine-engine/actions?query=workflow%3A"Unit+tests")
[![codecov](https://codecov.io/gh/spine-tools/spine-engine/branch/master/graph/badge.svg)](https://codecov.io/gh/spine-tools/spine-engine)
[![PyPI version](https://badge.fury.io/py/spine-engine.svg)](https://badge.fury.io/py/spine-engine)
@@ -27,11 +27,6 @@ To install Spine Engine into an existing Python environment, run
$ pip install spine_engine
-### Dependencies
-
-Spine Engine installation will install [dagster](https://dagster.readthedocs.io/en/master/index.html).
-
-
diff --git a/dev-requirements.txt b/dev-requirements.txt
index e5ec38da..f725d8a3 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,4 +1,8 @@
-black == 24.4.2
-pre-commit == 2.0.1
-pylint >= 2.3.0
--r "docs/requirements.txt"
+ipykernel
+coverage[toml]
+sphinx
+recommonmark
+sphinx_rtd_theme
+sphinx-autoapi
+pylint
+black == 21.12b
diff --git a/docs/requirements.txt b/docs/requirements.txt
deleted file mode 100644
index a4a65339..00000000
--- a/docs/requirements.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-sphinx >= 1.7.5
-sphinx_rtd_theme >= 0.4.0
-recommonmark >=0.5.0
-sphinx-autoapi >=1.1.0
-jinja2 < 3.0 # Dagster 0.12.8 requires Jinja2<3.0
-markupsafe < 2.1 # Jinja2<3.0 tries to import soft_unicode, which has been removed in markupsafe 2.1
diff --git a/docs/source/autoapi/index.rst b/docs/source/autoapi/index.rst
deleted file mode 100644
index 968dd4bf..00000000
--- a/docs/source/autoapi/index.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-API Reference
-=============
-
-This page contains auto-generated API reference documentation [#f1]_.
-
-.. toctree::
- :titlesonly:
-
- /autoapi/spine_engine/index
-
-.. [#f1] Created with `sphinx-autoapi `_
\ No newline at end of file
diff --git a/docs/source/autoapi/spine_engine/config/index.rst b/docs/source/autoapi/spine_engine/config/index.rst
deleted file mode 100644
index 2182a788..00000000
--- a/docs/source/autoapi/spine_engine/config/index.rst
+++ /dev/null
@@ -1,105 +0,0 @@
-:py:mod:`spine_engine.config`
-=============================
-
-.. py:module:: spine_engine.config
-
-.. autoapi-nested-parse::
-
- Application constants.
-
- :author: P. Savolainen (VTT)
- :date: 2.1.2018
-
-
-
-Module Contents
----------------
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.config._executable
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.config._on_windows
- spine_engine.config.GAMS_EXECUTABLE
- spine_engine.config.GAMSIDE_EXECUTABLE
- spine_engine.config.JULIA_EXECUTABLE
- spine_engine.config.PYTHON_EXECUTABLE
- spine_engine.config._frozen
- spine_engine.config._path_to_executable
- spine_engine.config.APPLICATION_PATH
- spine_engine.config.EMBEDDED_PYTHON
- spine_engine.config.TOOL_OUTPUT_DIR
- spine_engine.config.GIMLET_WORK_DIR_NAME
-
-
-.. py:data:: _on_windows
-
-
-
-
-.. py:function:: _executable(name)
-
- Appends a .exe extension to `name` on Windows platform.
-
-
-.. py:data:: GAMS_EXECUTABLE
-
-
-
-
-.. py:data:: GAMSIDE_EXECUTABLE
-
-
-
-
-.. py:data:: JULIA_EXECUTABLE
-
-
-
-
-.. py:data:: PYTHON_EXECUTABLE
-
-
-
-
-.. py:data:: _frozen
-
-
-
-
-.. py:data:: _path_to_executable
-
-
-
-
-.. py:data:: APPLICATION_PATH
-
-
-
-
-.. py:data:: EMBEDDED_PYTHON
-
-
-
-
-.. py:data:: TOOL_OUTPUT_DIR
- :annotation: = output
-
-
-
-.. py:data:: GIMLET_WORK_DIR_NAME
- :annotation: = work
-
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_manager/index.rst
deleted file mode 100644
index 16e61193..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_manager/index.rst
+++ /dev/null
@@ -1,164 +0,0 @@
-:py:mod:`spine_engine.execution_managers.conda_kernel_spec_manager`
-===================================================================
-
-.. py:module:: spine_engine.execution_managers.conda_kernel_spec_manager
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.conda_kernel_spec_manager.CondaKernelSpecManager
-
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.conda_kernel_spec_manager.CACHE_TIMEOUT
- spine_engine.execution_managers.conda_kernel_spec_manager.RUNNER_COMMAND
-
-
-.. py:data:: CACHE_TIMEOUT
- :annotation: = 60
-
-
-
-.. py:data:: RUNNER_COMMAND
- :annotation: = ['python', '-m', 'spine_engine.execution_managers.conda_kernel_spec_runner']
-
-
-
-.. py:class:: CondaKernelSpecManager(**kwargs)
-
- Bases: :py:obj:`jupyter_client.kernelspec.KernelSpecManager`
-
- A custom KernelSpecManager able to search for conda environments and
- create kernelspecs for them.
-
- Create a configurable given a config config.
-
- :param config: If this is empty, default values are used. If config is a
- :class:`Config` instance, it will be used to configure the
- instance.
- :type config: Config
- :param parent: The parent Configurable instance of this object.
- :type parent: Configurable instance, optional
-
- .. rubric:: Notes
-
- Subclasses of Configurable must call the :meth:`__init__` method of
- :class:`Configurable` *before* doing anything else and using
- :func:`super`::
-
- class MyConfigurable(Configurable):
- def __init__(self, config=None):
- super(MyConfigurable, self).__init__(config=config)
- # Then any other code you need to finish initialization.
-
- This ensures that instances will be configured properly.
-
- .. py:attribute:: conda_only
-
-
-
-
- .. py:attribute:: env_filter
-
-
-
-
- .. py:attribute:: kernelspec_path
-
-
-
-
- .. py:attribute:: name_format
-
-
-
-
- .. py:method:: _validate_kernelspec_path(self, proposal)
-
-
- .. py:method:: clean_kernel_name(kname)
- :staticmethod:
-
- Replaces invalid characters in the Jupyter kernelname, with
- a bit of effort to preserve readability.
-
-
- .. py:method:: _conda_info(self)
- :property:
-
- Get and parse the whole conda information output
-
- Caches the information for CACHE_TIMEOUT seconds, as this is
- relatively expensive.
-
-
- .. py:method:: _all_envs(self)
-
- Find all of the environments we should be checking. We skip
- environments in the conda-bld directory as well as environments
- that match our env_filter regex. Returns a dict with canonical
- environment names as keys, and full paths as values.
-
-
- .. py:method:: _all_specs(self)
-
- Find the all kernel specs in all environments.
-
- Returns a dict with unique env names as keys, and the kernel.json
- content as values, modified so that they can be run properly in
- their native environments.
-
- Caches the information for CACHE_TIMEOUT seconds, as this is
- relatively expensive.
-
-
- .. py:method:: _conda_kspecs(self)
- :property:
-
- Get (or refresh) the cache of conda kernels
-
-
-
- .. py:method:: find_kernel_specs(self)
-
- Returns a dict mapping kernel names to resource directories.
-
- The update process also adds the resource dir for the conda
- environments.
-
-
- .. py:method:: get_kernel_spec(self, kernel_name)
-
- Returns a :class:`KernelSpec` instance for the given kernel_name.
-
- Additionally, conda kernelspecs are generated on the fly
- accordingly with the detected environments.
-
-
- .. py:method:: get_all_specs(self)
-
- Returns a dict mapping kernel names to dictionaries with two
- entries: "resource_dir" and "spec". This was added to fill out
- the full public interface to KernelManagerSpec.
-
-
- .. py:method:: remove_kernel_spec(self, name)
-
- Remove a kernel spec directory by name.
-
- Returns the path that was deleted.
-
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_runner/index.rst b/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_runner/index.rst
deleted file mode 100644
index fc06cdad..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/conda_kernel_spec_runner/index.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-:py:mod:`spine_engine.execution_managers.conda_kernel_spec_runner`
-==================================================================
-
-.. py:module:: spine_engine.execution_managers.conda_kernel_spec_runner
-
-
-Module Contents
----------------
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.conda_kernel_spec_runner.exec_in_env
-
-
-
-.. py:function:: exec_in_env(conda_prefix, env_path, *command)
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/execution_manager_base/index.rst b/docs/source/autoapi/spine_engine/execution_managers/execution_manager_base/index.rst
deleted file mode 100644
index 9c94c0f2..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/execution_manager_base/index.rst
+++ /dev/null
@@ -1,52 +0,0 @@
-:py:mod:`spine_engine.execution_managers.execution_manager_base`
-================================================================
-
-.. py:module:: spine_engine.execution_managers.execution_manager_base
-
-.. autoapi-nested-parse::
-
- Contains the ExecutionManagerBase class.
-
- :authors: M. Marin (KTH)
- :date: 12.10.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase
-
-
-
-
-.. py:class:: ExecutionManagerBase(logger)
-
- Base class for all tool instance execution managers.
-
- Class constructor.
-
- :param logger: a logger instance
- :type logger: LoggerInterface
-
- .. py:method:: run_until_complete(self)
- :abstractmethod:
-
- Runs until completion.
-
- :returns: return code
- :rtype: int
-
-
- .. py:method:: stop_execution(self)
- :abstractmethod:
-
- Stops execution gracefully.
-
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/index.rst b/docs/source/autoapi/spine_engine/execution_managers/index.rst
deleted file mode 100644
index 93fe245d..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/index.rst
+++ /dev/null
@@ -1,21 +0,0 @@
-:py:mod:`spine_engine.execution_managers`
-=========================================
-
-.. py:module:: spine_engine.execution_managers
-
-
-Submodules
-----------
-.. toctree::
- :titlesonly:
- :maxdepth: 1
-
- conda_kernel_spec_manager/index.rst
- conda_kernel_spec_runner/index.rst
- execution_manager_base/index.rst
- kernel_execution_manager/index.rst
- persistent_execution_manager/index.rst
- process_execution_manager/index.rst
- spine_repl/index.rst
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/kernel_execution_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/kernel_execution_manager/index.rst
deleted file mode 100644
index 0bcf0c9a..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/kernel_execution_manager/index.rst
+++ /dev/null
@@ -1,165 +0,0 @@
-:py:mod:`spine_engine.execution_managers.kernel_execution_manager`
-==================================================================
-
-.. py:module:: spine_engine.execution_managers.kernel_execution_manager
-
-.. autoapi-nested-parse::
-
- Contains the KernelExecutionManager class and subclasses, and some convenience functions.
-
- :authors: M. Marin (KTH)
- :date: 12.10.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.kernel_execution_manager._KernelManagerFactory
- spine_engine.execution_managers.kernel_execution_manager.KernelExecutionManager
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.kernel_execution_manager.get_kernel_manager
- spine_engine.execution_managers.kernel_execution_manager.pop_kernel_manager
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.kernel_execution_manager._kernel_manager_factory
-
-
-.. py:class:: _KernelManagerFactory
-
- .. py:attribute:: _kernel_managers
-
-
- Maps tuples (kernel name, group id) to associated KernelManager.
-
- .. py:attribute:: _key_by_connection_file
-
-
- Maps connection file string to tuple (kernel_name, group_id). Mostly for fast lookup in ``restart_kernel()``
-
- .. py:method:: _make_kernel_manager(self, kernel_name, group_id)
-
- Creates a new kernel manager for given kernel and group id if none exists, and returns it.
-
- :param kernel_name: the kernel
- :type kernel_name: str
- :param group_id: item group that will execute using this kernel
- :type group_id: str
-
- :returns: KernelManager
-
-
- .. py:method:: new_kernel_manager(self, kernel_name, group_id, logger, extra_switches=None, environment='', **kwargs)
-
- Creates a new kernel manager for given kernel and group id if none exists.
- Starts the kernel if not started, and returns it.
-
- :param kernel_name: the kernel
- :type kernel_name: str
- :param group_id: item group that will execute using this kernel
- :type group_id: str
- :param logger: for logging
- :type logger: LoggerInterface
- :param extra_switches: List of additional switches to julia or python.
- These come before the 'programfile'.
- :type extra_switches: list, optional
- :param environment: "conda" to launch a Conda kernel spec. "" for a regular kernel spec
- :type environment: str
- :param `**kwargs`: optional. Keyword arguments passed to ``KernelManager.start_kernel()``
-
- :returns: KernelManager
-
-
- .. py:method:: get_kernel_manager(self, connection_file)
-
- Returns a kernel manager for given connection file if any.
-
- :param connection_file: path of connection file
- :type connection_file: str
-
- :returns: KernelManager or None
-
-
- .. py:method:: pop_kernel_manager(self, connection_file)
-
- Returns a kernel manager for given connection file if any.
- It also removes it from cache.
-
- :param connection_file: path of connection file
- :type connection_file: str
-
- :returns: KernelManager or None
-
-
-
-.. py:data:: _kernel_manager_factory
-
-
-
-
-.. py:function:: get_kernel_manager(connection_file)
-
-
-.. py:function:: pop_kernel_manager(connection_file)
-
-
-.. py:class:: KernelExecutionManager(logger, kernel_name, *commands, group_id=None, workdir=None, startup_timeout=60, extra_switches=None, environment='', **kwargs)
-
- Bases: :py:obj:`spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase`
-
- Base class for all tool instance execution managers.
-
- :param logger:
- :type logger: LoggerInterface
- :param kernel_name: the kernel
- :type kernel_name: str
- :param \*commands: Commands to execute in the kernel
- :param group_id: item group that will execute using this kernel
- :type group_id: str, optional
- :param workdir: item group that will execute using this kernel
- :type workdir: str, optional
- :param startup_timeout: How much to wait for the kernel, used in ``KernelClient.wait_for_ready()``
- :type startup_timeout: int, optional
- :param extra_switches: List of additional switches to launch julia.
- These come before the 'programfile'.
- :type extra_switches: list, optional
- :param environment: "conda" to launch a Conda kernel spec. "" for a regular kernel spec.
- :type environment: str
- :param \*\*kwargs: Keyword arguments passed to ``KernelManager.start_kernel()``
- :type \*\*kwargs: optional
-
- .. py:method:: run_until_complete(self)
-
- Runs until completion.
-
- :returns: return code
- :rtype: int
-
-
- .. py:method:: _do_run(self)
-
-
- .. py:method:: stop_execution(self)
-
- Stops execution gracefully.
-
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/persistent_execution_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/persistent_execution_manager/index.rst
deleted file mode 100644
index c54dadf4..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/persistent_execution_manager/index.rst
+++ /dev/null
@@ -1,528 +0,0 @@
-:py:mod:`spine_engine.execution_managers.persistent_execution_manager`
-======================================================================
-
-.. py:module:: spine_engine.execution_managers.persistent_execution_manager
-
-.. autoapi-nested-parse::
-
- Contains PersistentManagerBase, PersistentExecutionManagerBase classes and subclasses,
- as well as some convenience functions.
-
- :authors: M. Marin (KTH)
- :date: 12.10.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.persistent_execution_manager.PersistentManagerBase
- spine_engine.execution_managers.persistent_execution_manager.JuliaPersistentManager
- spine_engine.execution_managers.persistent_execution_manager.PythonPersistentManager
- spine_engine.execution_managers.persistent_execution_manager._PersistentManagerFactory
- spine_engine.execution_managers.persistent_execution_manager.PersistentExecutionManagerBase
- spine_engine.execution_managers.persistent_execution_manager.JuliaPersistentExecutionManager
- spine_engine.execution_managers.persistent_execution_manager.PythonPersistentExecutionManager
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.persistent_execution_manager.restart_persistent
- spine_engine.execution_managers.persistent_execution_manager.interrupt_persistent
- spine_engine.execution_managers.persistent_execution_manager.kill_persistent_processes
- spine_engine.execution_managers.persistent_execution_manager.issue_persistent_command
- spine_engine.execution_managers.persistent_execution_manager.get_persistent_completions
- spine_engine.execution_managers.persistent_execution_manager.get_persistent_history_item
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.persistent_execution_manager._persistent_manager_factory
-
-
-.. py:class:: PersistentManagerBase(args, cwd=None)
-
-
- :param args: the arguments to launch the persistent process
- :type args: list
- :param cwd: the directory where to start the process
- :type cwd: str, optional
-
- .. py:method:: language(self)
- :property:
-
- Returns the underlying language for UI customization in toolbox.
-
- :returns: str
-
-
- .. py:method:: _init_args(self)
- :abstractmethod:
-
- Returns init args for Popen.
-
- Subclasses must reimplement to include appropriate switches to ensure the
- process is interactive, or to load modules for internal communication.
-
- :returns: list
-
-
- .. py:method:: _sentinel_command(host, port)
- :staticmethod:
- :abstractmethod:
-
- Returns a command in the underlying language, that sends a sentinel to a socket listening on given
- host/port.
-
- Used to synchronize with the persistent process.
-
- :param host:
- :type host: str
- :param port:
- :type port: int
-
- :returns: str
-
-
- .. py:method:: _start_persistent(self)
-
- Starts the persistent process.
-
-
- .. py:method:: _log_stdout(self)
-
- Puts stdout from the process into the queue (it will be consumed by issue_command()).
-
-
- .. py:method:: _log_stderr(self)
-
- Puts stderr from the process into the queue (it will be consumed by issue_command()).
-
-
- .. py:method:: _is_complete(self, cmd)
-
-
- .. py:method:: issue_command(self, cmd, add_history=False)
-
- Issues cmd to the persistent process and returns an iterator of stdout and stderr messages.
-
- Each message is a dictionary with two keys:
-
- - "type": either "stdout" or "stderr"
- - "data": the actual message string.
-
- :param cmd:
- :type cmd: str
-
- :returns: generator
-
-
- .. py:method:: _make_complete_command(self)
-
-
- .. py:method:: _issue_command_and_wait_for_idle(self, cmd, add_history)
-
- Issues command and wait for idle.
-
- :param cmd: Command to pass to the persistent process
- :type cmd: str
- :param add_history: Whether or not to add the command to history
- :type add_history: bool
-
-
- .. py:method:: _issue_command(self, cmd, is_complete=True, add_history=False)
-
- Writes command to the process's stdin and flushes.
-
- :param cmd: Command to pass to the persistent process
- :type cmd: str
- :param is_complete: Whether or not the command is complete
- :type is_complete: bool
- :param add_history: Whether or not to add the command to history
- :type add_history: bool
-
-
- .. py:method:: _wait(self)
-
- Waits for the persistent process to become idle.
-
- This is implemented by writing the sentinel to stdin and waiting for the _SENTINEL to come out of stdout.
-
- :returns: bool
-
-
- .. py:method:: _listen_and_enqueue(host, port, queue)
- :staticmethod:
-
- Listens on the server and enqueues all data received.
-
-
- .. py:method:: _communicate(self, request, arg, receive=True)
-
- Sends a request to the persistent process with the given argument.
-
- :param request: One of the supported requests
- :type request: str
- :param arg: Request argument
- :param receive: If True (the default) also receives the response and returns it.
- :type receive: bool, optional
-
- :returns: response, or None if the ``receive`` argument is False
- :rtype: str or NoneType
-
-
- .. py:method:: get_completions(self, text)
-
- Returns a list of autocompletion options for given text.
-
- :param text: Text to complete
- :type text: str
-
- :returns: List of options
- :rtype: list(str)
-
-
- .. py:method:: get_history_item(self, index)
-
- Returns the history item given by index.
-
- :param index: Index to retrieve, one-based, 1 means most recent
- :type index: int
-
- :returns: str
-
-
- .. py:method:: restart_persistent(self)
-
- Restarts the persistent process.
-
-
- .. py:method:: interrupt_persistent(self)
-
- Interrupts the persistent process.
-
-
- .. py:method:: _do_interrupt_persistent(self)
-
-
- .. py:method:: is_persistent_alive(self)
-
- Whether or not the persistent is still alive and ready to receive commands.
-
- :returns: bool
-
-
- .. py:method:: kill_process(self)
-
-
-
-.. py:class:: JuliaPersistentManager(args, cwd=None)
-
- Bases: :py:obj:`PersistentManagerBase`
-
-
- :param args: the arguments to launch the persistent process
- :type args: list
- :param cwd: the directory where to start the process
- :type cwd: str, optional
-
- .. py:method:: language(self)
- :property:
-
- See base class.
-
-
- .. py:method:: _init_args(self)
-
- See base class.
-
-
- .. py:method:: _sentinel_command(host, port)
- :staticmethod:
-
- Returns a command in the underlying language, that sends a sentinel to a socket listening on given
- host/port.
-
- Used to synchronize with the persistent process.
-
- :param host:
- :type host: str
- :param port:
- :type port: int
-
- :returns: str
-
-
-
-.. py:class:: PythonPersistentManager(args, cwd=None)
-
- Bases: :py:obj:`PersistentManagerBase`
-
-
- :param args: the arguments to launch the persistent process
- :type args: list
- :param cwd: the directory where to start the process
- :type cwd: str, optional
-
- .. py:method:: language(self)
- :property:
-
- See base class.
-
-
- .. py:method:: _init_args(self)
-
- See base class.
-
-
- .. py:method:: _sentinel_command(host, port)
- :staticmethod:
-
- Returns a command in the underlying language, that sends a sentinel to a socket listening on given
- host/port.
-
- Used to synchronize with the persistent process.
-
- :param host:
- :type host: str
- :param port:
- :type port: int
-
- :returns: str
-
-
-
-.. py:class:: _PersistentManagerFactory
-
- .. py:attribute:: _persistent_managers
-
-
- Maps tuples (process args) to associated PersistentManagerBase.
-
- .. py:method:: new_persistent_manager(self, constructor, logger, args, group_id, cwd=None)
-
- Creates a new persistent for given args and group id if none exists.
-
- :param constructor: the persistent manager constructor
- :type constructor: function
- :param logger:
- :type logger: LoggerInterface
- :param args: the arguments to launch the persistent process
- :type args: list
- :param group_id: item group that will execute using this persistent
- :type group_id: str
- :param cwd: directory where to start the persistent
- :type cwd: str, optional
-
- :returns: persistent manager
- :rtype: PersistentManagerBase
-
-
- .. py:method:: restart_persistent(self, key)
-
- Restart a persistent process.
-
- :param key: persistent identifier
- :type key: tuple
-
-
- .. py:method:: interrupt_persistent(self, key)
-
- Interrupts a persistent process.
-
- :param key: persistent identifier
- :type key: tuple
-
-
- .. py:method:: issue_persistent_command(self, key, cmd)
-
- Issues a command to a persistent process.
-
- :param key: persistent identifier
- :type key: tuple
- :param cmd: command to issue
- :type cmd: str
-
- :returns: stdio and stderr messages (dictionaries with two keys: type, and data)
- :rtype: generator
-
-
- .. py:method:: get_persistent_completions(self, key, text)
-
- Returns a list of completion options.
-
- :param key: persistent identifier
- :type key: tuple
- :param text: text to complete
- :type text: str
-
- :returns: options that match given text
- :rtype: list of str
-
-
- .. py:method:: get_persistent_history_item(self, key, index)
-
- Issues a command to a persistent process.
-
- :param key: persistent identifier
- :type key: tuple
- :param index: index of the history item, most recen first
- :type index: int
-
- :returns: history item or empty string if none
- :rtype: str
-
-
- .. py:method:: kill_manager_processes(self)
-
- Kills persistent managers' Popen instances.
-
-
-
-.. py:data:: _persistent_manager_factory
-
-
-
-
-.. py:function:: restart_persistent(key)
-
- See _PersistentManagerFactory.
-
-
-.. py:function:: interrupt_persistent(key)
-
- See _PersistentManagerFactory.
-
-
-.. py:function:: kill_persistent_processes()
-
- Kills all persistent processes.
-
- On Windows systems the work directories are reserved by the ``Popen`` objects owned by the persistent managers.
- They need to be killed to before the directories can be modified, e.g. deleted or renamed.
-
-
-.. py:function:: issue_persistent_command(key, cmd)
-
- See _PersistentManagerFactory.
-
-
-.. py:function:: get_persistent_completions(key, text)
-
- See _PersistentManagerFactory.
-
-
-.. py:function:: get_persistent_history_item(key, index)
-
- See _PersistentManagerFactory.
-
-
-.. py:class:: PersistentExecutionManagerBase(logger, args, commands, alias, group_id=None, workdir=None)
-
- Bases: :py:obj:`spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase`
-
- Base class for managing execution of commands on a persistent process.
-
- Class constructor.
-
- :param logger: a logger instance
- :type logger: LoggerInterface
- :param args: List of args to start the persistent process
- :type args: list
- :param commands: List of commands to execute in the persistent process
- :type commands: list
- :param group_id: item group that will execute using this kernel
- :type group_id: str, optional
- :param workdir: item group that will execute using this kernel
- :type workdir: str, optional
-
- .. py:method:: alias(self)
- :property:
-
-
- .. py:method:: persistent_manager_factory()
- :staticmethod:
- :abstractmethod:
-
- Returns a function to create a persistent manager for this execution
- (a subclass of PersistentManagerBase)
-
- :returns: function
-
-
- .. py:method:: run_until_complete(self)
-
- See base class.
-
-
- .. py:method:: stop_execution(self)
-
- See base class.
-
-
-
-.. py:class:: JuliaPersistentExecutionManager(logger, args, commands, alias, group_id=None, workdir=None)
-
- Bases: :py:obj:`PersistentExecutionManagerBase`
-
- Manages execution of commands on a Julia persistent process.
-
- Class constructor.
-
- :param logger: a logger instance
- :type logger: LoggerInterface
- :param args: List of args to start the persistent process
- :type args: list
- :param commands: List of commands to execute in the persistent process
- :type commands: list
- :param group_id: item group that will execute using this kernel
- :type group_id: str, optional
- :param workdir: item group that will execute using this kernel
- :type workdir: str, optional
-
- .. py:method:: persistent_manager_factory()
- :staticmethod:
-
- See base class.
-
-
-
-.. py:class:: PythonPersistentExecutionManager(logger, args, commands, alias, group_id=None, workdir=None)
-
- Bases: :py:obj:`PersistentExecutionManagerBase`
-
- Manages execution of commands on a Python persistent process.
-
- Class constructor.
-
- :param logger: a logger instance
- :type logger: LoggerInterface
- :param args: List of args to start the persistent process
- :type args: list
- :param commands: List of commands to execute in the persistent process
- :type commands: list
- :param group_id: item group that will execute using this kernel
- :type group_id: str, optional
- :param workdir: item group that will execute using this kernel
- :type workdir: str, optional
-
- .. py:method:: persistent_manager_factory()
- :staticmethod:
-
- See base class.
-
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/process_execution_manager/index.rst b/docs/source/autoapi/spine_engine/execution_managers/process_execution_manager/index.rst
deleted file mode 100644
index 2a115542..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/process_execution_manager/index.rst
+++ /dev/null
@@ -1,62 +0,0 @@
-:py:mod:`spine_engine.execution_managers.process_execution_manager`
-===================================================================
-
-.. py:module:: spine_engine.execution_managers.process_execution_manager
-
-.. autoapi-nested-parse::
-
- Contains the ProcessExecutionManager class.
-
- :authors: M. Marin (KTH)
- :date: 12.10.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.process_execution_manager.ProcessExecutionManager
-
-
-
-
-.. py:class:: ProcessExecutionManager(logger, program, *args, workdir=None)
-
- Bases: :py:obj:`spine_engine.execution_managers.execution_manager_base.ExecutionManagerBase`
-
- Base class for all tool instance execution managers.
-
- Class constructor.
-
- :param logger: a logger instance
- :type logger: LoggerInterface
- :param program: Path to program to run in the subprocess (e.g. julia.exe)
- :type program: str
- :param args: List of argument for the program (e.g. path to script file)
- :type args: list
-
- .. py:method:: run_until_complete(self)
-
- Runs until completion.
-
- :returns: return code
- :rtype: int
-
-
- .. py:method:: stop_execution(self)
-
- Stops execution gracefully.
-
-
- .. py:method:: _log_stdout(self, stdout)
-
-
- .. py:method:: _log_stderr(self, stderr)
-
-
-
diff --git a/docs/source/autoapi/spine_engine/execution_managers/spine_repl/index.rst b/docs/source/autoapi/spine_engine/execution_managers/spine_repl/index.rst
deleted file mode 100644
index 038e1f94..00000000
--- a/docs/source/autoapi/spine_engine/execution_managers/spine_repl/index.rst
+++ /dev/null
@@ -1,104 +0,0 @@
-:py:mod:`spine_engine.execution_managers.spine_repl`
-====================================================
-
-.. py:module:: spine_engine.execution_managers.spine_repl
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.spine_repl.SpineDBServer
- spine_engine.execution_managers.spine_repl._RequestHandler
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.spine_repl.completions
- spine_engine.execution_managers.spine_repl.add_history
- spine_engine.execution_managers.spine_repl.history_item
- spine_engine.execution_managers.spine_repl.is_complete
- spine_engine.execution_managers.spine_repl.start_server
- spine_engine.execution_managers.spine_repl.send_sentinel
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.execution_managers.spine_repl.readline
-
-
-.. py:data:: readline
-
-
-
-
-.. py:class:: SpineDBServer(server_address, RequestHandlerClass, bind_and_activate=True)
-
- Bases: :py:obj:`socketserver.ThreadingMixIn`, :py:obj:`socketserver.TCPServer`
-
- Mix-in class to handle each request in a new thread.
-
- Constructor. May be extended, do not override.
-
- .. py:attribute:: allow_reuse_address
- :annotation: = True
-
-
-
-
-.. py:class:: _RequestHandler(request, client_address, server)
-
- Bases: :py:obj:`socketserver.BaseRequestHandler`
-
- Base class for request handler classes.
-
- This class is instantiated for each request to be handled. The
- constructor sets the instance variables request, client_address
- and server, and then calls the handle() method. To implement a
- specific service, all you need to do is to derive a class which
- defines a handle() method.
-
- The handle() method can find the request as self.request, the
- client address as self.client_address, and the server (in case it
- needs access to per-server information) as self.server. Since a
- separate instance is created for each request, the handle() method
- can define other arbitrary instance variables.
-
-
- .. py:method:: handle(self)
-
-
-
-.. py:function:: completions(text)
-
-
-.. py:function:: add_history(line)
-
-
-.. py:function:: history_item(index)
-
-
-.. py:function:: is_complete(cmd)
-
-
-.. py:function:: start_server(address)
-
- :param address: Server address
- :type address: tuple(str,int)
-
-
-.. py:function:: send_sentinel(host, port)
-
-
diff --git a/docs/source/autoapi/spine_engine/index.rst b/docs/source/autoapi/spine_engine/index.rst
deleted file mode 100644
index dae87444..00000000
--- a/docs/source/autoapi/spine_engine/index.rst
+++ /dev/null
@@ -1,394 +0,0 @@
-:py:mod:`spine_engine`
-======================
-
-.. py:module:: spine_engine
-
-
-Subpackages
------------
-.. toctree::
- :titlesonly:
- :maxdepth: 3
-
- execution_managers/index.rst
- multithread_executor/index.rst
- project_item/index.rst
- utils/index.rst
-
-
-Submodules
-----------
-.. toctree::
- :titlesonly:
- :maxdepth: 1
-
- config/index.rst
- load_project_items/index.rst
- project_item_loader/index.rst
- spine_engine/index.rst
- spine_engine_server/index.rst
- version/index.rst
-
-
-Package Contents
-----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.ExecutionDirection
- spine_engine.SpineEngine
- spine_engine.SpineEngineState
- spine_engine.ItemExecutionFinishState
-
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.__version__
-
-
-.. py:class:: ExecutionDirection
-
- Bases: :py:obj:`enum.Enum`
-
- Generic enumeration.
-
- Derive from this class to define new enumerations.
-
- .. py:attribute:: FORWARD
-
-
-
-
- .. py:attribute:: BACKWARD
-
-
-
-
- .. py:method:: __str__(self)
-
- Return str(self).
-
-
-
-.. py:class:: SpineEngine(items=None, specifications=None, connections=None, items_module_name='spine_items', settings=None, project_dir=None, execution_permits=None, node_successors=None, debug=False)
-
- An engine for executing a Spine Toolbox DAG-workflow.
-
- :param items: List of executable item dicts.
- :type items: list(dict)
- :param specifications: A mapping from item type to list of specification dicts.
- :type specifications: dict(str,list(dict))
- :param connections: List of connection dicts
- :type connections: list of dict
- :param items_module_name: name of the Python module that contains project items
- :type items_module_name: str
- :param settings: Toolbox execution settings.
- :type settings: dict
- :param project_dir: Path to project directory.
- :type project_dir: str
- :param execution_permits: A mapping from item name to a boolean value, False indicating that
- the item is not executed, only its resources are collected.
- :type execution_permits: dict(str,bool)
- :param node_successors: A mapping from item name to list of successor item names, dictating the dependencies.
- :type node_successors: dict(str,list(str))
- :param debug: Whether debug mode is active or not.
- :type debug: bool
-
- .. py:method:: _make_item_specifications(self, specifications, project_item_loader, items_module_name)
-
- Instantiates item specifications.
-
- :param specifications: A mapping from item type to list of specification dicts.
- :type specifications: dict
- :param project_item_loader: loader instance
- :type project_item_loader: ProjectItemLoader
- :param items_module_name: name of the Python module that contains the project items
- :type items_module_name: str
-
- :returns: mapping from item type to a dict that maps specification names to specification instances
- :rtype: dict
-
-
- .. py:method:: _make_item(self, item_name, direction)
-
- Recreates item from project item dictionary. Note that all items are created twice.
- One for the backward pipeline, the other one for the forward pipeline.
-
-
- .. py:method:: get_event(self)
-
- Returns the next event in the stream. Calling this after receiving the event of type "dag_exec_finished"
- will raise StopIterationError.
-
-
- .. py:method:: state(self)
-
-
- .. py:method:: _get_event_stream(self)
-
- Returns an iterator of tuples (event_type, event_data).
-
- TODO: Describe the events in depth.
-
-
- .. py:method:: answer_prompt(self, item_name, accepted)
-
-
- .. py:method:: run(self)
-
- Runs this engine.
-
-
- .. py:method:: _process_event(self, event)
-
- Processes events from a pipeline.
-
- :param event: an event
- :type event: DagsterEvent
-
-
- .. py:method:: stop(self)
-
- Stops the engine.
-
-
- .. py:method:: _stop_item(self, item)
-
-
- .. py:method:: _make_pipeline(self)
-
- Returns a PipelineDefinition for executing this engine.
-
- :returns: PipelineDefinition
-
-
- .. py:method:: _make_backward_solid_def(self, item_name)
-
- Returns a SolidDefinition for executing the given item in the backward sweep.
-
- :param item_name: The project item that gets executed by the solid.
- :type item_name: str
-
-
- .. py:method:: _make_forward_solid_def(self, item_name)
-
- Returns a SolidDefinition for executing the given item.
-
- :param item_name:
- :type item_name: str
-
- :returns: SolidDefinition
-
-
- .. py:method:: _execute_item(self, context, item_name, forward_resource_stacks, backward_resources)
-
- Executes the given item using the given forward resource stacks and backward resources.
- Returns list of output resource stacks.
-
- Called by ``_make_forward_solid_def.compute_fn``.
-
- For each element yielded by ``_filtered_resources_iterator``, spawns a thread that runs ``_execute_item_filtered``.
-
- :param context:
- :param item_name:
- :type item_name: str
- :param forward_resource_stacks:
- :type forward_resource_stacks: list(tuple(ProjectItemResource))
- :param backward_resources:
- :type backward_resources: list(ProjectItemResource)
-
- :returns: list(tuple(ProjectItemResource))
-
-
- .. py:method:: _execute_item_filtered(self, item, filtered_forward_resources, filtered_backward_resources, output_resources_list, success)
-
- Executes the given item using the given filtered resources. Target for threads in ``_execute_item``.
-
- :param item:
- :type item: ExecutableItemBase
- :param filtered_forward_resources:
- :type filtered_forward_resources: list(ProjectItemResource)
- :param filtered_backward_resources:
- :type filtered_backward_resources: list(ProjectItemResource)
- :param output_resources_list: A list of lists, to append the
- output resources generated by the item.
- :type output_resources_list: list(list(ProjectItemResource))
- :param success: A list of one element, to write the outcome of the execution.
- :type success: list
-
-
- .. py:method:: _filtered_resources_iterator(self, item_name, forward_resource_stacks, backward_resources, timestamp)
-
- Yields tuples of (filtered forward resources, filtered backward resources, filter id).
-
- Each tuple corresponds to a unique filter combination. Combinations are obtained by applying the cross-product
- over forward resource stacks as yielded by ``_forward_resource_stacks_iterator``.
-
- :param item_name:
- :type item_name: str
- :param forward_resource_stacks:
- :type forward_resource_stacks: list(tuple(ProjectItemResource))
- :param backward_resources:
- :type backward_resources: list(ProjectItemResource)
- :param timestamp: timestamp for the execution filter
- :type timestamp: str
-
- :returns: forward resources, backward resources, filter id
- :rtype: Iterator(tuple(list,list,str))
-
-
- .. py:method:: _expand_resource_stack(self, item_name, resource_stack)
-
- Expands a resource stack if possible.
-
- If the stack has more than one resource, returns the unaltered stack.
-
- Otherwise, if the stack has only one resource but there are no filters defined for that resource,
- again, returns the unaltered stack.
-
- Otherwise, returns an expanded stack of as many resources as filter stacks defined for the only one resource.
- Each resource in the expanded stack is a clone of the original, with one of the filter stacks
- applied to the URL.
-
- :param item_name: resource receiving item's name
- :type item_name: str
- :param resource_stack:
- :type resource_stack: tuple(ProjectItemResource)
-
- :returns: tuple(ProjectItemResource)
-
-
- .. py:method:: _filter_stacks(self, item_name, resource_label)
-
- Computes filter stacks.
-
- Stacks are computed as the cross-product of all individual filters defined for a resource.
-
- :param item_name: item's name
- :type item_name: str
- :param resource_label: resource's label
- :type resource_label: str
-
- :returns: filter stacks
- :rtype: list of list
-
-
- .. py:method:: _convert_forward_resources(self, item_name, resources)
-
- Converts resources as they're being forwarded to given item.
- The conversion is dictated by the connection the resources traverse in order to reach the item.
-
- :param item_name: receiving item's name
- :type item_name: str
- :param resources: resources to convert
- :type resources: list of ProjectItemResource
-
- :returns: converted resources
- :rtype: list of ProjectItemResource
-
-
- .. py:method:: _make_dependencies(self)
-
- Returns a dictionary of dependencies according to the given dictionaries of injectors.
-
- :returns: a dictionary to pass to the PipelineDefinition constructor as dependencies
- :rtype: dict
-
-
-
-.. py:class:: SpineEngineState
-
- Bases: :py:obj:`enum.Enum`
-
- Generic enumeration.
-
- Derive from this class to define new enumerations.
-
- .. py:attribute:: SLEEPING
- :annotation: = 1
-
-
-
- .. py:attribute:: RUNNING
- :annotation: = 2
-
-
-
- .. py:attribute:: USER_STOPPED
- :annotation: = 3
-
-
-
- .. py:attribute:: FAILED
- :annotation: = 4
-
-
-
- .. py:attribute:: COMPLETED
- :annotation: = 5
-
-
-
- .. py:method:: __str__(self)
-
- Return str(self).
-
-
-
-.. py:class:: ItemExecutionFinishState
-
- Bases: :py:obj:`enum.Enum`
-
- Generic enumeration.
-
- Derive from this class to define new enumerations.
-
- .. py:attribute:: SUCCESS
- :annotation: = 1
-
-
-
- .. py:attribute:: FAILURE
- :annotation: = 2
-
-
-
- .. py:attribute:: SKIPPED
- :annotation: = 3
-
-
-
- .. py:attribute:: EXCLUDED
- :annotation: = 4
-
-
-
- .. py:attribute:: STOPPED
- :annotation: = 5
-
-
-
- .. py:attribute:: NEVER_FINISHED
- :annotation: = 6
-
-
-
- .. py:method:: __str__(self)
-
- Return str(self).
-
-
-
-.. py:data:: __version__
- :annotation: = 0.10.1
-
-
-
diff --git a/docs/source/autoapi/spine_engine/load_project_items/index.rst b/docs/source/autoapi/spine_engine/load_project_items/index.rst
deleted file mode 100644
index fa761fff..00000000
--- a/docs/source/autoapi/spine_engine/load_project_items/index.rst
+++ /dev/null
@@ -1,50 +0,0 @@
-:py:mod:`spine_engine.load_project_items`
-=========================================
-
-.. py:module:: spine_engine.load_project_items
-
-.. autoapi-nested-parse::
-
- Functions to load project item modules.
-
- :author: A. Soininen (VTT)
- :date: 29.4.2020
-
-
-
-Module Contents
----------------
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.load_project_items.load_item_specification_factories
- spine_engine.load_project_items.load_executable_item_classes
-
-
-
-.. py:function:: load_item_specification_factories(items_package_name)
-
- Loads the project item specification factories in given project item package.
-
- :param items_package_name: name of the package that contains the project items
- :type items_package_name: str
-
- :returns: a map from item type to specification factory
- :rtype: dict
-
-
-.. py:function:: load_executable_item_classes(items_package_name)
-
- Loads the project item executable classes included in given project item package.
-
- :param items_package_name: name of the package that contains the project items
- :type items_package_name: str
-
- :returns: a map from item type to the executable item class
- :rtype: dict
-
-
diff --git a/docs/source/autoapi/spine_engine/multithread_executor/executor/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/executor/index.rst
deleted file mode 100644
index d3bf4471..00000000
--- a/docs/source/autoapi/spine_engine/multithread_executor/executor/index.rst
+++ /dev/null
@@ -1,52 +0,0 @@
-:py:mod:`spine_engine.multithread_executor.executor`
-====================================================
-
-.. py:module:: spine_engine.multithread_executor.executor
-
-.. autoapi-nested-parse::
-
- Module contains multithread_executor.
-
- :author: M. Marin (KTH)
- :date: 1.11.2020
-
-
-
-Module Contents
----------------
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.multithread_executor.executor.multithread_executor
-
-
-
-.. py:function:: multithread_executor(init_context)
-
- A custom multithread executor.
-
- This simple multithread executor borrows almost all the code from dagster's builtin multiprocess executor,
- but takes a twist to use threading instead of multiprocessing.
- To select the multithread executor, include a fragment
- such as the following in your config:
-
- .. code-block:: yaml
-
- execution:
- multithread:
- config:
- max_concurrent: 4
-
- The ``max_concurrent`` arg is optional and tells the execution engine how many threads may run
- concurrently. By default, or if you set ``max_concurrent`` to be 0, this is 100
- (in attendance of a better method).
-
- Execution priority can be configured using the ``dagster/priority`` tag via solid metadata,
- where the higher the number the higher the priority. 0 is the default and both positive
- and negative numbers can be used.
-
-
diff --git a/docs/source/autoapi/spine_engine/multithread_executor/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/index.rst
deleted file mode 100644
index 3a98581c..00000000
--- a/docs/source/autoapi/spine_engine/multithread_executor/index.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-:py:mod:`spine_engine.multithread_executor`
-===========================================
-
-.. py:module:: spine_engine.multithread_executor
-
-
-Submodules
-----------
-.. toctree::
- :titlesonly:
- :maxdepth: 1
-
- executor/index.rst
- multithread/index.rst
- thread_executor/index.rst
-
-
diff --git a/docs/source/autoapi/spine_engine/multithread_executor/multithread/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/multithread/index.rst
deleted file mode 100644
index 95c09324..00000000
--- a/docs/source/autoapi/spine_engine/multithread_executor/multithread/index.rst
+++ /dev/null
@@ -1,82 +0,0 @@
-:py:mod:`spine_engine.multithread_executor.multithread`
-=======================================================
-
-.. py:module:: spine_engine.multithread_executor.multithread
-
-.. autoapi-nested-parse::
-
- Module contains MultithreadExecutor.
-
- :author: M. Marin (KTH)
- :date: 1.11.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.multithread_executor.multithread.MultithreadExecutor
-
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.multithread_executor.multithread.DELEGATE_MARKER
-
-
-.. py:data:: DELEGATE_MARKER
- :annotation: = multithread_threat_init
-
-
-
-.. py:exception:: DagsterThreadError(*args, **kwargs)
-
- Bases: :py:obj:`dagster.core.errors.DagsterError`
-
- An exception has occurred in one or more of the threads dagster manages.
- This error forwards the message and stack trace for all of the collected errors.
-
- Initialize self. See help(type(self)) for accurate signature.
-
-
-.. py:class:: MultithreadExecutor(retries, max_concurrent=None)
-
- Bases: :py:obj:`dagster.core.executor.base.Executor`
-
- Helper class that provides a standard way to create an ABC using
- inheritance.
-
- .. py:method:: retries(self)
- :property:
-
- The Retries state / policy for this instance of the Executor. Executors should allow this to be
- controlled via configuration if possible.
-
- Returns: Retries
-
-
- .. py:method:: execute(self, pipeline_context, execution_plan)
-
- For the given context and execution plan, orchestrate a series of sub plan executions in a way that satisfies the whole plan being executed.
-
- :param pipeline_context: The pipeline execution context.
- :type pipeline_context: SystemPipelineExecutionContext
- :param execution_plan: The plan to execute.
- :type execution_plan: ExecutionPlan
-
- :returns: A stream of dagster events.
-
-
- .. py:method:: execute_step_in_thread(self, step_key, step_context, errors)
-
-
-
diff --git a/docs/source/autoapi/spine_engine/multithread_executor/thread_executor/index.rst b/docs/source/autoapi/spine_engine/multithread_executor/thread_executor/index.rst
deleted file mode 100644
index 90d94b0f..00000000
--- a/docs/source/autoapi/spine_engine/multithread_executor/thread_executor/index.rst
+++ /dev/null
@@ -1,133 +0,0 @@
-:py:mod:`spine_engine.multithread_executor.thread_executor`
-===========================================================
-
-.. py:module:: spine_engine.multithread_executor.thread_executor
-
-.. autoapi-nested-parse::
-
- Facilities for running arbitrary commands in child processes.
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.multithread_executor.thread_executor.ThreadEvent
- spine_engine.multithread_executor.thread_executor.ThreadStartEvent
- spine_engine.multithread_executor.thread_executor.ThreadDoneEvent
- spine_engine.multithread_executor.thread_executor.ThreadSystemErrorEvent
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.multithread_executor.thread_executor._execute_step_in_thread
- spine_engine.multithread_executor.thread_executor._poll_for_event
- spine_engine.multithread_executor.thread_executor.execute_thread_step
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.multithread_executor.thread_executor.TICK
- spine_engine.multithread_executor.thread_executor.THREAD_DEAD_AND_QUEUE_EMPTY
-
-
-.. py:class:: ThreadEvent
-
-
-.. py:class:: ThreadStartEvent
-
- Bases: :py:obj:`namedtuple`\ (\ :py:obj:`'ThreadStartEvent'`\ , :py:obj:`'tid'`\ ), :py:obj:`ThreadEvent`
-
- docstring
-
- Initialize self. See help(type(self)) for accurate signature.
-
-
-.. py:class:: ThreadDoneEvent
-
- Bases: :py:obj:`namedtuple`\ (\ :py:obj:`'ThreadDoneEvent'`\ , :py:obj:`'tid'`\ ), :py:obj:`ThreadEvent`
-
- docstring
-
- Initialize self. See help(type(self)) for accurate signature.
-
-
-.. py:class:: ThreadSystemErrorEvent
-
- Bases: :py:obj:`namedtuple`\ (\ :py:obj:`'ThreadSystemErrorEvent'`\ , :py:obj:`'tid error_info'`\ ), :py:obj:`ThreadEvent`
-
- docstring
-
- Initialize self. See help(type(self)) for accurate signature.
-
-
-.. py:exception:: ThreadCrashException
-
- Bases: :py:obj:`Exception`
-
- Thrown when the thread crashes.
-
- Initialize self. See help(type(self)) for accurate signature.
-
-
-.. py:function:: _execute_step_in_thread(event_queue, step_context, retries)
-
- Wraps the execution of a step.
-
- Handles errors and communicates across a queue with the parent process.
-
-
-.. py:data:: TICK
-
-
- The minimum interval at which to check for child process liveness -- default 20ms.
-
-.. py:data:: THREAD_DEAD_AND_QUEUE_EMPTY
- :annotation: = THREAD_DEAD_AND_QUEUE_EMPTY
-
- Sentinel value.
-
-.. py:function:: _poll_for_event(thread, event_queue)
-
-
-.. py:function:: execute_thread_step(step_context, retries)
-
- Execute a step in a new thread.
-
- This function starts a new thread whose execution target is the given step context wrapped by
- _execute_step_in_thread; polls the queue for events yielded by the thread
- until it dies and the queue is empty.
-
- This function yields a complex set of objects to enable having multiple thread
- executions in flight:
-
- * None - nothing has happened, yielded to enable cooperative multitasking other iterators
- * ThreadEvent - Family of objects that communicates state changes in the thread
- * KeyboardInterrupt - Yielded in the case that an interrupt was recieved while
- polling the thread. Yielded instead of raised to allow forwarding of the
- interrupt to the thread and completion of the iterator for this thread and
- any others that may be executing
- * The actual values yielded by the thread execution
-
- :param step_context: The step context to execute in the child process.
- :type step_context: SystemStepExecutionContext
- :param retries:
- :type retries: Retries
-
- Warning: if the thread is in an infinite loop, this will
- also infinitely loop.
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item/connection/index.rst b/docs/source/autoapi/spine_engine/project_item/connection/index.rst
deleted file mode 100644
index 51198c8c..00000000
--- a/docs/source/autoapi/spine_engine/project_item/connection/index.rst
+++ /dev/null
@@ -1,165 +0,0 @@
-:py:mod:`spine_engine.project_item.connection`
-==============================================
-
-.. py:module:: spine_engine.project_item.connection
-
-.. autoapi-nested-parse::
-
- Provides the :class:`Connection` class.
-
- :authors: A. Soininen (VTT)
- :date: 12.2.2021
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item.connection.Connection
-
-
-
-
-.. py:class:: Connection(source_name, source_position, destination_name, destination_position, resource_filters=None, options=None)
-
- Represents a connection between two project items.
-
- :param source_name: source project item's name
- :type source_name: str
- :param source_position: source anchor's position
- :type source_position: str
- :param destination_name: destination project item's name
- :type destination_name: str
- :param destination_position: destination anchor's position
- :type destination_position: str
- :param resource_filters: mapping from resource labels and filter types to
- database ids and activity flags
- :type resource_filters: dict, optional
- :param options: any options, at the moment only has "use_datapackage"
- :type options: dict, optional
-
- .. py:method:: __eq__(self, other)
-
- Return self==value.
-
-
- .. py:method:: name(self)
- :property:
-
-
- .. py:method:: destination_position(self)
- :property:
-
- Anchor's position on destination item.
-
-
- .. py:method:: source_position(self)
- :property:
-
- Anchor's position on source item.
-
-
- .. py:method:: database_resources(self)
- :property:
-
- Connection's database resources
-
-
- .. py:method:: has_filters(self)
-
- Return True if connection has filters.
-
- :returns: True if connection has filters, False otherwise
- :rtype: bool
-
-
- .. py:method:: resource_filters(self)
- :property:
-
- Connection's resource filters.
-
-
- .. py:method:: use_datapackage(self)
- :property:
-
-
- .. py:method:: id_to_name(self, id_, filter_type)
-
- Map from scenario/tool database id to name
-
-
- .. py:method:: receive_resources_from_source(self, resources)
-
- Receives resources from source item.
-
- :param resources: source item's resources
- :type resources: Iterable of ProjectItemResource
-
-
- .. py:method:: replace_resource_from_source(self, old, new)
-
- Replaces an existing resource.
-
- :param old: old resource
- :type old: ProjectItemResource
- :param new: new resource
- :type new: ProjectItemResource
-
-
- .. py:method:: fetch_database_items(self)
-
- Reads filter information from database.
-
-
- .. py:method:: set_online(self, resource, filter_type, online)
-
- Sets the given filters online or offline.
-
- :param resource: Resource label
- :type resource: str
- :param filter_type: Either SCENARIO_FILTER_TYPE or TOOL_FILTER_TYPE, for now.
- :type filter_type: str
- :param online: mapping from scenario/tool id to online flag
- :type online: dict
-
-
- .. py:method:: convert_resources(self, resources)
-
- Called when advertising resources through this connection *in the FORWARD direction*.
- Takes the initial list of resources advertised by the source item and returns a new list,
- which is the one finally advertised.
-
- At the moment it only packs CSVs into datapackage (and again, it's only used in the FORWARD direction).
-
- :param resources: Resources to convert
- :type resources: list of ProjectItemResource
-
- :returns: list of ProjectItemResource
-
-
- .. py:method:: to_dict(self)
-
- Returns a dictionary representation of this Connection.
-
- :returns: serialized Connection
- :rtype: dict
-
-
- .. py:method:: from_dict(connection_dict)
- :staticmethod:
-
- Restores a connection from dictionary.
-
- :param connection_dict: connection dictionary
- :type connection_dict: dict
-
- :returns: restored connection
- :rtype: Connection
-
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item/executable_item_base/index.rst b/docs/source/autoapi/spine_engine/project_item/executable_item_base/index.rst
deleted file mode 100644
index a68d6839..00000000
--- a/docs/source/autoapi/spine_engine/project_item/executable_item_base/index.rst
+++ /dev/null
@@ -1,186 +0,0 @@
-:py:mod:`spine_engine.project_item.executable_item_base`
-========================================================
-
-.. py:module:: spine_engine.project_item.executable_item_base
-
-.. autoapi-nested-parse::
-
- Contains ExecutableItem, a project item's counterpart in execution as well as support utilities.
-
- :authors: A. Soininen (VTT)
- :date: 30.3.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item.executable_item_base.ExecutableItemBase
-
-
-
-
-.. py:class:: ExecutableItemBase(name, project_dir, logger)
-
- The part of a project item that is executed by the Spine Engine.
-
- :param name: item's name
- :type name: str
- :param project_dir: absolute path to project directory
- :type project_dir: str
- :param logger: a logger
- :type logger: LoggerInterface
-
- .. py:method:: name(self)
- :property:
-
- Project item's name.
-
-
- .. py:method:: group_id(self)
- :property:
-
- Returns the id for group-execution.
- Items in the same group share a kernel, and also reuse the same kernel from past executions.
- By default each item is its own group, so it executes in isolation.
- NOTE: At the moment this is only used by Tool, but could be used by other items in the future?
-
- :returns: item's id within an execution group
- :rtype: str
-
-
- .. py:method:: filter_id(self)
- :property:
-
-
- .. py:method:: ready_to_execute(self, settings)
-
- Validates the internal state of this project item before execution.
-
- Subclasses can implement this method to do the appropriate work.
-
- :param settings: Application settings
- :type settings: AppSettings
-
- :returns: True if project item is ready for execution, False otherwise
- :rtype: bool
-
-
- .. py:method:: execute(self, forward_resources, backward_resources)
-
- Executes this item using the given resources and returns a boolean indicating the outcome.
-
- Subclasses can implement this method to do the appropriate work.
-
- :param forward_resources: a list of ProjectItemResources from predecessors (forward)
- :type forward_resources: list
- :param backward_resources: a list of ProjectItemResources from successors (backward)
- :type backward_resources: list
-
- :returns: State depending on operation success
- :rtype: ItemExecutionFinishState
-
-
- .. py:method:: exclude_execution(self, forward_resources, backward_resources)
-
- Excludes execution of this item.
-
- This method is called when the item is not selected (i.e EXCLUDED) for execution.
- Only lightweight bookkeeping or processing should be done in this case, e.g.
- forward input resources.
-
- Subclasses can implement this method to the appropriate work.
-
- :param forward_resources: a list of ProjectItemResources from predecessors (forward)
- :type forward_resources: list
- :param backward_resources: a list of ProjectItemResources from successors (backward)
- :type backward_resources: list
-
-
- .. py:method:: finish_execution(self, state)
-
- Does any work needed after execution given the execution success status.
-
- :param state: Item execution finish state
- :type state: ItemExecutionFinishState
-
-
- .. py:method:: item_type()
- :staticmethod:
- :abstractmethod:
-
- Returns the item's type identifier string.
-
-
- .. py:method:: output_resources(self, direction)
-
- Returns output resources in the given direction.
-
- Subclasses need to implement _output_resources_backward and/or _output_resources_forward
- if they want to provide resources in any direction.
-
- :param direction: Direction where output resources are passed
- :type direction: ExecutionDirection
-
- :returns: a list of ProjectItemResources
- :rtype: list
-
-
- .. py:method:: stop_execution(self)
-
- Stops executing this item.
-
-
- .. py:method:: _output_resources_forward(self)
-
- Returns output resources for forward execution.
-
- The default implementation returns an empty list.
-
- :returns: a list of ProjectItemResources
- :rtype: list
-
-
- .. py:method:: _output_resources_backward(self)
-
- Returns output resources for backward execution.
-
- The default implementation returns an empty list.
-
- :returns: a list of ProjectItemResources
- :rtype: list
-
-
- .. py:method:: from_dict(cls, item_dict, name, project_dir, app_settings, specifications, logger)
- :classmethod:
- :abstractmethod:
-
- Deserializes an executable item from item dictionary.
-
- :param item_dict: serialized project item
- :type item_dict: dict
- :param name: item's name
- :type name: str
- :param project_dir: absolute path to the project directory
- :type project_dir: str
- :param app_settings: Toolbox settings
- :type app_settings: QSettings
- :param specifications: mapping from item type to specification name to :class:`ProjectItemSpecification`
- :type specifications: dict
- :param logger: a logger
- :type logger: LoggingInterface
-
- :returns: deserialized executable item
- :rtype: ExecutableItemBase
-
-
- .. py:method:: _get_specification(name, item_type, specification_name, specifications, logger)
- :staticmethod:
-
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item/index.rst b/docs/source/autoapi/spine_engine/project_item/index.rst
deleted file mode 100644
index 20ec7975..00000000
--- a/docs/source/autoapi/spine_engine/project_item/index.rst
+++ /dev/null
@@ -1,28 +0,0 @@
-:py:mod:`spine_engine.project_item`
-===================================
-
-.. py:module:: spine_engine.project_item
-
-.. autoapi-nested-parse::
-
- This subpackage contains base classes for project items.
-
- :authors: M. Marin (KTH)
- :date: 8.10.2020
-
-
-
-Submodules
-----------
-.. toctree::
- :titlesonly:
- :maxdepth: 1
-
- connection/index.rst
- executable_item_base/index.rst
- project_item_info/index.rst
- project_item_resource/index.rst
- project_item_specification/index.rst
- project_item_specification_factory/index.rst
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_info/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_info/index.rst
deleted file mode 100644
index 35e27fa3..00000000
--- a/docs/source/autoapi/spine_engine/project_item/project_item_info/index.rst
+++ /dev/null
@@ -1,50 +0,0 @@
-:py:mod:`spine_engine.project_item.project_item_info`
-=====================================================
-
-.. py:module:: spine_engine.project_item.project_item_info
-
-.. autoapi-nested-parse::
-
- Provides the ProjectItemInfo class.
-
- :authors: A. Soininen (VTT)
- :date: 29.4.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item.project_item_info.ProjectItemInfo
-
-
-
-
-.. py:class:: ProjectItemInfo
-
- .. py:method:: item_category()
- :staticmethod:
- :abstractmethod:
-
- Returns the item category string, e.g., "Tools".
-
- :returns: item's category
- :rtype: str
-
-
- .. py:method:: item_type()
- :staticmethod:
- :abstractmethod:
-
- Returns the item type string, e.g., "Gimlet".
-
- :returns: item's type
- :rtype: str
-
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_resource/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_resource/index.rst
deleted file mode 100644
index 4338f57f..00000000
--- a/docs/source/autoapi/spine_engine/project_item/project_item_resource/index.rst
+++ /dev/null
@@ -1,198 +0,0 @@
-:py:mod:`spine_engine.project_item.project_item_resource`
-=========================================================
-
-.. py:module:: spine_engine.project_item.project_item_resource
-
-.. autoapi-nested-parse::
-
- Provides the ProjectItemResource class.
-
- :authors: M. Marin (KTH)
- :date: 29.4.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item.project_item_resource.ProjectItemResource
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item.project_item_resource.database_resource
- spine_engine.project_item.project_item_resource.file_resource
- spine_engine.project_item.project_item_resource.transient_file_resource
- spine_engine.project_item.project_item_resource.file_resource_in_pack
- spine_engine.project_item.project_item_resource.extract_packs
-
-
-
-.. py:class:: ProjectItemResource(provider_name, type_, label, url=None, metadata=None)
-
- Class to hold a resource made available by a project item and that may be consumed by another project item.
-
- .. attribute:: provider_name
-
- name of resource provider
-
- :type: str
-
- .. attribute:: type_
-
- resource's type
-
- :type: str
-
- .. attribute:: label
-
- an identifier string
-
- :type: str
-
- .. attribute:: metadata
-
- resource's metadata
-
- :type: dict
-
- :param provider_name: The name of the item that provides the resource
- :type provider_name: str
- :param type_: The resource type, currently available types:
-
- - "file": url points to the file's path
- - "file_pack": resource is part of a pack; url points to the file's path
- - "database": url is the databases url
- :type type_: str
- :param label: A label that identifies the resource.
- :type label: str
- :param url: The url of the resource.
- :type url: str, optional
- :param metadata: Additional metadata providing extra information about the resource.
- Currently available keys:
-
- - filter_stack (str): resource's filter stack
- - filter_id (str): filter id
- :type metadata: dict
-
- .. py:method:: clone(self, additional_metadata=None)
-
- Clones a resource and optionally updates the clone's metadata.
-
- :param additional_metadata: metadata to add to the clone
- :type additional_metadata: dict
-
- :returns: cloned resource
- :rtype: ProjectItemResource
-
-
- .. py:method:: __eq__(self, other)
-
- Return self==value.
-
-
- .. py:method:: __hash__(self)
-
- Return hash(self).
-
-
- .. py:method:: __repr__(self)
-
- Return repr(self).
-
-
- .. py:method:: url(self)
- :property:
-
- Resource URL.
-
-
- .. py:method:: path(self)
- :property:
-
- Returns the resource path in the local syntax, as obtained from parsing the url.
-
-
- .. py:method:: scheme(self)
- :property:
-
- Returns the resource scheme, as obtained from parsing the url.
-
-
- .. py:method:: hasfilepath(self)
- :property:
-
-
- .. py:method:: arg(self)
- :property:
-
-
-
-.. py:function:: database_resource(provider_name, url, label=None)
-
- Constructs a database resource.
-
- :param provider_name: resource provider's name
- :type provider_name: str
- :param url: database URL
- :type url: str
- :param label: resource label
- :type label: str, optional
-
-
-.. py:function:: file_resource(provider_name, file_path, label=None)
-
- Constructs a file resource.
-
- :param provider_name: resource provider's name
- :type provider_name: str
- :param file_path: path to file
- :type file_path: str
- :param label: resource label
- :type label: str, optional
-
-
-.. py:function:: transient_file_resource(provider_name, label, file_path=None)
-
- Constructs a transient file resource.
-
- :param provider_name: resource provider's name
- :type provider_name: str
- :param label: resource label
- :type label: str
- :param file_path: file path if the file exists
- :type file_path: str, optional
-
-
-.. py:function:: file_resource_in_pack(provider_name, label, file_path=None)
-
- Constructs a file resource that is part of a resource pack.
-
- :param provider_name: resource provider's name
- :type provider_name: str
- :param label: resource label
- :type label: str
- :param file_path: file path if the file exists
- :type file_path: str, optional
-
-
-.. py:function:: extract_packs(resources)
-
- Extracts file packs from resources.
-
- :param resources: resources to process
- :type resources: Iterable of ProjectItemResource
-
- :returns: list of non-pack resources and dictionary of packs keyed by label
- :rtype: tuple
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_specification/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_specification/index.rst
deleted file mode 100644
index a7d1626a..00000000
--- a/docs/source/autoapi/spine_engine/project_item/project_item_specification/index.rst
+++ /dev/null
@@ -1,101 +0,0 @@
-:py:mod:`spine_engine.project_item.project_item_specification`
-==============================================================
-
-.. py:module:: spine_engine.project_item.project_item_specification
-
-.. autoapi-nested-parse::
-
- Contains project item specification class.
-
- :authors: M. Marin (KTH)
- :date: 7.5.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item.project_item_specification.ProjectItemSpecification
-
-
-
-
-.. py:class:: ProjectItemSpecification(name, description=None, item_type='', item_category='')
-
- Class to hold a project item specification.
-
- .. attribute:: item_type
-
- type of the project item the specification is compatible with
-
- :type: str
-
- .. attribute:: definition_file_path
-
- specification's JSON file path
-
- :type: str
-
- :param name: specification name
- :type name: str
- :param description: description
- :type description: str
- :param item_type: Project item type
- :type item_type: str
- :param item_category: Project item category
- :type item_category: str
-
- .. py:method:: set_name(self, name)
-
- Set object name and short name.
- Note: Check conflicts (e.g. name already exists)
- before calling this method.
-
- :param name: New (long) name for this object
- :type name: str
-
-
- .. py:method:: set_description(self, description)
-
- Set object description.
-
- :param description: Object description
- :type description: str
-
-
- .. py:method:: save(self)
- :abstractmethod:
-
- Writes the specification to the path given by ``self.definition_file_path``
-
- :returns: True if the operation was successful, False otherwise
- :rtype: bool
-
-
- .. py:method:: to_dict(self)
- :abstractmethod:
-
- Returns a dict for the specification.
-
- :returns: specification dict
- :rtype: dict
-
-
- .. py:method:: is_equivalent(self, other)
- :abstractmethod:
-
- Returns True if two specifications are essentially the same.
-
- :param other: specification to compare to
- :type other: DataTransformerSpecification
-
- :returns: True if the specifications are equivalent, False otherwise
- :rtype: bool
-
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item/project_item_specification_factory/index.rst b/docs/source/autoapi/spine_engine/project_item/project_item_specification_factory/index.rst
deleted file mode 100644
index d423ff65..00000000
--- a/docs/source/autoapi/spine_engine/project_item/project_item_specification_factory/index.rst
+++ /dev/null
@@ -1,56 +0,0 @@
-:py:mod:`spine_engine.project_item.project_item_specification_factory`
-======================================================================
-
-.. py:module:: spine_engine.project_item.project_item_specification_factory
-
-.. autoapi-nested-parse::
-
- Contains project item specification factory.
-
- :authors: A. Soininen (VTT)
- :date: 6.5.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item.project_item_specification_factory.ProjectItemSpecificationFactory
-
-
-
-
-.. py:class:: ProjectItemSpecificationFactory
-
- A factory to make project item specifications.
-
- .. py:method:: item_type()
- :staticmethod:
- :abstractmethod:
-
- Returns the project item's type.
-
-
- .. py:method:: make_specification(definition, app_settings, logger)
- :staticmethod:
- :abstractmethod:
-
- Makes a project item specification.
-
- :param definition: specification's definition dictionary
- :type definition: dict
- :param app_settings: Toolbox settings
- :type app_settings: QSettings
- :param logger: a logger
- :type logger: LoggerInterface
-
- :returns: a specification built from the given definition
- :rtype: ProjectItemSpecification
-
-
-
diff --git a/docs/source/autoapi/spine_engine/project_item_loader/index.rst b/docs/source/autoapi/spine_engine/project_item_loader/index.rst
deleted file mode 100644
index e30b78ed..00000000
--- a/docs/source/autoapi/spine_engine/project_item_loader/index.rst
+++ /dev/null
@@ -1,74 +0,0 @@
-:py:mod:`spine_engine.project_item_loader`
-==========================================
-
-.. py:module:: spine_engine.project_item_loader
-
-.. autoapi-nested-parse::
-
- Contains :class:`ProjectItemLoader`.
-
- :author: A. Soininen (VTT)
- :date: 11.2.2021
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.project_item_loader.ProjectItemLoader
-
-
-
-
-.. py:class:: ProjectItemLoader
-
- A singleton class for loading project items from multiple processes simultaneously.
-
- .. py:attribute:: _specification_factories
-
-
-
-
- .. py:attribute:: _executable_item_classes
-
-
-
-
- .. py:attribute:: _specification_factories_lock
-
-
-
-
- .. py:attribute:: _executable_item_classes_lock
-
-
-
-
- .. py:method:: load_item_specification_factories(self, items_module_name)
-
- Loads the project item specification factories in the standard Toolbox package.
-
- :param items_module_name: name of the Python module that contains the project items
- :type items_module_name: str
-
- :returns: a map from item type to specification factory
- :rtype: dict
-
-
- .. py:method:: load_executable_item_classes(self, items_module_name)
-
- Loads the project item executable classes included in the standard Toolbox package.
-
- :param items_module_name: name of the Python module that contains the project items
- :type items_module_name: str
-
- :returns: a map from item type to the executable item class
- :rtype: dict
-
-
-
diff --git a/docs/source/autoapi/spine_engine/spine_engine/index.rst b/docs/source/autoapi/spine_engine/spine_engine/index.rst
deleted file mode 100644
index 9bad5745..00000000
--- a/docs/source/autoapi/spine_engine/spine_engine/index.rst
+++ /dev/null
@@ -1,395 +0,0 @@
-:py:mod:`spine_engine.spine_engine`
-===================================
-
-.. py:module:: spine_engine.spine_engine
-
-.. autoapi-nested-parse::
-
- Contains the SpineEngine class for running Spine Toolbox DAGs.
-
- :authors: M. Marin (KTH)
- :date: 20.11.2019
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.spine_engine.ExecutionDirection
- spine_engine.spine_engine.SpineEngineState
- spine_engine.spine_engine.ItemExecutionFinishState
- spine_engine.spine_engine.SpineEngine
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.spine_engine._shorten
- spine_engine.spine_engine._make_filter_id
- spine_engine.spine_engine._filter_names_from_stack
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.spine_engine.ED
-
-
-.. py:class:: ExecutionDirection
-
- Bases: :py:obj:`enum.Enum`
-
- Generic enumeration.
-
- Derive from this class to define new enumerations.
-
- .. py:attribute:: FORWARD
-
-
-
-
- .. py:attribute:: BACKWARD
-
-
-
-
- .. py:method:: __str__(self)
-
- Return str(self).
-
-
-
-.. py:data:: ED
-
-
-
-
-.. py:class:: SpineEngineState
-
- Bases: :py:obj:`enum.Enum`
-
- Generic enumeration.
-
- Derive from this class to define new enumerations.
-
- .. py:attribute:: SLEEPING
- :annotation: = 1
-
-
-
- .. py:attribute:: RUNNING
- :annotation: = 2
-
-
-
- .. py:attribute:: USER_STOPPED
- :annotation: = 3
-
-
-
- .. py:attribute:: FAILED
- :annotation: = 4
-
-
-
- .. py:attribute:: COMPLETED
- :annotation: = 5
-
-
-
- .. py:method:: __str__(self)
-
- Return str(self).
-
-
-
-.. py:class:: ItemExecutionFinishState
-
- Bases: :py:obj:`enum.Enum`
-
- Generic enumeration.
-
- Derive from this class to define new enumerations.
-
- .. py:attribute:: SUCCESS
- :annotation: = 1
-
-
-
- .. py:attribute:: FAILURE
- :annotation: = 2
-
-
-
- .. py:attribute:: SKIPPED
- :annotation: = 3
-
-
-
- .. py:attribute:: EXCLUDED
- :annotation: = 4
-
-
-
- .. py:attribute:: STOPPED
- :annotation: = 5
-
-
-
- .. py:attribute:: NEVER_FINISHED
- :annotation: = 6
-
-
-
- .. py:method:: __str__(self)
-
- Return str(self).
-
-
-
-.. py:class:: SpineEngine(items=None, specifications=None, connections=None, items_module_name='spine_items', settings=None, project_dir=None, execution_permits=None, node_successors=None, debug=False)
-
- An engine for executing a Spine Toolbox DAG-workflow.
-
- :param items: List of executable item dicts.
- :type items: list(dict)
- :param specifications: A mapping from item type to list of specification dicts.
- :type specifications: dict(str,list(dict))
- :param connections: List of connection dicts
- :type connections: list of dict
- :param items_module_name: name of the Python module that contains project items
- :type items_module_name: str
- :param settings: Toolbox execution settings.
- :type settings: dict
- :param project_dir: Path to project directory.
- :type project_dir: str
- :param execution_permits: A mapping from item name to a boolean value, False indicating that
- the item is not executed, only its resources are collected.
- :type execution_permits: dict(str,bool)
- :param node_successors: A mapping from item name to list of successor item names, dictating the dependencies.
- :type node_successors: dict(str,list(str))
- :param debug: Whether debug mode is active or not.
- :type debug: bool
-
- .. py:method:: _make_item_specifications(self, specifications, project_item_loader, items_module_name)
-
- Instantiates item specifications.
-
- :param specifications: A mapping from item type to list of specification dicts.
- :type specifications: dict
- :param project_item_loader: loader instance
- :type project_item_loader: ProjectItemLoader
- :param items_module_name: name of the Python module that contains the project items
- :type items_module_name: str
-
- :returns: mapping from item type to a dict that maps specification names to specification instances
- :rtype: dict
-
-
- .. py:method:: _make_item(self, item_name, direction)
-
- Recreates item from project item dictionary. Note that all items are created twice.
- One for the backward pipeline, the other one for the forward pipeline.
-
-
- .. py:method:: get_event(self)
-
- Returns the next event in the stream. Calling this after receiving the event of type "dag_exec_finished"
- will raise StopIterationError.
-
-
- .. py:method:: state(self)
-
-
- .. py:method:: _get_event_stream(self)
-
- Returns an iterator of tuples (event_type, event_data).
-
- TODO: Describe the events in depth.
-
-
- .. py:method:: answer_prompt(self, item_name, accepted)
-
-
- .. py:method:: run(self)
-
- Runs this engine.
-
-
- .. py:method:: _process_event(self, event)
-
- Processes events from a pipeline.
-
- :param event: an event
- :type event: DagsterEvent
-
-
- .. py:method:: stop(self)
-
- Stops the engine.
-
-
- .. py:method:: _stop_item(self, item)
-
-
- .. py:method:: _make_pipeline(self)
-
- Returns a PipelineDefinition for executing this engine.
-
- :returns: PipelineDefinition
-
-
- .. py:method:: _make_backward_solid_def(self, item_name)
-
- Returns a SolidDefinition for executing the given item in the backward sweep.
-
- :param item_name: The project item that gets executed by the solid.
- :type item_name: str
-
-
- .. py:method:: _make_forward_solid_def(self, item_name)
-
- Returns a SolidDefinition for executing the given item.
-
- :param item_name:
- :type item_name: str
-
- :returns: SolidDefinition
-
-
- .. py:method:: _execute_item(self, context, item_name, forward_resource_stacks, backward_resources)
-
- Executes the given item using the given forward resource stacks and backward resources.
- Returns list of output resource stacks.
-
- Called by ``_make_forward_solid_def.compute_fn``.
-
- For each element yielded by ``_filtered_resources_iterator``, spawns a thread that runs ``_execute_item_filtered``.
-
- :param context:
- :param item_name:
- :type item_name: str
- :param forward_resource_stacks:
- :type forward_resource_stacks: list(tuple(ProjectItemResource))
- :param backward_resources:
- :type backward_resources: list(ProjectItemResource)
-
- :returns: list(tuple(ProjectItemResource))
-
-
- .. py:method:: _execute_item_filtered(self, item, filtered_forward_resources, filtered_backward_resources, output_resources_list, success)
-
- Executes the given item using the given filtered resources. Target for threads in ``_execute_item``.
-
- :param item:
- :type item: ExecutableItemBase
- :param filtered_forward_resources:
- :type filtered_forward_resources: list(ProjectItemResource)
- :param filtered_backward_resources:
- :type filtered_backward_resources: list(ProjectItemResource)
- :param output_resources_list: A list of lists, to append the
- output resources generated by the item.
- :type output_resources_list: list(list(ProjectItemResource))
- :param success: A list of one element, to write the outcome of the execution.
- :type success: list
-
-
- .. py:method:: _filtered_resources_iterator(self, item_name, forward_resource_stacks, backward_resources, timestamp)
-
- Yields tuples of (filtered forward resources, filtered backward resources, filter id).
-
- Each tuple corresponds to a unique filter combination. Combinations are obtained by applying the cross-product
- over forward resource stacks as yielded by ``_forward_resource_stacks_iterator``.
-
- :param item_name:
- :type item_name: str
- :param forward_resource_stacks:
- :type forward_resource_stacks: list(tuple(ProjectItemResource))
- :param backward_resources:
- :type backward_resources: list(ProjectItemResource)
- :param timestamp: timestamp for the execution filter
- :type timestamp: str
-
- :returns: forward resources, backward resources, filter id
- :rtype: Iterator(tuple(list,list,str))
-
-
- .. py:method:: _expand_resource_stack(self, item_name, resource_stack)
-
- Expands a resource stack if possible.
-
- If the stack has more than one resource, returns the unaltered stack.
-
- Otherwise, if the stack has only one resource but there are no filters defined for that resource,
- again, returns the unaltered stack.
-
- Otherwise, returns an expanded stack of as many resources as filter stacks defined for the only one resource.
- Each resource in the expanded stack is a clone of the original, with one of the filter stacks
- applied to the URL.
-
- :param item_name: resource receiving item's name
- :type item_name: str
- :param resource_stack:
- :type resource_stack: tuple(ProjectItemResource)
-
- :returns: tuple(ProjectItemResource)
-
-
- .. py:method:: _filter_stacks(self, item_name, resource_label)
-
- Computes filter stacks.
-
- Stacks are computed as the cross-product of all individual filters defined for a resource.
-
- :param item_name: item's name
- :type item_name: str
- :param resource_label: resource's label
- :type resource_label: str
-
- :returns: filter stacks
- :rtype: list of list
-
-
- .. py:method:: _convert_forward_resources(self, item_name, resources)
-
- Converts resources as they're being forwarded to given item.
- The conversion is dictated by the connection the resources traverse in order to reach the item.
-
- :param item_name: receiving item's name
- :type item_name: str
- :param resources: resources to convert
- :type resources: list of ProjectItemResource
-
- :returns: converted resources
- :rtype: list of ProjectItemResource
-
-
- .. py:method:: _make_dependencies(self)
-
- Returns a dictionary of dependencies according to the given dictionaries of injectors.
-
- :returns: a dictionary to pass to the PipelineDefinition constructor as dependencies
- :rtype: dict
-
-
-
-.. py:function:: _shorten(resource)
-
-
-.. py:function:: _make_filter_id(resource_filter_stack)
-
-
-.. py:function:: _filter_names_from_stack(stack)
-
-
diff --git a/docs/source/autoapi/spine_engine/spine_engine_server/index.rst b/docs/source/autoapi/spine_engine/spine_engine_server/index.rst
deleted file mode 100644
index b2a04d99..00000000
--- a/docs/source/autoapi/spine_engine/spine_engine_server/index.rst
+++ /dev/null
@@ -1,143 +0,0 @@
-:py:mod:`spine_engine.spine_engine_server`
-==========================================
-
-.. py:module:: spine_engine.spine_engine_server
-
-.. autoapi-nested-parse::
-
- Contains the SpineEngineServer class.
-
- :authors: M. Marin (KTH)
- :date: 7.11.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.spine_engine_server.EngineRequestHandler
- spine_engine.spine_engine_server.SpineEngineServer
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.spine_engine_server.start_spine_engine_server
- spine_engine.spine_engine_server._shutdown_servers
-
-
-
-Attributes
-~~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.spine_engine_server._servers
-
-
-.. py:class:: EngineRequestHandler(request, client_address, server)
-
- Bases: :py:obj:`socketserver.BaseRequestHandler`
-
- The request handler class for our server.
-
- .. py:attribute:: _engines
-
-
-
-
- .. py:attribute:: _ENCODING
- :annotation: = ascii
-
-
-
- .. py:method:: _run_engine(self, data)
-
- Creates and engine and runs it.
-
- :param data: data to be passed as keyword arguments to SpineEngine()
- :type data: dict
-
- :returns: engine id, for further calls
- :rtype: str
-
-
- .. py:method:: _get_engine_event(self, engine_id)
-
- Gets the next event in the engine's execution stream.
-
- :param engine_id: the engine id, must have been returned by run.
- :type engine_id: str
-
- :returns: two element tuple: event type identifier string, and event data dictionary
- :rtype: tuple(str,dict)
-
-
- .. py:method:: _stop_engine(self, engine_id)
-
- Stops the engine.
-
- :param engine_id: the engine id, must have been returned by run.
- :type engine_id: str
-
-
- .. py:method:: _restart_kernel(self, connection_file)
-
- Restarts the jupyter kernel associated to given connection file.
-
- :param connection_file: path of connection file
- :type connection_file: str
-
-
- .. py:method:: _shutdown_kernel(self, connection_file)
-
- Shuts down the jupyter kernel associated to given connection file.
-
- :param connection_file: path of connection file
- :type connection_file: str
-
-
- .. py:method:: handle(self)
-
-
- .. py:method:: _recvall(self)
-
- Receives and returns all data in the request.
-
- :returns: str
-
-
-
-.. py:class:: SpineEngineServer(server_address, RequestHandlerClass, bind_and_activate=True)
-
- Bases: :py:obj:`socketserver.ThreadingMixIn`, :py:obj:`socketserver.TCPServer`
-
- Mix-in class to handle each request in a new thread.
-
- Constructor. May be extended, do not override.
-
- .. py:attribute:: allow_reuse_address
- :annotation: = True
-
-
-
-
-.. py:data:: _servers
- :annotation: = []
-
-
-
-.. py:function:: start_spine_engine_server(host, port)
-
-
-.. py:function:: _shutdown_servers()
-
-
diff --git a/docs/source/autoapi/spine_engine/utils/command_line_arguments/index.rst b/docs/source/autoapi/spine_engine/utils/command_line_arguments/index.rst
deleted file mode 100644
index 091c0a81..00000000
--- a/docs/source/autoapi/spine_engine/utils/command_line_arguments/index.rst
+++ /dev/null
@@ -1,42 +0,0 @@
-:py:mod:`spine_engine.utils.command_line_arguments`
-===================================================
-
-.. py:module:: spine_engine.utils.command_line_arguments
-
-.. autoapi-nested-parse::
-
- Split command line arguments.
-
- :authors: P. Savolainen (VTT)
- :date: 10.1.2018
-
-
-
-Module Contents
----------------
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.utils.command_line_arguments.split_cmdline_args
-
-
-
-.. py:function:: split_cmdline_args(arg_string)
-
- Splits a string of command line arguments into a list of tokens.
-
- Things in single ('') and double ("") quotes are kept as single tokens
- while the quotes themselves are stripped away.
- Thus, `--file="a long quoted 'file' name.txt` becomes ["--file=a long quoted 'file' name.txt"]
-
- :param arg_string: command line arguments as a string
- :type arg_string: str
-
- :returns: a list of tokens
- :rtype: list
-
-
diff --git a/docs/source/autoapi/spine_engine/utils/helpers/index.rst b/docs/source/autoapi/spine_engine/utils/helpers/index.rst
deleted file mode 100644
index a4b807e9..00000000
--- a/docs/source/autoapi/spine_engine/utils/helpers/index.rst
+++ /dev/null
@@ -1,167 +0,0 @@
-:py:mod:`spine_engine.utils.helpers`
-====================================
-
-.. py:module:: spine_engine.utils.helpers
-
-.. autoapi-nested-parse::
-
- Helpers functions and classes.
-
- :authors: M. Marin (KTH)
- :date: 20.11.2019
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.utils.helpers.Singleton
- spine_engine.utils.helpers.AppSettings
-
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.utils.helpers.shorten
- spine_engine.utils.helpers.create_log_file_timestamp
- spine_engine.utils.helpers.create_timestamp
- spine_engine.utils.helpers.resolve_conda_executable
- spine_engine.utils.helpers.resolve_python_interpreter
- spine_engine.utils.helpers.resolve_julia_executable
- spine_engine.utils.helpers.resolve_gams_executable
- spine_engine.utils.helpers.resolve_executable_from_path
- spine_engine.utils.helpers.inverted
- spine_engine.utils.helpers.get_julia_command
- spine_engine.utils.helpers.get_julia_env
-
-
-
-.. py:class:: Singleton
-
- Bases: :py:obj:`type`
-
- .. py:attribute:: _instances
-
-
-
-
- .. py:method:: __call__(cls, *args, **kwargs)
-
- Call self as a function.
-
-
-
-.. py:class:: AppSettings(settings)
-
- A QSettings replacement.
-
- Init.
-
- :param settings:
- :type settings: dict
-
- .. py:method:: value(self, key, defaultValue='')
-
-
-
-.. py:function:: shorten(name)
-
- Returns the 'short name' version of given name.
-
-
-.. py:function:: create_log_file_timestamp()
-
- Creates a new timestamp string that is used as Data Store and Importer error log file.
-
- :returns: Timestamp string or empty string if failed.
-
-
-.. py:function:: create_timestamp()
-
-
-.. py:function:: resolve_conda_executable(conda_path)
-
- If given conda_path is an empty str, returns current Conda
- executable from CONDA_EXE env variable if the app was started
- on Conda, otherwise returns an empty string.
-
-
-.. py:function:: resolve_python_interpreter(python_path)
-
- If given python_path is empty, returns the
- full path to Python interpreter depending on user's
- settings and whether the app is frozen or not.
-
-
-.. py:function:: resolve_julia_executable(julia_path)
-
- if given julia_path is empty, tries to find the path to Julia
- in user's PATH env variable. If Julia is not found in PATH,
- returns an empty string.
-
- Note: In the long run, we should decide whether this is something we want to do
- because adding julia-x.x./bin/ dir to the PATH is not recommended because this
- also exposes some .dlls to other programs on user's (windows) system. I.e. it
- may break other programs, and this is why the Julia installer does not
- add (and does not even offer the chance to add) Julia to PATH.
-
-
-.. py:function:: resolve_gams_executable(gams_path)
-
- if given gams_path is empty, tries to find the path to Gams
- in user's PATH env variable. If Gams is not found in PATH,
- returns an empty string.
-
-
-.. py:function:: resolve_executable_from_path(executable_name)
-
- Returns full path to executable name in user's
- PATH env variable. If not found, returns an empty string.
-
- Basically equivalent to 'where' and 'which' commands in
- cmd.exe and bash respectively.
-
- :param executable_name: Executable filename to find (e.g. python.exe, julia.exe)
- :type executable_name: str
-
- :returns: Full path or empty string
- :rtype: str
-
-
-.. py:function:: inverted(input_)
-
- Inverts a dictionary of list values.
-
- :param input_:
- :type input_: dict
-
- :returns: keys are list items, and values are keys listing that item from the input dictionary
- :rtype: dict
-
-
-.. py:function:: get_julia_command(settings)
-
- :param settings:
- :type settings: QSettings, AppSettings
-
- :returns: e.g. ["path/to/julia", "--project=path/to/project/"]
- :rtype: list
-
-
-.. py:function:: get_julia_env(settings)
-
- :param settings:
- :type settings: QSettings, AppSettings
-
- :returns: (julia_exe, julia_project), or None if none found
- :rtype: tuple, NoneType
-
-
diff --git a/docs/source/autoapi/spine_engine/utils/index.rst b/docs/source/autoapi/spine_engine/utils/index.rst
deleted file mode 100644
index 1128cce4..00000000
--- a/docs/source/autoapi/spine_engine/utils/index.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-:py:mod:`spine_engine.utils`
-============================
-
-.. py:module:: spine_engine.utils
-
-
-Submodules
-----------
-.. toctree::
- :titlesonly:
- :maxdepth: 1
-
- command_line_arguments/index.rst
- helpers/index.rst
- queue_logger/index.rst
- returning_process/index.rst
- serialization/index.rst
-
-
diff --git a/docs/source/autoapi/spine_engine/utils/queue_logger/index.rst b/docs/source/autoapi/spine_engine/utils/queue_logger/index.rst
deleted file mode 100644
index 2278fa0c..00000000
--- a/docs/source/autoapi/spine_engine/utils/queue_logger/index.rst
+++ /dev/null
@@ -1,69 +0,0 @@
-:py:mod:`spine_engine.utils.queue_logger`
-=========================================
-
-.. py:module:: spine_engine.utils.queue_logger
-
-.. autoapi-nested-parse::
-
- The QueueLogger class.
-
- :authors: M. Marin (KTH)
- :date: 3.11.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.utils.queue_logger._Message
- spine_engine.utils.queue_logger._Prompt
- spine_engine.utils.queue_logger._ExecutionMessage
- spine_engine.utils.queue_logger.QueueLogger
-
-
-
-
-.. py:class:: _Message(queue, event_type, msg_type, item_name)
-
- .. py:method:: filter_id(self)
- :property:
-
-
- .. py:method:: emit(self, msg_text)
-
-
-
-.. py:class:: _Prompt(queue, item_name, prompt_queue)
-
- .. py:method:: filter_id(self)
- :property:
-
-
- .. py:method:: emit(self, prompt)
-
-
-
-.. py:class:: _ExecutionMessage(queue, event_type, item_name)
-
- .. py:method:: filter_id(self)
- :property:
-
-
- .. py:method:: emit(self, msg)
-
-
-
-.. py:class:: QueueLogger(queue, item_name, prompt_queue)
-
- A :class:`LoggerInterface` compliant logger that puts messages into a Queue.
-
-
- .. py:method:: set_filter_id(self, filter_id)
-
-
-
diff --git a/docs/source/autoapi/spine_engine/utils/returning_process/index.rst b/docs/source/autoapi/spine_engine/utils/returning_process/index.rst
deleted file mode 100644
index e27006a8..00000000
--- a/docs/source/autoapi/spine_engine/utils/returning_process/index.rst
+++ /dev/null
@@ -1,54 +0,0 @@
-:py:mod:`spine_engine.utils.returning_process`
-==============================================
-
-.. py:module:: spine_engine.utils.returning_process
-
-.. autoapi-nested-parse::
-
- The ReturningProcess class.
-
- :authors: M. Marin (KTH)
- :date: 3.11.2020
-
-
-
-Module Contents
----------------
-
-Classes
-~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.utils.returning_process.ReturningProcess
-
-
-
-
-.. py:class:: ReturningProcess(*args, **kwargs)
-
- Bases: :py:obj:`multiprocessing.Process`
-
- Process objects represent activity that is run in a separate process
-
- The class is analogous to `threading.Thread`
-
- .. py:method:: run_until_complete(self)
-
- Starts the process and joins it after it has finished.
-
- :returns: Return value of the process where the first element is a status flag
- :rtype: tuple
-
-
- .. py:method:: run(self)
-
- Method to be run in sub-process; can be overridden in sub-class
-
-
- .. py:method:: terminate(self)
-
- Terminate process; sends SIGTERM signal or uses TerminateProcess()
-
-
-
diff --git a/docs/source/autoapi/spine_engine/utils/serialization/index.rst b/docs/source/autoapi/spine_engine/utils/serialization/index.rst
deleted file mode 100644
index c6081fb9..00000000
--- a/docs/source/autoapi/spine_engine/utils/serialization/index.rst
+++ /dev/null
@@ -1,82 +0,0 @@
-:py:mod:`spine_engine.utils.serialization`
-==========================================
-
-.. py:module:: spine_engine.utils.serialization
-
-.. autoapi-nested-parse::
-
- Functions to (de)serialize stuff.
-
- :authors: P. Savolainen (VTT)
- :date: 10.1.2018
-
-
-
-Module Contents
----------------
-
-
-Functions
-~~~~~~~~~
-
-.. autoapisummary::
-
- spine_engine.utils.serialization.path_in_dir
- spine_engine.utils.serialization.serialize_path
- spine_engine.utils.serialization.serialize_url
- spine_engine.utils.serialization.deserialize_path
- spine_engine.utils.serialization.deserialize_remote_path
-
-
-
-.. py:function:: path_in_dir(path, directory)
-
- Returns True if the given path is in the given directory.
-
-
-.. py:function:: serialize_path(path, project_dir)
-
- Returns a dict representation of the given path.
-
- If path is in project_dir, converts the path to relative.
-
- :param path: path to serialize
- :type path: str
- :param project_dir: path to the project directory
- :type project_dir: str
-
- :returns: Dictionary representing the given path
- :rtype: dict
-
-
-.. py:function:: serialize_url(url, project_dir)
-
- Return a dict representation of the given URL.
-
- If the URL is a file that is in project dir, the URL is converted to a relative path.
-
- :param url: a URL to serialize
- :type url: str
- :param project_dir: path to the project directory
- :type project_dir: str
-
- :returns: Dictionary representing the URL
- :rtype: dict
-
-
-.. py:function:: deserialize_path(serialized, project_dir)
-
- Returns a deserialized path or URL.
-
- :param serialized: a serialized path or URL
- :type serialized: dict
- :param project_dir: path to the project directory
- :type project_dir: str
-
- :returns: Path or URL as string
- :rtype: str
-
-
-.. py:function:: deserialize_remote_path(serialized, base_path)
-
-
diff --git a/docs/source/autoapi/spine_engine/version/index.rst b/docs/source/autoapi/spine_engine/version/index.rst
deleted file mode 100644
index 53d7485e..00000000
--- a/docs/source/autoapi/spine_engine/version/index.rst
+++ /dev/null
@@ -1,14 +0,0 @@
-:py:mod:`spine_engine.version`
-==============================
-
-.. py:module:: spine_engine.version
-
-
-Module Contents
----------------
-
-.. py:data:: __version__
- :annotation: = 0.10.1
-
-
-
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 8211b5ba..524c373c 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -67,7 +67,7 @@
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
diff --git a/pyproject.toml b/pyproject.toml
index 89c66b8f..444a4233 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,32 +7,22 @@ description = "A package to run Spine workflows."
keywords = ["energy system modelling", "workflow", "optimisation", "database"]
readme = {file = "README.md", content-type = "text/markdown"}
classifiers = [
- "Programming Language :: Python :: 3",
- "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
- "Operating System :: OS Independent",
+ "Programming Language :: Python :: 3",
+ "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
+ "Operating System :: OS Independent",
]
-requires-python = ">=3.8.1, <3.12"
+requires-python = ">=3.8.1"
dependencies = [
- # dagster versions lower that 1.5.7 do not support pendulum >= 3.0.0
- "pendulum < 3.0.0",
- # https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates
- "protobuf<3.21.0",
"networkx>2.5.1",
"datapackage>=1.15.2, <1.16",
"jupyter_client>=6.0",
"spinedb_api>=0.31.0",
"pyzmq >=21.0",
- # dagster 0.12.8 requires Jinja2<3.0, which tries to import
- # soft_unicode, which has been removed in markupsafe 2.1
- "markupsafe < 2.1",
]
[project.urls]
Repository = "https://github.com/spine-tools/spine-engine"
-[project.optional-dependencies]
-dev = ["coverage[toml]"]
-
[build-system]
requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2", "wheel", "build"]
build-backend = "setuptools.build_meta"
@@ -49,10 +39,10 @@ spine_engine = ["execution_managers/spine_repl.jl"]
[tool.setuptools.packages.find]
exclude = [
- "bin*",
- "docs*",
- "fig*",
- "tests*",
+ "bin*",
+ "docs*",
+ "fig*",
+ "tests*",
]
[tool.coverage.run]
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000..07160c84
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
+-e git+https://github.com/spine-tools/Spine-Database-API.git#egg=spinedb_api
+-e .
diff --git a/tests/execution_managers/test_kernel_execution_manager.py b/tests/execution_managers/test_kernel_execution_manager.py
index 0d79cb44..8226d850 100644
--- a/tests/execution_managers/test_kernel_execution_manager.py
+++ b/tests/execution_managers/test_kernel_execution_manager.py
@@ -52,8 +52,8 @@ def test_kernel_execution_manager(self):
self.assertEqual(0, retval)
self.assertTrue(exec_mngr._kernel_manager.is_alive())
connection_file = exec_mngr._kernel_manager.connection_file
- exec_mngr = self.release_exec_mngr_resources(exec_mngr)
- exec_mngr = None
+ self.release_exec_mngr_resources(exec_mngr)
+ del exec_mngr
self.assertEqual(1, _kernel_manager_factory.n_kernel_managers())
_kernel_manager_factory.shutdown_kernel_manager(connection_file)
self.assertEqual(0, _kernel_manager_factory.n_kernel_managers())