From 3ef83bc7f4d88fbb50e0b080ad4460df1705a1ec Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 29 Aug 2016 18:43:34 +0200 Subject: [PATCH] Feat: Default to no multiprocessing. Because multiprocessing is not open for public use, unless one really wants to use it. The test are incredibly coupled, so they needed some massaging too. Closes #314. --- .travis.yml | 4 ++-- docs/examples/Tutorial.ipynb | 24 ++++++++------------ qcodes/data/data_set.py | 34 +++++++++++++++------------- qcodes/instrument/base.py | 8 ++++--- qcodes/instrument/remote.py | 1 - qcodes/loops.py | 18 ++++++++++----- qcodes/process/helpers.py | 2 ++ qcodes/process/stream_queue.py | 3 ++- qcodes/tests/test_data.py | 8 +++---- qcodes/tests/test_driver_testcase.py | 2 +- qcodes/tests/test_instrument.py | 11 +++++---- qcodes/tests/test_loop.py | 28 +++++++++++++++-------- qcodes/tests/test_visa.py | 2 +- 13 files changed, 82 insertions(+), 63 deletions(-) diff --git a/.travis.yml b/.travis.yml index 515345b1480..b6645af1d70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,8 +18,8 @@ install: - python setup.py develop # command to run tests script: - - python qcodes/test.py --skip-coverage - - python qcodes/test.py --mp-spawn + - python qcodes/test.py --skip-coverage -f + - python qcodes/test.py --mp-spawn -f after_success: # install codacy coverage plugin only on traivs - pip install codacy-coverage diff --git a/docs/examples/Tutorial.ipynb b/docs/examples/Tutorial.ipynb index ce7b8b9748a..3d88bd96d7e 100644 --- a/docs/examples/Tutorial.ipynb +++ b/docs/examples/Tutorial.ipynb @@ -475,10 +475,10 @@ "# If you leave these out, you get a new timestamped DataSet each time.\n", "\n", "loop = qc.Loop(c0.sweep(0,20,0.1), delay=0.001).each(meter.amplitude)\n", - "data = loop.get_data_set(data_manager=False)\n", + "data = loop.get_data_set()\n", "plot = qc.QtPlot()\n", "plot.add(data.meter_amplitude)\n", - "_ = loop.with_bg_task(plot.update, 0.0005).run(name='testsweep',background=False)" + "_ = loop.with_bg_task(plot.update, 0.0005).run(name='testsweep')" ] }, { @@ -1386,7 +1386,7 @@ " meter.amplitude, # second measurement, at c2=1 -> amplitude_4 bcs it's action 4\n", " qc.Task(c2.set, 0)\n", " )\n", - "data = loop.get_data_set(data_manager=False)" + "data = loop.get_data_set()" ] }, { @@ -2199,7 +2199,7 @@ "source": [ "plot = qc.MatPlot(data.meter_amplitude_0, cmap=plt.cm.hot, figsize=(12, 4.5), subplots=(1, 2))\n", "plot.add(data.meter_amplitude_3, cmap=plt.cm.hot, subplot=2)\n", - "data2 = loop.with_bg_task(plot.update, 0.0005).run(name='2D_test',background=False)" + "data2 = loop.with_bg_task(plot.update, 0.0005).run(name='2D_test')" ] }, { @@ -2240,12 +2240,12 @@ " meter.amplitude, # second measurement, at c2=1 -> amplitude_4 bcs it's action 4\n", " qc.Task(c2.set, 0)\n", " )\n", - "data = loop.get_data_set(data_manager=False)\n", + "data = loop.get_data_set()\n", "\n", "plotQ = qc.QtPlot()\n", "plotQ.add(data.meter_amplitude_0, figsize=(1200, 500))\n", "plotQ.add(data.meter_amplitude_3, subplot=2)\n", - "data2 = loop.with_bg_task(plotQ.update, 0.0005).run(name='2D_test',background=False)" + "data2 = loop.with_bg_task(plotQ.update, 0.0005).run(name='2D_test')" ] }, { @@ -2275,7 +2275,7 @@ " qc.Loop(c2[-10:10:0.2], 0.001).each(meter.amplitude),\n", " AverageGetter(meter.amplitude, c2[-10:10:0.2], 0.001)\n", ")\n", - "data = loop3.get_data_set(data_manager=False) \n" + "data = loop3.get_data_set() " ] }, { @@ -2324,7 +2324,7 @@ "plotQ.add(data.meter_amplitude_3_0)\n", "plotQ.add(data.meter_amplitude_5_0, cmap='viridis', subplot=2)\n", "plotQ.add(data.meter_avg_amplitude, subplot=3)\n", - "data = loop3.with_bg_task(plotQ.update, 0.0005).run(name='TwoD_different_inner_test', background=False)" + "data = loop3.with_bg_task(plotQ.update, 0.0005).run(name='TwoD_different_inner_test')" ] }, { @@ -2365,11 +2365,11 @@ "loop4 = qc.Loop(c1[-15:15:1], 0.01).each(\n", " AverageAndRaw(meter.amplitude, c2[-10:10:0.2], 0.001)\n", ")\n", - "data4 = loop4.get_data_set(data_manager=False)\n", + "data4 = loop4.get_data_set()\n", "plotQ = qc.QtPlot()\n", "plotQ.add(data4.meter_amplitude, figsize=(1200, 500), cmap='viridis')\n", "plotQ.add(data4.meter_avg_amplitude, subplot=2)\n", - "data4 = loop4.with_bg_task(plotQ.update, 0.005).run(name='TwoD_average_test', background=False)" + "data4 = loop4.with_bg_task(plotQ.update, 0.005).run(name='TwoD_average_test')" ] } ], @@ -2391,10 +2391,6 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" - }, - "widgets": { - "state": {}, - "version": "1.1.1" } }, "nbformat": 4, diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 57d65093fef..c86504de936 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -27,7 +27,9 @@ class DataMode(Enum): def new_data(location=None, loc_record=None, name=None, overwrite=False, - io=None, data_manager=None, mode=DataMode.LOCAL, **kwargs): + io=None, data_manager=False, mode=DataMode.LOCAL, **kwargs): + # NOTE(giulioungaretti): leave this docstrings as it is, because + # documenting the types is silly in this case. """ Create a new DataSet. @@ -57,11 +59,11 @@ def new_data(location=None, loc_record=None, name=None, overwrite=False, says the root data directory is the current working directory, ie where you started the python session. - data_manager (DataManager or False, optional): manager for the - ``DataServer`` that offloads storage and syncing of this - ``DataSet``. Usually omitted (default None) to use the default - from ``get_data_manager()``. If ``False``, this ``DataSet`` will - store itself. + data_manager (Optional[bool]): use a manager for the + ``DataServer`` that offloads storage and syncing of this Defaults + to ``False`` i.e. this ``DataSet`` will store itself without extra + processes. Set to ``True`` to use the default from + ``get_data_manager()``. mode (DataMode, optional): connection type to the ``DataServer``. ``DataMode.LOCAL``: this DataSet doesn't communicate across @@ -105,11 +107,11 @@ def new_data(location=None, loc_record=None, name=None, overwrite=False, if location and (not overwrite) and io.list(location): raise FileExistsError('"' + location + '" already has data') - if data_manager is False: + if data_manager is True: + data_manager = get_data_manager() + else: if mode != DataMode.LOCAL: raise ValueError('DataSets without a data_manager must be local') - elif data_manager is None: - data_manager = get_data_manager() return DataSet(location=location, io=io, data_manager=data_manager, mode=mode, **kwargs) @@ -206,11 +208,11 @@ class DataSet(DelegateAttributes): says the root data directory is the current working directory, ie where you started the python session. - data_manager (DataManager or False, optional): manager for the - ``DataServer`` that offloads storage and syncing of this - ``DataSet``. Usually omitted (default None) to use the default - from ``get_data_manager()``. If ``False``, this ``DataSet`` will - store itself. + data_manager (Optional[bool]): use a manager for the + ``DataServer`` that offloads storage and syncing of this Defaults + to ``False`` i.e. this ``DataSet`` will store itself without extra + processes. Set to ``True`` to use the default from + ``get_data_manager()``. mode (DataMode, optional): connection type to the ``DataServer``. ``DataMode.LOCAL``: this DataSet doesn't communicate across @@ -258,7 +260,7 @@ class DataSet(DelegateAttributes): background_functions = OrderedDict() def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None, - data_manager=None, formatter=None, io=None, write_period=5): + data_manager=False, formatter=None, io=None, write_period=5): if location is False or isinstance(location, str): self.location = location else: @@ -281,7 +283,7 @@ def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None, for array in arrays: self.add_array(array) - if data_manager is None and mode in SERVER_MODES: + if data_manager is True and mode in SERVER_MODES: data_manager = get_data_manager() if mode == DataMode.LOCAL: diff --git a/qcodes/instrument/base.py b/qcodes/instrument/base.py index d12c713fee2..33691ca12d9 100644 --- a/qcodes/instrument/base.py +++ b/qcodes/instrument/base.py @@ -1,7 +1,8 @@ """Instrument base class.""" -import weakref -import time import logging +import time +import warnings +import weakref from qcodes.utils.metadata import Metadatable from qcodes.utils.helpers import DelegateAttributes, strip_attrs, full_class @@ -75,11 +76,12 @@ class Instrument(Metadatable, DelegateAttributes, NestedAttrAccess): _all_instruments = {} - def __new__(cls, *args, server_name='', **kwargs): + def __new__(cls, *args, server_name=None, **kwargs): """Figure out whether to create a base instrument or proxy.""" if server_name is None: return super().__new__(cls) else: + warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning) return RemoteInstrument(*args, instrument_class=cls, server_name=server_name, **kwargs) diff --git a/qcodes/instrument/remote.py b/qcodes/instrument/remote.py index d6540984556..f5e133e8160 100644 --- a/qcodes/instrument/remote.py +++ b/qcodes/instrument/remote.py @@ -43,7 +43,6 @@ class RemoteInstrument(DelegateAttributes): def __init__(self, *args, instrument_class=None, server_name='', **kwargs): - if server_name == '': server_name = instrument_class.default_server_name(**kwargs) diff --git a/qcodes/loops.py b/qcodes/loops.py index e4b89c3bf79..3a5e020f6c4 100644 --- a/qcodes/loops.py +++ b/qcodes/loops.py @@ -50,6 +50,7 @@ import multiprocessing as mp import time import numpy as np +import warnings from qcodes.station import Station from qcodes.data.data_set import new_data, DataMode @@ -62,7 +63,10 @@ from .actions import (_actions_snapshot, Task, Wait, _Measure, _Nest, BreakIf, _QcodesBreak) +# Switches off multiprocessing by default, cant' be altered after module import. +# TODO(giulioungaretti) use config. +USE_MP = False MP_NAME = 'Measurement' @@ -640,7 +644,7 @@ def _check_signal(self): else: raise ValueError('unknown signal', signal_) - def get_data_set(self, data_manager=None, *args, **kwargs): + def get_data_set(self, data_manager=False, *args, **kwargs): """ Return the data set for this loop. If no data set has been created yet, a new one will be created and returned. @@ -670,6 +674,7 @@ def get_data_set(self, data_manager=None, *args, **kwargs): if data_manager is False: data_mode = DataMode.LOCAL else: + warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning) data_mode = DataMode.PUSH_TO_SERVER data_set = new_data(arrays=self.containers(), mode=data_mode, @@ -688,20 +693,19 @@ def run_temp(self, **kwargs): return self.run(background=False, quiet=True, data_manager=False, location=False, **kwargs) - def run(self, background=True, use_threads=True, quiet=False, - data_manager=None, station=None, progress_interval=False, + def run(self, background=USE_MP, use_threads=False, quiet=False, + data_manager=USE_MP, station=None, progress_interval=False, *args, **kwargs): """ Execute this loop. - background: (default True) run this sweep in a separate process + background: (default False) run this sweep in a separate process so we can have live plotting and other analysis in the main process use_threads: (default True): whenever there are multiple `get` calls back-to-back, execute them in separate threads so they run in parallel (as long as they don't block each other) quiet: (default False): set True to not print anything except errors - data_manager: a DataManager instance (omit to use default, - False to store locally) + data_manager: set to True to use a DataManager. Default to False. station: a Station instance for snapshots (omit to use a previously provided Station, or the default Station) progress_interval (default None): show progress of the loop every x @@ -737,6 +741,7 @@ def run(self, background=True, use_threads=True, quiet=False, prev_loop.join() data_set = self.get_data_set(data_manager, *args, **kwargs) + self.set_common_attrs(data_set=data_set, use_threads=use_threads, signal_queue=self.signal_queue) @@ -762,6 +767,7 @@ def run(self, background=True, use_threads=True, quiet=False, flush=True) if background: + warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning) p = QcodesProcess(target=self._run_wrapper, name=MP_NAME) p.is_sweep = True p.signal_queue = self.signal_queue diff --git a/qcodes/process/helpers.py b/qcodes/process/helpers.py index 7c613bbed8e..25a91f3bea9 100644 --- a/qcodes/process/helpers.py +++ b/qcodes/process/helpers.py @@ -2,6 +2,7 @@ import multiprocessing as mp import time +import warnings MP_ERR = 'context has already been set' @@ -23,6 +24,7 @@ def set_mp_method(method, force=False): with the *same* method raises an error, but here we only raise the error if you *don't* force *and* the context changes """ + warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning) try: mp.set_start_method(method, force=force) except RuntimeError as err: diff --git a/qcodes/process/stream_queue.py b/qcodes/process/stream_queue.py index 86e2f3d675d..3dcbc5bff62 100644 --- a/qcodes/process/stream_queue.py +++ b/qcodes/process/stream_queue.py @@ -2,9 +2,10 @@ import multiprocessing as mp import sys -from datetime import datetime import time +from datetime import datetime + from .helpers import kill_queue diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index 0afe605af11..b49131f4e6e 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -449,14 +449,14 @@ def test_from_server(self, gdm_mock): mock_dm.live_data = MockLive() # wrong location or False location - converts to local - data = DataSet(location='Jupiter', mode=DataMode.PULL_FROM_SERVER) + data = DataSet(location='Jupiter', data_manager=True, mode=DataMode.PULL_FROM_SERVER) self.assertEqual(data.mode, DataMode.LOCAL) - data = DataSet(location=False, mode=DataMode.PULL_FROM_SERVER) + data = DataSet(location=False, data_manager=True, mode=DataMode.PULL_FROM_SERVER) self.assertEqual(data.mode, DataMode.LOCAL) # location matching server - stays in server mode - data = DataSet(location='Mars', mode=DataMode.PULL_FROM_SERVER, + data = DataSet(location='Mars', data_manager=True, mode=DataMode.PULL_FROM_SERVER, formatter=MockFormatter()) self.assertEqual(data.mode, DataMode.PULL_FROM_SERVER) self.assertEqual(data.arrays, MockLive.arrays) @@ -495,7 +495,7 @@ def test_to_server(self, gdm_mock): mock_dm.needs_restart = True gdm_mock.return_value = mock_dm - data = DataSet(location='Venus', mode=DataMode.PUSH_TO_SERVER) + data = DataSet(location='Venus', data_manager=True, mode=DataMode.PUSH_TO_SERVER) self.assertEqual(mock_dm.needs_restart, False, data) self.assertEqual(mock_dm.data_set, data) self.assertEqual(data.data_manager, mock_dm) diff --git a/qcodes/tests/test_driver_testcase.py b/qcodes/tests/test_driver_testcase.py index 185c0e07559..b91b04411c5 100644 --- a/qcodes/tests/test_driver_testcase.py +++ b/qcodes/tests/test_driver_testcase.py @@ -34,7 +34,7 @@ class TestDriverTestCase(DriverTestCase): @classmethod def setUpClass(cls): cls.an_empty_model = EmptyModel() - cls.an_instrument = MockMock('a', model=cls.an_empty_model) + cls.an_instrument = MockMock('a', model=cls.an_empty_model, server_name='') super().setUpClass() @classmethod diff --git a/qcodes/tests/test_instrument.py b/qcodes/tests/test_instrument.py index 7299c1c9bad..faa46a937ae 100644 --- a/qcodes/tests/test_instrument.py +++ b/qcodes/tests/test_instrument.py @@ -31,6 +31,7 @@ def __init__(self, *args, **kwargs): vals=Numbers(-10, 10), step=0.2, delay=0.01, max_delay='forever') + self.crahs() class GatesBadDelayValue(MockGates): @@ -51,9 +52,9 @@ class TestInstrument(TestCase): def setUpClass(cls): cls.model = AMockModel() - cls.gates = MockGates(model=cls.model) - cls.source = MockSource(model=cls.model) - cls.meter = MockMeter(model=cls.model, keep_history=False) + cls.gates = MockGates(model=cls.model, server_name='') + cls.source = MockSource(model=cls.model, server_name='') + cls.meter = MockMeter(model=cls.model, keep_history=False, server_name='') def setUp(self): # reset the model state via the gates function @@ -192,7 +193,7 @@ def test_remove_instance(self): with self.assertRaises(KeyError): Instrument.find_instrument('gates') - type(self).gates = MockGates(model=self.model) + type(self).gates = MockGates(model=self.model, server_name="") self.assertEqual(self.gates.instances(), [self.gates]) self.assertEqual(Instrument.find_instrument('gates'), self.gates) @@ -941,7 +942,7 @@ def setUp(self): name='testdummy', gates=['dac1', 'dac2', 'dac3'], server_name=None) def tearDown(self): - #TODO (giulioungaretti) remove ( does nothing ?) + # TODO (giulioungaretti) remove ( does nothing ?) pass def test_attr_access(self): diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index 2ae8ecd0fa5..d13d84b930a 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -33,9 +33,9 @@ def setUp(self): self.model = AMockModel() - self.gates = MockGates(model=self.model) - self.source = MockSource(model=self.model) - self.meter = MockMeter(model=self.model) + self.gates = MockGates(model=self.model, server_name='') + self.source = MockSource(model=self.model, server_name='') + self.meter = MockMeter(model=self.model, server_name='') self.location = '_loop_test_' self.location2 = '_loop_test2_' self.io = DiskIO('.') @@ -78,7 +78,7 @@ def test_background_and_datamanager(self): # you can only do in-memory loops if you set data_manager=False # TODO: this is the one place we don't do quiet=True - test that we # really print stuff? - data = self.loop.run(location=self.location) + data = self.loop.run(location=self.location, background=True) self.check_empty_data(data) # wait for process to finish (ensures that this was run in the bg, @@ -100,7 +100,9 @@ def test_local_instrument(self): # if spawn, pickle will happen if mp.get_start_method() == "spawn": with self.assertRaises(RuntimeError): - loop_local.run(location=self.location, quiet=True) + loop_local.run(location=self.location, + quiet=True, + background=True) # allow for *nix # TODO(giulioungaretti) see what happens ? # what is the expected beavhiour ? @@ -115,7 +117,9 @@ def test_local_instrument(self): self.check_loop_data(data) def test_background_no_datamanager(self): - data = self.loop.run(location=self.location, data_manager=False, + data = self.loop.run(location=self.location, + background=True, + data_manager=False, quiet=True) self.check_empty_data(data) @@ -168,12 +172,18 @@ def test_foreground_no_datamanager(self): def test_enqueue(self): c1 = self.gates.chan1 loop = Loop(c1[1:5:1], 0.01).each(c1) - data1 = loop.run(location=self.location, quiet=True) + data1 = loop.run(location=self.location, + quiet=True, + background=True, + data_manager=True) # second running of the loop should be enqueued, blocks until # the first one finishes. # TODO: check what it prints? - data2 = loop.run(location=self.location2, quiet=True) + data2 = loop.run(location=self.location2, + quiet=True, + background=True, + data_manager=True) data1.sync() data2.sync() @@ -646,7 +656,7 @@ def g(): }, 'loop': { 'background': False, - 'use_threads': True, + 'use_threads': False, 'use_data_manager': False, '__class__': 'qcodes.loops.ActiveLoop', 'sweep_values': { diff --git a/qcodes/tests/test_visa.py b/qcodes/tests/test_visa.py index c09e82f5ea8..93c56321482 100644 --- a/qcodes/tests/test_visa.py +++ b/qcodes/tests/test_visa.py @@ -118,7 +118,7 @@ def test_ask_write_local(self): def test_ask_write_server(self): # same thing as above but Joe is on a server now... - mv = MockVisa('Joe') + mv = MockVisa('Joe', server_name='') # test normal ask and write behavior mv.state.set(2)