Merge pull request #319 from qdev-dk/feature/remove_mp

Feat: Default to no multiprocessing.

.travis.yml

@@ -18,8 +18,8 @@ install:
     - python setup.py develop
 # command to run tests
 script: 
-    - python  qcodes/test.py --skip-coverage
-    - python  qcodes/test.py --mp-spawn
+    - python  qcodes/test.py --skip-coverage -f
+    - python  qcodes/test.py --mp-spawn -f
 after_success:
     # install codacy coverage plugin only on traivs
     - pip install codacy-coverage

docs/examples/Tutorial.ipynb

@@ -475,10 +475,10 @@
     "# If you leave these out, you get a new timestamped DataSet each time.\n",
     "\n",
     "loop = qc.Loop(c0.sweep(0,20,0.1), delay=0.001).each(meter.amplitude)\n",
-    "data = loop.get_data_set(data_manager=False)\n",
+    "data = loop.get_data_set()\n",
     "plot = qc.QtPlot()\n",
     "plot.add(data.meter_amplitude)\n",
-    "_ = loop.with_bg_task(plot.update, 0.0005).run(name='testsweep',background=False)"
+    "_ = loop.with_bg_task(plot.update, 0.0005).run(name='testsweep')"
    ]
   },
   {
@@ -1386,7 +1386,7 @@
     "    meter.amplitude, # second measurement, at c2=1 -> amplitude_4 bcs it's action 4\n",
     "    qc.Task(c2.set, 0)\n",
     "    )\n",
-    "data = loop.get_data_set(data_manager=False)"
+    "data = loop.get_data_set()"
    ]
   },
   {
@@ -2199,7 +2199,7 @@
    "source": [
     "plot = qc.MatPlot(data.meter_amplitude_0, cmap=plt.cm.hot, figsize=(12, 4.5), subplots=(1, 2))\n",
     "plot.add(data.meter_amplitude_3, cmap=plt.cm.hot, subplot=2)\n",
-    "data2 = loop.with_bg_task(plot.update, 0.0005).run(name='2D_test',background=False)"
+    "data2 = loop.with_bg_task(plot.update, 0.0005).run(name='2D_test')"
    ]
   },
   {
@@ -2240,12 +2240,12 @@
     "    meter.amplitude, # second measurement, at c2=1 -> amplitude_4 bcs it's action 4\n",
     "    qc.Task(c2.set, 0)\n",
     "    )\n",
-    "data = loop.get_data_set(data_manager=False)\n",
+    "data = loop.get_data_set()\n",
     "\n",
     "plotQ = qc.QtPlot()\n",
     "plotQ.add(data.meter_amplitude_0, figsize=(1200, 500))\n",
     "plotQ.add(data.meter_amplitude_3, subplot=2)\n",
-    "data2 = loop.with_bg_task(plotQ.update, 0.0005).run(name='2D_test',background=False)"
+    "data2 = loop.with_bg_task(plotQ.update, 0.0005).run(name='2D_test')"
    ]
   },
   {
@@ -2275,7 +2275,7 @@
     "    qc.Loop(c2[-10:10:0.2], 0.001).each(meter.amplitude),\n",
     "    AverageGetter(meter.amplitude, c2[-10:10:0.2], 0.001)\n",
     ")\n",
-    "data = loop3.get_data_set(data_manager=False) \n"
+    "data = loop3.get_data_set() "
    ]
   },
   {
@@ -2324,7 +2324,7 @@
     "plotQ.add(data.meter_amplitude_3_0)\n",
     "plotQ.add(data.meter_amplitude_5_0, cmap='viridis', subplot=2)\n",
     "plotQ.add(data.meter_avg_amplitude, subplot=3)\n",
-    "data = loop3.with_bg_task(plotQ.update, 0.0005).run(name='TwoD_different_inner_test', background=False)"
+    "data = loop3.with_bg_task(plotQ.update, 0.0005).run(name='TwoD_different_inner_test')"
    ]
   },
   {
@@ -2365,11 +2365,11 @@
     "loop4 = qc.Loop(c1[-15:15:1], 0.01).each(\n",
     "    AverageAndRaw(meter.amplitude, c2[-10:10:0.2], 0.001)\n",
     ")\n",
-    "data4 = loop4.get_data_set(data_manager=False)\n",
+    "data4 = loop4.get_data_set()\n",
     "plotQ = qc.QtPlot()\n",
     "plotQ.add(data4.meter_amplitude, figsize=(1200, 500), cmap='viridis')\n",
     "plotQ.add(data4.meter_avg_amplitude, subplot=2)\n",
-    "data4 = loop4.with_bg_task(plotQ.update, 0.005).run(name='TwoD_average_test', background=False)"
+    "data4 = loop4.with_bg_task(plotQ.update, 0.005).run(name='TwoD_average_test')"
    ]
   }
  ],
@@ -2391,10 +2391,6 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.5.2"
-  },
-  "widgets": {
-   "state": {},
-   "version": "1.1.1"
   }
  },
  "nbformat": 4,

qcodes/data/data_set.py

@@ -27,7 +27,9 @@ class DataMode(Enum):
 
 
 def new_data(location=None, loc_record=None, name=None, overwrite=False,
-             io=None, data_manager=None, mode=DataMode.LOCAL, **kwargs):
+             io=None, data_manager=False, mode=DataMode.LOCAL, **kwargs):
+    # NOTE(giulioungaretti): leave this docstrings as it is, because
+    # documenting the types is silly in this case.
     """
     Create a new DataSet.
 
@@ -57,11 +59,11 @@ def new_data(location=None, loc_record=None, name=None, overwrite=False,
             says the root data directory is the current working directory, ie
             where you started the python session.
 
-        data_manager (DataManager or False, optional): manager for the
-            ``DataServer`` that offloads storage and syncing of this
-            ``DataSet``. Usually omitted (default None) to use the default
-            from ``get_data_manager()``. If ``False``, this ``DataSet`` will
-            store itself.
+        data_manager (Optional[bool]): use a manager for the
+            ``DataServer`` that offloads storage and syncing of this Defaults
+            to  ``False`` i.e. this ``DataSet`` will store itself without extra
+            processes. Set to ``True`` to use the default from
+            ``get_data_manager()``.
 
         mode (DataMode, optional): connection type to the ``DataServer``.
             ``DataMode.LOCAL``: this DataSet doesn't communicate across
@@ -105,11 +107,11 @@ def new_data(location=None, loc_record=None, name=None, overwrite=False,
     if location and (not overwrite) and io.list(location):
         raise FileExistsError('"' + location + '" already has data')
 
-    if data_manager is False:
+    if data_manager is True:
+        data_manager = get_data_manager()
+    else:
         if mode != DataMode.LOCAL:
             raise ValueError('DataSets without a data_manager must be local')
-    elif data_manager is None:
-        data_manager = get_data_manager()
 
     return DataSet(location=location, io=io, data_manager=data_manager,
                    mode=mode, **kwargs)
@@ -206,11 +208,11 @@ class DataSet(DelegateAttributes):
             says the root data directory is the current working directory, ie
             where you started the python session.
 
-        data_manager (DataManager or False, optional): manager for the
-            ``DataServer`` that offloads storage and syncing of this
-            ``DataSet``. Usually omitted (default None) to use the default
-            from ``get_data_manager()``. If ``False``, this ``DataSet`` will
-            store itself.
+        data_manager (Optional[bool]): use a manager for the
+            ``DataServer`` that offloads storage and syncing of this Defaults
+            to  ``False`` i.e. this ``DataSet`` will store itself without extra
+            processes.  Set to ``True`` to use the default from
+            ``get_data_manager()``.
 
         mode (DataMode, optional): connection type to the ``DataServer``.
             ``DataMode.LOCAL``: this DataSet doesn't communicate across
@@ -258,7 +260,7 @@ class DataSet(DelegateAttributes):
     background_functions = OrderedDict()
 
     def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None,
-                 data_manager=None, formatter=None, io=None, write_period=5):
+                 data_manager=False, formatter=None, io=None, write_period=5):
         if location is False or isinstance(location, str):
             self.location = location
         else:
@@ -281,7 +283,7 @@ def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None,
             for array in arrays:
                 self.add_array(array)
 
-        if data_manager is None and mode in SERVER_MODES:
+        if data_manager is True and mode in SERVER_MODES:
             data_manager = get_data_manager()
 
         if mode == DataMode.LOCAL:

qcodes/instrument/base.py

@@ -1,7 +1,8 @@
 """Instrument base class."""
-import weakref
-import time
 import logging
+import time
+import warnings
+import weakref
 
 from qcodes.utils.metadata import Metadatable
 from qcodes.utils.helpers import DelegateAttributes, strip_attrs, full_class
@@ -75,11 +76,12 @@ class Instrument(Metadatable, DelegateAttributes, NestedAttrAccess):
 
     _all_instruments = {}
 
-    def __new__(cls, *args, server_name='', **kwargs):
+    def __new__(cls, *args, server_name=None, **kwargs):
         """Figure out whether to create a base instrument or proxy."""
         if server_name is None:
             return super().__new__(cls)
         else:
+            warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning)
             return RemoteInstrument(*args, instrument_class=cls,
                                     server_name=server_name, **kwargs)
 

qcodes/instrument/remote.py

@@ -43,7 +43,6 @@ class RemoteInstrument(DelegateAttributes):
 
     def __init__(self, *args, instrument_class=None, server_name='',
                  **kwargs):
-
         if server_name == '':
             server_name = instrument_class.default_server_name(**kwargs)
 

qcodes/loops.py

@@ -50,6 +50,7 @@
 import multiprocessing as mp
 import time
 import numpy as np
+import warnings
 
 from qcodes.station import Station
 from qcodes.data.data_set import new_data, DataMode
@@ -62,7 +63,10 @@
 from .actions import (_actions_snapshot, Task, Wait, _Measure, _Nest,
                       BreakIf, _QcodesBreak)
 
+# Switches off multiprocessing by default, cant' be altered after module import.
+# TODO(giulioungaretti) use config.
 
+USE_MP = False
 MP_NAME = 'Measurement'
 
 
@@ -640,7 +644,7 @@ def _check_signal(self):
             else:
                 raise ValueError('unknown signal', signal_)
 
-    def get_data_set(self, data_manager=None, *args, **kwargs):
+    def get_data_set(self, data_manager=False, *args, **kwargs):
         """
         Return the data set for this loop.
         If no data set has been created yet, a new one will be created and returned.
@@ -670,6 +674,7 @@ def get_data_set(self, data_manager=None, *args, **kwargs):
             if data_manager is False:
                 data_mode = DataMode.LOCAL
             else:
+                warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning)
                 data_mode = DataMode.PUSH_TO_SERVER
 
             data_set = new_data(arrays=self.containers(), mode=data_mode,
@@ -688,20 +693,19 @@ def run_temp(self, **kwargs):
         return self.run(background=False, quiet=True,
                         data_manager=False, location=False, **kwargs)
 
-    def run(self, background=True, use_threads=True, quiet=False,
-            data_manager=None, station=None, progress_interval=False,
+    def run(self, background=USE_MP, use_threads=False, quiet=False,
+            data_manager=USE_MP, station=None, progress_interval=False,
             *args, **kwargs):
         """
         Execute this loop.
 
-        background: (default True) run this sweep in a separate process
+        background: (default False) run this sweep in a separate process
             so we can have live plotting and other analysis in the main process
         use_threads: (default True): whenever there are multiple `get` calls
             back-to-back, execute them in separate threads so they run in
             parallel (as long as they don't block each other)
         quiet: (default False): set True to not print anything except errors
-        data_manager: a DataManager instance (omit to use default,
-            False to store locally)
+        data_manager: set to True to use a DataManager. Default to False.
         station: a Station instance for snapshots (omit to use a previously
             provided Station, or the default Station)
         progress_interval (default None): show progress of the loop every x
@@ -737,6 +741,7 @@ def run(self, background=True, use_threads=True, quiet=False,
             prev_loop.join()
 
         data_set = self.get_data_set(data_manager, *args, **kwargs)
+
         self.set_common_attrs(data_set=data_set, use_threads=use_threads,
                               signal_queue=self.signal_queue)
 
@@ -762,6 +767,7 @@ def run(self, background=True, use_threads=True, quiet=False,
                   flush=True)
 
         if background:
+            warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning)
             p = QcodesProcess(target=self._run_wrapper, name=MP_NAME)
             p.is_sweep = True
             p.signal_queue = self.signal_queue

qcodes/process/helpers.py

@@ -2,6 +2,7 @@
 
 import multiprocessing as mp
 import time
+import warnings
 
 MP_ERR = 'context has already been set'
 
@@ -23,6 +24,7 @@ def set_mp_method(method, force=False):
         with the *same* method raises an error, but here we only
         raise the error if you *don't* force *and* the context changes
     """
+    warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning)
     try:
         mp.set_start_method(method, force=force)
     except RuntimeError as err:

qcodes/process/stream_queue.py

@@ -2,9 +2,10 @@
 
 import multiprocessing as mp
 import sys
-from datetime import datetime
 import time
 
+from datetime import datetime
+
 from .helpers import kill_queue
 
 

qcodes/tests/test_data.py

@@ -449,14 +449,14 @@ def test_from_server(self, gdm_mock):
         mock_dm.live_data = MockLive()
 
         # wrong location or False location - converts to local
-        data = DataSet(location='Jupiter', mode=DataMode.PULL_FROM_SERVER)
+        data = DataSet(location='Jupiter', data_manager=True, mode=DataMode.PULL_FROM_SERVER)
         self.assertEqual(data.mode, DataMode.LOCAL)
 
-        data = DataSet(location=False, mode=DataMode.PULL_FROM_SERVER)
+        data = DataSet(location=False,  data_manager=True, mode=DataMode.PULL_FROM_SERVER)
         self.assertEqual(data.mode, DataMode.LOCAL)
 
         # location matching server - stays in server mode
-        data = DataSet(location='Mars', mode=DataMode.PULL_FROM_SERVER,
+        data = DataSet(location='Mars',  data_manager=True, mode=DataMode.PULL_FROM_SERVER,
                        formatter=MockFormatter())
         self.assertEqual(data.mode, DataMode.PULL_FROM_SERVER)
         self.assertEqual(data.arrays, MockLive.arrays)
@@ -495,7 +495,7 @@ def test_to_server(self, gdm_mock):
         mock_dm.needs_restart = True
         gdm_mock.return_value = mock_dm
 
-        data = DataSet(location='Venus', mode=DataMode.PUSH_TO_SERVER)
+        data = DataSet(location='Venus', data_manager=True, mode=DataMode.PUSH_TO_SERVER)
         self.assertEqual(mock_dm.needs_restart, False, data)
         self.assertEqual(mock_dm.data_set, data)
         self.assertEqual(data.data_manager, mock_dm)

qcodes/tests/test_driver_testcase.py

@@ -34,7 +34,7 @@ class TestDriverTestCase(DriverTestCase):
     @classmethod
     def setUpClass(cls):
         cls.an_empty_model = EmptyModel()
-        cls.an_instrument = MockMock('a', model=cls.an_empty_model)
+        cls.an_instrument = MockMock('a', model=cls.an_empty_model, server_name='')
         super().setUpClass()
 
     @classmethod

qcodes/tests/test_instrument.py

@@ -31,6 +31,7 @@ def __init__(self, *args, **kwargs):
                            vals=Numbers(-10, 10), step=0.2,
                            delay=0.01,
                            max_delay='forever')
+        self.crahs()
 
 
 class GatesBadDelayValue(MockGates):
@@ -51,9 +52,9 @@ class TestInstrument(TestCase):
     def setUpClass(cls):
         cls.model = AMockModel()
 
-        cls.gates = MockGates(model=cls.model)
-        cls.source = MockSource(model=cls.model)
-        cls.meter = MockMeter(model=cls.model, keep_history=False)
+        cls.gates = MockGates(model=cls.model, server_name='')
+        cls.source = MockSource(model=cls.model, server_name='')
+        cls.meter = MockMeter(model=cls.model, keep_history=False, server_name='')
 
     def setUp(self):
         # reset the model state via the gates function
@@ -192,7 +193,7 @@ def test_remove_instance(self):
         with self.assertRaises(KeyError):
             Instrument.find_instrument('gates')
 
-        type(self).gates = MockGates(model=self.model)
+        type(self).gates = MockGates(model=self.model, server_name="")
         self.assertEqual(self.gates.instances(), [self.gates])
         self.assertEqual(Instrument.find_instrument('gates'), self.gates)
 
@@ -941,7 +942,7 @@ def setUp(self):
             name='testdummy', gates=['dac1', 'dac2', 'dac3'], server_name=None)
 
     def tearDown(self):
-        #TODO (giulioungaretti) remove ( does nothing ?)
+        # TODO (giulioungaretti) remove ( does nothing ?)
         pass
 
     def test_attr_access(self):