diff --git a/gwcelery/data/__init__.py b/gwcelery/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/data/first2years/__init__.py b/gwcelery/data/first2years/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/data/first2years/2016/gstlal.xml.gz b/gwcelery/data/first2years/gstlal.xml.gz similarity index 100% rename from gwcelery/data/first2years/2016/gstlal.xml.gz rename to gwcelery/data/first2years/gstlal.xml.gz diff --git a/gwcelery/data/first2years/2016/psd.xml.gz b/gwcelery/data/first2years/psd.xml.gz similarity index 100% rename from gwcelery/data/first2years/2016/psd.xml.gz rename to gwcelery/data/first2years/psd.xml.gz diff --git a/gwcelery/tasks/em_bright.py b/gwcelery/tasks/em_bright.py index fa7d360e..bffef025 100644 --- a/gwcelery/tasks/em_bright.py +++ b/gwcelery/tasks/em_bright.py @@ -10,12 +10,12 @@ from ..import app from . import gracedb, lvalert from .p_astro import _format_prob -from ..util import NamedTemporaryFile, PromiseProxy, resource_pickle +from ..util import NamedTemporaryFile, PromiseProxy, read_pickle NS_CLASSIFIER = PromiseProxy( - resource_pickle, ('ligo.data', 'knn_ns_classifier.pkl')) + read_pickle, ('ligo.data', 'knn_ns_classifier.pkl')) EM_CLASSIFIER = PromiseProxy( - resource_pickle, ('ligo.data', 'knn_em_classifier.pkl')) + read_pickle, ('ligo.data', 'knn_em_classifier.pkl')) log = get_task_logger(__name__) diff --git a/gwcelery/tasks/first2years.py b/gwcelery/tasks/first2years.py index dc306593..22d4f58a 100644 --- a/gwcelery/tasks/first2years.py +++ b/gwcelery/tasks/first2years.py @@ -1,4 +1,5 @@ """Create mock events from the "First Two Years" paper.""" +from importlib import resources import io import random @@ -9,8 +10,8 @@ import lal from ligo.skymap.io.events.ligolw import ContentHandler import numpy as np -import pkg_resources +from ..data import first2years as data_first2years from ..import app from . import gracedb @@ -19,9 +20,8 @@ def pick_coinc(): """Pick a coincidence from the "First Two Years" paper.""" - filename = pkg_resources.resource_filename( - __name__, '../data/first2years/2016/gstlal.xml.gz') - xmldoc = utils.load_filename(filename, contenthandler=ContentHandler) + with resources.open_binary(data_first2years, 'gstlal.xml.gz') as f: + xmldoc, _ = utils.load_fileobj(f, contenthandler=ContentHandler) root, = xmldoc.childNodes # Remove unneeded tables @@ -139,8 +139,7 @@ def _vet_event(superevents): @gracedb.task(ignore_result=True, shared=False) def _upload_psd(graceid): - psd = pkg_resources.resource_string( - __name__, '../data/first2years/2016/psd.xml.gz') + psd = resources.read_binary(data_first2years, 'psd.xml.gz') gracedb.upload(psd, 'psd.xml.gz', graceid, 'Noise PSD', ['psd']) diff --git a/gwcelery/tasks/p_astro.py b/gwcelery/tasks/p_astro.py index 9a7fec3a..a88783cc 100644 --- a/gwcelery/tasks/p_astro.py +++ b/gwcelery/tasks/p_astro.py @@ -12,18 +12,16 @@ from . import gracedb, lvalert from .. import app -from ..util import PromiseProxy, resource_json +from ..util import PromiseProxy, read_json MEAN_VALUES_DICT = PromiseProxy( - resource_json, ('ligo.data', - 'H1L1V1-mean_counts-1126051217-61603201.json')) + read_json, ('ligo.data', 'H1L1V1-mean_counts-1126051217-61603201.json')) THRESHOLDS_DICT = PromiseProxy( - resource_json, ('ligo.data', - 'H1L1V1-pipeline-far_snr-thresholds.json')) + read_json, ('ligo.data', 'H1L1V1-pipeline-far_snr-thresholds.json')) P_ASTRO_LIVETIME = PromiseProxy( - resource_json, ('ligo.data', 'p_astro_livetime.json')) + read_json, ('ligo.data', 'p_astro_livetime.json')) log = get_task_logger(__name__) diff --git a/gwcelery/templates/index.jinja2 b/gwcelery/templates/index.jinja2 index 1ec9744b..2edcdbf0 100644 --- a/gwcelery/templates/index.jinja2 +++ b/gwcelery/templates/index.jinja2 @@ -294,8 +294,8 @@
diff --git a/gwcelery/tests/data/__init__.py b/gwcelery/tests/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/tests/data/llhoft/__init__.py b/gwcelery/tests/data/llhoft/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/tests/data/llhoft/fail/L1/__init__.py b/gwcelery/tests/data/llhoft/fail/L1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/tests/data/llhoft/fail/__init__.py b/gwcelery/tests/data/llhoft/fail/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/tests/data/llhoft/omegascan/__init__.py b/gwcelery/tests/data/llhoft/omegascan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/tests/data/llhoft/pass/H1/__init__.py b/gwcelery/tests/data/llhoft/pass/H1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/tests/data/llhoft/pass/__init__.py b/gwcelery/tests/data/llhoft/pass/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gwcelery/tests/test_tasks_bayestar.py b/gwcelery/tests/test_tasks_bayestar.py index 3d97d5b6..af85f14d 100644 --- a/gwcelery/tests/test_tasks_bayestar.py +++ b/gwcelery/tests/test_tasks_bayestar.py @@ -1,3 +1,4 @@ +from importlib import resources from unittest.mock import patch from xml.sax import SAXParseException @@ -5,9 +6,9 @@ from astropy.io import fits from celery.exceptions import Ignore import numpy as np -import pkg_resources import pytest +from . import data from ..tasks.bayestar import localize from ..util.tempfile import NamedTemporaryFile @@ -15,7 +16,7 @@ def test_localize_bad_psd(): """Test running BAYESTAR with a pad PSD file""" # Test data - coinc = pkg_resources.resource_string(__name__, 'data/coinc.xml') + coinc = resources.read_binary(data, 'coinc.xml') psd = b'' # Run function under test @@ -37,8 +38,8 @@ def mock_bayestar(event, *args, **kwargs): @pytest.fixture def coinc_psd(): - return (pkg_resources.resource_string(__name__, 'data/coinc.xml'), - pkg_resources.resource_string(__name__, 'data/psd.xml.gz')) + return (resources.read_binary(data, 'coinc.xml'), + resources.read_binary(data, 'psd.xml.gz')) @patch('ligo.skymap.bayestar.localize', mock_bayestar) diff --git a/gwcelery/tests/test_tasks_detchar.py b/gwcelery/tests/test_tasks_detchar.py index 209351a0..2dab091b 100644 --- a/gwcelery/tests/test_tasks_detchar.py +++ b/gwcelery/tests/test_tasks_detchar.py @@ -1,3 +1,4 @@ +from importlib import resources from io import BytesIO import logging from unittest.mock import call, patch @@ -6,29 +7,29 @@ from gwpy.timeseries import Bits import matplotlib.pyplot as plt import numpy as np -from pkg_resources import resource_filename import pytest from ..import app from ..import _version from ..tasks import detchar +from . import data @pytest.fixture def llhoft_glob_pass(): old = app.conf['llhoft_glob'] - app.conf['llhoft_glob'] = resource_filename( - __name__, 'data/llhoft/pass/{detector}/*.gwf') - yield + with resources.path(data, '') as path: + app.conf['llhoft_glob'] = str(path / 'llhoft/pass/{detector}/*.gwf') + yield app.conf['llhoft_glob'] = old @pytest.fixture def llhoft_glob_fail(): old = app.conf['llhoft_glob'] - app.conf['llhoft_glob'] = resource_filename( - __name__, 'data/llhoft/fail/{detector}/*.gwf') - yield + with resources.path(data, '') as path: + app.conf['llhoft_glob'] = str(path / 'llhoft/fail/{detector}/*.gwf') + yield app.conf['llhoft_glob'] = old @@ -86,8 +87,11 @@ def test_create_cache_old_data(mock_find, llhoft_glob_fail): mock_find.assert_called() -@patch('gwcelery.tasks.detchar.create_cache', return_value=[resource_filename( - __name__, 'data/llhoft/omegascan/scanme.gwf')]) +with resources.path(data, '') as expected_path: + expected_path = str(expected_path / 'llhoft/omegascan/scanme.gwf') + + +@patch('gwcelery.tasks.detchar.create_cache', return_value=[expected_path]) def test_make_omegascan_worked(mock_create_cache, scan_strainname): durs = [1, 1, 1] t0 = 1126259463 diff --git a/gwcelery/tests/test_tasks_external_skymaps.py b/gwcelery/tests/test_tasks_external_skymaps.py index e9ccdd90..eb8e0191 100644 --- a/gwcelery/tests/test_tasks_external_skymaps.py +++ b/gwcelery/tests/test_tasks_external_skymaps.py @@ -1,9 +1,10 @@ +from importlib import resources from unittest.mock import patch -import pkg_resources import pytest -from ..util import resource_json +from . import data +from ..util import read_json from .test_tasks_skymaps import toy_fits_filecontents # noqa: F401 from .test_tasks_skymaps import toy_3d_fits_filecontents # noqa: F401 from ..tasks import external_skymaps @@ -19,14 +20,14 @@ def mock_get_event(exttrig): def mock_get_superevent(graceid): - return resource_json(__name__, 'data/mock_superevent_object.json') + return read_json(data, 'mock_superevent_object.json') def mock_get_log(graceid): if graceid == 'S12345': - return resource_json(__name__, 'data/gracedb_setrigger_log.json') + return read_json(data, 'gracedb_setrigger_log.json') elif graceid == 'E12345': - return resource_json(__name__, 'data/gracedb_externaltrigger_log.json') + return read_json(data, 'gracedb_externaltrigger_log.json') else: raise ValueError @@ -41,9 +42,8 @@ def download(filename, graceid): elif (graceid == 'E12345' and filename == ('nasa.gsfc.gcn_Fermi%23GBM_Gnd_Pos_2017-08-17' + 'T12%3A41%3A06.47_524666471_57-431.xml')): - return pkg_resources.resource_string( - __name__, 'data/externaltrigger_original_data.xml' - ) + return resources.read_binary( + data, 'externaltrigger_original_data.xml') else: raise ValueError diff --git a/gwcelery/tests/test_tasks_external_triggers.py b/gwcelery/tests/test_tasks_external_triggers.py index 4a0e4f5b..295bdb9a 100644 --- a/gwcelery/tests/test_tasks_external_triggers.py +++ b/gwcelery/tests/test_tasks_external_triggers.py @@ -1,18 +1,18 @@ +from importlib.resources import read_binary from unittest.mock import patch, call import pytest -from pkg_resources import resource_string - +from . import data from ..tasks import external_triggers from ..tasks import detchar -from ..util import resource_json +from ..util import read_json @pytest.mark.parametrize('pipeline, path', - [['Fermi', 'data/fermi_grb_gcn.xml'], - ['INTEGRAL', 'data/integral_grb_gcn.xml'], - ['AGILE', 'data/agile_grb_gcn.xml']]) + [['Fermi', 'fermi_grb_gcn.xml'], + ['INTEGRAL', 'integral_grb_gcn.xml'], + ['AGILE', 'agile_grb_gcn.xml']]) @patch('gwcelery.tasks.external_skymaps.create_upload_external_skymap') @patch('gwcelery.tasks.external_skymaps.get_upload_external_skymap.run') @patch('gwcelery.tasks.detchar.dqr_json', return_value='dqrjson') @@ -29,7 +29,7 @@ def test_handle_create_grb_event(mock_create_event, mock_get_event, mock_get_upload_external_skymap, mock_create_upload_external_skymap, pipeline, path): - text = resource_string(__name__, path) + text = read_binary(data, path) external_triggers.handle_grb_gcn(payload=text) mock_create_event.assert_called_once_with(filecontents=text, search='GRB', @@ -98,11 +98,10 @@ def test_handle_create_subthreshold_grb_event(mock_get_upload_ext_skymap, mock_create_event, mock_get_event, mock_get_events): - text = resource_string(__name__, - 'data/fermi_subthresh_grb_lowconfidence.xml') + text = read_binary(data, 'fermi_subthresh_grb_lowconfidence.xml') external_triggers.handle_grb_gcn(payload=text) mock_create_event.assert_not_called() - text = resource_string(__name__, 'data/fermi_subthresh_grb_gcn.xml') + text = read_binary(data, 'fermi_subthresh_grb_gcn.xml') external_triggers.handle_grb_gcn(payload=text) mock_get_events.assert_called_once_with(query=( 'group: External pipeline: ' @@ -136,7 +135,7 @@ def test_handle_noise_fermi_event(mock_check_vectors, mock_get_event, mock_get_events, mock_get_upload_external_skymap): - text = resource_string(__name__, 'data/fermi_noise_gcn.xml') + text = read_binary(data, 'fermi_noise_gcn.xml') external_triggers.handle_grb_gcn(payload=text) mock_get_events.assert_called_once_with(query=( 'group: External pipeline: ' @@ -153,8 +152,8 @@ def test_handle_noise_fermi_event(mock_check_vectors, @pytest.mark.parametrize('filename', - ['data/fermi_grb_gcn.xml', - 'data/fermi_noise_gcn.xml']) + ['fermi_grb_gcn.xml', + 'fermi_noise_gcn.xml']) @patch('gwcelery.tasks.external_skymaps.get_upload_external_skymap.run') @patch('gwcelery.tasks.gracedb.create_label') @patch('gwcelery.tasks.gracedb.remove_label') @@ -175,7 +174,7 @@ def test_handle_replace_grb_event(mock_get_event, mock_get_events, mock_replace_event, mock_remove_label, mock_create_label, mock_get_upload_external_skymap, filename): - text = resource_string(__name__, filename) + text = read_binary(data, filename) external_triggers.handle_grb_gcn(payload=text) mock_replace_event.assert_called_once_with('E1', text) if 'grb' in filename: @@ -278,7 +277,7 @@ def test_handle_skymap_combine(mock_create_combined_skymap): @patch('gwcelery.tasks.gracedb.create_event') def test_handle_create_snews_event(mock_create_event, mock_get_event, mock_upload, mock_json): - text = resource_string(__name__, 'data/snews_gcn.xml') + text = read_binary(data, 'snews_gcn.xml') external_triggers.handle_snews_gcn(payload=text) mock_create_event.assert_called_once_with(filecontents=text, search='Supernova', @@ -310,7 +309,7 @@ def test_handle_create_snews_event(mock_create_event, mock_get_event, @patch('gwcelery.tasks.gracedb.replace_event') @patch('gwcelery.tasks.gracedb.get_events', return_value=[{'graceid': 'E1'}]) def test_handle_replace_snews_event(mock_get_events, mock_replace_event): - text = resource_string(__name__, 'data/snews_gcn.xml') + text = read_binary(data, 'snews_gcn.xml') external_triggers.handle_snews_gcn(payload=text) mock_replace_event.assert_called_once_with('E1', text) @@ -319,7 +318,7 @@ def test_handle_replace_snews_event(mock_get_events, mock_replace_event): def test_handle_grb_exttrig_creation(mock_raven_coincidence_search): """Test dispatch of an LVAlert message for an exttrig creation.""" # Test LVAlert payload. - alert = resource_json(__name__, 'data/lvalert_exttrig_creation.json') + alert = read_json(data, 'lvalert_exttrig_creation.json') # Run function under test external_triggers.handle_grb_lvalert(alert) @@ -334,7 +333,7 @@ def test_handle_grb_exttrig_creation(mock_raven_coincidence_search): def test_handle_subgrb_exttrig_creation(mock_raven_coincidence_search): """Test dispatch of an LVAlert message for an exttrig creation.""" # Test LVAlert payload. - alert = resource_json(__name__, 'data/lvalert_subgrb_creation.json') + alert = read_json(data, 'lvalert_subgrb_creation.json') # Run function under test external_triggers.handle_grb_lvalert(alert) @@ -352,8 +351,7 @@ def test_handle_subgrb_targeted_creation(mock_raven_coincidence_search, mock_create_upload_external_skymap): """Test dispatch of an LVAlert message for an exttrig creation.""" # Test LVAlert payload. - alert = resource_json(__name__, - 'data/lvalert_exttrig_subgrb_targeted_creation.json') + alert = read_json(data, 'lvalert_exttrig_subgrb_targeted_creation.json') # Run function under test external_triggers.handle_grb_lvalert(alert) @@ -371,13 +369,13 @@ def test_handle_subgrb_targeted_creation(mock_raven_coincidence_search, @pytest.mark.parametrize('calls, path', - [[False, 'data/lvalert_snews_test_creation.json'], - [True, 'data/lvalert_snews_creation.json']]) + [[False, 'lvalert_snews_test_creation.json'], + [True, 'lvalert_snews_creation.json']]) @patch('gwcelery.tasks.raven.coincidence_search') def test_handle_sntrig_creation(mock_raven_coincidence_search, calls, path): """Test dispatch of an LVAlert message for SNEWS alerts.""" # Test LVAlert payload. - alert = resource_json(__name__, path) + alert = read_json(data, path) # Run function under test external_triggers.handle_snews_lvalert(alert) @@ -399,7 +397,7 @@ def test_handle_superevent_cbc_creation(mock_raven_coincidence_search, mock_get_superevent): """Test dispatch of an LVAlert message for a CBC superevent creation.""" # Test LVAlert payload. - alert = resource_json(__name__, 'data/lvalert_superevent_creation.json') + alert = read_json(data, 'lvalert_superevent_creation.json') # Run function under test external_triggers.handle_grb_lvalert(alert) @@ -423,7 +421,7 @@ def test_handle_superevent_burst_creation(mock_raven_coincidence_search, mock_get_superevent): """Test dispatch of an LVAlert message for a burst superevent creation.""" # Test LVAlert payload. - alert = resource_json(__name__, 'data/lvalert_superevent_creation.json') + alert = read_json(data, 'lvalert_superevent_creation.json') # Run function under test external_triggers.handle_grb_lvalert(alert) diff --git a/gwcelery/tests/test_tasks_first2years.py b/gwcelery/tests/test_tasks_first2years.py index 58ebf5b3..f7dc53ce 100644 --- a/gwcelery/tests/test_tasks_first2years.py +++ b/gwcelery/tests/test_tasks_first2years.py @@ -1,13 +1,14 @@ +from importlib import resources import io from unittest.mock import call, patch from glue.ligolw import utils from glue.ligolw import lsctables from ligo.skymap.io.events.ligolw import ContentHandler -import pkg_resources import pytest from ..tasks.first2years import pick_coinc, upload_event +from ..data import first2years as data_first2years pytest.importorskip('lal') @@ -39,8 +40,7 @@ def test_pick_coinc(): def test_upload_event(mock_create_signoff, mock_get_superevents, mock_upload, mock_create_event): coinc = pick_coinc() - psd = pkg_resources.resource_string( - __name__, '../data/first2years/2016/psd.xml.gz') + psd = resources.read_binary(data_first2years, 'psd.xml.gz') upload_event() diff --git a/gwcelery/tests/test_tasks_gcn.py b/gwcelery/tests/test_tasks_gcn.py index 4b42fbfe..9386628d 100644 --- a/gwcelery/tests/test_tasks_gcn.py +++ b/gwcelery/tests/test_tasks_gcn.py @@ -1,20 +1,20 @@ +from importlib import resources import logging from unittest.mock import patch from comet.utility.xml import xml_document import lxml.etree -import pkg_resources import pytest from ..tasks import gcn +from . import data logging.basicConfig(level=logging.INFO) def fake_gcn(notice_type): # Check the real GCN notice, which is valid. - payload = pkg_resources.resource_string( - __name__, 'data/G298048-1-Initial.xml') + payload = resources.read_binary(data, 'G298048-1-Initial.xml') root = lxml.etree.fromstring(payload) notice_type = str(int(notice_type)) root.find(".//Param[@name='Packet_Type']").attrib['value'] = notice_type diff --git a/gwcelery/tests/test_tasks_gcn_validate.py b/gwcelery/tests/test_tasks_gcn_validate.py index ca6e8902..755065f9 100644 --- a/gwcelery/tests/test_tasks_gcn_validate.py +++ b/gwcelery/tests/test_tasks_gcn_validate.py @@ -1,10 +1,11 @@ -import json +from importlib import resources from unittest.mock import patch -import pkg_resources import pytest from ..tasks.gcn import validate +from ..util import read_json +from . import data @pytest.fixture @@ -13,12 +14,11 @@ def fake_gcn(celeryconf, monkeypatch): def mock_download(filename, graceid): assert filename == 'G298048-1-Initial.xml' assert graceid == 'G298048' - return pkg_resources.resource_string(__name__, 'data/' + filename) + return resources.read_binary(data, filename) def mock_get_log(graceid): assert graceid == 'G298048' - return json.loads( - pkg_resources.resource_string(__name__, 'data/G298048_log.json')) + return read_json(data, 'G298048_log.json') monkeypatch.setattr( 'gwcelery.tasks.gracedb.download', mock_download) @@ -26,8 +26,7 @@ def mock_get_log(graceid): 'gwcelery.tasks.gracedb.get_log', mock_get_log) # Get the VOEvent. - yield pkg_resources.resource_string( - __name__, 'data/G298048-1-Initial.xml') + yield resources.read_binary(data, 'G298048-1-Initial.xml') @patch('gwcelery.tasks.gracedb.create_tag.run') diff --git a/gwcelery/tests/test_tasks_gracedb.py b/gwcelery/tests/test_tasks_gracedb.py index 3e92e9ec..e90f3c24 100644 --- a/gwcelery/tests/test_tasks_gracedb.py +++ b/gwcelery/tests/test_tasks_gracedb.py @@ -1,8 +1,9 @@ from collections import defaultdict +from importlib import resources from unittest import mock -from pkg_resources import resource_string from ..tasks import gracedb +from . import data class DictMock(mock.MagicMock): @@ -156,7 +157,7 @@ def test_get_labels(mock_gracedb): @patch('gwcelery.tasks.gracedb.client') def test_replace_event(mock_gracedb): - text = resource_string(__name__, 'data/fermi_grb_gcn.xml') + text = resources.read_binary(data, 'fermi_grb_gcn.xml') gracedb.replace_event(graceid='G123456', payload=text) mock_gracedb.events.update.assert_called_once_with('G123456', filecontents=text) diff --git a/gwcelery/tests/test_tasks_lvalert.py b/gwcelery/tests/test_tasks_lvalert.py index 54f4e5cd..b6732ac1 100644 --- a/gwcelery/tests/test_tasks_lvalert.py +++ b/gwcelery/tests/test_tasks_lvalert.py @@ -1,3 +1,4 @@ +from importlib import resources import json import logging import os @@ -5,10 +6,10 @@ from unittest.mock import patch import lxml -import pkg_resources import pytest from ..tasks import lvalert +from . import data @pytest.fixture @@ -25,7 +26,7 @@ def netrc_lvalert(tmpdir): @pytest.fixture def fake_lvalert(): - with pkg_resources.resource_stream(__name__, 'data/lvalert_xmpp.xml') as f: + with resources.open_binary(data, 'lvalert_xmpp.xml') as f: root = lxml.etree.parse(f) node = root.find('.//{*}items').attrib['node'] payload = root.find('.//{*}entry').text diff --git a/gwcelery/tests/test_tasks_orchestrator.py b/gwcelery/tests/test_tasks_orchestrator.py index 7b5ae39c..5d4d7486 100644 --- a/gwcelery/tests/test_tasks_orchestrator.py +++ b/gwcelery/tests/test_tasks_orchestrator.py @@ -1,16 +1,16 @@ -import os +from importlib import resources import json from unittest.mock import call, Mock, patch -import pkg_resources import pytest from .. import app from ..tasks import inference from ..tasks import orchestrator from ..tasks import superevents -from ..util import resource_json +from ..util import read_json from .test_tasks_skymaps import toy_3d_fits_filecontents # noqa: F401 +from . import data @pytest.mark.parametrize( # noqa: F811 @@ -85,7 +85,8 @@ def download(filename, graceid): elif filename == 'em_bright.json' and group == 'CBC': return json.dumps({'HasNS': 0.0, 'HasRemnant': 0.0}) elif filename == 'psd.xml.gz': - return pkg_resources.resource_filename(__name__, 'data/psd.xml.gz') + with resources.path('psd.xml.gz') as p: + return str(p) elif filename == 'S1234-1-Preliminary.xml': return b'fake VOEvent file contents' elif filename == 'p_astro.json': @@ -323,8 +324,7 @@ def mock_download(filename, graceid, *args, **kwargs): filenames = {'coinc.xml': 'coinc.xml', 'psd.xml.gz': 'psd.xml.gz', 'ranking_data.xml.gz': 'ranking_data_G322589.xml.gz'} - return pkg_resources.resource_string( - __name__, os.path.join('data', filenames[filename])) + return resources.read_binary(data, filenames[filename]) @patch( @@ -340,7 +340,7 @@ def test_handle_cbc_event(mock_localize, mock_get_event): """Test that an LVAlert message for a newly uploaded PSD file triggers BAYESTAR. """ - alert = resource_json(__name__, 'data/lvalert_psd.json') + alert = read_json(data, 'lvalert_psd.json') orchestrator.handle_cbc_event(alert) mock_localize.assert_called_once() @@ -450,7 +450,7 @@ def test_handle_cbc_event_ignored(mock_localize, mock_classifier, mock_get_event): """Test that unrelated LVAlert messages do not trigger BAYESTAR.""" - alert = resource_json(__name__, 'data/lvalert_detchar.json') + alert = read_json(data, 'lvalert_detchar.json') orchestrator.handle_cbc_event(alert) mock_localize.assert_not_called() mock_classifier.assert_not_called() diff --git a/gwcelery/tests/test_tasks_skymaps.py b/gwcelery/tests/test_tasks_skymaps.py index c2a90db7..e20f06e4 100644 --- a/gwcelery/tests/test_tasks_skymaps.py +++ b/gwcelery/tests/test_tasks_skymaps.py @@ -1,20 +1,16 @@ import argparse import gzip +from importlib import resources import io import os from unittest.mock import patch from astropy.table import Table import numpy as np -import pkg_resources import pytest from ..tasks import skymaps - - -def resource_unicode(*args, **kwargs): - with open(pkg_resources.resource_filename(*args, **kwargs), 'r') as f: - return f.read() +from . import data def get_toy_fits_filecontents(): @@ -78,7 +74,7 @@ def test_fits_header(toy_fits_filecontents): html = skymaps.fits_header(toy_fits_filecontents, 'test.fits') # Check output - assert html == resource_unicode(__name__, 'data/fits_header_result.html') + assert html == resources.read_text(data, 'fits_header_result.html') @patch('ligo.skymap.tool.ligo_skymap_plot.main') @@ -136,7 +132,7 @@ def mock_skymap_from_samples(args): with open(os.path.join(args.outdir, args.fitsoutname), 'wb') as f: f.write(toy_3d_fits_filecontents) - inbytes = pkg_resources.resource_string(__name__, 'data/samples.hdf5') + inbytes = resources.read_binary(data, 'samples.hdf5') with patch('ligo.skymap.tool.ligo_skymap_from_samples.main', mock_skymap_from_samples): diff --git a/gwcelery/tests/test_tasks_superevents.py b/gwcelery/tests/test_tasks_superevents.py index a12cef1a..03d29012 100644 --- a/gwcelery/tests/test_tasks_superevents.py +++ b/gwcelery/tests/test_tasks_superevents.py @@ -6,7 +6,8 @@ from requests.models import Response from ..tasks import gracedb, superevents -from ..util import resource_json +from ..util import read_json +from . import data lvalert_content = { 'object': { @@ -163,15 +164,15 @@ def s100response(sid): def mock_db(monkeypatch): def get_superevents(*args, **kwargs): - return resource_json(__name__, 'data/superevents.json')['superevents'] + return read_json(data, 'superevents.json')['superevents'] def get_event(gid): if gid == "T0212": - return resource_json(__name__, 'data/T0212_S0039_preferred.json') + return read_json(data, 'T0212_S0039_preferred.json') elif gid == "G000003": return G000003_RESPONSE elif gid == "G000012": - return resource_json(__name__, 'data/G000012_S0040_preferred.json') + return read_json(data, 'G000012_S0040_preferred.json') elif gid == "G330308": return G330308_RESPONSE elif gid == "G330298": @@ -190,7 +191,7 @@ def get_event(gid): def test_select_preferred_event(): """Provide the list of trigger information of a sample event.""" - events = resource_json(__name__, 'data/sample_events.json') + events = read_json(data, 'sample_events.json') r = superevents.select_preferred_event(events) assert r['graceid'] == 'G3' @@ -225,8 +226,7 @@ def test_update_preferred_event(superevent_labels, new_event_labels, T1234. The new event T1234 passes FAR threshold and has higher SNR compared to the preferred event. """ - preferred_event_dictionary = resource_json( - __name__, 'data/T0212_S0039_preferred.json') + preferred_event_dictionary = read_json(data, 'T0212_S0039_preferred.json') preferred_event_dictionary['labels'] = preferred_event_labels if new_event_id == 'T1234': new_event_dictionary = dict( diff --git a/gwcelery/tools/condor.py b/gwcelery/tools/condor.py index 53e7963c..97a2d7b2 100644 --- a/gwcelery/tools/condor.py +++ b/gwcelery/tools/condor.py @@ -4,6 +4,7 @@ These commands apply to the GWCelery instance that is running in the current working directory. """ +from importlib import resources import json import os import shlex @@ -13,9 +14,11 @@ from celery.bin.base import Command import lxml.etree -import pkg_resources -SUBMIT_FILE = pkg_resources.resource_filename(__name__, '../data/gwcelery.sub') +from .. import data + +with resources.path(data, 'gwcelery.sub') as p: + SUBMIT_FILE = str(p) def get_constraints(): diff --git a/gwcelery/util/resources.py b/gwcelery/util/resources.py index 75eaedf2..0582a5f9 100644 --- a/gwcelery/util/resources.py +++ b/gwcelery/util/resources.py @@ -1,19 +1,18 @@ """Package data helpers.""" +from importlib import resources import json import pickle -import pkg_resources +__all__ = ('read_json', 'read_pickle') -__all__ = ('resource_json', 'resource_pickle') - -def resource_json(*args, **kwargs): +def read_json(*args, **kwargs): """Load a JSON file from package data.""" - with pkg_resources.resource_stream(*args, **kwargs) as f: + with resources.open_text(*args, **kwargs) as f: return json.load(f) -def resource_pickle(*args, **kwargs): +def read_pickle(*args, **kwargs): """Load a JSON file from package data.""" - with pkg_resources.resource_stream(*args, **kwargs) as f: + with resources.open_binary(*args, **kwargs) as f: return pickle.load(f) diff --git a/gwcelery/views.py b/gwcelery/views.py index cb87d504..dd50ee19 100644 --- a/gwcelery/views.py +++ b/gwcelery/views.py @@ -2,16 +2,24 @@ import datetime import re +try: + from importlib import metadata +except ImportError: + # FIXME Remove when we drop support for Python < 3.7 + import importlib_metadata as metadata + from astropy.time import Time from flask import flash, jsonify, redirect, render_template, request, url_for from flask import make_response from requests.exceptions import HTTPError -import pkg_resources from . import app as celery_app from ._version import get_versions from .flask import app, cache from .tasks import first2years, gracedb, orchestrator, circulars, superevents +from .util import PromiseProxy + +distributions = PromiseProxy(metadata.distributions) @app.route('/') @@ -20,7 +28,7 @@ def index(): return render_template( 'index.jinja2', conf=celery_app.conf, - packages=pkg_resources.working_set, + packages=distributions, versions=get_versions()) diff --git a/requirements.txt b/requirements.txt index db299c2c..525029c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,8 @@ gwpy healpy humanize == 0.5.1 # pinned by Flower, not respected by pipenv for some reason imapclient +importlib-metadata; python_version<'3.8' +importlib-resources; python_version<'3.7' jinja2 lalsuite ligo-followup-advocate >= 1.1.6 diff --git a/setup.cfg b/setup.cfg index 82235071..2af9e27d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,7 +64,7 @@ console_scripts = gwcelery-condor-submit-helper = gwcelery.tools.condor_submit_helper:main [options.package_data] -gwcelery = static/*.css, static/vega/*.html, templates/*.jinja2, data/*.sub, data/first2years/2016/*.xml.gz +gwcelery = static/*.css, static/vega/*.html, templates/*.jinja2, data/*.sub, data/first2years/*.xml.gz gwcelery.tests = data/*.html, data/*.json, data/*.xml, data/*.xml.gz, data/llhoft/*/*.gwf, data/llhoft/*/*/*.gwf, data/*.pickle, data/*.hdf5 [versioneer]