CSCFAIRMETA-776: [ADD] Bulk delete

src/metax_api/api/rest/base/views/dataset_view.py

@@ -403,8 +403,11 @@ def list_datasets(self, request):
         self.queryset_search_params = {"id__in": ids}
         return super(DatasetViewSet, self).list(request)
 
-    @action(detail=False, methods=["post"], url_path="flush_password")
-    def flush_password(self, request):  # pragma: no cover
+    def destroy_bulk(self, request, *args, **kwargs):
+        return self.service_class.destroy_bulk(request)
+
+    @action(detail=False, methods=['post'], url_path="flush_password")
+    def flush_password(self, request): # pragma: no cover
         """
         Set a password for flush api
         """

src/metax_api/management/commands/loadinitialdata.py

@@ -87,7 +87,7 @@ def _load_data_catalogs(self):
         try:
             with open("metax_api/initialdata/datacatalogs.json", "r") as f:
                 data_catalogs = json.load(f)
-        except FileNotFoundError:
+        except FileNotFoundError: # noqa
             raise CommandError("File initialdata/datacatalogs.json does not exist?")
         except json.decoder.JSONDecodeError as e:
             raise CommandError("Error loading data catalog json: %s" % str(e))
@@ -141,7 +141,7 @@ def _load_file_storages(self):
         try:
             with open("metax_api/initialdata/filestorages.json", "r") as f:
                 storages = json.load(f)
-        except FileNotFoundError:
+        except FileNotFoundError: # noqa
             raise CommandError("File initialdata/filestorages.json does not exist?")
         except json.decoder.JSONDecodeError as e:
             raise CommandError("Error loading file storage json: %s" % str(e))

src/metax_api/models/catalog_record.py

@@ -954,8 +954,9 @@ def delete(self, *args, **kwargs):
                 self.previous_dataset_version.next_dataset_version = None
                 super(Common, self.previous_dataset_version).save()
 
-            super(Common, self).delete()
-            return
+            crid = self.id
+            super().delete()
+            return crid
 
         elif self.state == self.STATE_PUBLISHED:
             if self.has_alternate_records():
@@ -981,13 +982,16 @@ def delete(self, *args, **kwargs):
             }
             if self.catalog_is_legacy():
                 # delete permanently instead of only marking as 'removed'
+                crid = self.id
                 super().delete()
+                return crid
             else:
                 super().remove(*args, **kwargs)
                 log_args['catalogrecord']['date_removed'] = datetime_to_str(self.date_removed)
                 log_args['catalogrecord']['date_modified'] = datetime_to_str(self.date_modified)
 
             self.add_post_request_callable(DelayedLog(**log_args))
+        return self.id
 
     def deprecate(self, timestamp=None):
         self.deprecated = True

src/metax_api/services/auth_service.py

@@ -83,7 +83,7 @@ def get_additional_user_projects_from_file(username):
         try:
             with open(settings.ADDITIONAL_USER_PROJECTS_PATH, 'r') as file:
                 additional_projects = json.load(file)
-        except FileNotFoundError:
+        except FileNotFoundError: # noqa
             _logger.info("No local file for user projects")
         except Exception as e:
             _logger.error(e)

src/metax_api/services/catalog_record_service.py

@@ -12,6 +12,8 @@
 import simplexquery as sxq
 import xmltodict
 from django.db.models import Q
+from rest_framework import status
+from rest_framework.response import Response
 from rest_framework.serializers import ValidationError
 
 from metax_api.exceptions import Http400, Http403, Http503
@@ -760,3 +762,33 @@ def get_research_dataset_license_url(rd):
         license = rd['access_rights']['license'][0]
 
         return license.get('identifier') or license.get('license')
+
+    @classmethod
+    def destroy_bulk(cls, request):
+        """
+        Mark datasets as deleted en masse. Parameter cr_identifiers can be a list of pk's
+        (integers), or file identifiers (strings).
+        """
+        _logger.info('Begin bulk delete datasets')
+
+        cr_ids = cls.identifiers_to_ids(request.data)
+        cr_deleted = []
+        no_access = []
+        for id in cr_ids:
+            try:
+                cr = CatalogRecord.objects.get(pk=id)
+                if cr.user_has_access(request):
+                    cr_deleted.append(cr.delete())
+                else:
+                    no_access.append(id)
+            except:
+                pass
+
+        if sorted(no_access) == sorted(cr_ids):
+            raise Http403({ 'detail': ['None of datasets exists or are permitted for users']})
+
+        if not cr_deleted:
+            return Response(cr_deleted, status=status.HTTP_404_NOT_FOUND)
+
+        _logger.info(f'Marked datasets {cr_deleted} as deleted')
+        return Response(cr_deleted, status=status.HTTP_200_OK)

src/metax_api/services/common_service.py

@@ -6,8 +6,9 @@
 # :license: MIT
 
 import logging
-from json import load as json_load
+from typing import List
 
+from json import load as json_load
 from django.db.models import Q
 from django.utils import timezone
 from rest_framework import status
@@ -480,7 +481,7 @@ def set_if_modified_since_filter(cls, request, filter_obj):
                 filter_obj['q_filters'] = [flter]
 
     @staticmethod
-    def identifiers_to_ids(identifiers, params=None):
+    def identifiers_to_ids(identifiers: List[any], params=None):
         """
         In case identifiers are identifiers (strings), which they probably are in real use,
         do a query to get a list of pk's instead, since they will be used quite a few times.

src/metax_api/services/redis_cache_service.py

@@ -142,8 +142,8 @@ def set(self, key, value, **kwargs):
         except (TimeoutError, ConnectionError, MasterNotFoundError) as e:
             if self._DEBUG:
                 d(
-                    "cache: master timed out or not found, or connection refused. no write instances available. "
-                    "error: %s" % str(e)
+                    f"cache: master timed out or not found, or connection refused. \
+                    No write instances available. error: {str(e)}"
                 )
             # no master available
             return
@@ -199,8 +199,8 @@ def delete(self, *keys):
         except (TimeoutError, ConnectionError, MasterNotFoundError) as e:
             if self._DEBUG:
                 d(
-                    "cache: master timed out or not found, or connection refused. no write instances available. "
-                    "error: %s" % str(e)
+                    f"cache: master timed out or not found, or connection refused. \
+                    No write instances available. error: {str(e)}"
                 )
             # no master available
             return

src/metax_api/tests/api/rest/base/views/datacatalogs/read.py

@@ -39,4 +39,4 @@ def test_allowed_read_methods(self):
             response = self.client.head(req)
             self.assertEqual(response.status_code, status.HTTP_200_OK)
             response = self.client.options(req)
-            self.assertEqual(response.status_code, status.HTTP_200_OK)
+            self.assertEqual(response.status_code, status.HTTP_200_OK)

src/metax_api/tests/api/rest/base/views/datasets/write.py

@@ -1314,6 +1314,81 @@ def test_delete_catalog_record_error_using_preferred_identifier(self):
         response = self.client.delete(url)
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
+    def test_bulk_delete_catalog_record_permissions(self):
+        # create catalog with 'metax' edit permissions and create dataset with this catalog as 'metax' user
+        cr = self._get_new_test_cr_data()
+        cr.pop('id')
+        catalog = self._get_object_from_test_data('datacatalog', requested_index=0)
+        catalog.pop('id')
+        catalog['catalog_json']['identifier'] = 'metax-catalog'
+        catalog['catalog_record_services_edit'] = 'metax'
+        catalog = self.client.post('/rest/datacatalogs', catalog, format="json")
+        cr['data_catalog'] = {'id': catalog.data['id'], 'identifier': catalog.data['catalog_json']['identifier']}
+
+        self._use_http_authorization(username='metax')
+        response = self.client.post('/rest/datasets/', cr, format="json")
+        metax_cr = response.data['id']
+
+        # create catalog with 'testuser' edit permissions and create dataset with this catalog as 'testuser' user
+        cr = self._get_new_test_cr_data()
+        cr.pop('id')
+        catalog = self._get_object_from_test_data('datacatalog', requested_index=1)
+        catalog.pop('id')
+        catalog['catalog_json']['identifier'] = 'testuser-catalog'
+        catalog['catalog_record_services_edit'] = 'testuser'
+        catalog = self.client.post('/rest/datacatalogs', catalog, format="json")
+        cr['data_catalog'] = {'id': catalog.data['id'], 'identifier': catalog.data['catalog_json']['identifier']}
+
+        self._use_http_authorization(username='testuser', password='testuserpassword')
+        response = self.client.post('/rest/datasets/', cr, format="json")
+        testuser_cr = response.data['id']
+
+        # after trying to delete as 'testuser' only one catalog is deleted
+        response = self.client.delete('/rest/datasets', [metax_cr, testuser_cr], format="json")
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data, [testuser_cr])
+        response = self.client.post('/rest/datasets/list?pagination=false', [metax_cr, testuser_cr], format="json")
+        self.assertTrue(len(response.data), 1)
+
+        response = self.client.delete('/rest/datasets', [metax_cr], format="json")
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+        response = self.client.post('/rest/datasets/list?pagination=false', [metax_cr, testuser_cr], format="json")
+        self.assertTrue(len(response.data), 1)
+
+    def test_bulk_delete_catalog_record(self):
+        ids = [1, 2, 3]
+        identifiers = CatalogRecord.objects.filter(pk__in=[4, 5, 6]).values_list('identifier', flat=True)
+
+        for crs in [ids, identifiers]:
+            response = self.client.delete('/rest/datasets', crs, format="json")
+            self.assertEqual(response.status_code, status.HTTP_200_OK)
+            self.assertTrue(response.data == [1, 2, 3] or response.data == [4, 5, 6])
+            response = self.client.post('/rest/datasets/list?pagination=false', crs, format="json")
+            self.assertFalse(response.data)
+
+            for cr in crs:
+                if isinstance(cr, int):
+                    deleted = CatalogRecord.objects_unfiltered.get(id=cr)
+                else:
+                    deleted = CatalogRecord.objects_unfiltered.get(identifier=cr)
+
+                self.assertEqual(deleted.removed, True)
+                self.assertEqual(deleted.date_modified, deleted.date_removed,
+                    'date_modified should be updated')
+
+        # failing tests
+        ids = [1000, 2000]
+        identifiers = ['1000', '2000']
+
+        for crs in [ids, identifiers]:
+            response = self.client.delete('/rest/datasets', ids, format="json")
+            self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
+
+        ids = []
+        response = self.client.delete('/rest/datasets', ids, format="json")
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertTrue('Received empty list of identifiers' in response.data['detail'][0])
+
 
 class CatalogRecordApiWritePreservationStateTests(CatalogRecordApiWriteCommon):
 

src/metax_api/tests/api/rest/v2/views/common/auth.py

@@ -5,12 +5,9 @@
 # :author: CSC - IT Center for Science Ltd., Espoo Finland <[email protected]>
 # :license: MIT
 
-import json
 import logging
-import os
 
 import responses
-from django.conf import settings
 from rest_framework import status
 
 from metax_api.tests.api.rest.base.views.datasets.write import CatalogRecordApiWriteCommon
@@ -153,85 +150,3 @@ def test_end_user_create_access_error(self):
         # end users should not have create access to files api.
         response = self.client.post('/rest/v2/files', {}, format='json')
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, response.data)
-
-    def test_removing_bearer_from_allowed_auth_methods_disables_oidc(self):
-        pass
-        # ALLOWED_AUTH_METHODS
-
-class ApiEndUserAdditionalProjects(CatalogRecordApiWriteCommon):
-
-    """
-    Test reading additional permissions from local file
-    """
-
-    def setUp(self):
-        super().setUp()
-        self._use_http_authorization(method='bearer', token=get_test_oidc_token(new_proxy=True))
-        self._mock_token_validation_succeeds()
-
-    def tearDown(self):
-        super().tearDown()
-        try:
-            os.remove(settings.ADDITIONAL_USER_PROJECTS_PATH)
-        except:
-            _logger.info("error removing file from %s" % settings.ADDITIONAL_USER_PROJECTS_PATH)
-
-    @responses.activate
-    def test_successful_read(self):
-        """
-        Ensures user's file projects are also fetched from local file.
-        """
-        testdata = { "testuser": ["some_project", "project_x"] }
-        with open(settings.ADDITIONAL_USER_PROJECTS_PATH, 'w+') as testfile:
-            json.dump(testdata, testfile, indent=4)
-            os.chmod(settings.ADDITIONAL_USER_PROJECTS_PATH, 0o400)
-
-        response = self.client.get('/rest/v2/files?project_identifier=project_x', format='json')
-        self.assertEqual(response.status_code, status.HTTP_200_OK, response.data)
-
-    @responses.activate
-    def test_no_file(self):
-        """
-        Projects are fetched from token when local file is not available.
-        """
-        response = self.client.get('/rest/v2/files?project_identifier=2001036', format='json')
-        self.assertEqual(response.status_code, status.HTTP_200_OK, response.data)
-
-    @responses.activate
-    def test_bad_file_keys(self):
-        """
-        Must return forbidden which indicates that code is run properly despite the bad local file.
-        """
-        testdata = { 123445: "project_x" }
-        with open(settings.ADDITIONAL_USER_PROJECTS_PATH, 'w+') as testfile:
-            json.dump(testdata, testfile, indent=4)
-            os.chmod(settings.ADDITIONAL_USER_PROJECTS_PATH, 0o400)
-
-        response = self.client.get('/rest/v2/files?project_identifier=project_x', format='json')
-        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, response.data)
-
-    @responses.activate
-    def test_bad_file_values(self):
-        """
-        Returns forbidden since values on local file are not list of strings.
-        """
-        testdata = { "testuser": "project_x" }
-        with open(settings.ADDITIONAL_USER_PROJECTS_PATH, 'w+') as testfile:
-            json.dump(testdata, testfile, indent=4)
-            os.chmod(settings.ADDITIONAL_USER_PROJECTS_PATH, 0o400)
-
-        response = self.client.get('/rest/v2/files?project_identifier=project_x', format='json')
-        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, response.data)
-
-    @responses.activate
-    def test_bad_file_successful(self):
-        """
-        Ensures that projects are read from token despite file reading is failed.
-        """
-        testdata = { "testuser": [151342, 236314] }
-        with open(settings.ADDITIONAL_USER_PROJECTS_PATH, 'w+') as testfile:
-            json.dump(testdata, testfile, indent=4)
-            os.chmod(settings.ADDITIONAL_USER_PROJECTS_PATH, 0o400)
-
-        response = self.client.get('/rest/v2/files?project_identifier=2001036', format='json')
-        self.assertEqual(response.status_code, status.HTTP_200_OK, response.data)