Production update 21/12/1 Mark files removed script

src/metax_api/management/commands/mark_files_removed.py

@@ -0,0 +1,49 @@
+import logging
+
+from django.core.management.base import BaseCommand
+from metax_api.models import File
+from metax_api.tests.utils import management_command_add_test_logs
+
+logger = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+	help = """Marks files which start with a given file path removed from a given project.
+	File path prefix can be given as command line parameter or they can be read from a file.
+	This command will produce duplicate prints, the first line is used in tests, but is not stored on logs.
+	The second line is stored on logs, but can't be used by the tests."""
+
+	def add_arguments(self, parser):
+		parser.add_argument("project_identifier", type=str, help="Identifier of the project where the files are removed")
+		parser.add_argument("--path_prefix", type=str, help="Prefix for the file path of the files that are removed")
+		parser.add_argument("--path_prefix_file", type=str, help="Name of the file where to read the path prefixes (Required if --path-prefix is not given)")
+
+	@management_command_add_test_logs(logger)
+	def handle(self, *args, **options):
+		logger.info("Remove files from database")
+		if options["path_prefix"] is None and options["path_prefix_file"] is None:
+			logger.error("path_prefix or path_prefix_file is required")
+			return
+
+		if options["path_prefix_file"]:
+			path_prefixes = self.read_prefixes_from_file(options["path_prefix_file"])
+		else:
+			path_prefixes = [options["path_prefix"]]
+
+
+		removed_files_sum = 0
+		for prefix in path_prefixes:
+			files = File.objects.filter(project_identifier = options["project_identifier"], file_path__startswith = prefix, removed = "f")
+			logger.info(f"Found {len(files)} files to remove in project: {options['project_identifier']} with path prefix: {prefix}")
+			for file in files:
+				file.delete()
+			removed_files_sum += len(files)
+
+		logger.info(f"Removed {removed_files_sum} files")
+
+
+
+	def read_prefixes_from_file(self, filename):
+		with open(filename) as file:
+			lines = file.readlines()
+			lines = [line.rstrip() for line in lines]
+		return lines

src/metax_api/tests/management/commands/mark_files_removed.py

@@ -0,0 +1,35 @@
+from io import StringIO
+import logging
+
+from django.core.management import call_command
+from django.test import TestCase
+
+from metax_api.models import File
+from metax_api.tests.utils import test_data_file_path
+from metax_api.management.commands.mark_files_removed import Command
+
+_logger = logging.getLogger(__name__)
+
+class RemoveFilesTest(TestCase):
+
+    def setUp(self):
+        call_command("loaddata", test_data_file_path, verbosity=0)
+
+    def test_command_output(self):
+        project_identifier = "project_x"
+        path_prefix = "/project_x_FROZEN/Experiment_X/Phase_1/2017"
+        path_prefix_file = None
+
+        out = StringIO()
+        args = [project_identifier]
+        options = {"path_prefix": path_prefix, "stdout": out}
+        call_command('mark_files_removed', *args, **options)
+
+        self.assertIn(f'Found 10 files to remove in project: {project_identifier} with path prefix: {path_prefix}', out.getvalue())
+        self.assertIn('Removed 10 files', out.getvalue())
+
+        files = File.objects_unfiltered.filter(project_identifier = project_identifier, file_path__startswith = path_prefix)
+
+        self.assertEqual(10, len(files))
+        for file in files:
+            self.assertTrue(file.removed)

src/metax_api/tests/utils.py

@@ -9,6 +9,7 @@
 from base64 import b64encode
 from contextlib import contextmanager
 from json import load as json_load
+from io import StringIO
 from os import path
 
 import jwt
@@ -456,10 +457,15 @@ def _get_new_file_data(
 
 
 @contextmanager
-def streamhandler_to_console(lggr):
+def streamhandler_to_console(lggr, command_obj = None):
     # Use 'up to date' value of sys.stdout for StreamHandler,
     # as set by test runner.
-    stream_handler = logging.StreamHandler(sys.stdout)
+    # If command_obj is given, use OutputWrapper of the given 
+    # Command object
+    if command_obj == None:
+        stream_handler = logging.StreamHandler(sys.stdout)
+    else:
+        stream_handler = logging.StreamHandler(command_obj.stdout)
     lggr.addHandler(stream_handler)
     yield
     lggr.removeHandler(stream_handler)
@@ -474,3 +480,17 @@ def testcase_log_console(*args, **kwargs):
         return testcase_log_console
 
     return testcase_decorator
+
+def management_command_add_test_logs(lggr):
+    # This decorator can be used by management commands to capture
+    # the logs of the command in a format that can be used in tests
+    def management_command_log_decorator(func):
+        def management_command_add_test_logs(*args, **kwargs):
+            # args[0] is the object which uses this decorator (typically a command object)
+            with streamhandler_to_console(lggr, args[0]):
+                return func(*args, **kwargs)
+
+        return management_command_add_test_logs
+
+    return management_command_log_decorator
+