Merge branch 'master' of github.com:myushen/HPCell

Pipeline_benchmarking_script.R

@@ -45,7 +45,8 @@ for(core in Cores) {
     slurm_memory_gigabytes_per_cpu = 20,
     slurm_cpus_per_task = 1, 
     workers = total_workers, 
-    verbose = FALSE
+    verbose = FALSE, 
+    seconds_idle = 30
   )
   # Time and run your pipeline function
   time_taken <- system.time({

R/functions.R

@@ -582,7 +582,7 @@ alive_identification <- function(input_read_RNA_assay,
       input_read_RNA_assay |>
       left_join(empty_droplets_tbl, by=".cell") |>
       dplyr::filter(!empty_droplet)
-  } else if (is.null(empty_droplets_tbl)) {return(NULL)}
+  } 
 
   # Calculate nFeature_RNA and nCount_RNA if not exist in the data
   nFeature_name <- paste0("nFeature_", assay)
@@ -818,7 +818,7 @@ doublet_identification <- function(input_read_RNA_assay,
       input_read_RNA_assay = input_read_RNA_assay |>
         left_join(alive_identification_tbl |> select(.cell, alive), by = ".cell") |>
         filter(alive)
-  } else if (is.null(empty_droplets_tbl)) {return(NULL)} 
+  } 
 
   # Condition as scDblFinder only accept assay "counts"
   if (!"counts" %in% (SummarizedExperiment::assays(filter_empty_droplets) |> names())){
@@ -909,7 +909,7 @@ cell_cycle_scoring <- function(input_read_RNA_assay,
     filtered_counts <- input_read_RNA_assay |>
       left_join(empty_droplets_tbl, by = ".cell") |>
       dplyr::filter(!empty_droplet)
-  } else if (is.null(empty_droplets_tbl)) {return(NULL)}
+  } 
 
   counts <- filtered_counts |>
     # Normalise needed
@@ -982,7 +982,7 @@ non_batch_variation_removal <- function(input_read_RNA_assay,
     filtered_counts <- input_read_RNA_assay_transform |>
       left_join(empty_droplets_tbl, by = ".cell") |>
       dplyr::filter(!empty_droplet)
-  } else if (is.null(empty_droplets_tbl)) {return(NULL)}
+  } 
 
   counts =
     filtered_counts |>
@@ -1103,9 +1103,7 @@ preprocessing_output <- function(input_read_RNA_assay,
       input_read_RNA_assay |>
       left_join(empty_droplets_tbl, by = ".cell") |>
       filter(!empty_droplet)
-  } else if (empty_droplets_tbl |> is.null()) {
-    return(NULL)
-  }
+  } 
 
   # Add normalisation
   if(!is.null(non_batch_variation_removal_S)){

R/modules_grammar_hpc.R

@@ -61,7 +61,6 @@ initialise_hpc <- function(input_hpc,
   if(input_hpc |> names() |> is.null())
     input_hpc = input_hpc |> set_names(seq_len(length(input_hpc)))
 
-  input_hpc |> names() |> saveRDS("sample_names.rds")
 
   # Optionally, you can evaluate the arguments if they are expressions
   args_list <- lapply(args_list, eval, envir = parent.frame())
@@ -71,6 +70,8 @@ initialise_hpc <- function(input_hpc,
   data_file_names = glue("{store}/{names(input_hpc)}.rds")
 
   input_hpc |> as.list() |>  saveRDS("input_file.rds")
+  input_hpc |> names() |> saveRDS("sample_names.rds")
+
   gene_nomenclature |> saveRDS("temp_gene_nomenclature.rds")
   data_container_type |> saveRDS("data_container_type.rds")
   computing_resources |> saveRDS("temp_computing_resources.rds")

R_scripts/de_parallel.R

@@ -20,7 +20,8 @@ se |>
       slurm_memory_gigabytes_per_cpu = 5,
       slurm_cpus_per_task = 2,
       workers = 200,
-      verbose = T
+      verbose = T, 
+      seconds_idle = 30
     )
   )
 
@@ -357,7 +358,8 @@ slurm = crew.cluster::crew_controller_slurm(
   slurm_memory_gigabytes_per_cpu = 5,
   slurm_cpus_per_task = 1,
   workers = 200,
-  verbose = T
+  verbose = T, 
+  seconds_idle = 30
 )
 
 
@@ -488,7 +490,8 @@ se =
          slurm_memory_gigabytes_per_cpu = 5,
          slurm_cpus_per_task = 1,
          workers = 200,
-         verbose = T
+         verbose = T, 
+         seconds_idle = 30
        )
    ) 
 

tests/testthat/test_single_functions.R

@@ -510,15 +510,17 @@ computing_resources = crew_controller_local(workers = 8) #resource_tuned_slurm
     slurm_cpus_per_task = 1,
     workers = 50,
     tasks_max = 5,
-    verbose = T
+    verbose = T, 
+    seconds_idle = 30
   ),
   crew_controller_slurm(
     name = "tier_2",
     slurm_memory_gigabytes_per_cpu = 10,
     slurm_cpus_per_task = 1,
     workers = 50,
     tasks_max = 5,
-    verbose = T
+    verbose = T, 
+    seconds_idle = 30
   )
 )
 
@@ -564,7 +566,7 @@ file_list |>
   ) |> 
 
   # ONLY APPLICABLE TO SCE FOR NOW
-  tranform_assay(fx = file_list |> purrr::map(~identity), target_output = "sce_transformed") |> 
+  transform_assay(fx = file_list |> purrr::map(~identity), target_output = "sce_transformed") |> 
 
   # hpc_report(
   #   "empty_report",