Added CPU/RAM monitoring, but UNTESTED

prepare_vm.sh

@@ -47,6 +47,12 @@ echo "Installing R..."
 sudo apt install r-base
 sudo apt install libcurl4-openssl-dev libxml2-dev
 
+echo "Installing psrecord for monitoring"
+sudo apt install python-pip
+pip install psrecord
+
+export PATH="~/.local/bin/:$PATH" 
+
 #R packages
 #----------
 #

wgbs-pipeline.sh

@@ -22,8 +22,11 @@ samtools_path="/opt/samtools-1.10"
 ext_chr_srt=$( realpath ./apps/ext_chr_srt.py )
 meth_prop=$( realpath ./apps/methylation_proportion.py )
 
+psrecord="~/.local/bin/psrecord"
 
-
+# TO ACTIVATE MONITORING SET TO true
+MONITOR=false
+#MONITOR=true
 
 
 
@@ -90,6 +93,9 @@ fi
 mkdir -p ${output_dir}
 log_dir=${output_dir}/logs
 mkdir -p ${log_dir}
+# Create TMP folder
+tmp_dir=${output_dir}/tmp
+mkdir -p ${tmp_dir}
 
 # Print input data :)
 start_date=$( date )
@@ -97,9 +103,9 @@ echo
 echo "Analysis started:"
 echo ${start_date}
 echo
-echo "=========================="
+echo "=================================================="
 echo "SAMPLE ID: \"${sample_id}\""
-echo "=========================="
+echo "=================================================="
 echo "R1 FILES:"
 for f in ${R1[@]}; do
     echo "  $f"
@@ -109,7 +115,7 @@ for f in ${R2[@]}; do
     echo "  $f"
 done
 echo "OUTPUT DIR: \"${output_dir}\""
-echo "=========================="
+echo "=================================================="
 echo
 
 # Comma-separate input files for Bismark command
@@ -139,7 +145,11 @@ CMD="${bismark} ${b38_ref} -1 ${R1} -2 ${R2} --multicore 8 \
     -o ${output_dir} --prefix b38 --path_to_bowtie ${bowtie_path} \
     --samtools_path ${samtools_path} > ${log_dir}/1_1_aln.stdout 2> ${log_dir}/1_1_aln.stderr"
 echo -e ${CMD} > ${log_dir}/1_1_aln.cmd
-bash ${log_dir}/1_1_aln.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/1_1_aln.cmd" --log ${log_dir}/1_1_aln.resources --interval 60 --include-children
+else
+    bash ${log_dir}/1_1_aln.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo ;
@@ -162,7 +172,11 @@ CMD="${samtools} merge -n -r -@ 32 ${output_dir}/b38_${sample_id}.bam ${bamfile_
     ${log_dir}/1_2_merge.stdout 2> \
     ${log_dir}/1_2_merge.stderr"
 echo -e ${CMD} > ${log_dir}/1_2_merge.cmd
-bash ${log_dir}/1_2_merge.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/1_2_aln.cmd" --log ${log_dir}/1_2_merge.resources --interval 60 --include-children
+else
+    bash ${log_dir}/1_2_merge.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo ;
@@ -183,7 +197,11 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
     ${log_dir}/1_3_dedup.stdout 2> \
     ${log_dir}/1_3_dedup.stderr"
 echo -e ${CMD} > ${log_dir}/1_3_dedup.cmd
-bash ${log_dir}/1_3_dedup.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/1_3_dedup.cmd" --log ${log_dir}/1_3_dedup.resources --interval 60 --include-children
+else
+    bash ${log_dir}/1_3_dedup.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo "ERROR IN STEP 1.3"
@@ -205,7 +223,11 @@ CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
     ${log_dir}/1_4_meth.stdout 2> \
     ${log_dir}/1_4_meth.stderr"
 echo -e ${CMD} > ${log_dir}/1_4_meth.cmd
-bash ${log_dir}/1_4_meth.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/1_4_meth.cmd" --log ${log_dir}/1_4_meth.resources --interval 60 --include-children
+else
+    bash ${log_dir}/1_4_meth.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo "ERROR IN STEP 1.4"
@@ -233,7 +255,11 @@ cat ${output_dir}/CHG_context_b38_${sample_id}.dedup.txt | python3 ${ext_chr_srt
     zcat ${output_dir}/C??_context_LAMBDA_${sample_id}.dedup.txt.gz | python3 ${meth_prop} > \
     ${output_dir}/Lambda_meth_prop_${sample_id}.txt"
 echo -e ${CMD} > ${log_dir}/1_5_extract_MT.cmd
-bash ${log_dir}/1_5_extract_MT.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/1_5_extract_MT.cmd" --log ${log_dir}/1_5_extract_MT.resources --interval 60 --include-children
+else
+    bash ${log_dir}/1_5_extract_MT.cmd
+fi
     
 if [ "$?" != "0" ]; then
     echo "ERROR IN STEP 1.5"
@@ -264,7 +290,11 @@ CMD="${bismark} ${mt_ref} -1 ${R1} -2 ${R2} --multicore 8 \
     -o ${output_dir} --prefix MT --path_to_bowtie ${bowtie_path} \
     --samtools_path ${samtools_path} > ${log_dir}/2_1_aln.stdout 2> ${log_dir}/2_1_aln.stderr"
 echo -e ${CMD} > ${log_dir}/2_1_aln.cmd
-bash ${log_dir}/2_1_aln.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/2_1_aln.cmd" --log ${log_dir}/2_1_aln.resources --interval 60 --include-children
+else
+    bash ${log_dir}/2_1_aln.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo ;
@@ -287,7 +317,11 @@ CMD="${samtools} merge -n -r -@ 32 ${output_dir}/MT_${sample_id}.bam ${bamfile_l
     ${log_dir}/2_2_merge.stdout 2> \
     ${log_dir}/2_2_merge.stderr"
 echo -e ${CMD} > ${log_dir}/2_2_merge.cmd
-bash ${log_dir}/2_2_merge.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/2_2_merge.cmd" --log ${log_dir}/2_2_merge.resources --interval 60 --include-children
+else
+    bash ${log_dir}/2_2_merge.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo ;
@@ -308,7 +342,11 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
     ${log_dir}/2_3_dedup.stdout 2> \
     ${log_dir}/2_3_dedup.stderr"
 echo -e ${CMD} > ${log_dir}/2_3_dedup.cmd
-bash ${log_dir}/2_3_dedup.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/2_3_dedup.cmd" --log ${log_dir}/2_3_dedup.resources --interval 60 --include-children
+else
+    bash ${log_dir}/2_3_dedup.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo "ERROR IN STEP 2.3"
@@ -330,7 +368,11 @@ CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
     ${log_dir}/2_4_meth.stdout 2> \
     ${log_dir}/2_4_meth.stderr"
 echo -e ${CMD} > ${log_dir}/2_4_meth.cmd
-bash ${log_dir}/2_4_meth.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/2_4_meth.cmd" --log ${log_dir}/2_4_meth.resources --interval 60 --include-children
+else
+    bash ${log_dir}/2_4_meth.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo "ERROR IN STEP 2.4"
@@ -364,7 +406,11 @@ CMD="${bismark} ${nuc_ref} -1 ${R1} -2 ${R2} --multicore 8 \
     --un --ambiguous -o ${output_dir} --prefix b38_noMT --path_to_bowtie ${bowtie_path} \
     --samtools_path ${samtools_path} > ${log_dir}/3_1_aln.stdout 2> ${log_dir}/3_1_aln.stderr"
 echo -e ${CMD} > ${log_dir}/3_1_aln.cmd
-bash ${log_dir}/3_1_aln.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/3_1_aln.cmd" --log ${log_dir}/3_1_aln.resources --interval 60 --include-children
+else
+    bash ${log_dir}/3_1_aln.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo ;
@@ -396,7 +442,11 @@ CMD="cat ${unmapped_list_r1} > ${output_dir}/b38_noMT_${sample_id}_unmapped.R1.f
     rm ${unmapped_list_r2} \n\
     rm ${output_dir}/b38_noMT.*_ambiguous_reads_?.fq.gz"
 echo -e ${CMD} > ${log_dir}/3_2_merge_unmapped.cmd
-bash ${log_dir}/3_2_merge_unmapped.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/3_2_merge.cmd" --log ${log_dir}/3_2_merge.resources --interval 60 --include-children
+else
+    bash ${log_dir}/3_2_merge_unmapped.cmd
+fi
  
 if [ "$?" != "0" ]; then
     echo ;
@@ -419,8 +469,12 @@ for element in "${array[@]}"; do
 done
 
 CMD="rm ${bamfile_list}"
-echo ${CMD} > ${log_dir}/3_3_merge.cmd
-bash ${log_dir}/3_3_merge.cmd
+echo ${CMD} > ${log_dir}/3_3_delete.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/3_3_delete.cmd" --log ${log_dir}/3_3_delete.resources --interval 60 --include-children
+else
+    bash ${log_dir}/3_3_delete.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo ;
@@ -443,7 +497,11 @@ CMD="(${bismark} ${mt_ref} -1 ${output_dir}/b38_noMT_${sample_id}_unmapped.R1.fa
     ${log_dir}/3_4_realign_to_mt.stdout 2> \
     ${log_dir}/3_4_realign_to_mt.stderr"
 echo ${CMD} > ${log_dir}/3_4_realign_to_mt.cmd
-bash ${log_dir}/3_4_realign_to_mt.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/3_4_realign_to_mt.cmd" --log ${log_dir}/3_4_realign_to_mt.resources --interval 60 --include-children
+else
+    bash ${log_dir}/3_4_realign_to_mt.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo "ERROR IN STEP 3.4"
@@ -460,7 +518,11 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
     ${log_dir}/3_5_dedup.stdout 2> \
     ${log_dir}/3_5_dedup.stderr"
 echo -e ${CMD} > ${log_dir}/3_5_dedup.cmd
-bash ${log_dir}/3_5_dedup.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/3_5_dedup.cmd" --log ${log_dir}/3_5_dedup.resources --interval 60 --include-children
+else
+    bash ${log_dir}/3_5_dedup.cmd
+fi
 
 if [ "$?" != "0" ]; then
     echo "ERROR IN STEP 3.5"
@@ -476,18 +538,22 @@ fi
 
 
 
-echo -ne "     3.7 Extracting methylation in MT..."
+echo -ne "     3.6 Extracting methylation in MT..."
 CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
     --gzip --multicore 16 \
     --samtools_path ${samtools_path} \
     ${output_dir}/umap_to_MT_${sample_id}.dedup.bam > \
     ${log_dir}/3_7_meth.stdout 2> \
     ${log_dir}/3_7_meth.stderr"
-echo -e ${CMD} > ${log_dir}/3_7_meth.cmd
-bash ${log_dir}/3_7_meth.cmd
+echo -e ${CMD} > ${log_dir}/3_6_meth.cmd
+if $MONITOR; then
+    ${psrecord} "bash ${log_dir}/3_6_meth.cmd" --log ${log_dir}/3_6_meth.resources --interval 60 --include-children
+else
+    bash ${log_dir}/3_6_meth.cmd
+fi
 
 if [ "$?" != "0" ]; then
-    echo "ERROR IN STEP 3.7"
+    echo "ERROR IN STEP 3.6"
     exit 1
 else
     echo "DONE"