Commit 7825a455 authored by Gonzalo S Nido's avatar Gonzalo S Nido
Browse files

Added CPU/RAM monitoring, but UNTESTED

parent a0b4bd07
......@@ -47,6 +47,12 @@ echo "Installing R..."
sudo apt install r-base
sudo apt install libcurl4-openssl-dev libxml2-dev
echo "Installing psrecord for monitoring"
sudo apt install python-pip
pip install psrecord
export PATH="~/.local/bin/:$PATH"
#R packages
#----------
#
......
......@@ -22,8 +22,11 @@ samtools_path="/opt/samtools-1.10"
ext_chr_srt=$( realpath ./apps/ext_chr_srt.py )
meth_prop=$( realpath ./apps/methylation_proportion.py )
psrecord="~/.local/bin/psrecord"
# TO ACTIVATE MONITORING SET TO true
MONITOR=false
#MONITOR=true
......@@ -90,6 +93,9 @@ fi
mkdir -p ${output_dir}
log_dir=${output_dir}/logs
mkdir -p ${log_dir}
# Create TMP folder
tmp_dir=${output_dir}/tmp
mkdir -p ${tmp_dir}
# Print input data :)
start_date=$( date )
......@@ -97,9 +103,9 @@ echo
echo "Analysis started:"
echo ${start_date}
echo
echo "=========================="
echo "=================================================="
echo "SAMPLE ID: \"${sample_id}\""
echo "=========================="
echo "=================================================="
echo "R1 FILES:"
for f in ${R1[@]}; do
echo " $f"
......@@ -109,7 +115,7 @@ for f in ${R2[@]}; do
echo " $f"
done
echo "OUTPUT DIR: \"${output_dir}\""
echo "=========================="
echo "=================================================="
echo
# Comma-separate input files for Bismark command
......@@ -139,7 +145,11 @@ CMD="${bismark} ${b38_ref} -1 ${R1} -2 ${R2} --multicore 8 \
-o ${output_dir} --prefix b38 --path_to_bowtie ${bowtie_path} \
--samtools_path ${samtools_path} > ${log_dir}/1_1_aln.stdout 2> ${log_dir}/1_1_aln.stderr"
echo -e ${CMD} > ${log_dir}/1_1_aln.cmd
bash ${log_dir}/1_1_aln.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/1_1_aln.cmd" --log ${log_dir}/1_1_aln.resources --interval 60 --include-children
else
bash ${log_dir}/1_1_aln.cmd
fi
if [ "$?" != "0" ]; then
echo ;
......@@ -162,7 +172,11 @@ CMD="${samtools} merge -n -r -@ 32 ${output_dir}/b38_${sample_id}.bam ${bamfile_
${log_dir}/1_2_merge.stdout 2> \
${log_dir}/1_2_merge.stderr"
echo -e ${CMD} > ${log_dir}/1_2_merge.cmd
bash ${log_dir}/1_2_merge.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/1_2_aln.cmd" --log ${log_dir}/1_2_merge.resources --interval 60 --include-children
else
bash ${log_dir}/1_2_merge.cmd
fi
if [ "$?" != "0" ]; then
echo ;
......@@ -183,7 +197,11 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
${log_dir}/1_3_dedup.stdout 2> \
${log_dir}/1_3_dedup.stderr"
echo -e ${CMD} > ${log_dir}/1_3_dedup.cmd
bash ${log_dir}/1_3_dedup.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/1_3_dedup.cmd" --log ${log_dir}/1_3_dedup.resources --interval 60 --include-children
else
bash ${log_dir}/1_3_dedup.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 1.3"
......@@ -205,7 +223,11 @@ CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
${log_dir}/1_4_meth.stdout 2> \
${log_dir}/1_4_meth.stderr"
echo -e ${CMD} > ${log_dir}/1_4_meth.cmd
bash ${log_dir}/1_4_meth.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/1_4_meth.cmd" --log ${log_dir}/1_4_meth.resources --interval 60 --include-children
else
bash ${log_dir}/1_4_meth.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 1.4"
......@@ -233,7 +255,11 @@ cat ${output_dir}/CHG_context_b38_${sample_id}.dedup.txt | python3 ${ext_chr_srt
zcat ${output_dir}/C??_context_LAMBDA_${sample_id}.dedup.txt.gz | python3 ${meth_prop} > \
${output_dir}/Lambda_meth_prop_${sample_id}.txt"
echo -e ${CMD} > ${log_dir}/1_5_extract_MT.cmd
bash ${log_dir}/1_5_extract_MT.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/1_5_extract_MT.cmd" --log ${log_dir}/1_5_extract_MT.resources --interval 60 --include-children
else
bash ${log_dir}/1_5_extract_MT.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 1.5"
......@@ -264,7 +290,11 @@ CMD="${bismark} ${mt_ref} -1 ${R1} -2 ${R2} --multicore 8 \
-o ${output_dir} --prefix MT --path_to_bowtie ${bowtie_path} \
--samtools_path ${samtools_path} > ${log_dir}/2_1_aln.stdout 2> ${log_dir}/2_1_aln.stderr"
echo -e ${CMD} > ${log_dir}/2_1_aln.cmd
bash ${log_dir}/2_1_aln.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/2_1_aln.cmd" --log ${log_dir}/2_1_aln.resources --interval 60 --include-children
else
bash ${log_dir}/2_1_aln.cmd
fi
if [ "$?" != "0" ]; then
echo ;
......@@ -287,7 +317,11 @@ CMD="${samtools} merge -n -r -@ 32 ${output_dir}/MT_${sample_id}.bam ${bamfile_l
${log_dir}/2_2_merge.stdout 2> \
${log_dir}/2_2_merge.stderr"
echo -e ${CMD} > ${log_dir}/2_2_merge.cmd
bash ${log_dir}/2_2_merge.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/2_2_merge.cmd" --log ${log_dir}/2_2_merge.resources --interval 60 --include-children
else
bash ${log_dir}/2_2_merge.cmd
fi
if [ "$?" != "0" ]; then
echo ;
......@@ -308,7 +342,11 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
${log_dir}/2_3_dedup.stdout 2> \
${log_dir}/2_3_dedup.stderr"
echo -e ${CMD} > ${log_dir}/2_3_dedup.cmd
bash ${log_dir}/2_3_dedup.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/2_3_dedup.cmd" --log ${log_dir}/2_3_dedup.resources --interval 60 --include-children
else
bash ${log_dir}/2_3_dedup.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 2.3"
......@@ -330,7 +368,11 @@ CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
${log_dir}/2_4_meth.stdout 2> \
${log_dir}/2_4_meth.stderr"
echo -e ${CMD} > ${log_dir}/2_4_meth.cmd
bash ${log_dir}/2_4_meth.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/2_4_meth.cmd" --log ${log_dir}/2_4_meth.resources --interval 60 --include-children
else
bash ${log_dir}/2_4_meth.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 2.4"
......@@ -364,7 +406,11 @@ CMD="${bismark} ${nuc_ref} -1 ${R1} -2 ${R2} --multicore 8 \
--un --ambiguous -o ${output_dir} --prefix b38_noMT --path_to_bowtie ${bowtie_path} \
--samtools_path ${samtools_path} > ${log_dir}/3_1_aln.stdout 2> ${log_dir}/3_1_aln.stderr"
echo -e ${CMD} > ${log_dir}/3_1_aln.cmd
bash ${log_dir}/3_1_aln.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/3_1_aln.cmd" --log ${log_dir}/3_1_aln.resources --interval 60 --include-children
else
bash ${log_dir}/3_1_aln.cmd
fi
if [ "$?" != "0" ]; then
echo ;
......@@ -396,7 +442,11 @@ CMD="cat ${unmapped_list_r1} > ${output_dir}/b38_noMT_${sample_id}_unmapped.R1.f
rm ${unmapped_list_r2} \n\
rm ${output_dir}/b38_noMT.*_ambiguous_reads_?.fq.gz"
echo -e ${CMD} > ${log_dir}/3_2_merge_unmapped.cmd
bash ${log_dir}/3_2_merge_unmapped.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/3_2_merge.cmd" --log ${log_dir}/3_2_merge.resources --interval 60 --include-children
else
bash ${log_dir}/3_2_merge_unmapped.cmd
fi
if [ "$?" != "0" ]; then
echo ;
......@@ -419,8 +469,12 @@ for element in "${array[@]}"; do
done
CMD="rm ${bamfile_list}"
echo ${CMD} > ${log_dir}/3_3_merge.cmd
bash ${log_dir}/3_3_merge.cmd
echo ${CMD} > ${log_dir}/3_3_delete.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/3_3_delete.cmd" --log ${log_dir}/3_3_delete.resources --interval 60 --include-children
else
bash ${log_dir}/3_3_delete.cmd
fi
if [ "$?" != "0" ]; then
echo ;
......@@ -443,7 +497,11 @@ CMD="(${bismark} ${mt_ref} -1 ${output_dir}/b38_noMT_${sample_id}_unmapped.R1.fa
${log_dir}/3_4_realign_to_mt.stdout 2> \
${log_dir}/3_4_realign_to_mt.stderr"
echo ${CMD} > ${log_dir}/3_4_realign_to_mt.cmd
bash ${log_dir}/3_4_realign_to_mt.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/3_4_realign_to_mt.cmd" --log ${log_dir}/3_4_realign_to_mt.resources --interval 60 --include-children
else
bash ${log_dir}/3_4_realign_to_mt.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 3.4"
......@@ -460,7 +518,11 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
${log_dir}/3_5_dedup.stdout 2> \
${log_dir}/3_5_dedup.stderr"
echo -e ${CMD} > ${log_dir}/3_5_dedup.cmd
bash ${log_dir}/3_5_dedup.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/3_5_dedup.cmd" --log ${log_dir}/3_5_dedup.resources --interval 60 --include-children
else
bash ${log_dir}/3_5_dedup.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 3.5"
......@@ -476,18 +538,22 @@ fi
echo -ne " 3.7 Extracting methylation in MT..."
echo -ne " 3.6 Extracting methylation in MT..."
CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
--gzip --multicore 16 \
--samtools_path ${samtools_path} \
${output_dir}/umap_to_MT_${sample_id}.dedup.bam > \
${log_dir}/3_7_meth.stdout 2> \
${log_dir}/3_7_meth.stderr"
echo -e ${CMD} > ${log_dir}/3_7_meth.cmd
bash ${log_dir}/3_7_meth.cmd
echo -e ${CMD} > ${log_dir}/3_6_meth.cmd
if $MONITOR; then
${psrecord} "bash ${log_dir}/3_6_meth.cmd" --log ${log_dir}/3_6_meth.resources --interval 60 --include-children
else
bash ${log_dir}/3_6_meth.cmd
fi
if [ "$?" != "0" ]; then
echo "ERROR IN STEP 3.7"
echo "ERROR IN STEP 3.6"
exit 1
else
echo "DONE"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment