Commit 4123e0f2 authored by Gonzalo S Nido's avatar Gonzalo S Nido
Browse files

Fixed some bugs and tested the monitoring

parent 32e21880
......@@ -15,17 +15,17 @@ cd samtools-1.10
./configure
make
sudo mkdir /opt/samtools-1.10
sudo cp ./samtools /opt/samtools-1.10/samtools-1.10
sudo ln -s /opt/samtools-1.10/samtools-1.10 /usr/local/bin/samtools-1.10
sudo cp ./samtools /opt/samtools-1.10/samtools
sudo ln -s /opt/samtools-1.10/samtools /usr/local/bin/samtools-1.10
echo "#### Installing bowtie2-2.4.41..."
echo "#### Installing bowtie2-2.4.1..."
cd ~
wget https://github.com/BenLangmead/bowtie2/releases/download/v2.4.1/bowtie2-2.4.1-linux-x86_64.zip
unzip bowtie2-2.4.41-linux-x86_64.zip
rm bowtie2-2.4.41-linux-x86_64.zip
sudo mkdir /opt/bowtie2-2.4.41
sudo cp -r bowtie2-2.4.41-linux-x86_64/* /opt/bowtie2-2.4.41/
sudo ln -s /opt/bowtie2-2.4.41/bowtie2 /usr/local/bin/bowtie2-2.4.41
unzip bowtie2-2.4.1-linux-x86_64.zip
rm bowtie2-2.4.1-linux-x86_64.zip
sudo mkdir /opt/bowtie2-2.4.1
sudo cp -r bowtie2-2.4.1-linux-x86_64/* /opt/bowtie2-2.4.1/
sudo ln -s /opt/bowtie2-2.4.1/bowtie2 /usr/local/bin/bowtie2-2.4.1
echo "#### Installing Bismark 0.22.3..."
wget https://github.com/FelixKrueger/Bismark/archive/0.22.3.tar.gz
......@@ -44,14 +44,14 @@ sudo ln -s /opt/Bismark-0.22.3/bismark /usr/local/bin/bismark-0.22.3
sudo ln -s /opt/Bismark-0.22.3/bam2nuc /usr/local/bin/bam2nuc-0.22.3
echo "Installing R..."
sudo apt install r-base
sudo apt install libcurl4-openssl-dev libxml2-dev
sudo apt -y install r-base
sudo apt -y install libcurl4-openssl-dev libxml2-dev
echo "Installing psrecord for monitoring"
sudo apt install python-pip
sudo apt -y install python-pip
pip install psrecord
export PATH="~/.local/bin/:$PATH"
echo "export PATH=~/.local/bin/:$PATH" >> ~/.bashrc
#R packages
#----------
......
......@@ -19,15 +19,15 @@ mt_ref="${ref_dir}/chrM"
bismark="/usr/local/bin/bismark-0.22.3"
deduplicate_bismark="/usr/local/bin/deduplicate_bismark-0.22.3"
bismark_meth_extract="/usr/local/bin/bismark_methylation_extractor-0.22.3"
bowtie2="/usr/local/bin/bowtie2-2.4.41"
bowtie_path="/opt/bowtie2-2.4.41/"
bowtie2="/usr/local/bin/bowtie2-2.4.1"
bowtie_path="/opt/bowtie2-2.4.1/"
samtools="/usr/local/bin/samtools-1.10"
samtools_path="/opt/samtools-1.10"
ext_chr_srt="${PIPELINEPATH}/apps/ext_chr_srt.py"
meth_prop="${PIPELINEPATH}/apps/methylation_proportion.py"
generate_bs_indices="${PIPELINEPATH}/apps/generate_bs_indices.sh"
psrecord="~/.local/bin/psrecord"
psrecord="psrecord"
# TO ACTIVATE MONITORING SET TO true
MONITOR=false
......@@ -36,8 +36,8 @@ MONITOR=false
# exit when any command fails
set -e
#trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
#trap 'echo "\"${last_command}\" command filed with exit code $?"' EXIT
# trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
# trap 'echo "\"${last_command}\" command filed with exit code $?"' EXIT
# Ensure reference files/indices are there, download and index otherwise
......@@ -139,6 +139,10 @@ R2=$( echo "${R2%,}" )
if $MONITOR; then
echo "MONITORING CPU AND RAM..."
echo
fi
......@@ -153,13 +157,14 @@ echo "##################################################"
echo -ne " 1.1 Aligning... "
#CMD="${bismark} ${b38_ref} -1 ${R1} -2 ${R2} -u 10000 \
#CMD="${bismark} ${b38_ref} -1 ${R1} -2 ${R2} --multicore 2 -u 100000 \
CMD="${bismark} ${b38_ref} -1 ${R1} -2 ${R2} --multicore 8 \
-o ${output_dir} --prefix b38 --path_to_bowtie ${bowtie_path} \
--samtools_path ${samtools_path} --temp_dir ${tmp_dir} > \
${log_dir}/1_1_aln.stdout 2> ${log_dir}/1_1_aln.stderr"
echo -e ${CMD} > ${log_dir}/1_1_aln.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/1_1_aln.cmd" --log ${log_dir}/1_1_aln.resources --interval 60 --include-children
else
bash ${log_dir}/1_1_aln.cmd
......@@ -187,6 +192,7 @@ CMD="${samtools} merge -n -r -@ 32 ${output_dir}/b38_${sample_id}.bam ${bamfile_
${log_dir}/1_2_merge.stderr"
echo -e ${CMD} > ${log_dir}/1_2_merge.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/1_2_aln.cmd" --log ${log_dir}/1_2_merge.resources --interval 60 --include-children
else
bash ${log_dir}/1_2_merge.cmd
......@@ -212,6 +218,7 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
${log_dir}/1_3_dedup.stderr"
echo -e ${CMD} > ${log_dir}/1_3_dedup.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/1_3_dedup.cmd" --log ${log_dir}/1_3_dedup.resources --interval 60 --include-children
else
bash ${log_dir}/1_3_dedup.cmd
......@@ -238,6 +245,7 @@ CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
${log_dir}/1_4_meth.stderr"
echo -e ${CMD} > ${log_dir}/1_4_meth.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/1_4_meth.cmd" --log ${log_dir}/1_4_meth.resources --interval 60 --include-children
else
bash ${log_dir}/1_4_meth.cmd
......@@ -270,6 +278,7 @@ cat ${output_dir}/CHG_context_b38_${sample_id}.dedup.txt | python3 ${ext_chr_srt
${output_dir}/Lambda_meth_prop_${sample_id}.txt"
echo -e ${CMD} > ${log_dir}/1_5_extract_MT.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/1_5_extract_MT.cmd" --log ${log_dir}/1_5_extract_MT.resources --interval 60 --include-children
else
bash ${log_dir}/1_5_extract_MT.cmd
......@@ -299,13 +308,14 @@ echo "##################################################"
echo -ne " 2.1 Aligning... "
#CMD="${bismark} ${mt_ref} -1 ${R1} -2 ${R2} -u 10000 \
#CMD="${bismark} ${mt_ref} -1 ${R1} -2 ${R2} --multicore 2 -u 100000 \
CMD="${bismark} ${mt_ref} -1 ${R1} -2 ${R2} --multicore 8 \
-o ${output_dir} --prefix MT --path_to_bowtie ${bowtie_path} \
--samtools_path ${samtools_path} --temp_dir ${tmp_dir} > \
${log_dir}/2_1_aln.stdout 2> ${log_dir}/2_1_aln.stderr"
echo -e ${CMD} > ${log_dir}/2_1_aln.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/2_1_aln.cmd" --log ${log_dir}/2_1_aln.resources --interval 60 --include-children
else
bash ${log_dir}/2_1_aln.cmd
......@@ -333,6 +343,7 @@ CMD="${samtools} merge -n -r -@ 32 ${output_dir}/MT_${sample_id}.bam ${bamfile_l
${log_dir}/2_2_merge.stderr"
echo -e ${CMD} > ${log_dir}/2_2_merge.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/2_2_merge.cmd" --log ${log_dir}/2_2_merge.resources --interval 60 --include-children
else
bash ${log_dir}/2_2_merge.cmd
......@@ -358,6 +369,7 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
${log_dir}/2_3_dedup.stderr"
echo -e ${CMD} > ${log_dir}/2_3_dedup.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/2_3_dedup.cmd" --log ${log_dir}/2_3_dedup.resources --interval 60 --include-children
else
bash ${log_dir}/2_3_dedup.cmd
......@@ -384,6 +396,7 @@ CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
${log_dir}/2_4_meth.stderr"
echo -e ${CMD} > ${log_dir}/2_4_meth.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/2_4_meth.cmd" --log ${log_dir}/2_4_meth.resources --interval 60 --include-children
else
bash ${log_dir}/2_4_meth.cmd
......@@ -416,13 +429,14 @@ echo "##################################################"
echo -ne " 3.1 Aligning vs nuclear DNA... "
#CMD="${bismark} ${nuc_ref} -1 ${R1} -2 ${R2} -u 10000 \
#CMD="${bismark} ${nuc_ref} -1 ${R1} -2 ${R2} --multicore 2 -u 100000 \
CMD="${bismark} ${nuc_ref} -1 ${R1} -2 ${R2} --multicore 8 \
--un --ambiguous -o ${output_dir} --prefix b38_noMT --path_to_bowtie ${bowtie_path} \
--samtools_path ${samtools_path} --temp_dir ${tmp_dir} > \
${log_dir}/3_1_aln.stdout 2> ${log_dir}/3_1_aln.stderr"
echo -e ${CMD} > ${log_dir}/3_1_aln.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/3_1_aln.cmd" --log ${log_dir}/3_1_aln.resources --interval 60 --include-children
else
bash ${log_dir}/3_1_aln.cmd
......@@ -459,6 +473,7 @@ CMD="cat ${unmapped_list_r1} > ${output_dir}/b38_noMT_${sample_id}_unmapped.R1.f
rm ${output_dir}/b38_noMT.*_ambiguous_reads_?.fq.gz"
echo -e ${CMD} > ${log_dir}/3_2_merge_unmapped.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/3_2_merge.cmd" --log ${log_dir}/3_2_merge.resources --interval 60 --include-children
else
bash ${log_dir}/3_2_merge_unmapped.cmd
......@@ -487,6 +502,7 @@ done
CMD="rm ${bamfile_list}"
echo ${CMD} > ${log_dir}/3_3_delete.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/3_3_delete.cmd" --log ${log_dir}/3_3_delete.resources --interval 60 --include-children
else
bash ${log_dir}/3_3_delete.cmd
......@@ -514,6 +530,7 @@ CMD="(${bismark} ${mt_ref} -1 ${output_dir}/b38_noMT_${sample_id}_unmapped.R1.fa
${log_dir}/3_4_realign_to_mt.stderr"
echo ${CMD} > ${log_dir}/3_4_realign_to_mt.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/3_4_realign_to_mt.cmd" --log ${log_dir}/3_4_realign_to_mt.resources --interval 60 --include-children
else
bash ${log_dir}/3_4_realign_to_mt.cmd
......@@ -527,6 +544,9 @@ else
fi
rm ${output_dir}/b38_noMT_${sample_id}_unmapped.R?.fastq.gz
echo -ne " 3.5 Dedupping alignments... "
CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
......@@ -535,6 +555,7 @@ CMD="${deduplicate_bismark} --paired --samtools_path ${samtools_path} \
${log_dir}/3_5_dedup.stderr"
echo -e ${CMD} > ${log_dir}/3_5_dedup.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/3_5_dedup.cmd" --log ${log_dir}/3_5_dedup.resources --interval 60 --include-children
else
bash ${log_dir}/3_5_dedup.cmd
......@@ -563,6 +584,7 @@ CMD="${bismark_meth_extract} -p --comprehensive --output ${output_dir} \
${log_dir}/3_7_meth.stderr"
echo -e ${CMD} > ${log_dir}/3_6_meth.cmd
if $MONITOR; then
echo -e "\n "
${psrecord} "bash ${log_dir}/3_6_meth.cmd" --log ${log_dir}/3_6_meth.resources --interval 60 --include-children
else
bash ${log_dir}/3_6_meth.cmd
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment