From 9826333334e63eae85cf53aafcf30b28623f7dd3 Mon Sep 17 00:00:00 2001 From: PengNi <543943952@qq.com> Date: Tue, 15 Jan 2019 18:27:28 -0500 Subject: [PATCH 1/4] change download_url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2cdcdae..93cf673 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ def run_tests(self): keywords=['methylation', 'nanopore', 'neural network'], version=find_version('deepsignal', '__init__.py'), url='https://github.com/bioinfomaticsCSU/deepsignal', - download_url='https://github.com/bioinfomaticsCSU/deepsignal/archive/v0.1.1.tar.gz', + download_url='https://github.com/bioinfomaticsCSU/deepsignal/archive/v0.1.2.tar.gz', license='GNU General Public License v3 (GPLv3)', author='Peng Ni, Neng Huang', # tests_require=['pytest'], From 096edc488a55238b4f17821b6b0ff929fa4cf7fd Mon Sep 17 00:00:00 2001 From: PengNi <543943952@qq.com> Date: Tue, 15 Jan 2019 18:59:09 -0500 Subject: [PATCH 2/4] modify README after v1.0.2 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e0eece0..3bdfa63 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@ DeepSignal constructs a BiLSTM+Inception structure to detect DNA methylation sta built with **Tensorflow 1.8** and Python 3. ## Contents -- [Install](#Install) +- [Installation](#Installation) - [Trained models](#Trained-models) - [Example data](#Example-data) - [Usage](#Usage) -## Install +## Installation deepsignal is built on Python3. [tombo](https://github.com/nanoporetech/tombo) is required to re-squiggle the raw signals from nanopore reads before running deepsignal. - Prerequisites:\ [Python 3.*](https://www.python.org/)\ @@ -30,7 +30,7 @@ conda activate deepsignalenv The virtual environment can also be created by using [*virtualenv*](https://github.com/pypa/virtualenv/). #### Install deepsignal -After creating the environment, download and install the **lastest** deepsignl from github: +After creating the environment, download and install deepsignal (**lastest version**) from github: ```bash git clone https://github.com/bioinfomaticsCSU/deepsignal.git cd deepsignal From 8cb4ebd58e5036aae8399e2ed810df2730799dc4 Mon Sep 17 00:00:00 2001 From: PengNi <543943952@qq.com> Date: Thu, 17 Jan 2019 11:37:27 -0500 Subject: [PATCH 3/4] fix the deadlock issue --- README.md | 9 +++++++-- deepsignal/call_modifications.py | 32 ++++++++++++++++++++++++-------- deepsignal/deepsignal.py | 2 +- deepsignal/extract_features.py | 20 +++++++++++++------- requirements.txt | 2 +- setup.py | 4 +++- 6 files changed, 49 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3bdfa63..db85b28 100644 --- a/README.md +++ b/README.md @@ -22,15 +22,19 @@ deepsignal is built on Python3. [tombo](https://github.com/nanoporetech/tombo) i [tensorflow v1.8.0](https://www.tensorflow.org/) #### Create an environment -We highly recommend to use a virtual environment for the installation of deepsignal and its dependencies. A virtual environment can be created and activated as follows by using [conda](https://conda.io/docs/): +We highly recommend to use a virtual environment for the installation of deepsignal and its dependencies. A virtual environment can be created and (de)activated as follows by using [conda](https://conda.io/docs/): ```bash +# create conda create -n deepsignalenv python=3.6 +# activate conda activate deepsignalenv +# deactivate +conda deactivate ``` The virtual environment can also be created by using [*virtualenv*](https://github.com/pypa/virtualenv/). #### Install deepsignal -After creating the environment, download and install deepsignal (**lastest version**) from github: +After creating and activating the environment, download and install deepsignal (**lastest version**) from github: ```bash git clone https://github.com/bioinfomaticsCSU/deepsignal.git cd deepsignal @@ -48,6 +52,7 @@ conda install -c bioconda ont-tombo # or install using pip pip install ont-tombo[full] ``` + If a GPU-machine is used, the gpu version of tensorflow is required: ```bash # install using conda diff --git a/deepsignal/call_modifications.py b/deepsignal/call_modifications.py index bf22d18..f0cdcdd 100644 --- a/deepsignal/call_modifications.py +++ b/deepsignal/call_modifications.py @@ -54,7 +54,7 @@ def _read_features_file(features_file, features_batch_q, batch_num=512): if len(sampleinfo) == batch_num: features_batch_q.put((sampleinfo, kmers, base_means, base_stds, base_signal_lens, cent_signals, labels)) - while features_batch_q.qsize() >= queen_size_border: + while features_batch_q.qsize() > queen_size_border: time.sleep(time_wait) sampleinfo = [] kmers = [] @@ -118,7 +118,7 @@ def _read_features_fast5s_q(fast5s_q, features_batch_q, errornum_q, corrected_gr errornum_q.put(error) for features_batch in features_batches: features_batch_q.put(features_batch) - while features_batch_q.qsize() >= queen_size_border: + while features_batch_q.qsize() > queen_size_border: time.sleep(time_wait) @@ -267,15 +267,22 @@ def _call_mods_from_fast5s_cpu(motif_seqs, chrom2len, fast5s_q, len_fast5s, p_w.daemon = True p_w.start() + errornum_sum = 0 + while True: + running = any(p.is_alive() for p in pred_str_procs) + while not errornum_q.empty(): + errornum_sum += errornum_q.get() + if not running: + break + for p in pred_str_procs: p.join() + + print("finishing the write_process..") pred_str_q.put("kill") p_w.join() - errornum_sum = 0 - while not errornum_q.empty(): - errornum_sum += errornum_q.get() print("%d of %d fast5 files failed.." % (errornum_sum, len_fast5s)) @@ -316,18 +323,25 @@ def _call_mods_from_fast5s_gpu(motif_seqs, chrom2len, fast5s_q, len_fast5s, p_w.daemon = True p_w.start() + errornum_sum = 0 + while True: + running = any(p.is_alive() for p in features_batch_procs) + while not errornum_q.empty(): + errornum_sum += errornum_q.get() + if not running: + break + for p in features_batch_procs: p.join() features_batch_q.put("kill") p_call_mods_gpu.join() + + print("finishing the write_process..") pred_str_q.put("kill") p_w.join() - errornum_sum = 0 - while not errornum_q.empty(): - errornum_sum += errornum_q.get() print("%d of %d fast5 files failed.." % (errornum_sum, len_fast5s)) @@ -391,6 +405,8 @@ def call_mods(input_path, model_path, result_file, kmer_len, cent_signals_len, for p in predstr_procs: p.join() + + print("finishing the write_process..") pred_str_q.put("kill") p_rf.join() diff --git a/deepsignal/deepsignal.py b/deepsignal/deepsignal.py index 6805d88..99b19f8 100644 --- a/deepsignal/deepsignal.py +++ b/deepsignal/deepsignal.py @@ -14,7 +14,7 @@ def display_args(args): arg_vars = vars(args) print("# ===============================================") - print("parameters: ") + print("## parameters: ") for arg_key in arg_vars.keys(): if arg_key != 'func': print("{}:\n\t{}".format(arg_key, arg_vars[arg_key])) diff --git a/deepsignal/extract_features.py b/deepsignal/extract_features.py index 28a2df7..8b2ade3 100644 --- a/deepsignal/extract_features.py +++ b/deepsignal/extract_features.py @@ -298,7 +298,7 @@ def _write_featurestr_to_file(write_fp, featurestr_q): def _extract_preprocess(fast5_dir, is_recursive, motifs, is_dna, reference_path, f5_batch_num): fast5_files = get_fast5s(fast5_dir, is_recursive) - print("{} fast5 files in total".format(len(fast5_files))) + print("{} fast5 files in total..".format(len(fast5_files))) print("parse the motifs string..") motif_seqs = get_motif_seqs(motifs, is_dna) @@ -342,19 +342,25 @@ def extract_features(fast5_dir, is_recursive, reference_path, is_dna, p_w.daemon = True p_w.start() + errornum_sum = 0 + while True: + running = any(p.is_alive() for p in featurestr_procs) + while not errornum_q.empty(): + errornum_sum += errornum_q.get() + if not running: + break + for p in featurestr_procs: p.join() + print("finishing the write_process..") featurestr_q.put("kill") - time.sleep(1) p_w.join() - errornum_sum = 0 - while not errornum_q.empty(): - errornum_sum += errornum_q.get() - print("%d of %d failed, extract_features costs %.1f seconds.." % (errornum_sum, len_fast5s, - time.time() - start)) + print("%d of %d fast5 files failed..\n" + "extract_features costs %.1f seconds.." % (errornum_sum, len_fast5s, + time.time() - start)) def main(): diff --git a/requirements.txt b/requirements.txt index 6b9108f..1b9186a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ numpy>=1.15.3 h5py>=2.8.0 statsmodels>=0.9.0 scikit-learn>=0.20.1 -tensorflow==1.8.0 \ No newline at end of file +tensorflow>=1.8.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 93cf673..2d2f540 100644 --- a/setup.py +++ b/setup.py @@ -48,11 +48,13 @@ def run_tests(self): license='GNU General Public License v3 (GPLv3)', author='Peng Ni, Neng Huang', # tests_require=['pytest'], + # TODO: when use python setup.py install, tensorflow>=1.8.0 may not work, + # TODO: but it looks fine when using pip install_requires=['numpy>=1.15.3', 'h5py>=2.8.0', 'statsmodels>=0.9.0', 'scikit-learn>=0.20.1', - 'tensorflow==1.8.0', + 'tensorflow>=1.8.0', ], # cmdclass={'test': PyTest}, author_email='543943952@qq.com', From 2724efd345e3da31ed88814643ed7d10ab19e655 Mon Sep 17 00:00:00 2001 From: PengNi <543943952@qq.com> Date: Thu, 17 Jan 2019 11:47:12 -0500 Subject: [PATCH 4/4] bump the version to 0.1.3 --- README.rst | 3 +++ deepsignal/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c2a1789..ddc8810 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,9 @@ deepsignal Documentation ------------- +v0.1.3 +------------- +fix the deadlock issue in multiprocessing v0.1.2 ------------- diff --git a/deepsignal/__init__.py b/deepsignal/__init__.py index 4c10cf5..a9fb1d1 100644 --- a/deepsignal/__init__.py +++ b/deepsignal/__init__.py @@ -2,4 +2,4 @@ DNA methylation state from Nanopore reads.""" -__version__ = '0.1.2' +__version__ = '0.1.3' diff --git a/setup.py b/setup.py index 2d2f540..c5cf53a 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ def run_tests(self): keywords=['methylation', 'nanopore', 'neural network'], version=find_version('deepsignal', '__init__.py'), url='https://github.com/bioinfomaticsCSU/deepsignal', - download_url='https://github.com/bioinfomaticsCSU/deepsignal/archive/v0.1.2.tar.gz', + download_url='https://github.com/bioinfomaticsCSU/deepsignal/archive/v0.1.3.tar.gz', license='GNU General Public License v3 (GPLv3)', author='Peng Ni, Neng Huang', # tests_require=['pytest'],