Merge branch 'release/0.1.3'

bioinfomaticsCSU · Jan 17, 2019 · e5c7740 · e5c7740
2 parents 7dcebae + 2724efd
commit e5c7740
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -4,12 +4,12 @@ DeepSignal constructs a BiLSTM+Inception structure to detect DNA methylation sta
 built with **Tensorflow 1.8** and Python 3.
 
 ## Contents
-- [Install](#Install)
+- [Installation](#Installation)
 - [Trained models](#Trained-models)
 - [Example data](#Example-data)
 - [Usage](#Usage)
 
-## Install
+## Installation
 deepsignal is built on Python3. [tombo](https://github.com/nanoporetech/tombo) is required to re-squiggle the raw signals from nanopore reads before running deepsignal.
    - Prerequisites:\
        [Python 3.*](https://www.python.org/)\
@@ -22,15 +22,19 @@ deepsignal is built on Python3. [tombo](https://github.com/nanoporetech/tombo) i
        [tensorflow v1.8.0](https://www.tensorflow.org/)
 
 #### Create an environment
-We highly recommend to use a virtual environment for the installation of deepsignal and its dependencies. A virtual environment can be created and activated as follows by using [conda](https://conda.io/docs/):
+We highly recommend to use a virtual environment for the installation of deepsignal and its dependencies. A virtual environment can be created and (de)activated as follows by using [conda](https://conda.io/docs/):
 ```bash
+# create
 conda create -n deepsignalenv python=3.6
+# activate
 conda activate deepsignalenv
+# deactivate
+conda deactivate
 ```
 The virtual environment can also be created by using [*virtualenv*](https://github.com/pypa/virtualenv/).
 
 #### Install deepsignal
-After creating the environment, download and install the **lastest** deepsignl from github:
+After creating and activating the environment, download and install deepsignal (**lastest version**) from github:
 ```bash
 git clone https://github.com/bioinfomaticsCSU/deepsignal.git
 cd deepsignal
@@ -48,6 +52,7 @@ conda install -c bioconda ont-tombo
 # or install using pip
 pip install ont-tombo[full]
 ``` 
+
 If a GPU-machine is used, the gpu version of tensorflow is required:
 ```bash
 # install using conda

diff --git a/README.rst b/README.rst
@@ -5,6 +5,9 @@ deepsignal
 Documentation
 -------------
 
+v0.1.3
+-------------
+fix the deadlock issue in multiprocessing
 
 v0.1.2
 -------------

diff --git a/deepsignal/__init__.py b/deepsignal/__init__.py
@@ -2,4 +2,4 @@
 DNA methylation state from Nanopore reads."""
 
 
-__version__ = '0.1.2'
+__version__ = '0.1.3'
diff --git a/deepsignal/call_modifications.py b/deepsignal/call_modifications.py
@@ -54,7 +54,7 @@ def _read_features_file(features_file, features_batch_q, batch_num=512):
             if len(sampleinfo) == batch_num:
                 features_batch_q.put((sampleinfo, kmers, base_means, base_stds,
                                       base_signal_lens, cent_signals, labels))
-                while features_batch_q.qsize() >= queen_size_border:
+                while features_batch_q.qsize() > queen_size_border:
                     time.sleep(time_wait)
                 sampleinfo = []
                 kmers = []
@@ -118,7 +118,7 @@ def _read_features_fast5s_q(fast5s_q, features_batch_q, errornum_q, corrected_gr
         errornum_q.put(error)
         for features_batch in features_batches:
             features_batch_q.put(features_batch)
-        while features_batch_q.qsize() >= queen_size_border:
+        while features_batch_q.qsize() > queen_size_border:
             time.sleep(time_wait)
 
 
@@ -267,15 +267,22 @@ def _call_mods_from_fast5s_cpu(motif_seqs, chrom2len, fast5s_q, len_fast5s,
     p_w.daemon = True
     p_w.start()
 
+    errornum_sum = 0
+    while True:
+        running = any(p.is_alive() for p in pred_str_procs)
+        while not errornum_q.empty():
+            errornum_sum += errornum_q.get()
+        if not running:
+            break
+
     for p in pred_str_procs:
         p.join()
+
+    print("finishing the write_process..")
     pred_str_q.put("kill")
 
     p_w.join()
 
-    errornum_sum = 0
-    while not errornum_q.empty():
-        errornum_sum += errornum_q.get()
     print("%d of %d fast5 files failed.." % (errornum_sum, len_fast5s))
 
 
@@ -316,18 +323,25 @@ def _call_mods_from_fast5s_gpu(motif_seqs, chrom2len, fast5s_q, len_fast5s,
     p_w.daemon = True
     p_w.start()
 
+    errornum_sum = 0
+    while True:
+        running = any(p.is_alive() for p in features_batch_procs)
+        while not errornum_q.empty():
+            errornum_sum += errornum_q.get()
+        if not running:
+            break
+
     for p in features_batch_procs:
         p.join()
     features_batch_q.put("kill")
 
     p_call_mods_gpu.join()
+
+    print("finishing the write_process..")
     pred_str_q.put("kill")
 
     p_w.join()
 
-    errornum_sum = 0
-    while not errornum_q.empty():
-        errornum_sum += errornum_q.get()
     print("%d of %d fast5 files failed.." % (errornum_sum, len_fast5s))
 
 
@@ -391,6 +405,8 @@ def call_mods(input_path, model_path, result_file, kmer_len, cent_signals_len,
 
         for p in predstr_procs:
             p.join()
+
+        print("finishing the write_process..")
         pred_str_q.put("kill")
 
         p_rf.join()

diff --git a/deepsignal/deepsignal.py b/deepsignal/deepsignal.py
@@ -14,7 +14,7 @@
 def display_args(args):
     arg_vars = vars(args)
     print("# ===============================================")
-    print("parameters: ")
+    print("## parameters: ")
     for arg_key in arg_vars.keys():
         if arg_key != 'func':
             print("{}:\n\t{}".format(arg_key, arg_vars[arg_key]))

diff --git a/deepsignal/extract_features.py b/deepsignal/extract_features.py
@@ -298,7 +298,7 @@ def _write_featurestr_to_file(write_fp, featurestr_q):
 def _extract_preprocess(fast5_dir, is_recursive, motifs, is_dna, reference_path, f5_batch_num):
 
     fast5_files = get_fast5s(fast5_dir, is_recursive)
-    print("{} fast5 files in total".format(len(fast5_files)))
+    print("{} fast5 files in total..".format(len(fast5_files)))
 
     print("parse the motifs string..")
     motif_seqs = get_motif_seqs(motifs, is_dna)
@@ -342,19 +342,25 @@ def extract_features(fast5_dir, is_recursive, reference_path, is_dna,
     p_w.daemon = True
     p_w.start()
 
+    errornum_sum = 0
+    while True:
+        running = any(p.is_alive() for p in featurestr_procs)
+        while not errornum_q.empty():
+            errornum_sum += errornum_q.get()
+        if not running:
+            break
+
     for p in featurestr_procs:
         p.join()
 
+    print("finishing the write_process..")
     featurestr_q.put("kill")
-    time.sleep(1)
 
     p_w.join()
 
-    errornum_sum = 0
-    while not errornum_q.empty():
-        errornum_sum += errornum_q.get()
-    print("%d of %d failed, extract_features costs %.1f seconds.." % (errornum_sum, len_fast5s,
-                                                                      time.time() - start))
+    print("%d of %d fast5 files failed..\n"
+          "extract_features costs %.1f seconds.." % (errornum_sum, len_fast5s,
+                                                     time.time() - start))
 
 
 def main():

diff --git a/requirements.txt b/requirements.txt
@@ -2,4 +2,4 @@ numpy>=1.15.3
 h5py>=2.8.0
 statsmodels>=0.9.0
 scikit-learn>=0.20.1
-tensorflow==1.8.0
+tensorflow>=1.8.0
diff --git a/setup.py b/setup.py
@@ -44,15 +44,17 @@ def run_tests(self):
     keywords=['methylation', 'nanopore', 'neural network'],
     version=find_version('deepsignal', '__init__.py'),
     url='https://github.com/bioinfomaticsCSU/deepsignal',
-    download_url='https://github.com/bioinfomaticsCSU/deepsignal/archive/v0.1.1.tar.gz',
+    download_url='https://github.com/bioinfomaticsCSU/deepsignal/archive/v0.1.3.tar.gz',
     license='GNU General Public License v3 (GPLv3)',
     author='Peng Ni, Neng Huang',
     # tests_require=['pytest'],
+    # TODO: when use python setup.py install, tensorflow>=1.8.0 may not work,
+    # TODO: but it looks fine when using pip
     install_requires=['numpy>=1.15.3',
                       'h5py>=2.8.0',
                       'statsmodels>=0.9.0',
                       'scikit-learn>=0.20.1',
-                      'tensorflow==1.8.0',
+                      'tensorflow>=1.8.0',
                       ],
     # cmdclass={'test': PyTest},
     author_email='[email protected]',