Merge branch 'bug_fixes'

.github/workflows/bowtie2.yml

@@ -52,3 +52,4 @@ jobs:
         run: |
           make allall
           make simple-test
+          make random-test

BOWTIE2_VERSION

@@ -1 +1 @@
-2.4.4
+2.4.5

MANUAL

@@ -1924,6 +1924,10 @@ format:
 Fields are separated by tabs. Colorspace is always set to 0 for Bowtie
 2.
 
+    -o/--output <filename>
+
+Save output to user-specified filename (default: stdout)
+
     -v/--verbose
 
 Print verbose output (for debugging).

MANUAL.markdown

@@ -2584,6 +2584,14 @@ names and lengths of the input sequences.  The summary has this format:
 
 Fields are separated by tabs.  Colorspace is always set to 0 for Bowtie 2.
 
+</td></tr><tr><td id="bowtie2-inspect-options-o">
+
+    -o/--output <filename>
+
+</td><td>
+
+Save output to user-specified filename (default: stdout)
+
 </td></tr><tr><td>
 
     -v/--verbose
@@ -2913,6 +2921,7 @@ warnings due to the case insensitive nature of markdown URLs -->
 [`-m`]:                                               #bowtie2-options-m
 [`-n`/`--names`]:                                     #bowtie2-inspect-options-n
 [`-o`/`--offrate`]:                                   #bowtie2-options-o
+[`-o`/`--output`]:                                    #bowtie2-inspect-options-o
 [`-o`]:                                               #bowtie2-options-o
 [`-p`/`--packed`]:                                    #bowtie2-build-options-p
 [`-p`/`--threads`]:                                   #bowtie2-options-p

Makefile

@@ -43,6 +43,9 @@ ifneq (,$(findstring MINGW,$(shell uname)))
   WINDOWS := 1
   MINGW := 1
   # POSIX memory-mapped files not currently supported on Windows
+endif
+
+ifeq (1, $(WINDOWS))
   BOWTIE_MM :=
   BOWTIE_SHARED_MEM :=
 endif
@@ -252,7 +255,7 @@ GENERAL_LIST := $(wildcard scripts/*.sh) \
 
 ifeq (1,$(WINDOWS))
   BOWTIE2_BIN_LIST := $(BOWTIE2_BIN_LIST) bowtie2.bat bowtie2-build.bat bowtie2-inspect.bat
-  CXXFLAGS += -static-libgcc -static-libstdc++
+  CXXFLAGS += -static-libgcc -static-libstdc++ -static
 endif
 
 # This is helpful on Windows under MinGW/MSYS, where Make might go for

README.md

@@ -1,8 +1,8 @@
 
 <!-- badges: start -->
 ![Github Actions](https://github.com/BenLangmead/bowtie2/actions/workflows/bowtie2.yml/badge.svg)
-[![Generic badge](https://img.shields.io/badge/version-2.4.4-green.svg)](https://shields.io/)
-[![Build Status](https://travis-ci.org/BenLangmead/bowtie2.svg?branch=master)](https://travis-ci.org/BenLangmead/bowtie2)
+[![Generic badge](https://img.shields.io/badge/version-2.4.5-green.svg)](https://shields.io/)
+<!-- [![Build Status](https://travis-ci.org/BenLangmead/bowtie2.svg?branch=master)](https://travis-ci.org/BenLangmead/bowtie2) -->
 [![License: GPL v3](https://img.shields.io/badge/license-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 <!--badges: end -->
 

aln_sink.cpp

@@ -2114,6 +2114,12 @@ void AlnSinkSam::appendMate(
 	samc_.printPreservedOptFlags(o, rd);
 	samc_.printComment(o, rd.name);
 	o.append('\n');
+	if(samc_.passthrough()) {
+		// Original read string
+		samc_.printOptFieldNewlineEscapedZ(o, rd.readOrigBuf);
+		o.append('\n');
+	}
+
 }
 
 #ifdef ALN_SINK_MAIN

blockwise_sa.h

@@ -22,6 +22,7 @@
 
 #include <stdint.h>
 #include <stdlib.h>
+#include <future>
 #include <iostream>
 #include <sstream>
 #include <thread>
@@ -38,6 +39,7 @@
 #include "ds.h"
 #include "mem_ids.h"
 #include "word_io.h"
+#include "threadpool.h"
 
 using namespace std;
 
@@ -198,6 +200,7 @@ class KarkkainenBlockwiseSA : public InorderBlockwiseSA<TStr> {
 	KarkkainenBlockwiseSA(const TStr& __text,
 			      TIndexOffU __bucketSz,
 			      int __nthreads,
+			      thread_pool& pool,
 			      uint32_t __dcV,
 			      uint32_t __seed = 0,
 			      bool __sanityCheck = false,
@@ -208,6 +211,7 @@ class KarkkainenBlockwiseSA : public InorderBlockwiseSA<TStr> {
 		InorderBlockwiseSA<TStr>(__text, __bucketSz, __sanityCheck, __passMemExc, __verbose, __logger),
 		_sampleSuffs(EBWTB_CAT),
 		_nthreads(__nthreads),
+		_pool(pool),
 		_itrBucketIdx(0),
 		_cur(0),
 		_dcV(__dcV),
@@ -218,17 +222,7 @@ class KarkkainenBlockwiseSA : public InorderBlockwiseSA<TStr> {
 		_done(NULL)
 		{ _randomSrc.init(__seed); reset(); }
 
-	~KarkkainenBlockwiseSA() throw()
-		{
-			if(_threads.size() > 0) {
-				for (size_t tid = 0; tid < _threads.size(); tid++) {
-					_threads[tid]->join();
-					delete _threads[tid];
-				}
-			}
-			if (_done != NULL)
-				delete[] _done;
-		}
+	~KarkkainenBlockwiseSA() throw() {}
 
 	/**
 	 * Allocate an amount of memory that simulates the peak memory
@@ -253,7 +247,7 @@ class KarkkainenBlockwiseSA : public InorderBlockwiseSA<TStr> {
 		{
 			// Launch threads if not
 			if(this->_nthreads > 1) {
-				if(_threads.size() == 0) {
+				if(_tparams.size() == 0) {
 					_done = new volatile bool[_sampleSuffs.size() + 1];
 					for (size_t i = 0; i < _sampleSuffs.size() + 1; i++) {
 						_done[i] = false;
@@ -263,9 +257,11 @@ class KarkkainenBlockwiseSA : public InorderBlockwiseSA<TStr> {
 					for(int tid = 0; tid < this->_nthreads; tid++) {
 						_tparams[tid].first = this;
 						_tparams[tid].second = tid;
-						_threads.push_back(new thread(nextBlock_Worker((void*)&_tparams[tid])));
+						if (tid == _nthreads - 1)
+							nextBlock_Worker((void *)&_tparams[tid]);
+						else
+							_pool.submit(nextBlock_Worker((void *)&_tparams[tid]));
 					}
-					assert_eq(_threads.size(), (size_t)this->_nthreads);
 				}
 			}
 			if(this->_itrPushedBackSuffix != OFF_MASK) {
@@ -396,7 +392,7 @@ class KarkkainenBlockwiseSA : public InorderBlockwiseSA<TStr> {
 		assert(_dc.get() == NULL);
 		if(_dcV != 0) {
 			_dc.init(new TDC(this->text(), _dcV, this->verbose(), this->sanityCheck()));
-			_dc.get()->build(this->_nthreads);
+			_dc.get()->build(_pool, this->_nthreads);
 		}
 		// Calculate sample suffixes
 		if(this->bucketSz() <= this->text().length()) {
@@ -436,6 +432,7 @@ class KarkkainenBlockwiseSA : public InorderBlockwiseSA<TStr> {
 
 	EList<TIndexOffU>  _sampleSuffs; /// sample suffixes
 	int                _nthreads;    /// # of threads
+	thread_pool& _pool;
 	TIndexOffU         _itrBucketIdx;
 	TIndexOffU         _cur;         /// offset to 1st elt of next block
 	const uint32_t   _dcV;         /// difference-cover periodicity
@@ -522,7 +519,7 @@ struct BinarySortingParam {
 };
 
 template<typename TStr>
-class BinarySorting_worker {
+class  BinarySorting_worker {
         void *vp;
 
 public:
@@ -638,7 +635,7 @@ void KarkkainenBlockwiseSA<TStr>::buildSamples() {
 	// Iterate until all buckets are less than
 	while(--limit >= 0) {
 		TIndexOffU numBuckets = (TIndexOffU)_sampleSuffs.size()+1;
-		AutoArray<std::thread*> threads(this->_nthreads);
+		std::vector<std::future<void> > threads(_pool.size());
 		EList<BinarySortingParam<TStr> > tparams;
 		tparams.resize(this->_nthreads);
 		for(int tid = 0; tid < this->_nthreads; tid++) {
@@ -665,19 +662,17 @@ void KarkkainenBlockwiseSA<TStr>::buildSamples() {
 			tparams[tid].sampleSuffs = &_sampleSuffs;
 			tparams[tid].begin = (tid == 0 ? 0 : len / this->_nthreads * tid);
 			tparams[tid].end = (tid + 1 == this->_nthreads ? len : len / this->_nthreads * (tid + 1));
-			if(this->_nthreads == 1) {
+			if(this->_nthreads == 1 || tid == _nthreads - 1) {
 				BinarySorting_worker<TStr>((void*)&tparams[tid])();
 			} else {
-				threads[tid] = new std::thread(BinarySorting_worker<TStr>(((void*)&tparams[tid])));
+				threads[tid] = _pool.submit(BinarySorting_worker<TStr>(((void*)&tparams[tid])));
 			}
 		}
 
 		if(this->_nthreads > 1) {
-			for (int tid = 0; tid < this->_nthreads; tid++) {
-				threads[tid]->join();
+			for (int tid = 0; tid < _pool.size(); tid++) {
+				threads[tid].get();
 			}
-			for (int tid = 0; tid < this->_nthreads; tid++)
-				delete threads[tid];
 		}
 		EList<TIndexOffU>& bucketSzs = tparams[0].bucketSzs;
 		EList<TIndexOffU>& bucketReps = tparams[0].bucketReps;

bowtie2-build

@@ -19,14 +19,24 @@
  along with Bowtie 2.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-
 import os
 import sys
+import gzip
+import struct
 import inspect
 import logging
 import argparse
 import subprocess
-from collections import deque
+
+def get_gunzip_size(fn):
+    size = 0
+    with gzip.open(fn) as f:
+        while True:
+            data = f.read(8192)
+            size += len(data)
+            if not data:
+                break
+    return size
 
 def main():
     parser = argparse.ArgumentParser(add_help = False)
@@ -51,11 +61,6 @@ def main():
     build_bin_spec      = os.path.join(ex_path,build_bin_s)
 
     script_options, argv = parser.parse_known_args()
-    print_help = False
-    argv = deque(argv)
-
-    if '-h' in argv or '--help' in argv:
-        print_help = True
 
     if script_options.verbose:
         logging.getLogger().setLevel(logging.INFO)
@@ -68,23 +73,41 @@ def main():
         build_bin_spec += '-sanitized'
         build_bin_l += '-sanitized'
 
+    fastas = []
+    if '-c' not in argv and len(argv) >= 2:
+        for index in reversed(range(len(argv) - 1)):
+            arg = argv[index]
+            if arg.startswith('-') or arg.isdigit():
+                break
+            fastas.insert(0, arg)
+            argv.remove(arg)
+        if fastas:
+            fastas = ','.join(fastas)
+            argv.insert(len(argv) - 1, fastas)
+
     if script_options.large_index:
         build_bin_spec = os.path.join(ex_path,build_bin_l)
-    elif len(argv) >= 2:
-        ref_fnames = argv[-2]
+    elif fastas:
         tot_size = 0
-        for fn in ref_fnames.split(','):
+        for fn in fastas.split(','):
             if os.path.exists(fn):
-                statinfo = os.stat(fn)
-                tot_size += statinfo.st_size
+                if fn.endswith('.gz') or fn.endswith(".Z"):
+                    tot_size += get_gunzip_size(fn)
+                else:
+                    statinfo = os.stat(fn)
+                    tot_size += statinfo.st_size
         if tot_size > small_index_max_size:
             build_bin_spec = os.path.join(ex_path,build_bin_l)
 
-    argv.appendleft('basic-0')
-    argv.appendleft('--wrapper')
-    argv.appendleft(build_bin_spec)
+    if not os.path.exists(build_bin_spec):
+        sys.stderr.write('{0} does not exist, try running `[g]make {0}\'\n'.format(os.path.basename(build_bin_spec)))
+        sys.exit(1)
+
+    argv.insert(0, 'basic-0')
+    argv.insert(0, '--wrapper')
+    argv.insert(0, build_bin_spec)
     logging.info('Command: %s' % ' '.join(argv))
-    sys.exit(subprocess.call(list(argv)))
+    sys.exit(subprocess.call(argv))
 
 if __name__ == '__main__':
     main()

bt2_build.cpp

@@ -138,17 +138,17 @@ static void printUsage(ostream& out) {
 	out << "Usage: " << tool_name << " [options]* <reference_in> <bt2_index_base>" << endl
 	    << "    reference_in            comma-separated list of files with ref sequences" << endl
 	    << "    bt2_index_base          write " + gEbwt_ext + " data to files with this dir/basename" << endl
-	    << "*** Bowtie 2 indexes work only with v2 (not v1).  Likewise for v1 indexes. ***" << endl
+	    << "*** Bowtie 2 indexes will work with Bowtie v1.2.3 and later. ***" << endl
 	    << "Options:" << endl
 	    << "    -f                      reference files are Fasta (default)" << endl
 	    << "    -c                      reference sequences given on cmd line (as" << endl
 	    << "                            <reference_in>)" << endl;
 	if(wrapper == "basic-0") {
 	out << "    --large-index           force generated index to be 'large', even if ref" << endl
-		<< "                            has fewer than 4 billion nucleotides" << endl
-		<< "    --debug                 use the debug binary; slower, assertions enabled" << endl
-		<< "    --sanitized             use sanitized binary; slower, uses ASan and/or UBSan" << endl
-		<< "    --verbose               log the issued command" << endl;
+	    << "                            has fewer than 4 billion nucleotides" << endl
+	    << "    --debug                 use the debug binary; slower, assertions enabled" << endl
+	    << "    --sanitized             use sanitized binary; slower, uses ASan and/or UBSan" << endl
+	    << "    --verbose               log the issued command" << endl;
 	}
 	out << "    -a/--noauto             disable automatic -p/--bmax/--dcv memory-fitting" << endl
 	    << "    -p/--packed             use packed strings internally; slower, less memory" << endl
@@ -166,16 +166,15 @@ static void printUsage(ostream& out) {
 	    //<< (currentlyBigEndian()? "big":"little") << ")" << endl
 	    << "    --seed <int>            seed for random number generator" << endl
 	    << "    -q/--quiet              verbose output (for debugging)" << endl
-	    << "    -h/--help               print detailed description of tool and its options" << endl
-	    << "    --usage                 print this usage message" << endl
+	    << "    --h/--help              print this message and quit" << endl
 	    << "    --version               print version information and quit" << endl
 	    ;
 	if(wrapper.empty()) {
 		cerr << endl
 		     << "*** Warning ***" << endl
-			 << "'" << tool_name << "' was run directly.  It is recommended "
-			 << "that you run the wrapper script 'bowtie2-build' instead."
-			 << endl << endl;
+		     << "'" << tool_name << "' was run directly.  It is recommended "
+		     << "that you run the wrapper script 'bowtie2-build' instead."
+		     << endl << endl;
 	}
 }
 
@@ -360,6 +359,14 @@ static void deleteIdxFiles(
 	}
 }
 
+static void renameIdxFiles() {
+	for (size_t i = 0; i < filesWritten.size(); i++) {
+		std::string oldName = filesWritten[i] + ".tmp";
+		std::cerr << "Renaming " << oldName << " to " << filesWritten[i] << std::endl;
+		std::rename(oldName.c_str(), filesWritten[i].c_str());
+	}
+}
+
 /**
  * Drive the index construction process and optionally sanity-check the
  * result.
@@ -686,7 +693,6 @@ int bowtie_build(int argc, const char **argv) {
 		if(packed) {
 			driver<S2bDnaString>(infile, infiles, outfile + ".rev", true, reverseType);
 		}
-		return 0;
 	} catch(std::exception& e) {
 		cerr << "Error: Encountered exception: '" << e.what() << "'" << endl;
 		cerr << "Command: ";
@@ -704,5 +710,7 @@ int bowtie_build(int argc, const char **argv) {
 		deleteIdxFiles(outfile, writeRef || justRef, justRef);
 		return e;
 	}
+	renameIdxFiles();
+	return 0;
 }
 }