GPU-Optimized RMI-Based Seeding

Makefile

@@ -1,11 +1,47 @@
+OPT_FLAGS=-DALIGN_AVX -DAPPLY_AVX2
+OPT_FLAGS+= ${CHAIN_FLAG} 
+OPT_FLAGS+=$(COMP_FLAG)
 CFLAGS_NDEBUG = -DNDEBUG -O3 
-CDEBUG_FLAGS= -g -O2 #-Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare -Wno-unused-function -Wno-c++17-extensions -Wno-\#warnings #-O0 -DNDEBUG
+CDEBUG_FLAGS= -g -Wall -O2 -Wc++-compat #-Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare -Wno-unused-function -Wno-c++17-extensions -Wno-\#warnings #-O0 -DNDEBUG
 CPPFLAGS=	-DHAVE_KALLOC -D__AMD_SPLIT_KERNELS__ # -Wno-unused-but-set-variable -Wno-unused-variable
 CPPFLAGS+= 	$(if $(MAX_MICRO_BATCH),-DMAX_MICRO_BATCH=\($(MAX_MICRO_BATCH)\))
-INCLUDES=	-I .
+COMP_FLAG = -march=native
+CHAIN_FLAG = -DPARALLEL_CHAINING
+
+
+ifneq ($(portable),)
+	STATIC_GCC=-static-libgcc -static-libstdc++
+endif 
+ifeq ($(CXX), icpc)
+	CC= icc
+else ifeq ($(CXX), g++)
+	CC=gcc
+endif	
+
+ifeq ($(lhash_index), 1)	
+	CPPFLAGS+=	-DLISA_INDEX 
+endif
+ifeq ($(lhash), 1)	
+	OPT_FLAGS+=	-DLISA_HASH -DUINT64 -DVECTORIZE 
+endif
+
+ifeq ($(manual_profile), 1)
+	CPPFLAGS+= -DMANUAL_PROFILING 
+endif
+
+ifeq ($(disable_output), 1)
+	CPPFLAGS+= -DDISABLE_OUTPUT
+endif
+
+ifeq ($(no_opt),)
+	CPPFLAGS+= $(OPT_FLAGS)
+endif
+
+
+INCLUDES=	-I . -I./ext/TAL/src/LISA-hash
 OBJS=		kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o options.o index.o \
 			lchain.o align.o hit.o seed.o map.o format.o pe.o esterr.o splitidx.o \
-			ksw2_ll_sse.o
+			ksw2_ll_sse.o ./ext/TAL/src/LISA-hash/lisa_hash.o
 # PROG=		minimap2-zerobranch-debug
 # PROG=		minimap2-nobalance-debug
 PROG=		minimap2$(SUFFIX)
@@ -159,4 +195,5 @@ pe.o: mmpriv.h minimap.h bseq.h kseq.h kvec.h kalloc.h ksort.h
 sdust.o: kalloc.h kdq.h kvec.h sdust.h
 seed.o: mmpriv.h minimap.h bseq.h kseq.h kalloc.h ksort.h
 sketch.o: kvec.h kalloc.h mmpriv.h minimap.h bseq.h kseq.h
-splitidx.o: mmpriv.h minimap.h bseq.h kseq.h
+splitidx.o: mmpriv.h minimap.h bseq.h kseq.h
+./ext/TAL/src/LISA-hash/lisa_hash.o: ./ext/TAL/src/LISA-hash/lisa_hash.h

README.md

@@ -1,6 +1,68 @@
+# GPU-Optimized RMI-Based Seeding
+
+## Introduction
+In this work, we focused on optimizing the seeding look-up process. We aimed to improve the performance of mm2-gb by implementing the RMI-based seeding on GPUs, inspired by [mm2-fast](https://github.com/bwa-mem2/mm2-fast). By leveraging the parallel processing capabilities of GPU, our implementation aimed to significantly improve both speed and efficiency. Finally,  we integrated the GPU-optimized RMI-based seeding into the mm2-gb repository and conducted thorough performance evaluations.
+![architecture](https://github.com/chenchiii/mm2-gb/assets/114895836/7f9908ee-62cb-4752-9bd7-52b13de8e8fa)
+
+## Hardware Specifications
+GPU: mm2-gb was tested on AMD Instinct™ MI100 GPU running ROCm-5.7.1
+## Installation
+### mm2-gb Integration
+1. Clone the mm2-gb GitHub repo.
+```
+git clone --recursive git@github.com:chenchiii/mm2-gb.git mm2-gb     
+cd mm2-gb
+```
+2. Build the executable using the `make` command, specifying the debug level and GPU type at compile time:
+```
+# DEBUG levels:
+#   <empty>   : minimap2 prints only
+#   info      : print chaining kernel stats (except throughput in anchor pairs/s)
+#   analyze   : calculate kernel throughput in anchor pairs/s (requires additional device synchronization)
+#   verbose   : print kernel launch/debug information
+
+# Example:build for AMD GPUs with RMI-based seeding on CPU
+make GPU=AMD DEBUG=analyze lhash=1
+```
+### Standalone Seeding Lookup Process
+To install the standalone kernel function, follow these steps:
+1. Navigate to the rmi-seeding-lookup directory:
+```sh
+cd rmi-seeding-lookup
+```
+2. Build the kernel function using the provided build script:
+```sh
+./build <implementation.cu> <name>
+```
+3. Run the test for the installed kernel function:
+```sh
+./run_test <name>
+```
+For a more detailed description, refer to the [README.md](https://github.com/chenchiii/mm2-gb/tree/gpu_kernel/rmi-seeding-lookup) in the rmi-seeding-lookup directory.
+## Usage
+The usage of mm2-gb is similar to original mm2-gb. Command0line flags includes:
+- `--gpu-chain` flag enables GPU chaining.
+- `--gpu-cfg <filename.json>` specifies the GPU configuration json file. 
+## Profiling
+To profile the execution of mm2-gb using `rocprof`, use the following command:
+```
+rocprof --hip-trace ./minimap2 -t 1 --gpu-chain --gpu-cfg gpu/gpu_config.json test/MT-human.fa test/MT-orang.fa > mm2-gb_out.paf
+```
+## Performance
+Our GPU optimized ML-based seeding implementation demonstrates better performance compared to mm2-gb on an AMD Instinct MI100. Specifically, we observed a 30x increase in performance of lookup in the seeding step.  
+
+## Furture work
+- Merge our tested standalone GPU kernel functions to mm2-gb
+- Further optimization of GPU kernels with different strategies
+- Develop hybrid CPU-GPU architectures
+
+--------------------------------------------------------------------------------------------------
+## The original README contents of mm2-gb follow.
+
 # mm2-gb
 mm2-gb is based on minimap2-v2.24 with GPU accelerated chaining kernel for high throughput accurate mapping of ultra-long reads. 
 ![mm2-workflow (1)](https://github.com/Minimap2onGPU/minimap2/assets/42312167/5ae47c34-a8e0-487a-9ebf-fc2bf7d30154)
+
 ## System Requirements
 OS: linux. tested on Ubuntu 20.04, Ubuntu 22.04
 

ext/TAL/.gitmodules

@@ -0,0 +1,6 @@
+[submodule "ext/safestringlib"]
+	path = ext/safestringlib
+	url = https://github.com/intel/safestringlib.git
+[submodule "ext/build-rmi/learned-systems-rmi"]
+	path = ext/build-rmi/learned-systems-rmi
+	url = https://github.com/learnedsystems/RMI.git

ext/TAL/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Intel Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ext/TAL/LISA/Makefile

@@ -0,0 +1,72 @@
+##/*************************************************************************************
+##                           The MIT License
+##
+##   Copyright (C) 2020 Intel Labs.
+##
+##   Permission is hereby granted, free of charge, to any person obtaining a copy
+##   of this software and associated documentation files (the "Software"), to deal
+##   in the Software without restriction, including without limitation the rights
+##   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+##   copies of the Software, and to permit persons to whom the Software is
+##   furnished to do so, subject to the following conditions:
+##
+##   The above copyright notice and this permission notice shall be included in all
+##   copies or substantial portions of the Software.
+##   
+##   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+##   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+##   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+##   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+##   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+##   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+##   SOFTWARE.
+##
+##Contacts: Sanchit Misra <sanchit.misra@intel.com>; Vasimuddin Md <vasimuddin.md@intel.com>
+##*****************************************************************************************/
+
+HP= -DNO_HUGE_PAGE
+ifeq ($(huge_page), 1)
+	HP= -DHUGE_PAGE
+endif
+
+LISA_CC = icpc 
+LISA_CFLAGS = -DSAIS=1 -std=c++17 -xCORE-AVX512 -Ofast -fopenmp -Wall -Wshadow -Wno-char-subscripts
+LISA_CPPFLAGS = -DOUTPUT ${HP} -DNO_DNA_ORD -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK -DVECTORIZE  -DENABLE_PREFETCH -D_64BIT
+LISA_CPPFLAGS_EXACT_SEARCH = ${HP} -DNO_DNA_ORD -DBWA_MEM_BUG -DNOCHUNK -DVECTORIZE  -DENABLE_PREFETCH -D_64BIT
+LISA_BUILD_RMI_FLAGS = -DBWA_MEM_BUG -DNOCHUNK -xCORE-AVX512  -DVECTORIZE -DENABLE_PREFETCH -D_64BIT 
+LISA_INCLUDE = -I src/ -I ../ext/ -I ../ext/safestringlib/include/
+LISA_LDLIBS = -lz -L./.. -ltal -L ../ext/safestringlib/ -lsafestring 
+
+
+
+
+VTUNE = -DVTUNE_ANALYSIS -I/swtools/intel/vtune_amplifier/include/ -littnotify -L/swtools/intel/vtune_amplifier/lib64/ 
+
+lisa: smem-lisa exact-search-lisa build-index-forward-only-lisa build-index-with-rev-complement-lisa
+smem-lisa: benchmarks/bench-smem-lisa.cpp
+	${LISA_CC} ${LISA_CFLAGS} ${LISA_CPPFLAGS} ${LISA_INCLUDE} benchmarks/bench-smem-lisa.cpp ${LISA_LDLIBS} -DPRINT_OUTPUT -o smem-lisa.o
+
+
+exact-search-lisa: benchmarks/bench-fixed-len-e2e-match-lisa.cpp
+	${LISA_CC} ${LISA_CFLAGS} ${LISA_CPPFLAGS_EXACT_SEARCH} ${LISA_INCLUDE} -DENABLE_PREFETCH_OPT benchmarks/bench-fixed-len-e2e-match-lisa.cpp ${LISA_LDLIBS} -DPRINT_OUTPUT -o exact-search-lisa.o 
+
+
+build-index-forward-only-lisa: benchmarks/build-lisa-index.cpp
+	${LISA_CC} ${LISA_CFLAGS} ${LISA_BUILD_RMI_FLAGS} ${LISA_INCLUDE} benchmarks/build-lisa-index.cpp ${LISA_LDLIBS} -o build-index-forward-only-lisa.o
+
+build-index-with-rev-complement-lisa: benchmarks/build-lisa-index.cpp
+	${LISA_CC} ${LISA_CFLAGS} ${LISA_BUILD_RMI_FLAGS} -DREV_COMP ${LISA_INCLUDE} benchmarks/build-lisa-index.cpp ${LISA_LDLIBS} -o build-index-with-rev-complement-lisa.o
+
+clean:
+	rm *.o 
+
+
+
+# Experimental compilation flags
+#TEMP_LISA_CFLAGS = -DSAIS=1 -std=c++17  -Ofast -fopenmp -Wall -Wshadow -Wno-char-subscripts
+#TEMP_LISA_CPPFLAGS = -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK   -DENABLE_PREFETCH -D_64BIT
+#LISA_CPPFLAGS_SCALAR = -DOUTPUT ${HP} -DNO_DNA_ORD -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK -DNo_VECTORIZE  -DENABLE_PREFETCH -D_64BIT
+#LISA_CPPFLAGS_NO_OPT = -DOUTPUT -DNO_DNA_ORD -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK -DNo_VECTORIZE  -DENABLE_PREFETCH -D_64BIT
+#LISA_CPPFLAGS_EXACT_SEARCH_SCALAR = ${HP} -DNO_DNA_ORD -DBWA_MEM_BUG -DNOCHUNK -DNO_VECTORIZE  -DENABLE_PREFETCH -D_64BIT
+#LISA_CPPFLAGS_EXACT_SEARCH_UNOPT = -DVECTORIZE  -DENABLE_PREFETCH -DNO_DNA_ORD -DBWA_MEM_BUG -DNOCHUNK -D_64BIT
+#LISA_OUTPUT = qbwt-rmi-parallel-make.o

ext/TAL/LISA/README

@@ -0,0 +1,25 @@
+## Introduction
+LISA provides learned indexes based optimized implementations of two key kernels: exact-search and smem-search.
+
+The contributors to LISA are Saurabh Kalikar, Sanchit Misra and Darryl Ho.
+
+## Getting Started
+```sh
+git clone --recursive <REPO URL>
+make
+cd LISA
+make lisa
+```
+
+## LISA index building
+Learned hash-table uses an external training library that runs on Rust.
+Install Rust <i> https://rustup.rs/ </i> and add installation path to .bashrc file. 
+
+1. Build forward-only LISA index (for exact search).
+./build-index-forward-only-lisa.o full-path-to-ref_file K num_rmi_leaf_nodes
+
+2. Build LISA index for SMEM search.
+scripts/build-lisa-smem-index.sh full-path-to-ref_file K num_rmi_leaf_nodes
+
+
+