diff --git a/Makefile b/Makefile index f8e05134..9cf2015c 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,47 @@ +OPT_FLAGS=-DALIGN_AVX -DAPPLY_AVX2 +OPT_FLAGS+= ${CHAIN_FLAG} +OPT_FLAGS+=$(COMP_FLAG) CFLAGS_NDEBUG = -DNDEBUG -O3 -CDEBUG_FLAGS= -g -O2 #-Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare -Wno-unused-function -Wno-c++17-extensions -Wno-\#warnings #-O0 -DNDEBUG +CDEBUG_FLAGS= -g -Wall -O2 -Wc++-compat #-Wall -Wextra -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare -Wno-unused-function -Wno-c++17-extensions -Wno-\#warnings #-O0 -DNDEBUG CPPFLAGS= -DHAVE_KALLOC -D__AMD_SPLIT_KERNELS__ # -Wno-unused-but-set-variable -Wno-unused-variable CPPFLAGS+= $(if $(MAX_MICRO_BATCH),-DMAX_MICRO_BATCH=\($(MAX_MICRO_BATCH)\)) -INCLUDES= -I . +COMP_FLAG = -march=native +CHAIN_FLAG = -DPARALLEL_CHAINING + + +ifneq ($(portable),) + STATIC_GCC=-static-libgcc -static-libstdc++ +endif +ifeq ($(CXX), icpc) + CC= icc +else ifeq ($(CXX), g++) + CC=gcc +endif + +ifeq ($(lhash_index), 1) + CPPFLAGS+= -DLISA_INDEX +endif +ifeq ($(lhash), 1) + OPT_FLAGS+= -DLISA_HASH -DUINT64 -DVECTORIZE +endif + +ifeq ($(manual_profile), 1) + CPPFLAGS+= -DMANUAL_PROFILING +endif + +ifeq ($(disable_output), 1) + CPPFLAGS+= -DDISABLE_OUTPUT +endif + +ifeq ($(no_opt),) + CPPFLAGS+= $(OPT_FLAGS) +endif + + +INCLUDES= -I . -I./ext/TAL/src/LISA-hash OBJS= kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o options.o index.o \ lchain.o align.o hit.o seed.o map.o format.o pe.o esterr.o splitidx.o \ - ksw2_ll_sse.o + ksw2_ll_sse.o ./ext/TAL/src/LISA-hash/lisa_hash.o # PROG= minimap2-zerobranch-debug # PROG= minimap2-nobalance-debug PROG= minimap2$(SUFFIX) @@ -159,4 +195,5 @@ pe.o: mmpriv.h minimap.h bseq.h kseq.h kvec.h kalloc.h ksort.h sdust.o: kalloc.h kdq.h kvec.h sdust.h seed.o: mmpriv.h minimap.h bseq.h kseq.h kalloc.h ksort.h sketch.o: kvec.h kalloc.h mmpriv.h minimap.h bseq.h kseq.h -splitidx.o: mmpriv.h minimap.h bseq.h kseq.h \ No newline at end of file +splitidx.o: mmpriv.h minimap.h bseq.h kseq.h +./ext/TAL/src/LISA-hash/lisa_hash.o: ./ext/TAL/src/LISA-hash/lisa_hash.h \ No newline at end of file diff --git a/README.md b/README.md index 24b998ad..190abfcc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,68 @@ +# GPU-Optimized RMI-Based Seeding + +## Introduction +In this work, we focused on optimizing the seeding look-up process. We aimed to improve the performance of mm2-gb by implementing the RMI-based seeding on GPUs, inspired by [mm2-fast](https://github.com/bwa-mem2/mm2-fast). By leveraging the parallel processing capabilities of GPU, our implementation aimed to significantly improve both speed and efficiency. Finally, we integrated the GPU-optimized RMI-based seeding into the mm2-gb repository and conducted thorough performance evaluations. +![architecture](https://github.com/chenchiii/mm2-gb/assets/114895836/7f9908ee-62cb-4752-9bd7-52b13de8e8fa) + +## Hardware Specifications +GPU: mm2-gb was tested on AMD Instinct™ MI100 GPU running ROCm-5.7.1 +## Installation +### mm2-gb Integration +1. Clone the mm2-gb GitHub repo. +``` +git clone --recursive git@github.com:chenchiii/mm2-gb.git mm2-gb +cd mm2-gb +``` +2. Build the executable using the `make` command, specifying the debug level and GPU type at compile time: +``` +# DEBUG levels: +# : minimap2 prints only +# info : print chaining kernel stats (except throughput in anchor pairs/s) +# analyze : calculate kernel throughput in anchor pairs/s (requires additional device synchronization) +# verbose : print kernel launch/debug information + +# Example:build for AMD GPUs with RMI-based seeding on CPU +make GPU=AMD DEBUG=analyze lhash=1 +``` +### Standalone Seeding Lookup Process +To install the standalone kernel function, follow these steps: +1. Navigate to the rmi-seeding-lookup directory: +```sh +cd rmi-seeding-lookup +``` +2. Build the kernel function using the provided build script: +```sh +./build +``` +3. Run the test for the installed kernel function: +```sh +./run_test +``` +For a more detailed description, refer to the [README.md](https://github.com/chenchiii/mm2-gb/tree/gpu_kernel/rmi-seeding-lookup) in the rmi-seeding-lookup directory. +## Usage +The usage of mm2-gb is similar to original mm2-gb. Command0line flags includes: +- `--gpu-chain` flag enables GPU chaining. +- `--gpu-cfg ` specifies the GPU configuration json file. +## Profiling +To profile the execution of mm2-gb using `rocprof`, use the following command: +``` +rocprof --hip-trace ./minimap2 -t 1 --gpu-chain --gpu-cfg gpu/gpu_config.json test/MT-human.fa test/MT-orang.fa > mm2-gb_out.paf +``` +## Performance +Our GPU optimized ML-based seeding implementation demonstrates better performance compared to mm2-gb on an AMD Instinct MI100. Specifically, we observed a 30x increase in performance of lookup in the seeding step. + +## Furture work +- Merge our tested standalone GPU kernel functions to mm2-gb +- Further optimization of GPU kernels with different strategies +- Develop hybrid CPU-GPU architectures + +-------------------------------------------------------------------------------------------------- +## The original README contents of mm2-gb follow. + # mm2-gb mm2-gb is based on minimap2-v2.24 with GPU accelerated chaining kernel for high throughput accurate mapping of ultra-long reads. ![mm2-workflow (1)](https://github.com/Minimap2onGPU/minimap2/assets/42312167/5ae47c34-a8e0-487a-9ebf-fc2bf7d30154) + ## System Requirements OS: linux. tested on Ubuntu 20.04, Ubuntu 22.04 diff --git a/ext/TAL/.gitmodules b/ext/TAL/.gitmodules new file mode 100644 index 00000000..b6ccec59 --- /dev/null +++ b/ext/TAL/.gitmodules @@ -0,0 +1,6 @@ +[submodule "ext/safestringlib"] + path = ext/safestringlib + url = https://github.com/intel/safestringlib.git +[submodule "ext/build-rmi/learned-systems-rmi"] + path = ext/build-rmi/learned-systems-rmi + url = https://github.com/learnedsystems/RMI.git diff --git a/ext/TAL/LICENSE b/ext/TAL/LICENSE new file mode 100644 index 00000000..4d410204 --- /dev/null +++ b/ext/TAL/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ext/TAL/LISA/Makefile b/ext/TAL/LISA/Makefile new file mode 100644 index 00000000..c2349003 --- /dev/null +++ b/ext/TAL/LISA/Makefile @@ -0,0 +1,72 @@ +##/************************************************************************************* +## The MIT License +## +## Copyright (C) 2020 Intel Labs. +## +## Permission is hereby granted, free of charge, to any person obtaining a copy +## of this software and associated documentation files (the "Software"), to deal +## in the Software without restriction, including without limitation the rights +## to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +## copies of the Software, and to permit persons to whom the Software is +## furnished to do so, subject to the following conditions: +## +## The above copyright notice and this permission notice shall be included in all +## copies or substantial portions of the Software. +## +## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +## SOFTWARE. +## +##Contacts: Sanchit Misra ; Vasimuddin Md +##*****************************************************************************************/ + +HP= -DNO_HUGE_PAGE +ifeq ($(huge_page), 1) + HP= -DHUGE_PAGE +endif + +LISA_CC = icpc +LISA_CFLAGS = -DSAIS=1 -std=c++17 -xCORE-AVX512 -Ofast -fopenmp -Wall -Wshadow -Wno-char-subscripts +LISA_CPPFLAGS = -DOUTPUT ${HP} -DNO_DNA_ORD -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK -DVECTORIZE -DENABLE_PREFETCH -D_64BIT +LISA_CPPFLAGS_EXACT_SEARCH = ${HP} -DNO_DNA_ORD -DBWA_MEM_BUG -DNOCHUNK -DVECTORIZE -DENABLE_PREFETCH -D_64BIT +LISA_BUILD_RMI_FLAGS = -DBWA_MEM_BUG -DNOCHUNK -xCORE-AVX512 -DVECTORIZE -DENABLE_PREFETCH -D_64BIT +LISA_INCLUDE = -I src/ -I ../ext/ -I ../ext/safestringlib/include/ +LISA_LDLIBS = -lz -L./.. -ltal -L ../ext/safestringlib/ -lsafestring + + + + +VTUNE = -DVTUNE_ANALYSIS -I/swtools/intel/vtune_amplifier/include/ -littnotify -L/swtools/intel/vtune_amplifier/lib64/ + +lisa: smem-lisa exact-search-lisa build-index-forward-only-lisa build-index-with-rev-complement-lisa +smem-lisa: benchmarks/bench-smem-lisa.cpp + ${LISA_CC} ${LISA_CFLAGS} ${LISA_CPPFLAGS} ${LISA_INCLUDE} benchmarks/bench-smem-lisa.cpp ${LISA_LDLIBS} -DPRINT_OUTPUT -o smem-lisa.o + + +exact-search-lisa: benchmarks/bench-fixed-len-e2e-match-lisa.cpp + ${LISA_CC} ${LISA_CFLAGS} ${LISA_CPPFLAGS_EXACT_SEARCH} ${LISA_INCLUDE} -DENABLE_PREFETCH_OPT benchmarks/bench-fixed-len-e2e-match-lisa.cpp ${LISA_LDLIBS} -DPRINT_OUTPUT -o exact-search-lisa.o + + +build-index-forward-only-lisa: benchmarks/build-lisa-index.cpp + ${LISA_CC} ${LISA_CFLAGS} ${LISA_BUILD_RMI_FLAGS} ${LISA_INCLUDE} benchmarks/build-lisa-index.cpp ${LISA_LDLIBS} -o build-index-forward-only-lisa.o + +build-index-with-rev-complement-lisa: benchmarks/build-lisa-index.cpp + ${LISA_CC} ${LISA_CFLAGS} ${LISA_BUILD_RMI_FLAGS} -DREV_COMP ${LISA_INCLUDE} benchmarks/build-lisa-index.cpp ${LISA_LDLIBS} -o build-index-with-rev-complement-lisa.o + +clean: + rm *.o + + + +# Experimental compilation flags +#TEMP_LISA_CFLAGS = -DSAIS=1 -std=c++17 -Ofast -fopenmp -Wall -Wshadow -Wno-char-subscripts +#TEMP_LISA_CPPFLAGS = -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK -DENABLE_PREFETCH -D_64BIT +#LISA_CPPFLAGS_SCALAR = -DOUTPUT ${HP} -DNO_DNA_ORD -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK -DNo_VECTORIZE -DENABLE_PREFETCH -D_64BIT +#LISA_CPPFLAGS_NO_OPT = -DOUTPUT -DNO_DNA_ORD -DREV_COMP -DBWA_MEM_BUG -DNOCHUNK -DNo_VECTORIZE -DENABLE_PREFETCH -D_64BIT +#LISA_CPPFLAGS_EXACT_SEARCH_SCALAR = ${HP} -DNO_DNA_ORD -DBWA_MEM_BUG -DNOCHUNK -DNO_VECTORIZE -DENABLE_PREFETCH -D_64BIT +#LISA_CPPFLAGS_EXACT_SEARCH_UNOPT = -DVECTORIZE -DENABLE_PREFETCH -DNO_DNA_ORD -DBWA_MEM_BUG -DNOCHUNK -D_64BIT +#LISA_OUTPUT = qbwt-rmi-parallel-make.o diff --git a/ext/TAL/LISA/README b/ext/TAL/LISA/README new file mode 100644 index 00000000..28312f7f --- /dev/null +++ b/ext/TAL/LISA/README @@ -0,0 +1,25 @@ +## Introduction +LISA provides learned indexes based optimized implementations of two key kernels: exact-search and smem-search. + +The contributors to LISA are Saurabh Kalikar, Sanchit Misra and Darryl Ho. + +## Getting Started +```sh +git clone --recursive +make +cd LISA +make lisa +``` + +## LISA index building +Learned hash-table uses an external training library that runs on Rust. +Install Rust https://rustup.rs/ and add installation path to .bashrc file. + +1. Build forward-only LISA index (for exact search). +./build-index-forward-only-lisa.o full-path-to-ref_file K num_rmi_leaf_nodes + +2. Build LISA index for SMEM search. +scripts/build-lisa-smem-index.sh full-path-to-ref_file K num_rmi_leaf_nodes + + + diff --git a/ext/TAL/LISA/benchmarks/bench-fixed-len-e2e-match-lisa.cpp b/ext/TAL/LISA/benchmarks/bench-fixed-len-e2e-match-lisa.cpp new file mode 100644 index 00000000..81e91cd3 --- /dev/null +++ b/ext/TAL/LISA/benchmarks/bench-fixed-len-e2e-match-lisa.cpp @@ -0,0 +1,312 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +#ifdef VTUNE_ANALYSIS +#include +#endif + + +#include +#include "common.h" +#include "read.h" +#include +#include "sais.h" +#include "ipbwt_rmi.h" +#include +#ifdef _64BIT + typedef int64_t index_t; +#else + typedef uint32_t index_t; +#endif +/* +struct Info { + const char* p; + int q_len; + int l, r; //[l, r) + pair intv, next_intv; + uint64_t nxt_ext; +}; +*/ +int64_t one_calls = 0; + +int64_t bin_search_walk = 0; + + + + + +// TODO try only do chunk-based on the first SMEM? + +int main(int argc, char** argv) { +#ifdef VTUNE_ANALYSIS + __itt_pause(); +#endif + + + if(!(argc == 5 || argc == 6)) { + error_quit("Need 5 args: ref_file query_set K num_rmi_leaf_nodes num_threads"); + } + + int K = atoi(argv[3]); + eprintln("using K = %d", K); + + if(K < 0 || K > 30) return 0; + + int64_t num_rmi_leaf_nodes = atol(argv[4]); + eprintln("using num_rmi_leaf_nodes = %ld", num_rmi_leaf_nodes); + + string seq;// = read_seq(argv[1]); + read_seq_lisa(argv[1], seq); + eprintln("Read ref file done."); + eprintln("seq.size() = %lu", seq.size()); + string queries; int max_query_len = 0; + tie(queries, max_query_len) = read_query_separated_with_dot(argv[2]); + + + eprintln("Read query file done."); + + + +#ifdef REV_COMP + eprintln("No char placed between ref seq and reverse complement, to replicate BWA-MEM bug."); + // appending reverse complement + for(int64_t i=(index_t)seq.size()-1-(seq.back()=='@');i>=0;i--) { +#ifndef NO_DNA_ORD + seq.push_back(dna[3-dna_ord(seq[i])]); +#else + seq.push_back(dna[3-(__lg(seq[i]-'A'+2)-1)]); +#endif + } +#endif + + seq.push_back('$'); + + + + + + +string ref_seq_filename = argv[1]; +#ifdef REV_COMP + string rmi_filename = ref_seq_filename + ".qbwt4.walg.rev_comp"; +#else + string rmi_filename = ref_seq_filename + ".qbwt4.walg"; +#endif + + + //IPBWT_RMI rmi(seq, rmi_filename, sa.data()); + + + + IPBWT_RMI rmi(seq, seq.size(), rmi_filename, K, num_rmi_leaf_nodes, NULL); +// IPBWT_RMI rmi(seq, 3101804740, rmi_filename, K, num_rmi_leaf_nodes, NULL); + + int64_t numMatches = 0; + //int64_t sorting_ticks = 0; + //int64_t one_ticks = 0; + //int64_t chunk_ticks = 0; + int64_t totalTicks = 0; + vector qs; + { + for(int64_t i=0, j; i<(int64_t)queries.size(); i = j + 1) { + for(j=i; queries[j] != '.' && queries[j] != ';'; j++){} + if(j == i) continue; + Info q; + q.p = queries.c_str() + i; + // q.q_len = j-i; + q.l = q.r = j - i;//q.q_len; + q.intv = {0, rmi.n}; +#ifdef NO_DNA_ORD + for(int k=0; k 0123 + } +#endif + qs.push_back(q); + } + } + + uint64_t *str_enc = (uint64_t *)malloc(qs.size() * sizeof(uint64_t)); + + int64_t q_size = qs.size(); + int64_t *intv_all = (int64_t *)malloc(q_size * 2 * sizeof(int64_t)); + //int64_t readIntv; + + assert(str_enc != NULL && intv_all != NULL); + + int numThreads = atoi(argv[5]);//56; +// int numThreads = 1; + + #pragma omp parallel num_threads(numThreads) + { + + int id = omp_get_thread_num(); + if(id == 0) + eprintln("Thread created"); + + } + + + + for(int64_t j = 0; j < q_size; j++) + { + intv_all[2 * j] = 0; + intv_all[2 * j + 1] = rmi.n; + } + + + + +#ifdef VTUNE_ANALYSIS + __itt_resume(); +#endif + +totalTicks -= __rdtsc(); +uint64_t matchCount = 0; +int optimal_num_threads = min(34, numThreads); + + +int num_iter = max_query_len; +while(num_iter) +{ + num_iter -= K; +#pragma omp parallel num_threads(optimal_num_threads) +//#pragma omp parallel num_threads(1) +{ + int64_t workTicks = 0; + int64_t q_processed = 0; + + #pragma omp for schedule (static) + for(int64_t j = 0; j < q_size; j++) + { + +// int64_t start = __rdtsc(); + uint64_t nxt_ext = 0; + Info &q = qs[j]; + + +#ifndef NO_DNA_ORD + for(int it = q.l-K; it!=q.l; it++) { + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[it]); + } +#else + for(int it = q.l-K; it != q.l; it++) { + nxt_ext = (nxt_ext<<2) | (q.p[it]); + } +#endif + str_enc[j] = nxt_ext; + q.l = q.l - K; + q_processed ++; + } + +} + + + + + +#if ENABLE_PREFETCH_OPT + + int64_t parallel_batch_size = ceil((q_size/numThreads + 1)/80); + #pragma omp parallel num_threads(numThreads) + { + int64_t workTicks = 0; + int64_t q_processed = 0; + #pragma omp for schedule(dynamic, 1) + for(int64_t i = 0; i < q_size; i = i + parallel_batch_size){ + //int tid = omp_get_thread_num(); +// + int64_t qs_sz = ((i + parallel_batch_size) <= q_size)? parallel_batch_size: q_size - i; + + +// rmi.backward_extend_chunk_batched(&str_enc[0], q_size, intv_all); + rmi.backward_extend_chunk_batched(&str_enc[i], qs_sz, &intv_all[i*2]); + q_processed += qs_sz; + } + } + + + +#else + #pragma omp parallel for num_threads(numThreads) + for(int64_t i = 0; i < q_size; i++) + { + auto q = qs[i]; + auto q_intv = rmi.backward_extend_chunk(str_enc[i], {intv_all[2 * i], intv_all[2 * i + 1]});//}q.intv); + + intv_all[2 * i] = q_intv.first; + intv_all[2*i + 1] = q_intv.second; + } +#endif + + +} + +totalTicks += __rdtsc(); + +#ifdef VTUNE_ANALYSIS + __itt_pause(); +#endif + + + + for(int64_t i = 0; i < q_size; i++) + { + int64_t nm = intv_all[2 * i + 1] - intv_all[2 * i]; + if(nm > 0) + { + numMatches += nm; + matchCount++; + } + } + + + + int64_t num_queries = count(queries.begin(), queries.end(), ';'); + assert(num_queries > 0); + eprintln("Search Done."); + eprintln("Number of exact matchs = %lld, match counnt %lld num queries %lld qs_size %lld", (long long)numMatches, (long long)matchCount, (long long)num_queries, (long long)q_size); + eprintln("totalTicks = %lld", (long long)totalTicks); + eprintln("Ticks per query = %.3f", (double)(totalTicks * 1.0 / num_queries)); + eprintln("%lld: Binary search per query = %.3f", num_rmi_leaf_nodes,(double)(bin_search_walk * 1.0 / num_queries)); + + +#ifdef PRINT_OUTPUT + for(int64_t i = 0; i < q_size; i++) + { + int64_t nm = intv_all[2 * i + 1] - intv_all[2 * i]; + printf("%ld: ",i); + if(nm > 0) + { + printf("%ld, %ld", intv_all[2 * i], intv_all[2 * i + 1]); + } + printf("\n"); + } +#endif + free(str_enc); free(intv_all); + return 0; +} +#undef flip +#undef rev_comp + diff --git a/ext/TAL/LISA/benchmarks/bench-smem-lisa.cpp b/ext/TAL/LISA/benchmarks/bench-smem-lisa.cpp new file mode 100644 index 00000000..eb629659 --- /dev/null +++ b/ext/TAL/LISA/benchmarks/bench-smem-lisa.cpp @@ -0,0 +1,230 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +#include "qbwt-rmi-batched.h" +#include +// Batch pools +#define pool_size 30000 +int batch_size = 10000; + + +bool smem_sort(SMEM_out a, SMEM_out b) { + return a.id < b.id || (a.id == b.id && a.q_l < b.q_l); +} + +int smem_qsort(const void* a_p, const void* b_p) { + + SMEM_out a = *((SMEM_out*) a_p); + SMEM_out b = *((SMEM_out*) b_p); + + if(a.id < b.id || (a.id == b.id && a.q_l < b.q_l)) return -1; + else return 1; +} + +int main(int argc, char** argv) { +#ifdef VTUNE_ANALYSIS + __itt_pause(); +#endif + if(argc != 7) { + error_quit("Need 6 args: ref_file query_set k num_rmi_leaf_nodes number_of_threads min_seed_length"); + } + uint64_t totalTicks = 0; + int K = atoi(argv[3]); + eprintln("using K = %d", K); + + int64_t num_rmi_leaf_nodes = atol(argv[4]); + eprintln("using num_rmi_leaf_nodes = %ld", num_rmi_leaf_nodes); + + + + string seq; +// read_seq_lisa(argv[1], seq); + eprintln("Read ref file done."); + eprintln("seq.size() = %lu", seq.size()); + string queries; int max_query_len = 0; + tie(queries, max_query_len) = read_query_separated_with_dot(argv[2]); + eprintln("Read query file done."); + + + int numThreads = atoi(argv[5]); + int min_seed_len = atoi(argv[6]); + if(numThreads < 0 || numThreads > 128) return 0; + if(min_seed_len < 0 || min_seed_len > 151) return 0; + assert(numThreads > 0 && 128); + + threadData* v_td = (threadData*) malloc(numThreads * sizeof(threadData)); + assert(v_td != NULL); + for(int i = 0; i < numThreads; i++){ + threadData td(pool_size); + v_td[i] = td; + } + + int seq_size = seq.size(); + string size_file_name = (string) argv[1] + "_SIZE"; + ifstream fi(size_file_name.c_str()); + int64_t size_file; + fi>>size_file; + eprintln("Read ref file done. %lld", size_file); + + + QBWT_HYBRID qbwt(seq, size_file, argv[1], K, num_rmi_leaf_nodes); + + + int64_t qs_size ; + vector qs; + int idCount = 0; + { + for(int64_t i=0, j; i<(int64_t)queries.size(); i = j + 1) { + for(j=i; queries[j] != '.' && queries[j] != ';'; j++); + if(j == i) continue; + Info q; + q.p = queries.c_str() + i; + q.l = q.r = j-i; + q.intv = {0, qbwt.n}; + +#ifdef NO_DNA_ORD + for(int k=0; k 0123 + } +#endif + if(j - i >= min_seed_len) + { + q.id = idCount++; + qs.push_back(q); + } + } + } + + + qs_size = qs.size(); + +#pragma omp parallel num_threads(numThreads) + { + eprintln("Thread created"); + } + +#ifdef VTUNE_ANALYSIS + __itt_resume(); +#endif + totalTicks -= __rdtsc(); + + int64_t TotalSMEM = 0; + int64_t parallel_batch_size = ceil((qs_size/numThreads + 1)/80); + + Output *output = (Output*) malloc(numThreads * sizeof(Output)); + int64_t perThreadQuota = qs.size()/numThreads + 1; + + int64_t num_batches = ceil((double)qs_size/parallel_batch_size); + SMEM_out** batch_start = (SMEM_out**)malloc(num_batches * sizeof(SMEM_out*)); + int64_t* num_smem_per_batch = (int64_t*)malloc(num_batches * sizeof(int64_t)); + + +#pragma omp parallel num_threads(numThreads) + { + int64_t workTicks = 0; + int64_t q_processed = 0; + int64_t vAnsAllocation = perThreadQuota * 20; + int tid = omp_get_thread_num(); + + output[tid] = Output(tid); + output[tid].smem = (SMEM_out*) malloc(vAnsAllocation * sizeof(SMEM_out)); + +#pragma omp for schedule(dynamic, 1) + for(int64_t i = 0; i < qs_size; i = i + parallel_batch_size){ + + int64_t qs_sz = ((i + parallel_batch_size) <= qs_size)? parallel_batch_size: qs_size - i; + int64_t batch_id = i/parallel_batch_size; + int64_t prev_smem_count = v_td[tid].numSMEMs; + batch_start[batch_id] = output[tid].smem + prev_smem_count; + + // Ensures thread local memory is sufficient + if((vAnsAllocation - v_td[tid].numSMEMs < max_query_len * qs_sz)){ + eprintln("Insufficient memory!! Allocating more memory for stroing SMEMs"); + vAnsAllocation *= 2; + + output[tid].smem = (SMEM_out *)realloc(output[tid].smem, vAnsAllocation * sizeof(SMEM_out)); + } + // SMEM search + smem_rmi_batched(&qs[i], qs_sz, batch_size, qbwt, v_td[tid], &output[tid], min_seed_len); + + num_smem_per_batch[batch_id] = v_td[tid].numSMEMs - prev_smem_count; + + //Sort SMEM to rearrange the SMEMs within a batch + sort(batch_start[batch_id], batch_start[batch_id] + num_smem_per_batch[batch_id], smem_sort); + //qsort(batch_start[batch_id], num_smem_per_batch[batch_id], sizeof(SMEM_out), smem_qsort); + } + } + totalTicks += __rdtsc(); + +#ifdef VTUNE_ANALYSIS + __itt_pause(); +#endif + + + for(int i = 0; i < numThreads; i++) + TotalSMEM += v_td[i].numSMEMs; + + int64_t num_queries = count(queries.begin(), queries.end(), ';'); + assert(num_queries > 0); + eprintln("Search Done."); + eprintln("numSMEMs = %lld", (long long)TotalSMEM); + eprintln("SMEMs per query = %.3f", TotalSMEM * 1.0 / num_queries); + eprintln("totalTicks = %lld", (long long)totalTicks); + + + +#ifdef PRINT_OUTPUT + + + int64_t prev_qid = 0; + for(int i = 0; i< num_batches; i++) + { + int64_t num_smem = num_smem_per_batch[i]; + + for(int j = 0; j < num_smem; j++){ + while (prev_qid <= batch_start[i][j].id){ + printf("%lld:\n", prev_qid); + prev_qid++; + } + printf("[%d,%d][%lld, %lld]\n", batch_start[i][j].q_l, batch_start[i][j].q_r, batch_start[i][j].ref_l, batch_start[i][j].ref_r - batch_start[i][j].ref_l); + } + } + +#endif + + +#if 0 + for (int i = 0; i < output.size(); i++){ + eprintln("%d:", output[i].id); + for(int j = output[i].qPos.size() - 1; j >= 0; j--) + eprintln("[%d,%d][%lld, %lld]", output[i].qPos[j].first, output[i].qPos[j].second, (long long)output[i].refPos[j].first, (long long)(output[i].refPos[j].second - output[i].refPos[j].first)); + } + +#endif + free(v_td); + return 0; +} +#undef flip +#undef rev_comp diff --git a/ext/TAL/LISA/benchmarks/build-lisa-index.cpp b/ext/TAL/LISA/benchmarks/build-lisa-index.cpp new file mode 100644 index 00000000..35aa05dc --- /dev/null +++ b/ext/TAL/LISA/benchmarks/build-lisa-index.cpp @@ -0,0 +1,110 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +#ifdef VTUNE_ANALYSIS +#include +#endif + +#include +#include +#include "qbwt-rmi-batched.h" +#include +#include "sais.h" +#include +#include +#include +#include +#include +#include "bntseq.h" +#include +#ifdef _64BIT + typedef int64_t index_t; +#else + typedef uint32_t index_t; +#endif + +int main(int argc, char** argv) { +#ifdef VTUNE_ANALYSIS + __itt_pause(); +#endif + + + if(!(argc == 4)) { + error_quit("Need 3 args: ref_file K num_rmi_leaf_nodes"); + } + + int K = atoi(argv[2]); + eprintln("using K = %d", K); + + int64_t num_rmi_leaf_nodes = atol(argv[3]); + eprintln("using num_rmi_leaf_nodes = %ld", num_rmi_leaf_nodes); + + string seq;// = read_seq(argv[1]); + const char *ref_seq_file = argv[1]; + gzFile fp = xzopen(ref_seq_file, "r"); + bns_fasta2bntseq(fp, ref_seq_file, 1); + read_seq_lisa(argv[1], seq); + eprintln("Read ref file done."); + eprintln("seq.size() = %lu", seq.size()); + + string seq_forward_only = seq; +#ifdef REV_COMP + eprintln("No char placed between ref seq and reverse complement, to replicate BWA-MEM bug."); + // appending reverse complement + for(int64_t i=(index_t)seq.size()-1-(seq.back()=='@');i>=0;i--) { +#ifndef NO_DNA_ORD + seq.push_back(dna[3-dna_ord(seq[i])]); +#else + seq.push_back(dna[3-(__lg(seq[i]-'A'+2)-1)]); +#endif + } +#endif + + seq.push_back('$'); + string ref_seq_filename = argv[1]; +#ifdef REV_COMP + string rmi_filename = ref_seq_filename + ".qbwt4.walg.rev_comp"; +#else + string rmi_filename = ref_seq_filename + ".qbwt4.walg"; +#endif + +{ + IPBWT_RMI rmi(seq, seq.size(), rmi_filename, K, num_rmi_leaf_nodes, NULL); +} +#ifdef REV_COMP +{ + QBWT_HYBRID qbwt(seq_forward_only, seq_forward_only.size(), argv[1], K, num_rmi_leaf_nodes); +} + + + string size_file_name = (string) argv[1] + "_SIZE"; + ofstream f_sz(size_file_name.c_str()); + f_sz<<(seq_forward_only.size()); + +#endif + + return 0; +} + diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/COPYING b/ext/TAL/LISA/ext/build-rmi/RMI/COPYING new file mode 100644 index 00000000..b7560c37 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/COPYING @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Ryan Marcus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/Cargo.toml b/ext/TAL/LISA/ext/build-rmi/RMI/Cargo.toml new file mode 100644 index 00000000..db7c7925 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rmi" +version = "0.1.0" +authors = ["Ryan Marcus "] +edition = "2018" +license = "MIT" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "rmi" +path = "src/main.rs" + +[profile.release] +debug = true + +[dependencies] +rayon = "1.1" +memmap = "0.7.0" +byteorder = "1.3.2" +log = "0.4.8" +env_logger = "0.6.2" +clap = "2.33.0" +json = "0.12.0" +indicatif = "0.13.0" +rmi_lib = { path = "rmi_lib" } diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/LICENSE b/ext/TAL/LISA/ext/build-rmi/RMI/LICENSE new file mode 100644 index 00000000..b7560c37 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Ryan Marcus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/README.md b/ext/TAL/LISA/ext/build-rmi/RMI/README.md new file mode 100644 index 00000000..86cd0bfc --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/README.md @@ -0,0 +1,99 @@ +# RMI + +[![Build Status](https://drone.rm.cab/api/badges/learnedsystems/RMI/status.svg)](https://drone.rm.cab/learnedsystems/RMI) + +This is a reference implementation of recursive model indexes (RMIs). A prototype RMI was initially described in [The Case for Learned Index Structures](https://arxiv.org/abs/1712.01208) by Kraska et al. in 2017. + +![Fig 1 from the Case for Learned Index Structures](http://people.csail.mit.edu/ryanmarcus/rmi.png) + +## RMI basics + +Like binary search trees, an RMI is a structure to help search through sorted data. Given a sorted array, an RMI is a function that maps a key to an approximate index. This approximate index can be used as a starting point for a linear, exponential, or binary search. The [SOSD benchmark](https://learned.systems/sosd) demonstrates that RMIs can outperform binary search and many other standard approaches as well. + +Unlike a binary search tree, an RMI uses machine learning techniques to build this approximation function. The result is normally a small, compact mathematical function that can be evaluated quickly. RMIs are a good tool when you need to search the same sorted data many times. Compared to other structures, RMIs: + +* (➕) Offer faster lookup times (when properly tuned) +* (➕) Are generally much smaller than traditional structures like B-Trees or radix trees +* (➖) Must be trained ahead of time on a dataset +* (➖) Do not support inserts (without retraining the model) + +Many more details can be found in [the original paper](https://arxiv.org/abs/1712.01208). + +## Using this implementation + +To use the reference implementation, clone this repository and [install Rust](https://rustup.rs/). + +The reference RMI implementation is a *compiler.* It takes a dataset as input, and produces C/C++ source files as outputs. The data input file must be a binary file containing: + +1. The number of items, as a 64-bit unsigned integer (little endian) +2. The data items, either 32-bit or 64-bit unsigned integers (little endian) + +If the input file contains 32-bit integers, the filename must end with `uint32`. If the input file contains 64-bit integers, the filename must end with `uint64`. If the input file contains 64-bit floats, the filename must end with `f64`. + +In addition to the input dataset, you must also provide a model structure. For example, to build a 2-layer RMI on the data file `books_200M_uint32` (available from [the Harvard Dataverse](https://dataverse.harvard.edu/file.xhtml?persistentId=doi:10.7910/DVN/JGVF9A/MZZUP2&version=4.0)) with a branching factor of 100, we could run: + +``` +cargo run --release -- books_200M_uint32 my_first_rmi linear,linear 100 +``` + +Logging useful diagnostic information can be enabled by setting the `RUST_LOG` environmental variable to `trace`: `export RUST_LOG=trace`. + + +## Generated code +The RMI generator produces C/C++ source files in the current directory. The command directly above, for example, produces the following output. The C/C++ sources contain a few publicly-exposed fields: + +```C++ +#include +#include +namespace wiki { + bool load(char const* dataPath); + void cleanup(); + const size_t RMI_SIZE = 50331680; + const uint64_t BUILD_TIME_NS = 14288421237; + const char NAME[] = "wiki"; + uint64_t lookup(uint64_t key, size_t* err); +} + +``` + +* The `RMI_SIZE` constant represents the size of the constructed model in bytes. +* The `BUILD_TIME_NS` field records how long it took to build the RMI, in nanoseconds. +* The `NAME` field is a constant you specify (and always matches the namespace name). +* The `load` function will need to be called before any calls to `lookup`. The `dataPath` parameter must the path to the directory containing the RMI data (`rmi_data` in this example / the default). +* The `lookup` function takes in an unsigned, 64-bit integer key and produces an estimate of the offset. The `err` parameter will be populated with the maximum error from the RMI's prediction to the target key. This lookup error can be used to perform a bounded binary search. If the error of the trained RMI is low enough, linear search may give better performance. + +If you run the compiler with the `--no-errors` flag, the API will change to no longer report the maximum possible error of each lookup, saving some space. + +```c++ +uint64_t lookup(uint64_t key); +``` + + +## RMI Layers and Tuning + +Currently, the following types of RMI layers are supported: + +* `linear`, simple linear regression +* `linear_spline`, connected linear spline segments +* `cubic`, connected cubic spline segments +* `loglinear`, simple linear regression with a log transform +* `normal`, normal CDF with tuned mean, variance, and scale. +* `lognormal`, normal CDF with log transform +* `radix`, eliminates common prefixes and returns a fixed number of significant bits based on the branching factor +* `bradix`, same as radix, but attempts to choose the number of bits based on balancing the dataset +* `histogram`, partitions the data into several even-sized blocks (based on the branching factor) + +Tuning an RMI is critical to getting good performance. A good place to start is a `cubic` layer followed by a large linear layer, for example: `cubic,linear 262144`. For automatic tuning, try the RMI optimizer using the `--optimize` flag: + +``` +cargo run --release -- --optimize optimizer_out.json books_200M_uint64 +``` + +By default, the optimizer will use 4 threads. If you have a big machine, consider increasing this with the `--threads` option. + + +This work is freely available under the terms of the MIT license. + +## Contributors + +* [Ryan Marcus](https://rmarcus.info) diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/example_param_grid.json b/ext/TAL/LISA/ext/build-rmi/RMI/example_param_grid.json new file mode 100644 index 00000000..935d5294 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/example_param_grid.json @@ -0,0 +1,114 @@ +{ "configs": + [ + {"layers": "linear,linear", "branching factor": 256}, + {"layers": "linear,linear", "branching factor": 512}, + {"layers": "linear,linear", "branching factor": 1024}, + {"layers": "linear,linear", "branching factor": 2048}, + {"layers": "linear,linear", "branching factor": 4096}, + {"layers": "linear,linear", "branching factor": 8192}, + {"layers": "linear,linear", "branching factor": 16384}, + {"layers": "linear,linear", "branching factor": 32768}, + {"layers": "linear,linear", "branching factor": 65536}, + {"layers": "linear,linear", "branching factor": 131072}, + {"layers": "linear,linear", "branching factor": 262144}, + + {"layers": "loglinear,linear", "branching factor": 256}, + {"layers": "loglinear,linear", "branching factor": 1024}, + {"layers": "loglinear,linear", "branching factor": 4096}, + {"layers": "loglinear,linear", "branching factor": 16384}, + {"layers": "loglinear,linear", "branching factor": 65536}, + {"layers": "loglinear,linear", "branching factor": 262144}, + {"layers": "loglinear,cubic", "branching factor": 256}, + {"layers": "loglinear,cubic", "branching factor": 1024}, + {"layers": "loglinear,cubic", "branching factor": 4096}, + {"layers": "loglinear,cubic", "branching factor": 16384}, + {"layers": "loglinear,cubic", "branching factor": 65536}, + {"layers": "loglinear,cubic", "branching factor": 262144}, + + {"layers": "normal,linear", "branching factor": 256}, + {"layers": "normal,linear", "branching factor": 1024}, + {"layers": "normal,linear", "branching factor": 4096}, + {"layers": "normal,linear", "branching factor": 16384}, + {"layers": "normal,linear", "branching factor": 65536}, + {"layers": "normal,linear", "branching factor": 262144}, + {"layers": "normal,cubic", "branching factor": 256}, + {"layers": "normal,cubic", "branching factor": 1024}, + {"layers": "normal,cubic", "branching factor": 4096}, + {"layers": "normal,cubic", "branching factor": 16384}, + {"layers": "normal,cubic", "branching factor": 65536}, + {"layers": "normal,cubic", "branching factor": 262144}, + + {"layers": "lognormal,linear", "branching factor": 256}, + {"layers": "lognormal,linear", "branching factor": 1024}, + {"layers": "lognormal,linear", "branching factor": 4096}, + {"layers": "lognormal,linear", "branching factor": 16384}, + {"layers": "lognormal,linear", "branching factor": 65536}, + {"layers": "lognormal,linear", "branching factor": 262144}, + {"layers": "lognormal,cubic", "branching factor": 256}, + {"layers": "lognormal,cubic", "branching factor": 1024}, + {"layers": "lognormal,cubic", "branching factor": 4096}, + {"layers": "lognormal,cubic", "branching factor": 16384}, + {"layers": "lognormal,cubic", "branching factor": 65536}, + {"layers": "lognormal,cubic", "branching factor": 262144}, + + {"layers": "linear,cubic", "branching factor": 256}, + {"layers": "linear,cubic", "branching factor": 512}, + {"layers": "linear,cubic", "branching factor": 1024}, + {"layers": "linear,cubic", "branching factor": 2048}, + {"layers": "linear,cubic", "branching factor": 4096}, + {"layers": "linear,cubic", "branching factor": 8192}, + {"layers": "linear,cubic", "branching factor": 16384}, + {"layers": "linear,cubic", "branching factor": 32768}, + {"layers": "linear,cubic", "branching factor": 65536}, + {"layers": "linear,cubic", "branching factor": 131072}, + {"layers": "linear,cubic", "branching factor": 262144}, + + {"layers": "radix,linear", "branching factor": 256}, + {"layers": "radix,linear", "branching factor": 512}, + {"layers": "radix,linear", "branching factor": 1024}, + {"layers": "radix,linear", "branching factor": 2048}, + {"layers": "radix,linear", "branching factor": 4096}, + {"layers": "radix,linear", "branching factor": 8192}, + {"layers": "radix,linear", "branching factor": 16384}, + {"layers": "radix,linear", "branching factor": 32768}, + {"layers": "radix,linear", "branching factor": 65536}, + {"layers": "radix,linear", "branching factor": 131072}, + {"layers": "radix,linear", "branching factor": 262144}, + + {"layers": "radix,cubic", "branching factor": 256}, + {"layers": "radix,cubic", "branching factor": 512}, + {"layers": "radix,cubic", "branching factor": 1024}, + {"layers": "radix,cubic", "branching factor": 2048}, + {"layers": "radix,cubic", "branching factor": 4096}, + {"layers": "radix,cubic", "branching factor": 8192}, + {"layers": "radix,cubic", "branching factor": 16384}, + {"layers": "radix,cubic", "branching factor": 32768}, + {"layers": "radix,cubic", "branching factor": 65536}, + {"layers": "radix,cubic", "branching factor": 131072}, + {"layers": "radix,cubic", "branching factor": 262144}, + + {"layers": "bradix,linear", "branching factor": 256}, + {"layers": "bradix,linear", "branching factor": 512}, + {"layers": "bradix,linear", "branching factor": 1024}, + {"layers": "bradix,linear", "branching factor": 2048}, + {"layers": "bradix,linear", "branching factor": 4096}, + {"layers": "bradix,linear", "branching factor": 8192}, + {"layers": "bradix,linear", "branching factor": 16384}, + {"layers": "bradix,linear", "branching factor": 32768}, + {"layers": "bradix,linear", "branching factor": 65536}, + {"layers": "bradix,linear", "branching factor": 131072}, + {"layers": "bradix,linear", "branching factor": 262144}, + + {"layers": "cubic,cubic", "branching factor": 256}, + {"layers": "cubic,cubic", "branching factor": 512}, + {"layers": "cubic,cubic", "branching factor": 1024}, + {"layers": "cubic,cubic", "branching factor": 2048}, + {"layers": "cubic,cubic", "branching factor": 4096}, + {"layers": "cubic,cubic", "branching factor": 8192}, + {"layers": "cubic,cubic", "branching factor": 16384}, + {"layers": "cubic,cubic", "branching factor": 32768}, + {"layers": "cubic,cubic", "branching factor": 65536}, + {"layers": "cubic,cubic", "branching factor": 131072}, + {"layers": "cubic,cubic", "branching factor": 262144} + ] +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/Cargo.toml b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/Cargo.toml new file mode 100644 index 00000000..df21e0cb --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "rmi_lib" +version = "0.1.0" +authors = ["Ryan Marcus "] +edition = "2018" +license = "MIT" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[profile.release] +debug = true + +[dependencies] +rayon = "1.1" +byteorder = "1.3.2" +log = "0.4.8" +env_logger = "0.6.2" +bytesize = "1.0.0" +approx = "0.3.2" +superslice = "1.0.0" +json = "0.12.0" +indicatif = "0.13.0" +tabular = "0.1.4" diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/cache_fix.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/cache_fix.rs new file mode 100644 index 00000000..7f96773c --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/cache_fix.rs @@ -0,0 +1,150 @@ +use log::*; +use crate::models::TrainingKey; +use crate::RMITrainingData; + +#[derive(Debug)] +pub struct Spline { + from_x: u64, + from_y: usize, + to_x: u64, + to_y: usize +} + +impl Spline { + fn from(pt1: (u64, usize), pt2: (u64, usize)) -> Spline { + assert!(pt1.0 <= pt2.0, + "Cannot construct spline from {:?} to {:?}", pt1, pt2); + assert!(pt1.1 <= pt2.1, + "Cannot construct spline from {:?} to {:?}", pt1, pt2); + return Spline { from_x: pt1.0, from_y: pt1.1, + to_x: pt2.0, to_y: pt2.1 }; + } + + fn with_new_dest(&self, dest: (u64, usize)) -> Spline { + assert!(dest.0 >= self.from_x, + "When source x is {}, cannot set dest x to {}", + self.from_x, dest.0); + assert!(dest.1 >= self.from_y); + return Spline { from_x: self.from_x, from_y: self.from_y, + to_x: dest.0, to_y: dest.1 }; + } + + fn end(&self) -> (u64, usize) { + return (self.to_x, self.to_y); + } + + fn predict(&self, inp: u64) -> usize { + let v0 = self.from_y as f64; + let v1 = self.to_y as f64; + let t = ((inp - self.from_x) as f64) / (self.to_x - self.from_x) as f64; + + return (1.0 - t).mul_add(v0, t * v1) as usize; + } +} + +struct SplineFit { + spline: Option, + curr_pts: Vec<(u64, usize)>, + line_size: usize +} + +impl SplineFit { + + pub fn new(line_size: usize) -> SplineFit { + return SplineFit { + spline: None, + curr_pts: Vec::new(), + line_size + }; + } + + pub fn add_point(&mut self, point: (u64, usize)) -> Option<(u64, usize)> { + if self.spline.is_none() { + self.spline = Some(Spline::from(point, point)); + return Some(point); + } + + // check to see if the current spline can include this point + let last_spline = self.spline.as_ref().unwrap(); + let proposed_spline = last_spline.with_new_dest(point); + + self.curr_pts.push(last_spline.end()); + if self.check_spline(&proposed_spline) { + // accept this proposal, it works. + self.spline = Some(proposed_spline); + return None; + } else { + // reject this proposal, start a new spline. + let prev_pt = last_spline.end(); + assert!(point.0 > prev_pt.0, + "new point: {:?} prev point: {:?}", + point, prev_pt); + + self.spline = Some(Spline::from(prev_pt, point)); + self.curr_pts.clear(); + self.curr_pts.push(point); + return Some(prev_pt); + } + } + + + pub fn finish(self) -> Option<(u64, usize)> { + return self.spline.map(|s| s.end()); + } + + + fn check_spline(&self, spline: &Spline) -> bool { + return self.curr_pts.iter() + .all(|pt| { + let predicted_line: usize = spline.predict(pt.0) / self.line_size; + let correct_line: usize = pt.1 / self.line_size; + predicted_line == correct_line + }); + } +} + +pub fn cache_fix(data: &RMITrainingData, line_size: usize) -> Vec<(u64, usize)> { + assert!(data.len() > line_size, + "Cannot apply a cachefix with fewer items than the line size"); + info!("Fitting cachefix spline to {} datapoints", data.len()); + + let mut fit = SplineFit::new(line_size); + let mut spline = Vec::new(); + + // Potential speedup here by carefully building a spline over the first + // and last element of each cache line. Requires careful handling of duplicates, + // especially when they cross cache lines. + let mut last_key = 0; + for (key, offset) in data.iter_unique() { + assert!(key.minus_epsilon() >= last_key, + "key: {:?} last key: {:?}, key - e: {:?}", + key, last_key, key.minus_epsilon()); + + if key.minus_epsilon() != last_key { + match fit.add_point((key.minus_epsilon(), offset)) { + None => {}, + Some(p) => spline.push(p) + }; + } + + match fit.add_point((key, offset)) { + None => {}, + Some(p) => spline.push(p) + }; + + last_key = key; + } + + + match fit.finish() { + None => {}, + Some(p) => spline.push(p) + }; + + info!("Bounded spline compressed data to {}% of original ({} points, constructed from {} points).", + ((spline.len() as f64 / data.len() as f64)*100.0).round(), + spline.len(), data.len()); + + + return spline; +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/codegen.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/codegen.rs new file mode 100644 index 00000000..6193b394 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/codegen.rs @@ -0,0 +1,788 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + + +use crate::models::Model; +use crate::models::*; +use bytesize::ByteSize; +use log::*; +use std::collections::HashSet; +use std::io::Write; +use std::str; +use crate::train::TrainedRMI; +use std::fs::File; +use std::io::BufWriter; +use std::path::Path; +use std::fmt; + + +enum LayerParams { + Constant(usize, Vec), + Array(usize, usize, Vec), + MixedArray(usize, usize, Vec) +} + +macro_rules! constant_name { + ($layer:expr, $idx: expr) => { + format!("L{}_PARAMETER{}", $layer, $idx) + }; +} + + +macro_rules! array_name { + ($layer: expr) => { + format!("L{}_PARAMETERS", $layer) + } +} + +impl LayerParams { + + fn new(idx: usize, + array_access: bool, + params_per_model: usize, + params: Vec) -> LayerParams { + // first, if the underlying data is mixed, we can only support array mode. + let first_param = params.first().unwrap(); + let mixed = !params.iter().all(|p| first_param.is_same_type(p)); + + if mixed { + return LayerParams::MixedArray(idx, params_per_model, params); + } + + let param_size_bytes: usize = params.iter().map(|p| p.size()).sum(); + if array_access || param_size_bytes > 4096 { + return LayerParams::Array(idx, params_per_model, params); + } + + return LayerParams::Constant(idx, params); + } + + fn to_code(&self, target: &mut T) -> Result<(), std::io::Error> { + match self { + LayerParams::Constant(idx, params) => { + for (p_idx, param) in params.iter().enumerate() { + writeln!( + target, + "const {} {}{} = {};", + param.c_type(), + constant_name!(idx, p_idx), + param.c_type_mod(), + param.c_val() + )?; + } + } + + LayerParams::Array(idx, _, params) => { + write!( + target, + "const {} {}[] = {{", + params[0].c_type(), + array_name!(idx) + )?; + + let (last, rest) = params.split_last().unwrap(); + for param in rest { + write!(target, "{},", param.c_val())?; + } + write!(target, "{}", last.c_val())?; + writeln!(target, "}};")?; + }, + + LayerParams::MixedArray(_, _, _) => { + panic!("Cannot hardcode mixed array."); + } + }; + + return Result::Ok(()); + } + + fn requires_malloc(&self) -> bool { + return match self { + LayerParams::Array(_, _, params) => { + let array_size: usize = params.iter().map(|p| p.size()).sum(); + return array_size >= 4 * 1024; + }, + LayerParams::MixedArray(_, _, _) => true, + LayerParams::Constant(_, _) => false, + }; + } + + fn pointer_type(&self) -> &'static str { + assert!(self.requires_malloc()); + return match self { + LayerParams::Array(_, _, params) => params[0].c_type(), + LayerParams::MixedArray(_, _, _) => "char", + LayerParams::Constant(_, _) => panic!("No pointer type for constant params") + }; + } + + fn to_decl(&self, target: &mut T) -> Result<(), std::io::Error> { + match self { + LayerParams::Constant(_, _) => { + panic!("Cannot forward-declare constants"); + } + + LayerParams::Array(idx, _, params) => { + if !self.requires_malloc() { + let num_items: usize = params.iter().map(|p| p.len()).sum(); + writeln!( + target, + "{} {}[{}];", + params[0].c_type(), + array_name!(idx), + num_items + )?; + } else { + writeln!( + target, + "{}* {};", + params[0].c_type(), + array_name!(idx) + )?; + } + }, + + LayerParams::MixedArray(idx, _, _) => { + assert!(self.requires_malloc()); + writeln!( + target, + "char* {};", + array_name!(idx) + )?; + } + }; + + return Result::Ok(()); + } + + + fn write_to(&self, target: &mut T) -> Result<(), std::io::Error> { + match self { + LayerParams::Array(_idx, _, params) | + LayerParams::MixedArray(_idx, _, params) => { + let (first, rest) = params.split_first().unwrap(); + + first.write_to(target)?; + for itm in rest { + if let LayerParams::Array(_, _, _) = self { + assert!(first.is_same_type(itm)); + } + itm.write_to(target)?; + } + return Ok(()); + }, + LayerParams::Constant(_, _) => + panic!("Cannot write constant parameters to binary file.") + }; + } + + fn params(&self) -> &[ModelParam] { + return match self { + LayerParams::Array(_, _, params) | + LayerParams::MixedArray(_, _, params) + => params, + LayerParams::Constant(_, params) => params + }; + } + + fn index(&self) -> usize { + return match self { + LayerParams::Array(idx, _, _) | + LayerParams::MixedArray(idx, _, _) + => *idx, + LayerParams::Constant(idx, _) => *idx + }; + } + + fn params_per_model(&self) -> usize { + return match self { + LayerParams::Array(_idx, ppm, _params) | + LayerParams::MixedArray(_idx, ppm, _params) + => *ppm, + LayerParams::Constant(_, params) => params.len() + }; + } + + fn size(&self) -> usize { + return self.params().iter().map(|p| p.size()).sum(); + } + + + fn access_by_const( + &self, + target: &mut T, + parameter_index: usize, + ) -> Result<(), std::io::Error> { + if let LayerParams::Constant(idx, _) = self { + write!(target, "{}", constant_name!(idx, parameter_index))?; + return Result::Ok(()); + } + return self.access_by_ref(target, "0", parameter_index); + } + + fn access_by_ref( + &self, + target: &mut T, + model_index: &str, + parameter_index: usize + ) -> Result<(), std::io::Error> { + + if self.params()[0].is_array() { + assert_eq!(self.params().len(), 1, + "Layer params with array had more than one member."); + write!(target, "{}", array_name!(self.index()))?; + return Result::Ok(()); + } + + match self { + LayerParams::Constant(idx, _) => { + panic!( + "Cannot access constant parameters by reference on layer {}", + idx + ); + } + + LayerParams::Array(idx, params_per_model, params) => { + if params[0].is_array() { + assert_eq!(params.len(), 1); + } + let expr = format!("{}*{} + {}", + params_per_model, model_index, parameter_index); + write!(target, "{}[{}]", array_name!(idx), expr)?; + }, + + LayerParams::MixedArray(idx, params_per_model, params) => { + // determine the number of bytes for each model + let mut bytes_per_model = 0; + for item in params.iter().take(*params_per_model) { + bytes_per_model += item.size(); + } + // determine the byte offset of this parameter + let mut offset = 0; + for item in params.iter().take(parameter_index) { + offset += item.size(); + } + + // we have to determine the type of the index being accessed + // and add the appropiate cast. + let c_type = params[parameter_index].c_type(); + let ptr_expr = format!("{} + ({} * {}) + {}", + array_name!(idx), + model_index, bytes_per_model, + offset); + + write!(target, "*(({new_type}*) ({ptr_expr}))", + new_type=c_type, ptr_expr=ptr_expr)?; + + } + }; + + return Result::Ok(()); + } + + fn with_zipped_errors(&self, lle: &[u64]) -> LayerParams { + + let params = self.params(); + // integrate the errors into the model parameters of the last + // layer to save a cache miss. + + // TODO we should add padding to make sure each of these are + // cache-aligned. Also a lot of unneeded copying going on here... + let combined_lle_params: Vec = + params.chunks(self.params_per_model()) + .zip(lle) + .flat_map(|(mod_params, err)| { + let mut to_r: Vec = Vec::new(); + to_r.extend_from_slice(mod_params); + to_r.push(ModelParam::Int(*err)); + to_r + }).collect(); + + let is_constant = if let LayerParams::Constant(_, _) = self { + true + } else { + false + }; + + return LayerParams::new(self.index(), is_constant, self.params_per_model() + 1, + combined_lle_params); + + } +} + +impl fmt::Display for LayerParams { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + LayerParams::Constant(idx, params) => + write!(f, "Constant(idx: {}, len: {}, malloc: {})", + idx, params.len(), self.requires_malloc()), + LayerParams::Array(idx, ppm, params) => + write!(f, "Array(idx: {}, ppm: {}, len: {}, malloc: {})", + idx, ppm, params.len(), self.requires_malloc()), + LayerParams::MixedArray(idx, ppm, params) => + write!(f, "MixedArray(idx: {}, ppm: {}, len: {}, malloc: {})", + idx, ppm, params.len(), self.requires_malloc()) + + } + } +} + +fn params_for_layer(layer_idx: usize, + models: &[Box]) + -> LayerParams { + let params_per_model = models[0].params().len(); + let params = models.iter().flat_map(|m| m.params()).collect(); + return LayerParams::new(layer_idx, + models.len() > 1, // array access on non-singleton layers + params_per_model, + params); +} + +macro_rules! model_index_from_output { + ($from: expr, $bound: expr, $needs_check: expr) => { + match $from { + ModelDataType::Float => { + if $needs_check { + format!("FCLAMP(fpred, {}.0 - 1.0)", $bound) + } else { + format!("(uint64_t) fpred") + } + } + ModelDataType::Int => { + if $needs_check { + format!("(ipred > {0} - 1 ? {0} - 1 : ipred)", $bound) + } else { + format!("ipred") + } + } + ModelDataType::Int128 => { + if $needs_check { + format!("(i128pred > {0} - 1 ? {0} - 1 : i128pred)", $bound) + } else { + format!("i128pred") + } + } + + } + }; +} + +pub fn rmi_size(rmi: &TrainedRMI) -> u64 { + // compute the RMI size (used in the header, compute here before consuming) + let mut num_total_bytes = 0; + for layer in rmi.rmi.iter() { + let model_on_this_layer_size: usize = layer[0].params().iter().map(|p| p.size()).sum(); + + // assume all models on this layer have the same size + num_total_bytes += model_on_this_layer_size * layer.len(); + } + + if !rmi.last_layer_max_l1s.is_empty() { + num_total_bytes += rmi.rmi.last().unwrap().len() * 8; + } + + if rmi.cache_fix.is_some() { + num_total_bytes += rmi.cache_fix.as_ref().unwrap().1.len() * 16; + } + + return num_total_bytes as u64; +} + +fn generate_cache_fix_code( + target: &mut T, + rmi: &TrainedRMI, + array_name: String) -> Result<(), std::io::Error> { + + let num_splines = rmi.cache_fix.as_ref().unwrap().1.len(); + let line_size = rmi.cache_fix.as_ref().unwrap().0; + let total_keys = rmi.num_data_rows; + + writeln!(target, + " +struct __attribute__((packed)) SplinePoint {{ + uint64_t key; + uint64_t value; +}}; + +uint64_t lookup(uint64_t key, size_t* err) {{ + const uint64_t num_spline_pts = {}; + const uint64_t total_keys = {}; + size_t error_on_spline_search; + + struct SplinePoint* begin = (struct SplinePoint*) {}; + + *err = {}; + uint64_t start = _rmi_lookup_pre_cachefix(key, &error_on_spline_search); + + size_t upper = (start + error_on_spline_search > num_spline_pts + ? num_spline_pts : start + error_on_spline_search); + size_t lower = (error_on_spline_search > start + ? 0 : start - error_on_spline_search); + + + struct SplinePoint* res = std::lower_bound(begin + lower, + begin + upper, + key, + [](const auto& lhs, const auto rhs) {{ return lhs.key < rhs; }}); + + if (res == begin + num_spline_pts) + // we've searched for something past the last point + return total_keys - 1; + + auto pt1 = *(res - 1); + auto pt2 = *res; + + auto v0 = (double)pt1.value; + auto v1 = (double)pt2.value; + auto t = ((double)(key - pt1.key)) / (double)(pt2.key - pt1.key); + return (((uint64_t) std::fma(1.0 - t, v0, t * v1)) / {3}) * {3}; +}}", num_splines, total_keys, array_name, line_size)?; + + + return Ok(()); +} + +fn generate_code( + code_output: &mut T, + data_output: &mut T, + header_output: &mut T, + namespace: &str, + rmi: TrainedRMI, + data_dir: &str, + key_type: KeyType +) -> Result<(), std::io::Error> { + // construct the code for the model parameters. + let mut layer_params: Vec = rmi.rmi + .iter() + .enumerate() + .map(|(layer_idx, models)| params_for_layer(layer_idx, models)) + .collect(); + + let report_last_layer_errors = !rmi.last_layer_max_l1s.is_empty(); + + let mut report_lle: Vec = Vec::new(); + if report_last_layer_errors { + let lle = &rmi.last_layer_max_l1s; + if lle.len() > 1 { + let old_last = layer_params.pop().unwrap(); + let new_last = old_last.with_zipped_errors(lle); + + write!(report_lle, " *err = ")?; + new_last.access_by_ref(&mut report_lle, "modelIndex", + new_last.params_per_model() - 1)?; + writeln!(report_lle, ";")?; + + layer_params.push(new_last); + + } else { + write!(report_lle, " *err = {};", lle[0])?; + } + } + + if rmi.cache_fix.is_some() { + let cfv: Vec = rmi.cache_fix.as_ref().unwrap().1.iter() + .flat_map(|(mi, offset)| vec![(*mi).into(), (*offset).into()]) + .collect(); + let cache_fix_params = LayerParams::new( + layer_params.len(), true, 2, cfv + ); + + layer_params.push(cache_fix_params); + } + + trace!("Layer parameters:"); + for lps in layer_params.iter() { + trace!("{}", lps); + } + + writeln!(data_output, "namespace {} {{", namespace)?; + + let mut read_code = Vec::new(); + read_code.push("bool load(char const* dataPath) {".to_string()); + + for lp in layer_params.iter() { + match lp { + // constants are put directly in the header + LayerParams::Constant(_idx, _) => lp.to_code(data_output)?, + + LayerParams::Array(idx, _, _) | + LayerParams::MixedArray(idx, _, _) => { + let data_path = Path::new(&data_dir) + .join(format!("{}_{}", namespace, array_name!(idx))); + let f = File::create(data_path) + .expect("Could not write data file to RMI directory"); + let mut bw = BufWriter::new(f); + + lp.write_to(&mut bw)?; // write to data file + lp.to_decl(data_output)?; // write to source code + + read_code.push(" {".to_string()); + read_code.push(format!(" std::ifstream infile(std::filesystem::path(dataPath) / \"{ns}_{fn}\", std::ios::in | std::ios::binary);", + ns=namespace, fn=array_name!(idx))); + read_code.push(" if (!infile.good()) return false;".to_string()); + if lp.requires_malloc() { + read_code.push(format!(" {} = ({}*) malloc({});", + array_name!(idx), lp.pointer_type(), lp.size())); + read_code.push(format!(" if ({} == NULL) return false;", + array_name!(idx))); + } + read_code.push(format!(" infile.read((char*){fn}, {size});", + fn=array_name!(idx), size=lp.size())); + read_code.push(" if (!infile.good()) return false;".to_string()); + read_code.push(" }".to_string()); + } + } + } + read_code.push(" return true;".to_string()); + read_code.push("}".to_string()); + + + + let mut free_code = Vec::new(); + free_code.push("void cleanup() {".to_string()); + // generate free code + for lp in layer_params.iter() { + if !lp.requires_malloc() { continue; } + if let LayerParams::Array(idx, _, _) | LayerParams::MixedArray(idx, _, _) = lp { + free_code.push(format!(" free({});", array_name!(idx))); + continue; + } + panic!(); + } + + free_code.push("}".to_string()); + + writeln!(data_output, "}} // namespace")?; + + // get all of the required stdlib function signatures together + // TODO assumes all layers are homogenous + let mut decls = HashSet::new(); + let mut sigs = HashSet::new(); + for layer in rmi.rmi.iter() { + for stdlib in layer[0].standard_functions() { + decls.insert(stdlib.decl().to_string()); + sigs.insert(stdlib.code().to_string()); + } + } + + writeln!(code_output, "#include \"{}.h\"", namespace)?; + writeln!(code_output, "#include \"{}_data.h\"", namespace)?; + writeln!(code_output, "#include ")?; + writeln!(code_output, "#include ")?; + writeln!(code_output, "#include ")?; + writeln!(code_output, "#include ")?; + writeln!(code_output, "#include ")?; + if rmi.cache_fix.is_some() { + writeln!(code_output, "#include ")?; + } + + writeln!(code_output, "namespace {} {{", namespace)?; + + for ln in read_code { + writeln!(code_output, "{}", ln)?; + } + + for ln in free_code { + writeln!(code_output, "{}", ln)?; + } + + for decl in decls { + writeln!(code_output, "{}", decl)?; + } + + for sig in sigs { + writeln!(code_output, "{}", sig)?; + } + + // next, the model sigs + sigs = HashSet::new(); + for layer in rmi.rmi.iter() { + sigs.insert(layer[0].code()); + } + + for sig in sigs { + writeln!(code_output, "{}", sig)?; + } + + writeln!( + code_output, + " +inline size_t FCLAMP(double inp, double bound) {{ + if (inp < 0.0) return 0; + return (inp > bound ? bound : (size_t)inp); +}}\n" + )?; + + let rmi_lookup_name = if rmi.cache_fix.is_none() { + "lookup" + } else { + "_rmi_lookup_pre_cachefix" + }; + + let lookup_sig = if report_last_layer_errors { + format!("uint64_t {}({} key, size_t* err)", rmi_lookup_name, key_type.c_type()) + } else { + format!("uint64_t {}({} key)", rmi_lookup_name, key_type.c_type()) + }; + writeln!(code_output, "{} {{", lookup_sig)?; + + let mut needed_vars = HashSet::new(); + if rmi.rmi.len() > 1 { + needed_vars.insert("size_t modelIndex;"); + } + + // determine if we have any layers with float (fpred) or int (ipred) outputs + for layer in rmi.rmi.iter() { + match layer[0].output_type() { + ModelDataType::Int => needed_vars.insert("uint64_t ipred;"), + ModelDataType::Float => needed_vars.insert("double fpred;"), + ModelDataType::Int128 => needed_vars.insert("uint128_t i128pred;"), + }; + } + + for var in needed_vars { + writeln!(code_output, " {}", var)?; + } + + let model_size_bytes = rmi_size(&rmi); + info!("Generated model size: {:?} ({} bytes)", ByteSize(model_size_bytes), model_size_bytes); + + let mut last_model_output = key_type.to_model_data_type(); + let mut needs_bounds_check = true; + + for (layer_idx, layer) in rmi.rmi.iter().enumerate() { + let layer_param = &layer_params[layer_idx]; + let required_type = layer[0].input_type(); + + let current_model_output = layer[0].output_type(); + + let var_name = match current_model_output { + ModelDataType::Int => "ipred", + ModelDataType::Float => "fpred", + ModelDataType::Int128 => "i128pred" + }; + + let num_parameters = layer[0].params().len(); + if layer.len() == 1 { + // use constant indexing, only one model + write!( + code_output, + " {} = {}(", + var_name, + layer[0].function_name() + )?; + + for pidx in 0..num_parameters { + layer_param.access_by_const(code_output, pidx)?; + write!(code_output, ", ")?; + } + } else { + // we need to get the model index based on the previous + // prediction, and then use ref accessing + writeln!( + code_output, + " modelIndex = {};", + model_index_from_output!(last_model_output, layer.len(), needs_bounds_check) + )?; + + write!( + code_output, + " {} = {}(", + var_name, + layer[0].function_name() + )?; + + for pidx in 0..num_parameters { + layer_param.access_by_ref(code_output, "modelIndex", pidx)?; + write!(code_output, ", ")?; + } + } + writeln!(code_output, "({})key);", required_type.c_type())?; + + last_model_output = layer[0].output_type(); + needs_bounds_check = layer[0].needs_bounds_check(); + } + + writeln!(code_output, "{}", str::from_utf8(&report_lle).unwrap())?; + + writeln!( + code_output, + " return {};", + model_index_from_output!(last_model_output, rmi.num_rmi_rows, true) + )?; // always bounds check the last level + writeln!(code_output, "}}")?; + + if rmi.cache_fix.is_some() { + generate_cache_fix_code(code_output, &rmi, array_name!(layer_params.len()-1))?; + } + + writeln!(code_output, "}} // namespace")?; + + // write out our forward declarations + writeln!(header_output, "#include ")?; + writeln!(header_output, "#include ")?; + writeln!(header_output, "namespace {} {{", namespace)?; + + writeln!(header_output, "bool load(char const* dataPath);")?; + writeln!(header_output, "void cleanup();")?; + + writeln!( + header_output, + "const size_t RMI_SIZE = {};", + model_size_bytes + )?; + assert!(rmi.build_time <= u128::from(std::u64::MAX)); + writeln!( + header_output, + "const uint64_t BUILD_TIME_NS = {};", + rmi.build_time + )?; + writeln!(header_output, "const char NAME[] = \"{}\";", namespace)?; + if rmi.cache_fix.is_none() { + writeln!(header_output, "{};", lookup_sig)?; + } else { + writeln!(header_output, "uint64_t lookup(uint64_t key, size_t* err);")?; + } + writeln!(header_output, "}}")?; + + return Result::Ok(()); +} + + +pub fn output_rmi(namespace: &str, + mut trained_model: TrainedRMI, + data_dir: &str, + key_type: KeyType, + include_errors: bool) -> Result<(), std::io::Error> { + + let f1 = File::create(format!("{}.cpp", namespace)).expect("Could not write RMI CPP file"); + let mut bw1 = BufWriter::new(f1); + + let f2 = + File::create(format!("{}_data.h", namespace)).expect("Could not write RMI data file"); + let mut bw2 = BufWriter::new(f2); + + let f3 = File::create(format!("{}.h", namespace)).expect("Could not write RMI header file"); + let mut bw3 = BufWriter::new(f3); + + if !include_errors { + trained_model.last_layer_max_l1s.clear(); + } + + return generate_code( + &mut bw1, + &mut bw2, + &mut bw3, + namespace, + trained_model, + data_dir, + key_type + ); + + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/lib.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/lib.rs new file mode 100644 index 00000000..0dcc925f --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/lib.rs @@ -0,0 +1,12 @@ +mod codegen; +mod models; +mod train; +mod cache_fix; + +pub mod optimizer; +pub use models::{RMITrainingData, RMITrainingDataIteratorProvider, ModelInput}; +pub use models::KeyType; +pub use optimizer::find_pareto_efficient_configs; +pub use train::{train, train_for_size, train_bounded}; +pub use codegen::rmi_size; +pub use codegen::output_rmi; diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/balanced_radix.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/balanced_radix.rs new file mode 100644 index 00000000..a8e6633d --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/balanced_radix.rs @@ -0,0 +1,181 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +use crate::models::utils::{common_prefix_size, num_bits}; +use crate::models::*; +use log::*; +use std::f64; + +#[derive(Debug)] +pub struct BalancedRadixModel { + params: (u8, u8, u64), + high: bool, +} + +fn chi2(data: &RMITrainingData, + max_bin: u64, + model: &BalancedRadixModel) -> f64 { + // compute the x^2 value of the distribution + // induced by this model. + let mut counts = vec![0; max_bin as usize]; + + for (x, _y) in data.iter_model_input() { + counts[model.predict_to_int(&x) as usize] += 1; + } + + let expected = data.len() as f64 / max_bin as f64; + + return counts + .into_iter() + .map(|c| (c as f64 - expected).powf(2.0) / expected) + .sum(); +} + +fn bradix(data: &RMITrainingData, max_output: u64) -> BalancedRadixModel { + let bits = num_bits(max_output); + let common_prefix = common_prefix_size(data); + trace!("Bradix layer common prefix: {}", common_prefix); + + let mut best_result_score = f64::INFINITY; + let mut best_result = None; + for test_bits in bits..u8::min(bits + 2, 64) { + let bits_max = (1 << (test_bits + 1)) - 1; + + let high = BalancedRadixModel { + params: (common_prefix, test_bits, max_output - 1), + high: true, + }; + let high_score = chi2(data, max_output, &high); + + trace!( + "Bradix high with {} bits had score {}", + test_bits, + high_score + ); + if high_score < best_result_score { + best_result_score = high_score; + best_result = Some(high); + } + + let low = BalancedRadixModel { + params: (common_prefix, test_bits, max_output - bits_max), + high: false, + }; + let low_score = chi2(data, max_output, &low); + + trace!("Bradix low with {} bits had score {}", test_bits, low_score); + if low_score < best_result_score { + best_result_score = low_score; + best_result = Some(low); + } + } + + trace!( + "Best bradix setup: {:?} with score {}", + best_result, + best_result_score + ); + + return best_result.unwrap(); +} + +impl BalancedRadixModel { + pub fn new(data: &RMITrainingData) -> BalancedRadixModel { + if data.len() == 0 { + return BalancedRadixModel { + params: (0, 0, 0), + high: true, + }; + } + + let largest_value = data.iter().map(|(_x, y)| y).max().unwrap(); + + return bradix(data, largest_value as u64); + } +} + +impl Model for BalancedRadixModel { + fn predict_to_int(&self, inp: &ModelInput) -> u64 { + let (left_shift, num_bits, clamp) = self.params; + + let as_int: u64 = inp.as_int(); + let res = (as_int << left_shift) >> (64 - num_bits); + + if self.high { + return u64::min(res, clamp); + } else { + return if res < clamp { 0 } else { res - clamp }; + } + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Int; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Int; + } + + fn params(&self) -> Vec { + return vec![ + self.params.0.into(), + self.params.1.into(), + self.params.2.into(), + ]; + } + + fn code(&self) -> String { + if self.high { + return String::from( + " +inline uint64_t bradix_clamp_high(uint64_t prefix_length, + uint64_t bits, uint64_t clamp, uint64_t inp) { + uint64_t tmp = (inp << prefix_length) >> (64 - bits); + return (tmp > clamp ? clamp : tmp); + +} +", + ); + } else { + return String::from( + " +inline uint64_t bradix_clamp_low(uint64_t prefix_length, + uint64_t bits, uint64_t clamp, uint64_t inp) { + uint64_t tmp = (inp << prefix_length) >> (64 - bits); + return (tmp < clamp ? 0 : tmp - clamp); +} +", + ); + } + } + + fn function_name(&self) -> String { + return if self.high { + String::from("bradix_clamp_high") + } else { + String::from("bradix_clamp_low") + }; + } + + fn needs_bounds_check(&self) -> bool { + return false; + } + fn restriction(&self) -> ModelRestriction { + return ModelRestriction::MustBeTop; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty() { + BalancedRadixModel::new(&ModelData::empty()); + } + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/cubic_spline.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/cubic_spline.rs new file mode 100644 index 00000000..55cef04b --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/cubic_spline.rs @@ -0,0 +1,252 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +use crate::models::*; + +macro_rules! scale { + ($val: expr, $min: expr, $max: expr) => { + ($val - $min) / ($max - $min) + }; +} + +#[allow(clippy::float_cmp)] +fn cubic(data: &RMITrainingData) -> (f64, f64, f64, f64) { + if data.len() == 0 { + return (0.0, 0.0, 1.0, 0.0); + } + + if data.len() == 1 { + return (0.0, 0.0, 0.0, data.get(0).1 as f64); + } + + // ensure we have at least two unique values + { + let candidate = data.get(0).0; + let uniq = data.iter().any(|(x, _y)| x != candidate); + + if !uniq { + // all the same value! + return (0.0, 0.0, 0.0, data.get(0).1 as f64); + } + } + + let first_pt = data.get(0); + let last_pt = data.get(data.len() - 1); + let (xmin, ymin) = (first_pt.0.as_float(), first_pt.1 as f64); + let (xmax, ymax) = (last_pt.0.as_float(), last_pt.1 as f64); + + let (x1, y1) = (0.0, 0.0); + let (x2, y2) = (1.0, 1.0); + + let mut m1 = { + let (xn, yn) = data + .iter() + .find(|&(tx, _ty)| scale!(tx.as_float(), xmin, xmax) > 0.0) + .unwrap(); + + let (sxn, syn) = (scale!(xn.as_float(), xmin, xmax), scale!(yn as f64, ymin, ymax)); + (syn - y1) / (sxn - x1) + }; + + let mut m2 = { + let (xp, yp) = (0..data.len()) + .rev() + .map(|idx| data.get(idx)) + .find(|&(tx, _ty)| scale!(tx.as_float(), xmin, xmax) < 1.0) + .unwrap(); + + let (sxp, syp) = (scale!(xp.as_float(), xmin, xmax), scale!(yp as f64, ymin, ymax)); + (y2 - syp) / (x2 - sxp) + }; + + // keep it monotonic + if m1.powf(2.0) + m2.powf(2.0) > 9.0 { + let tau = 3.0 / (m1.powf(2.0) + m2.powf(2.0)).sqrt(); + m1 *= tau; + m2 *= tau; + } + + // from sympy, the first (a) term is: + // '(m1 + m2 - 2)/(xmax - xmin)**3' + let mut a = (m1 + m2 - 2.0) / (xmax - xmin).powf(3.0); + + // the second (b) term is: + // '-(xmax*(2*m1 + m2 - 3) + xmin*(m1 + 2*m2 - 3))/(xmax - xmin)**3' + let mut b = + -(xmax * (2.0 * m1 + m2 - 3.0) + xmin * (m1 + 2.0 * m2 - 3.0)) / (xmax - xmin).powf(3.0); + + // the third (c) term is: + // '(m1*xmax**2 + m2*xmin**2 + xmax*xmin*(2*m1 + 2*m2 - 6)) + // /(xmax - xmin)**3' + let mut c = + (m1 * xmax.powf(2.0) + m2 * xmin.powf(2.0) + xmax * xmin * (2.0 * m1 + 2.0 * m2 - 6.0)) + / (xmax - xmin).powf(3.0); + + // the fourth (d) term is: + // '-xmin*(m1*xmax**2 + xmax*xmin*(m2 - 3) + xmin**2)/(xmax - xmin)**3' + let mut d = -xmin * (m1 * xmax.powf(2.0) + xmax * xmin * (m2 - 3.0) + xmin.powf(2.0)) + / (xmax - xmin).powf(3.0); + + a *= ymax - ymin; + b *= ymax - ymin; + c *= ymax - ymin; + d *= ymax - ymin; + d += ymin; + return (a, b, c, d); +} + +pub struct CubicSplineModel { + params: (f64, f64, f64, f64), +} + +impl CubicSplineModel { + pub fn new(data: &RMITrainingData) -> CubicSplineModel { + let cubic = CubicSplineModel { + params: cubic(data), + }; + + // check our error against a linear model -- + // sometimes the slope really doesn't work out. + let linear = LinearSplineModel::new(data); + + let mut our_error = 0.0; + let mut lin_error = 0.0; + + for (x, y) in data.iter_model_input() { + let c_pred = cubic.predict_to_float(&x); + let l_pred = linear.predict_to_float(&x); + + our_error += (c_pred - (y as f64)).abs(); + lin_error += (l_pred - (y as f64)).abs(); + } + + if lin_error < our_error { + let lp = linear.params(); + return CubicSplineModel { + params: (0.0, 0.0, lp[1].as_float(), lp[0].as_float()), + }; + } + + return cubic; + } +} + +impl Model for CubicSplineModel { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + let (a, b, c, d) = self.params; + let val = inp.as_float(); + + // use mul_add here so we get the same FMA behavior as we do + // in C. + let v1 = a.mul_add(val, b); + let v2 = v1.mul_add(val, c); + let v3 = v2.mul_add(val, d); + return v3; + + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + + fn params(&self) -> Vec { + return vec![ + self.params.0.into(), + self.params.1.into(), + self.params.2.into(), + self.params.3.into(), + ]; + } + + fn code(&self) -> String { + return String::from( + " +inline double cubic(double a, double b, double c, double d, double x) { + auto v1 = std::fma(a, x, b); + auto v2 = std::fma(v1, x, c); + auto v3 = std::fma(v2, x, d); + return v3; +}", + ); + } + + fn function_name(&self) -> String { + return String::from("cubic"); + } + fn needs_bounds_check(&self) -> bool { + return false; + } + + fn set_to_constant_model(&mut self, constant: u64) -> bool { + self.params = (0.0, 0.0, 0.0, constant as f64); + return true; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use approx::*; + + #[test] + fn test_cubic() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2), (2, 3), (3, 8), (4, 20)]); + + let cubic_mod = CubicSplineModel::new(&md); + + assert_abs_diff_eq!(cubic_mod.predict_to_float(1.into()), 2.0, epsilon = 0.5); + assert_abs_diff_eq!(cubic_mod.predict_to_float(4.into()), 20.0, epsilon = 0.5); + } + + #[test] + fn test_cubic2() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2), (2, 3), (3, 8), (4, 20), (5, 80)]); + + let cubic_mod = CubicSplineModel::new(&md); + + assert_abs_diff_eq!(cubic_mod.predict_to_float(1.into()), 2.0, epsilon = 0.5); + assert_abs_diff_eq!(cubic_mod.predict_to_float(5.into()), 80.0, epsilon = 0.5); + } + + #[test] + fn test_cubic_dup() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2), (1, 2), (3, 8), (4, 20), (5, 80)]); + + let cubic_mod = CubicSplineModel::new(&md); + + assert_abs_diff_eq!(cubic_mod.predict_to_float(1.into()), 2.0, epsilon = 0.5); + assert_abs_diff_eq!(cubic_mod.predict_to_float(5.into()), 80.0, epsilon = 0.5); + } + + #[test] + fn test_cubic_all_dup() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2), (1, 2), (1, 2)]); + + let cubic_mod = CubicSplineModel::new(&md); + + assert_abs_diff_eq!(cubic_mod.predict_to_float(1.into()), 2.0, epsilon = 0.5); + } + + #[test] + fn test_linear_spline_single() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2)]); + + let cubic_mod = CubicSplineModel::new(&md); + + assert_eq!(cubic_mod.predict_to_int(1.into()), 2); + } + + #[test] + fn test_empty() { + CubicSplineModel::new(&ModelData::empty()); + } + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/histogram.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/histogram.rs new file mode 100644 index 00000000..bc73a489 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/histogram.rs @@ -0,0 +1,134 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +use crate::models::*; +use crate::models::utils::radix_index; +use superslice::*; +use log::*; + +pub struct EquidepthHistogramModel { + params: Vec, + radix: Vec +} + + +fn equidepth_histogram(data: &RMITrainingData) -> Vec { + assert!(data.len() > 0); + + let mut splits: Vec = Vec::new(); + let num_bins = data.get(data.len()-1).1 as usize; + let items_per_bin = data.len() / num_bins; + + assert!(items_per_bin >= 1, "not enough items for equidepth histogram"); + info!("Equidepth histogram using {} bins", num_bins); + + for bin_idx in 0..num_bins { + let start_idx = bin_idx * items_per_bin; + let start_val = data.get_key(start_idx).as_uint(); + splits.push(start_val); + } + + return splits; +} + + + +impl EquidepthHistogramModel { + pub fn new(data: &RMITrainingData) -> EquidepthHistogramModel { + if data.len() == 0 { + return EquidepthHistogramModel { params: Vec::new(), radix: Vec::new() }; + } + + let params = equidepth_histogram(data); + let radix = radix_index(¶ms, 20); + return EquidepthHistogramModel { + params, radix + }; + } +} + +impl Model for EquidepthHistogramModel { + + fn predict_to_int(&self, inp: &ModelInput) -> u64 { + let val = inp.as_int(); + + let val = self.params.upper_bound(&val) - 1; + return val as u64; + + /*for (idx, &split) in self.params.iter().enumerate() { + if val <= split { return (idx - 1) as u64; } + } + + return self.params.len() as u64 - 1;*/ + } + + fn input_type(&self) -> ModelDataType { return ModelDataType::Int; } + fn output_type(&self) -> ModelDataType { return ModelDataType::Int; } + + fn params(&self) -> Vec { + return vec![ + ModelParam::Int(self.params.len() as u64), + ModelParam::IntArray(self.radix.clone()), + ModelParam::IntArray(self.params.clone()) + ]; + } + fn code(&self) -> String { + return String::from(" +inline uint64_t ed_histogram(const uint64_t length, + const uint64_t radix[], + const uint64_t pivots[], + uint64_t key) { + uint64_t key_radix = key >> (64 - 20); + unsigned int radix_lb = radix[key_radix]; + unsigned int radix_ub = radix[key_radix+1]; + uint64_t li = bs_upper_bound(pivots + radix_lb, radix_ub - radix_lb, key) + radix_lb - 1; + return li; +} +"); + } + + fn standard_functions(&self) -> HashSet { + let mut to_r = HashSet::new(); + to_r.insert(StdFunctions::BinarySearch); + return to_r; + } + + fn function_name(&self) -> String { return String::from("ed_histogram"); } + fn restriction(&self) -> ModelRestriction { return ModelRestriction::MustBeTop; } + fn needs_bounds_check(&self) -> bool { return false; } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ed_hist1() { + let mut test_data: Vec<(u64, u64)> = Vec::new(); + + for i in 0..1000 { + test_data.push((i*3, i/3)); + } + + let md = ModelData::IntKeyToIntPos(test_data); + + let ed_mod = EquidepthHistogramModel::new(&md); + + assert_eq!(ed_mod.predict_to_int((0).into()), 0); + assert_eq!(ed_mod.predict_to_int((1*3).into()), 0); + assert_eq!(ed_mod.predict_to_int((4*3).into()), 1); + assert_eq!(ed_mod.predict_to_int((500*3).into()), 166); + assert_eq!(ed_mod.predict_to_int((5000*3).into()), 333); + } + + #[test] + fn test_empty() { + EquidepthHistogramModel::new(&ModelData::empty()); + } + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/linear.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/linear.rs new file mode 100644 index 00000000..a564d13a --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/linear.rs @@ -0,0 +1,297 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + + +use crate::models::*; + +fn slr>(loc_data: T) -> (f64, f64) { + + // compute the covariance of x and y as well as the variance of x in + // a single pass. + + let mut mean_x = 0.0; + let mut mean_y = 0.0; + let mut c = 0.0; + let mut n: u64 = 0; + let mut m2 = 0.0; + + let mut data_size = 0; + for (x, y) in loc_data { + n += 1; + let dx = x - mean_x; + mean_x += dx / (n as f64); + mean_y += (y - mean_y) / (n as f64); + c += dx * (y - mean_y); + + let dx2 = x - mean_x; + m2 += dx * dx2; + data_size += 1; + } + + // special case when we have 0 or 1 items + if data_size == 0 { + return (0.0, 0.0); + } + + if data_size == 1 { + return (mean_y, 0.0); + } + + + let cov = c / ((n - 1) as f64); + let var = m2 / ((n - 1) as f64); + assert!(var >= 0.0, "variance of model with {} data items was negative", n); + + if var == 0.0 { + // variance is zero. pick the mean (only) value. + return (mean_y, 0.0); + } + + let beta: f64 = cov / var; + let alpha = mean_y - beta * mean_x; + + return (alpha, beta); +} + +fn loglinear_slr(data: &RMITrainingData) -> (f64, f64) { + // log all of the outputs, omit any item that doesn't have a valid log + let transformed_data: Vec<(f64, f64)> = data + .iter() + .map(|(x, y)| (x.as_float(), (y as f64).ln())) + .filter(|(_, y)| y.is_finite()) + .collect(); + + // TODO this currently creates a copy of the data and then calls + // slr... we can probably do better by moving the log into the slr. + return slr(transformed_data.into_iter()); +} + +pub struct LinearModel { + params: (f64, f64), +} + +impl LinearModel { + pub fn new(data: &RMITrainingData) -> LinearModel { + let params = slr(data.iter() + .map(|(inp, offset)| (inp.as_float(), offset as f64))); + return LinearModel { params }; + } +} + +impl Model for LinearModel { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + let (intercept, slope) = self.params; + return slope.mul_add(inp.as_float(), intercept); + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + + fn params(&self) -> Vec { + return vec![self.params.0.into(), self.params.1.into()]; + } + + fn code(&self) -> String { + return String::from( + " +inline double linear(double alpha, double beta, double inp) { + return std::fma(beta, inp, alpha); +}", + ); + } + + fn function_name(&self) -> String { + return String::from("linear"); + } + + fn set_to_constant_model(&mut self, constant: u64) -> bool { + self.params = (constant as f64, 0.0); + return true; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_linear1() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2), (2, 3), (3, 4)]); + + let lin_mod = LinearModel::new(&md); + + assert_eq!(lin_mod.predict_to_int(1.into()), 2); + assert_eq!(lin_mod.predict_to_int(6.into()), 7); + } + + #[test] + fn test_linear_single() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2)]); + + let lin_mod = LinearModel::new(&md); + + assert_eq!(lin_mod.predict_to_int(1.into()), 2); + } + + #[test] + fn test_empty() { + LinearModel::new(&ModelData::empty()); + } + +} + +pub struct LogLinearModel { + params: (f64, f64), +} + +fn exp1(inp: f64) -> f64 { + let mut x = inp; + x = 1.0 + x / 64.0; + x *= x; + x *= x; + x *= x; + x *= x; + x *= x; + x *= x; + return x; +} + +impl LogLinearModel { + pub fn new(data: &RMITrainingData) -> LogLinearModel { + return LogLinearModel { + params: loglinear_slr(&data), + }; + } +} + +impl Model for LogLinearModel { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + let (alpha, beta) = self.params; + return exp1(beta.mul_add(inp.as_float(), alpha)); + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + + fn params(&self) -> Vec { + return vec![self.params.0.into(), self.params.1.into()]; + } + + fn code(&self) -> String { + return String::from( + " +inline double loglinear(double alpha, double beta, double inp) { + return exp1(std::fma(beta, inp, alpha)); +}", + ); + } + + fn function_name(&self) -> String { + return String::from("loglinear"); + } + fn standard_functions(&self) -> HashSet { + let mut to_r = HashSet::new(); + to_r.insert(StdFunctions::EXP1); + return to_r; + } +} + +#[cfg(test)] +mod loglin_tests { + use super::*; + + #[test] + fn test_loglinear1() { + let md = ModelData::IntKeyToIntPos(vec![(2, 2), (3, 4), (4, 16)]); + + let loglin_mod = LogLinearModel::new(&md); + + assert_eq!(loglin_mod.predict_to_int(2.into()), 1); + assert_eq!(loglin_mod.predict_to_int(4.into()), 13); + } + + #[test] + fn test_empty() { + LogLinearModel::new(&ModelData::empty()); + } +} + + +pub struct RobustLinearModel { + params: (f64, f64), +} + + +impl RobustLinearModel { + pub fn new(data: &RMITrainingData) -> RobustLinearModel { + let total_items = data.len(); + if data.len() == 0 { + return RobustLinearModel { + params: (0.0, 0.0) + }; + } + + let bnd = usize::max(1, ((total_items as f64) * 0.0001) as usize); + assert!(bnd*2+1 < data.len()); + + let iter = data.iter() + .skip(bnd) + .take(data.len() - 2*bnd); + + let robust_params = slr(iter + .map(|(inp, offset)| (inp.as_float(), offset as f64))); + + return RobustLinearModel { + params: robust_params + }; + } +} + +impl Model for RobustLinearModel { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + let (alpha, beta) = self.params; + return beta.mul_add(inp.as_float(), alpha); + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + + fn params(&self) -> Vec { + return vec![self.params.0.into(), self.params.1.into()]; + } + + fn code(&self) -> String { + return String::from( + " +inline double linear(double alpha, double beta, double inp) { + return std::fma(beta, inp, alpha); +}", + ); + } + + fn function_name(&self) -> String { + return String::from("linear"); + } + + fn set_to_constant_model(&mut self, constant: u64) -> bool { + self.params = (constant as f64, 0.0); + return true; + } +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/linear_spline.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/linear_spline.rs new file mode 100644 index 00000000..8a20f1f3 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/linear_spline.rs @@ -0,0 +1,113 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + + +use crate::models::*; + +#[allow(clippy::float_cmp)] +fn linear_splines(data: &RMITrainingData) -> (f64, f64) { + if data.len() == 0 { + return (0.0, 0.0); + } + + if data.len() == 1 { + return (data.get(0).1 as f64, 0.0); + } + + let first_pt = data.get(0); + let last_pt = data.get(data.len() - 1); + + if first_pt.0 == last_pt.0 { + // data is all duplicates! + return (data.get(0).1 as f64, 0.0); + } + + let slope = (first_pt.1 as f64 - last_pt.1 as f64) + / (first_pt.0.as_float() - last_pt.0.as_float()); + let intercept = first_pt.1 as f64 - slope * first_pt.0.as_float(); + + return (intercept, slope); +} + +pub struct LinearSplineModel { + params: (f64, f64), +} + +impl LinearSplineModel { + pub fn new(data: &RMITrainingData) -> LinearSplineModel { + return LinearSplineModel { + params: linear_splines(data), + }; + } +} + +impl Model for LinearSplineModel { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + let (alpha, beta) = self.params; + return beta.mul_add(inp.as_float(), alpha); + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + + fn params(&self) -> Vec { + return vec![self.params.0.into(), self.params.1.into()]; + } + + fn code(&self) -> String { + return String::from( + " +inline double linear(double alpha, double beta, double inp) { + return std::fma(beta, inp, alpha); +}", + ); + } + + fn function_name(&self) -> String { + return String::from("linear"); + } + + fn set_to_constant_model(&mut self, constant: u64) -> bool { + self.params = (constant as f64, 0.0); + return true; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_linear_spline1() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2), (2, 3), (3, 8)]); + + let lin_mod = LinearSplineModel::new(&md); + + assert_eq!(lin_mod.predict_to_int(1.into()), 2); + assert_eq!(lin_mod.predict_to_int(3.into()), 8); + } + + #[test] + fn test_linear_spline_single() { + let md = ModelData::IntKeyToIntPos(vec![(1, 2)]); + + let lin_mod = LinearSplineModel::new(&md); + + assert_eq!(lin_mod.predict_to_int(1.into()), 2); + } + + #[test] + fn test_empty() { + LinearSplineModel::new(&ModelData::empty()); + } + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/mod.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/mod.rs new file mode 100644 index 00000000..1b086d49 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/mod.rs @@ -0,0 +1,792 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + + +mod balanced_radix; +mod cubic_spline; +mod histogram; +mod linear; +mod linear_spline; +mod normal; +mod radix; +mod stdlib; +mod utils; + +pub use balanced_radix::BalancedRadixModel; +pub use cubic_spline::CubicSplineModel; +pub use histogram::EquidepthHistogramModel; +pub use linear::LinearModel; +pub use linear::RobustLinearModel; +pub use linear::LogLinearModel; +pub use linear_spline::LinearSplineModel; +pub use normal::LogNormalModel; +pub use normal::NormalModel; +pub use radix::RadixModel; +pub use radix::RadixTable; +pub use stdlib::StdFunctions; + +use std::cmp::Ordering; +use std::collections::HashSet; +use std::sync::Arc; +use std::io::Write; +use byteorder::{WriteBytesExt, LittleEndian}; + + +#[derive(Clone, Copy)] +pub enum KeyType { + U32, U64, F64, U128 +} + +impl KeyType { + pub fn c_type(&self) -> &'static str { + match self { + KeyType::U32 => "uint32_t", + KeyType::U64 => "uint64_t", + KeyType::F64 => "double", + KeyType::U128 => "uint128_t" + } + } + + pub fn to_model_data_type(self) -> ModelDataType { + match self { + KeyType::U32 => ModelDataType::Int, + KeyType::U64 => ModelDataType::Int, + KeyType::U128 => ModelDataType::Int128, + KeyType::F64 => ModelDataType::Float + } + } +} + +pub trait TrainingKey: PartialEq + Copy + Send + Sync + std::fmt::Debug + 'static { + fn minus_epsilon(&self) -> Self; + fn zero_value() -> Self; + fn plus_epsilon(&self) -> Self; + fn max_value() -> Self; + + fn as_float(&self) -> f64; + fn as_uint(&self) -> u64; + + fn to_model_input(&self) -> ModelInput; +} + +impl TrainingKey for u64 { + fn minus_epsilon(&self) -> Self { *self - 1 } + fn zero_value() -> Self { 0 } + fn plus_epsilon(&self) -> Self { *self + 1 } + fn max_value() -> Self { std::u64::MAX } + + fn as_float(&self) -> f64 { *self as f64 } + fn as_uint(&self) -> u64 { *self } + + fn to_model_input(&self) -> ModelInput { (*self).into() } +} + +impl TrainingKey for u32 { + fn minus_epsilon(&self) -> Self { *self - 1 } + fn zero_value() -> Self { 0 } + fn plus_epsilon(&self) -> Self { *self + 1 } + fn max_value() -> Self { std::u32::MAX } + + fn as_float(&self) -> f64 { *self as f64 } + fn as_uint(&self) -> u64 { *self as u64 } + + fn to_model_input(&self) -> ModelInput { (*self).into() } +} + +impl TrainingKey for f64 { + fn minus_epsilon(&self) -> Self { *self - std::f64::EPSILON } + fn zero_value() -> Self { 0.0 } + fn plus_epsilon(&self) -> Self { *self + std::f64::EPSILON } + fn max_value() -> Self { std::f64::MAX } + + fn as_float(&self) -> f64 { *self } + fn as_uint(&self) -> u64 { *self as u64 } + + fn to_model_input(&self) -> ModelInput { (*self).into() } +} + +pub trait RMITrainingDataIteratorProvider: Send + Sync { + type InpType: TrainingKey; + + fn len(&self) -> usize; + fn cdf_iter(&self) -> Box + '_>; + fn key_type(&self) -> KeyType; + fn get(&self, idx: usize) -> Option<(Self::InpType, usize)> { + return Some(self.cdf_iter().nth(idx).unwrap()); + } +} + +impl RMITrainingDataIteratorProvider for Vec<(K, usize)> { + type InpType = K; + fn len(&self) -> usize { + return Vec::len(&self); + } + + fn cdf_iter(&self) -> Box + '_> { + return Box::new(self.iter() + .cloned() + .map(|(key, offset)| (key.into(), offset))); + } + + fn key_type(&self) -> KeyType { return KeyType::U64; } + fn get(&self, idx: usize) -> Option<(Self::InpType, usize)> { + self.as_slice().get(idx).map(|(key, offset)| ((*key).into(), *offset)) + } +} + + +struct FixDupsIter> { + iter: T, + last_item: Option<(K, usize)> +} + +impl > FixDupsIter { + fn new(iter: T) -> FixDupsIter { + return FixDupsIter { iter: iter, last_item: None }; + } +} + +impl Iterator for FixDupsIter where + T: Iterator, + K: TrainingKey { + type Item = (K, usize); + + fn next(&mut self) -> Option { + match self.last_item { + None => { + match self.iter.next() { + None => { return None }, + Some(itm) => { + self.last_item = Some(itm); + return Some(itm); + } + } + }, + Some(last) => { + match self.iter.next() { + Some(nxt) => { + if nxt.0 == last.0 { + Some((nxt.0, last.1)) + } else { + self.last_item = Some(nxt); + return Some(nxt); + } + } + None => { self.last_item.take() } + } + } + } + } +} + +struct DedupIter> { + iter: T, + last_item: Option<(K, usize)> +} + +impl > DedupIter { + fn new(iter: T) -> DedupIter { + return DedupIter { iter: iter, last_item: None }; + } +} + +impl Iterator for DedupIter where + T: Iterator, + K: TrainingKey { + type Item = (K, usize); + + fn next(&mut self) -> Option { + match self.last_item { + None => { + match self.iter.next() { + None => { return None; } + Some(nxt) => { + self.last_item = Some(nxt); + return Some(nxt) + } + } + }, + Some(last) => { + loop { + match self.iter.next() { + Some(nxt) => { + if nxt.0 == last.0 { + continue; + } else { + self.last_item = Some(nxt); + return Some(nxt); + } + } + None => { return None; } + } + } + } + } + } +} + +pub struct RMITrainingData { + iterable: Arc>>, + scale: f64 +} + +macro_rules! map_scale { + ($self: expr, $inp: expr) => {{ + let sf = ($self).scale; + let use_sf = (sf - 1.0).abs() > std::f64::EPSILON; + ($inp).map(move |(key, offset)| { + if use_sf { + (key, (offset as f64 * sf) as usize) + } else { + (key, offset) + } + }) + }} +} + +impl RMITrainingData { + pub fn new(iterable: Box>) + -> RMITrainingData { + return RMITrainingData { iterable: Arc::new(iterable), scale: 1.0 }; + } + + pub fn empty() -> RMITrainingData { + return RMITrainingData::::new(Box::new(vec![])); + } + + pub fn len(&self) -> usize { return self.iterable.len(); } + + pub fn set_scale(&mut self, scale: f64) { + self.scale = scale; + } + + pub fn get(&self, idx: usize) -> (T, usize) { + return map_scale!(self, self.iterable.get(idx)).unwrap(); + } + + pub fn get_key(&self, idx: usize) -> T { + return map_scale!(self, self.iterable.get(idx)).unwrap().0 + } + + pub fn iter(&self) -> impl Iterator + '_ { + map_scale!(self, FixDupsIter::new(self.iterable.cdf_iter())) + } + + pub fn iter_model_input(&self) -> impl Iterator + '_ { + return map_scale!(self, FixDupsIter::new(self.iterable.cdf_iter())) + .map(|(k, o)| (k.to_model_input(), o)); + } + + + pub fn iter_unique(&self) -> impl Iterator + '_ { + map_scale!(self, DedupIter::new(self.iterable.cdf_iter())) + } + + + // Code adapted from superslice, + // https://docs.rs/superslice/1.0.0/src/superslice/lib.rs.html + // which was copyright 2017 Alkis Evlogimenos under the Apache License. + pub fn lower_bound_by(&self, f: F) -> usize + where F: Fn((T, usize)) -> Ordering { + let mut size = self.len(); + if size == 0 { return 0; } + + let mut base = 0usize; + while size > 1 { + let half = size / 2; + let mid = base + half; + let cmp = f(self.get(mid)); + base = if cmp == Ordering::Less { mid } else { base }; + size -= half; + } + let cmp = f(self.get(base)); + base + (cmp == Ordering::Less) as usize + } + + pub fn soft_copy(&self) -> RMITrainingData { + return RMITrainingData { + scale: self.scale, + iterable: Arc::clone(&self.iterable) + }; + } +} + +/*struct RMITrainingDataIteratorProviderWrapper { + orig: Arc> +} + +impl RMITrainingDataIteratorProvider for RMITrainingDataIteratorProviderWrapper { + +}*/ + + +/*impl PartialEq for ModelInput { + fn eq(&self, other: &Self) -> bool { + match self { + ModelInput::Int(x) => { + match other { + ModelInput::Int(y) => x == y, + ModelInput::Float(_) => false + } + } + + ModelInput::Float(x) => { + match other { + ModelInput::Int(_) => false, + ModelInput::Float(y) => x == y // exact equality is intentional + } + } + } + } +} + +impl Eq for ModelInput { } + +impl PartialOrd for ModelInput { + fn partial_cmp(&self, other: &ModelInput) -> Option { + match self { + ModelInput::Int(x) => { + match other { + ModelInput::Int(y) => x.partial_cmp(y), + ModelInput::Float(_) => None + } + } + ModelInput::Float(x) => { + match other { + ModelInput::Int(_) => None, + ModelInput::Float(y) => x.partial_cmp(y) + } + } + } + } +}*/ + + + +#[derive(Clone, Copy, Debug)] +pub enum ModelInput { + Int(u64), + Float(f64), +} + +impl PartialEq for ModelInput { + fn eq(&self, other: &Self) -> bool { + match self { + ModelInput::Int(x) => { + match other { + ModelInput::Int(y) => x == y, + ModelInput::Float(_) => false + } + } + + ModelInput::Float(x) => { + match other { + ModelInput::Int(_) => false, + ModelInput::Float(y) => x == y // exact equality is intentional + } + } + } + } +} + +impl Eq for ModelInput { } + +impl PartialOrd for ModelInput { + fn partial_cmp(&self, other: &ModelInput) -> Option { + match self { + ModelInput::Int(x) => { + match other { + ModelInput::Int(y) => x.partial_cmp(y), + ModelInput::Float(_) => None + } + } + ModelInput::Float(x) => { + match other { + ModelInput::Int(_) => None, + ModelInput::Float(y) => x.partial_cmp(y) + } + } + } + } +} + + + +impl ModelInput { + pub fn as_float(&self) -> f64 { + return match self { + ModelInput::Int(x) => *x as f64, + ModelInput::Float(x) => *x, + }; + } + + pub fn as_int(&self) -> u64 { + return match self { + ModelInput::Int(x) => *x, + ModelInput::Float(x) => *x as u64, + }; + } + + pub fn max_value(&self) -> ModelInput { + return match self { + ModelInput::Int(_) => std::u64::MAX.into(), + ModelInput::Float(_) => std::f64::MAX.into() + }; + } + + pub fn min_value(&self) -> ModelInput { + return match self { + ModelInput::Int(_) => 0.into(), + ModelInput::Float(_) => std::f64::MIN.into() + }; + } + + pub fn minus_epsilon(&self) -> ModelInput { + return match self { + ModelInput::Int(x) => if *x > 0 { (x - 1).into() } else { 0.into() } + ModelInput::Float(x) => (x - std::f64::EPSILON).into() + }; + } + + pub fn plus_epsilon(&self) -> ModelInput { + return match self { + ModelInput::Int(x) => if *x < std::u64::MAX { + (x + 1).into() + } else { + std::u64::MAX.into() + } + ModelInput::Float(x) => (x + std::f64::EPSILON).into() + }; + } +} + +impl From for ModelInput { + fn from(i: u64) -> Self { + ModelInput::Int(i) + } +} + +impl From for ModelInput { + fn from(i: u32) -> Self { + ModelInput::Int(i as u64) + } +} + +impl From for ModelInput { + fn from(i: i32) -> Self { + assert!(i >= 0); + ModelInput::Int(i as u64) + } +} + + +impl From for ModelInput { + fn from(f: f64) -> Self { + ModelInput::Float(f) + } +} +pub enum ModelDataType { + Int, + Int128, + Float, +} + +impl ModelDataType { + pub fn c_type(&self) -> &'static str { + match self { + ModelDataType::Int => "uint64_t", + ModelDataType::Int128 => "uint128_t", + ModelDataType::Float => "double", + } + } +} + +#[derive(Debug, Clone)] +pub enum ModelParam { + Int(u64), + Float(f64), + ShortArray(Vec), + IntArray(Vec), + Int32Array(Vec), + FloatArray(Vec), +} + +impl ModelParam { + // size in bytes + pub fn size(&self) -> usize { + match self { + ModelParam::Int(_) => 8, + ModelParam::Float(_) => 8, + ModelParam::ShortArray(a) => 2 * a.len(), + ModelParam::IntArray(a) => 8 * a.len(), + ModelParam::Int32Array(a) => 4 * a.len(), + ModelParam::FloatArray(a) => 8 * a.len(), + } + } + + pub fn c_type(&self) -> &'static str { + match self { + ModelParam::Int(_) => "uint64_t", + ModelParam::Float(_) => "double", + ModelParam::ShortArray(_) => "short", + ModelParam::IntArray(_) => "uint64_t", + ModelParam::Int32Array(_) => "uint32_t", + ModelParam::FloatArray(_) => "double", + } + } + + pub fn is_array(&self) -> bool { + match self { + ModelParam::Int(_) => false, + ModelParam::Float(_) => false, + ModelParam::ShortArray(_) => true, + ModelParam::IntArray(_) => true, + ModelParam::Int32Array(_) => true, + ModelParam::FloatArray(_) => true + } + } + + pub fn c_type_mod(&self) -> &'static str { + match self { + ModelParam::Int(_) => "", + ModelParam::Float(_) => "", + ModelParam::ShortArray(_) => "[]", + ModelParam::IntArray(_) => "[]", + ModelParam::Int32Array(_) => "[]", + ModelParam::FloatArray(_) => "[]", + } + } + + pub fn c_val(&self) -> String { + match self { + ModelParam::Int(v) => format!("{}UL", v), + ModelParam::Float(v) => { + let mut tmp = format!("{:.}", v); + if !tmp.contains('.') { + tmp.push_str(".0"); + } + return tmp; + }, + ModelParam::ShortArray(arr) => { + let itms: Vec = arr.iter().map(|i| format!("{}", i)).collect(); + return format!("{{ {} }}", itms.join(", ")); + }, + ModelParam::IntArray(arr) => { + let itms: Vec = arr.iter().map(|i| format!("{}UL", i)).collect(); + return format!("{{ {} }}", itms.join(", ")); + }, + ModelParam::Int32Array(arr) => { + let itms: Vec = arr.iter().map(|i| format!("{}UL", i)).collect(); + return format!("{{ {} }}", itms.join(", ")); + }, + ModelParam::FloatArray(arr) => { + let itms: Vec = arr + .iter() + .map(|i| format!("{:.}", i)) + .map(|s| if !s.contains('.') { s + ".0" } else { s }) + .collect(); + return format!("{{ {} }}", itms.join(", ")); + } + } + } + + /* useful for debugging floating point issues + pub fn as_bits(&self) -> u64 { + return match self { + ModelParam::Int(v) => *v, + ModelParam::Float(v) => v.to_bits(), + ModelParam::ShortArray(_) => panic!("Cannot treat a short array parameter as a float"), + ModelParam::IntArray(_) => panic!("Cannot treat an int array parameter as a float"), + ModelParam::FloatArray(_) => panic!("Cannot treat an float array parameter as a float"), + }; + }*/ + + pub fn is_same_type(&self, other: &ModelParam) -> bool { + return std::mem::discriminant(self) == std::mem::discriminant(other); + } + + pub fn write_to(&self, target: &mut T) -> Result<(), std::io::Error> { + match self { + ModelParam::Int(v) => target.write_u64::(*v), + ModelParam::Float(v) => target.write_f64::(*v), + ModelParam::ShortArray(arr) => { + for v in arr { + target.write_u16::(*v)?; + } + + Ok(()) + }, + + ModelParam::IntArray(arr) => { + for v in arr { + target.write_u64::(*v)?; + } + + Ok(()) + }, + + ModelParam::Int32Array(arr) => { + for v in arr { + target.write_u32::(*v)?; + } + + Ok(()) + }, + + ModelParam::FloatArray(arr) => { + for v in arr { + target.write_f64::(*v)?; + } + + Ok(()) + + } + + } + } + + pub fn as_float(&self) -> f64 { + match self { + ModelParam::Int(v) => *v as f64, + ModelParam::Float(v) => *v, + ModelParam::ShortArray(_) => panic!("Cannot treat a short array parameter as a float"), + ModelParam::IntArray(_) => panic!("Cannot treat an int array parameter as a float"), + ModelParam::Int32Array(_) => panic!("Cannot treat an int32 array parameter as a float"), + ModelParam::FloatArray(_) => panic!("Cannot treat an float array parameter as a float"), + } + } + + pub fn len(&self) -> usize { + match self { + ModelParam::Int(_) => 1, + ModelParam::Float(_) => 1, + ModelParam::ShortArray(p) => p.len(), + ModelParam::IntArray(p) => p.len(), + ModelParam::Int32Array(p) => p.len(), + ModelParam::FloatArray(p) => p.len() + } + } +} + +impl From for ModelParam { + fn from(i: usize) -> Self { + ModelParam::Int(i as u64) + } +} + +impl From for ModelParam { + fn from(i: u64) -> Self { + ModelParam::Int(i) + } +} + +impl From for ModelParam { + fn from(i: u8) -> Self { + ModelParam::Int(u64::from(i)) + } +} + +impl From for ModelParam { + fn from(f: f64) -> Self { + ModelParam::Float(f) + } +} + +impl From> for ModelParam { + fn from(f: Vec) -> Self { + ModelParam::ShortArray(f) + } +} + +impl From> for ModelParam { + fn from(f: Vec) -> Self { + ModelParam::IntArray(f) + } +} + +impl From> for ModelParam { + fn from(f: Vec) -> Self { + ModelParam::Int32Array(f) + } +} + +impl From> for ModelParam { + fn from(f: Vec) -> Self { + ModelParam::FloatArray(f) + } +} + +pub enum ModelRestriction { + None, + MustBeTop, + MustBeBottom, +} + +pub trait Model: Sync + Send { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + return self.predict_to_int(inp) as f64; + } + + fn predict_to_int(&self, inp: &ModelInput) -> u64 { + return f64::max(0.0, self.predict_to_float(inp).floor()) as u64; + } + + fn input_type(&self) -> ModelDataType; + fn output_type(&self) -> ModelDataType; + + fn params(&self) -> Vec; + + fn code(&self) -> String; + fn function_name(&self) -> String; + + fn standard_functions(&self) -> HashSet { + return HashSet::new(); + } + + fn needs_bounds_check(&self) -> bool { + return true; + } + fn restriction(&self) -> ModelRestriction { + return ModelRestriction::None; + } + fn error_bound(&self) -> Option { + return None; + } + + fn set_to_constant_model(&mut self, _constant: u64) -> bool { + return false; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_scale() { + let mut v = ModelData::IntKeyToIntPos(vec![(0, 0), (1, 1), (3, 2), (100, 3)]); + + v.scale_targets_to(50, 4); + + let results = v.as_int_int(); + assert_eq!(results[0].1, 0); + assert_eq!(results[1].1, 12); + assert_eq!(results[2].1, 25); + assert_eq!(results[3].1, 37); + } + + #[test] + fn test_iter() { + let data = vec![(0, 1), (1, 2), (3, 3), (100, 4)]; + + let v = ModelData::IntKeyToIntPos(data.clone()); + + let iterated: Vec<(u64, u64)> = v.iter_uint_uint().collect(); + assert_eq!(data, iterated); + } +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/normal.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/normal.rs new file mode 100644 index 00000000..1940e6da --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/normal.rs @@ -0,0 +1,223 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +use crate::models::*; +use std::f64; + +fn exp1(inp: f64) -> f64 { + let mut x = inp; + x = 1.0 + x / 64.0; + x *= x; + x *= x; + x *= x; + x *= x; + x *= x; + x *= x; + return x; +} + +fn phi(x: f64) -> f64 { + return 1.0 / (1.0 + exp1(-1.65451 * x)); +} + +fn ncdf(loc_data: &RMITrainingData) -> (f64, f64, f64) { + let mut scale = -f64::INFINITY; + let mut mean = 0.0; + let mut stdev = 0.0; + + let n = loc_data.len() as f64; + + for (inp, y) in loc_data.iter() { + let x = inp.as_float(); + mean += x / n; + scale = f64::max(scale, y as f64); + } + + for (inp, _y) in loc_data.iter() { + let x = inp.as_float(); + stdev += (x - mean).powf(2.0) + } + + stdev /= n; + stdev = stdev.sqrt(); + + return (mean, stdev, scale); +} + +fn lncdf(loc_data: &RMITrainingData) -> (f64, f64, f64) { + let mut scale = -f64::INFINITY; + let mut mean = 0.0; + let mut stdev = 0.0; + + let n = loc_data.len() as f64; + for (inp, y) in loc_data.iter() { + let x = inp.as_float(); + let lnx = if !f64::is_finite(x.ln()) { 0.0 } else { x.ln() }; + mean += lnx / n; + scale = f64::max(scale, y as f64); + } + + for (inp, _y) in loc_data.iter() { + let x = inp.as_float(); + let lnx = if !f64::is_finite(x.ln()) { 0.0 } else { x.ln() }; + stdev += (lnx - mean).powf(2.0) + } + + + stdev /= n; + stdev = stdev.sqrt(); + + return (mean, stdev, scale); +} + +pub struct NormalModel { + params: (f64, f64, f64), +} + +impl NormalModel { + pub fn new(data: &RMITrainingData) -> NormalModel { + return NormalModel { params: ncdf(data) }; + } +} + +impl Model for NormalModel { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + let (mean, stdev, scale) = self.params; + return phi((inp.as_float() - mean) / stdev) * scale; + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + + fn params(&self) -> Vec { + return vec![ + self.params.0.into(), + self.params.1.into(), + self.params.2.into(), + ]; + } + + fn code(&self) -> String { + return String::from( + " +inline double ncdf(double mean, double stdev, double scale, double inp) { + return phi((inp - mean) / stdev) * scale; +}", + ); + } + + fn function_name(&self) -> String { + return String::from("ncdf"); + } + fn standard_functions(&self) -> HashSet { + let mut to_r = HashSet::new(); + to_r.insert(StdFunctions::EXP1); + to_r.insert(StdFunctions::PHI); + return to_r; + } +} + +#[cfg(test)] +mod ncdf_tests { + use super::*; + + #[test] + fn test_ncdf1() { + let md = ModelData::IntKeyToIntPos(vec![(1, 1), (2, 3), (3, 5)]); + + let ncdf_mod = NormalModel::new(&md); + + assert_eq!(ncdf_mod.predict_to_int(2.into()), 2); + assert_eq!(ncdf_mod.predict_to_int(1.into()), 0); + } + + #[test] + fn test_empty() { + NormalModel::new(&ModelData::empty()); + } + +} + +pub struct LogNormalModel { + params: (f64, f64, f64), +} + +impl LogNormalModel { + pub fn new(data: &RMITrainingData) -> LogNormalModel { + return LogNormalModel { + params: lncdf(data), + }; + } +} + +impl Model for LogNormalModel { + fn predict_to_float(&self, inp: &ModelInput) -> f64 { + let (mean, stdev, scale) = self.params; + let data = inp.as_float(); + return phi((f64::max(data.ln(), 0.0) - mean) / stdev) * scale; + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Float; + } + + fn params(&self) -> Vec { + return vec![ + self.params.0.into(), + self.params.1.into(), + self.params.2.into(), + ]; + } + + fn code(&self) -> String { + return String::from( + " +inline double lncdf(double mean, double stdev, double scale, double inp) { + return phi((fmax(0.0, log(inp)) - mean) / stdev) * scale; +}", + ); + } + + fn function_name(&self) -> String { + return String::from("lncdf"); + } + fn standard_functions(&self) -> HashSet { + let mut to_r = HashSet::new(); + to_r.insert(StdFunctions::EXP1); + to_r.insert(StdFunctions::PHI); + return to_r; + } +} + +#[cfg(test)] +mod lncdf_tests { + use super::*; + + #[test] + fn test_lncdf1() { + let md = ModelData::IntKeyToIntPos(vec![(1, 1), (2, 2), (3, 20)]); + + let lncdf_mod = LogNormalModel::new(&md); + + assert_eq!(lncdf_mod.predict_to_int(2.into()), 11); + assert_eq!(lncdf_mod.predict_to_int(1.into()), 2); + } + + #[test] + fn test_empty() { + LogNormalModel::new(&ModelData::empty()); + } + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/radix.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/radix.rs new file mode 100644 index 00000000..558c4196 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/radix.rs @@ -0,0 +1,181 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +use crate::models::utils::{common_prefix_size, num_bits}; +use crate::models::*; +use log::*; + +pub struct RadixModel { + params: (u8, u8), +} + +impl RadixModel { + pub fn new(data: &RMITrainingData) -> RadixModel { + if data.len() == 0 { + return RadixModel { params: (0, 0) }; + } + + let largest_value = data.iter().map(|(_x, y)| y).max().unwrap() as u64; + let bits = num_bits(largest_value); + trace!( + "Radix layer using {} bits, from largest value {} (max layers: {})", + bits, + largest_value, + (1 << (bits + 1)) - 1 + ); + + + let common_prefix = common_prefix_size(data); + trace!("Radix layer common prefix: {}", common_prefix); + + return RadixModel { + params: (common_prefix, bits), + }; + } +} + +impl Model for RadixModel { + fn predict_to_int(&self, inp: &ModelInput) -> u64 { + let (left_shift, num_bits) = self.params; + + let as_int: u64 = inp.as_int(); + let res = (as_int << left_shift) >> (64 - num_bits); + + return res; + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Int; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Int; + } + + fn params(&self) -> Vec { + return vec![self.params.0.into(), self.params.1.into()]; + } + + fn code(&self) -> String { + return String::from( + " +inline uint64_t radix(uint64_t prefix_length, uint64_t bits, uint64_t inp) { + return (inp << prefix_length) >> (64 - bits); +}", + ); + } + + fn function_name(&self) -> String { + return String::from("radix"); + } + fn needs_bounds_check(&self) -> bool { + return false; + } + fn restriction(&self) -> ModelRestriction { + return ModelRestriction::MustBeTop; + } +} + +pub struct RadixTable { + prefix_bits: u8, + table_bits: u8, + hint_table: Vec +} + +impl RadixTable { + pub fn new(data: &RMITrainingData, bits: u8) -> RadixTable { + let prefix = common_prefix_size(data); + let mut hint_table: Vec = vec![0 ; 1 << bits]; + + let mut last_radix = 0; + for (inp, y) in data.iter_model_input() { + let x = inp.as_int(); + let num_bits = if prefix + bits > 64 { 0 } else { 64 - (prefix + bits) }; + let current_radix = ((x << prefix) >> prefix) >> num_bits; + if current_radix == last_radix { continue; } + assert!(current_radix < hint_table.len() as u64); + + hint_table[current_radix as usize] = y as u32; + + for i in (last_radix + 1)..current_radix { + hint_table[i as usize] = y as u32; + } + + last_radix = current_radix; + } + + for i in (last_radix as usize + 1)..hint_table.len() { + hint_table[i as usize] = hint_table.len() as u32; + } + + return RadixTable { + prefix_bits: prefix, + table_bits: bits, + hint_table + }; + } +} + +impl Model for RadixTable { + fn predict_to_int(&self, inp: &ModelInput) -> u64 { + let as_int: u64 = inp.as_int(); + let prefix = self.prefix_bits; + let bits = self.table_bits; + let num_bits = if prefix + bits > 64 { 0 } else { 64 - (prefix + bits) }; + let res = ((as_int << prefix) >> prefix) >> num_bits; + let idx = self.hint_table[res as usize] as u64; + + return idx; + } + + fn input_type(&self) -> ModelDataType { + return ModelDataType::Int; + } + fn output_type(&self) -> ModelDataType { + return ModelDataType::Int; + } + + fn params(&self) -> Vec { + return vec![self.hint_table.clone().into()]; + } + + fn code(&self) -> String { + let num_bits = if self.prefix_bits + self.table_bits > 64 { + 0 + } else { + 64 - (self.prefix_bits + self.table_bits) + }; + + return format!( + " +inline uint64_t radix_table(const uint32_t* table, const uint64_t inp) {{ + return table[((inp << {0}) >> {0}) >> {1}]; +}}", self.prefix_bits, num_bits + ); + } + + fn function_name(&self) -> String { + return String::from("radix_table"); + } + fn needs_bounds_check(&self) -> bool { + return false; + } + fn restriction(&self) -> ModelRestriction { + return ModelRestriction::None; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty() { + RadixModel::new(&ModelData::empty()); + } + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/stdlib.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/stdlib.rs new file mode 100644 index 00000000..53dd80b0 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/stdlib.rs @@ -0,0 +1,66 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +#[derive(Debug, PartialEq, Eq, Hash)] +pub enum StdFunctions { + EXP1, + PHI, + BinarySearch, +} + +impl StdFunctions { + pub fn decl(&self) -> &'static str { + match self { + StdFunctions::EXP1 => "inline double exp1(double x);", + StdFunctions::PHI => "inline double phi(double x);", + StdFunctions::BinarySearch => { + "uint64_t bs_lower_bound(const uint64_t a[], uint64_t n, uint64_t x);" + } + } + } + + pub fn code(&self) -> &'static str { + match self { + StdFunctions::EXP1 => { + " +inline double exp1(double x) { + x = 1.0 + x / 64.0; + x *= x; x *= x; x *= x; x *= x; + x *= x; x *= x; + return x; +} +" + } + StdFunctions::PHI => { + " +inline double phi(double x) { + return 1.0 / (1.0 + exp1(- 1.65451 * x)); +} +" + } + StdFunctions::BinarySearch => { + " +uint64_t bs_upper_bound(const uint64_t a[], uint64_t n, uint64_t x) { + int l = 0; + int h = n; // Not n - 1 + while (l < h) { + int mid = (l + h) / 2; + if (x >= a[mid]) { + l = mid + 1; + } else { + h = mid; + } + } + return l; +} + +" + } + } + } +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/utils.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/utils.rs new file mode 100644 index 00000000..98075244 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/models/utils.rs @@ -0,0 +1,128 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + +use crate::models::*; +use superslice::*; +use log::*; + + +pub fn num_bits(largest_target: u64) -> u8 { + let mut nbits = 0; + while (1 << (nbits+1)) - 1 <= largest_target { + nbits += 1; + } + nbits -= 1; + assert!((1 << (nbits+1)) - 1 <= largest_target); + + return nbits; +} + +pub fn common_prefix_size(data: &RMITrainingData) -> u8 { + let mut any_ones: u64 = 0; + let mut no_ones: u64 = !0; + + for (x, _y) in data.iter_model_input() { + any_ones |= x.as_int(); + no_ones &= x.as_int(); + } + + let any_zeros = !no_ones; + + let prefix_bits = any_zeros ^ any_ones; + return (!prefix_bits).leading_zeros() as u8; +} + +fn common_prefix_size2(data: &[u64]) -> u8 { + let mut any_ones: u64 = 0; + let mut no_ones: u64 = !0; + + for x in data { + any_ones |= x; + no_ones &= x; + } + + let any_zeros = !no_ones; + + let prefix_bits = any_zeros ^ any_ones; + return (!prefix_bits).leading_zeros() as u8; +} + + + +pub fn radix_index(points: &[u64], num_bits: u8) -> Vec { + // build the radix index + let cps = common_prefix_size2(points); + if cps != 0 { + warn!("Radix index currently assumes the common prefix size is 0, but it was {}", + cps); + } + + let mut radix_index: Vec = vec![0 ; 1 << num_bits]; + + let mut last_radix = 0; + for (idx, p) in points.iter().enumerate() { + let radix = p >> (64 - num_bits); + assert!(radix < radix_index.len() as u64); + + if radix == last_radix { continue; } + + for i in last_radix+1..radix { + radix_index[i as usize] = idx as u64; //radix_index[last_radix as usize] + 1; + } + radix_index[radix as usize] = idx as u64; + last_radix = radix; + } + + for i in last_radix+1..radix_index.len() as u64 { + radix_index[i as usize] = points.len() as u64; + } + + // end point + radix_index.push(points.len() as u64); + + // verify the radix construction + for p in points { + let radix = p >> (64 - num_bits); + let radix_lb = radix_index[radix as usize]; + let radix_ub = radix_index[radix as usize + 1]; + + let correct_idx = (points.upper_bound(p) - 1) as u64; + assert!(radix_lb <= correct_idx, + "On key {} with radix {}, correct index was {}, but radix LB = {} and UB = {}", + p, radix, correct_idx, radix_lb, radix_ub); + assert!(radix_ub > correct_idx, + "On key {} with radix {}, correct index was {}, but radix LB = {} and UB = {}", + p, radix, correct_idx, radix_lb, radix_ub); + } + + return radix_index; +} + + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_common_prefix1() { + let data = ModelData::IntKeyToIntPos(vec![ + (1, 0), (4, 4), (8, 8) + ]); + + assert_eq!(common_prefix_size(&data), 64-4); + } + + #[test] + fn test_common_prefix2() { + let data = ModelData::IntKeyToIntPos(vec![ + (1, 0), (8, 1), (9, 4), (12, 8) + ]); + + assert_eq!(common_prefix_size(&data), 64-4); + } +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/optimizer.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/optimizer.rs new file mode 100644 index 00000000..92d6e479 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/optimizer.rs @@ -0,0 +1,249 @@ +use crate::models::*; +use crate::train; +use crate::codegen; +use log::*; +use json::*; +use indicatif::{ProgressBar}; +use rayon::prelude::*; +use std::collections::BTreeSet; +use tabular::{Table, row}; + +//const TOP_ONLY_LAYERS: &[&str] = &["radix", "radix18", "radix22", "robust_linear"]; +//const ANYWHERE_LAYERS: &[&str] = &["linear", "cubic", "linear_spline"]; +//const SPECIALTY_TOP_LAYERS: &[&str] = &["histogram", "loglinear", "normal", "lognormal", "bradix"]; + +fn top_only_layers() -> Vec<&'static str> { + return match std::env::var_os("RMI_OPTIMIZER_PROFILE") { + None => vec!["radix", "radix18", "radix22", "robust_linear"], + Some(x) => { + match x.to_str().unwrap() { + "fast" => vec!["robust_linear"], + "memory" => vec!["radix", "radix18", "radix22", "robust_linear"], + "disk" => vec!["radix", "radix18", "radix22", "robust_linear", + "normal", "lognormal", "loglinear"], + _ => panic!("Invalid optimizer profile {}", x.to_str().unwrap()) + } + } + }; +} + +fn anywhere_layers() -> Vec<&'static str> { + return match std::env::var_os("RMI_OPTIMIZER_PROFILE") { + None => vec!["linear", "cubic", "linear_spline"], + Some(x) => { + match x.to_str().unwrap() { + "fast" => vec!["linear", "cubic"], + "memory" | "disk" => vec!["linear", "cubic", "linear_spline"], + _ => panic!("Invalid optimizer profile {}", x.to_str().unwrap()) + } + } + }; +} + +fn get_branching_factors() -> Vec { + let range = match std::env::var_os("RMI_OPTIMIZER_PROFILE") { + None => (6..25).step_by(1), + Some(x) => { + match x.to_str().unwrap() { + "fast" => (6..25).step_by(2), + "memory" => (6..25).step_by(1), + "disk" => (6..28).step_by(1), + _ => panic!("Invalid optimizer profile {}", x.to_str().unwrap()) + } + } + }; + + return range.map(|i| (2 as u64).pow(i)).collect(); +} + +fn pareto_front(results: &[RMIStatistics]) -> Vec { + let mut on_front: Vec = Vec::new(); + + for result in results.iter() { + if results.iter().any(|v| result.dominated_by(v)) { + // not on the front + continue; + } + + on_front.push(result.clone()); + } + + return on_front; +} + +fn narrow_front(results: &[RMIStatistics], desired_size: usize) -> Vec { + assert!(desired_size >= 2); + if results.len() <= desired_size { + return results.to_vec(); + } + + let mut tmp = results.to_vec(); + tmp.sort_by( + |a, b| a.size.partial_cmp(&b.size).unwrap() + ); + + let best_mod = tmp.remove(0); + while tmp.len() > desired_size - 1 { + // find the two items closest in size and remove less accuracte one. + let smallest_gap = + (0..tmp.len()-1).zip(1..tmp.len()) + .map(|(idx1, idx2)| (idx1, idx2, + (tmp[idx2].size as f64) / (tmp[idx1].size as f64))) + .min_by(|(_, _, v1), (_, _, v2)| v1.partial_cmp(v2).unwrap()).unwrap(); + + let err1 = tmp[smallest_gap.0].average_log2_error; + let err2 = tmp[smallest_gap.1].average_log2_error; + if err1 > err2 { + tmp.remove(smallest_gap.0); + } else { + tmp.remove(smallest_gap.1); + } + } + tmp.insert(0, best_mod); + + return tmp; + + + +} + +fn first_phase_configs() -> Vec<(String, u64)> { + let mut results = Vec::new(); + let mut all_top_models = Vec::new(); + all_top_models.extend(top_only_layers()); + all_top_models.extend(anywhere_layers()); + + for top_model in all_top_models { + for bottom_model in anywhere_layers() { + for branching_factor in get_branching_factors().iter().step_by(5) { + results.push((format!("{},{}", top_model, bottom_model), *branching_factor)); + } + } + } + + return results; +} + +fn second_phase_configs(first_phase: &[RMIStatistics]) -> Vec<(String, u64)> { + let qualifying_model_configs = { + let on_front = pareto_front(first_phase); + let mut qualifying = BTreeSet::new(); + for result in on_front { + qualifying.insert(result.models.clone()); + } + qualifying + }; + + info!("Qualifying model types for phase 2: {:?}", qualifying_model_configs); + let mut results = Vec::new(); + + for model in qualifying_model_configs.iter() { + for branching_factor in get_branching_factors() { + if first_phase.iter().any(|v| v.has_config(&model, branching_factor)) { + continue; + } + + results.push((model.clone(), branching_factor)); + } + } + + return results; +} + +#[derive(Clone, Debug)] +pub struct RMIStatistics { + pub models: String, + pub branching_factor: u64, + pub average_log2_error: f64, + pub max_log2_error: f64, + pub size: u64 +} + +impl RMIStatistics { + fn from_trained(rmi: &train::TrainedRMI) -> RMIStatistics { + return RMIStatistics { + average_log2_error: rmi.model_avg_log2_error, + max_log2_error: rmi.model_max_log2_error, + size: codegen::rmi_size(&rmi), + models: rmi.models.clone(), + branching_factor: rmi.branching_factor + }; + } + + fn dominated_by(&self, other: &RMIStatistics) -> bool { + if self.size < other.size { return false; } + if self.average_log2_error < other.average_log2_error { return false; } + + if self.size == other.size && self.average_log2_error <= other.average_log2_error { + return false; + } + + let log2_diff = (self.average_log2_error - other.average_log2_error).abs(); + if self.size <= other.size && log2_diff < std::f64::EPSILON { + return false; + } + + return true; + } + + fn has_config(&self, models: &str, branching_factor: u64) -> bool { + return self.models == models && self.branching_factor == branching_factor; + } + + pub fn display_table(itms: &[RMIStatistics]) { + let mut table = Table::new("{:<} {:>} {:>} {:>} {:>}"); + table.add_row(row!("Models", "Branch", " AvgLg2", + " MaxLg2", " Size (b)")); + for itm in itms { + table.add_row(row!(itm.models.clone(), + format!("{:10}", itm.branching_factor), + format!(" {:2.5}", itm.average_log2_error), + format!(" {:2.5}", itm.max_log2_error), + format!(" {}", itm.size))); + } + + print!("{}", table); + } + + pub fn to_grid_spec(&self, namespace: &str) -> JsonValue { + return object!( + "layers" => self.models.clone(), + "branching factor" => self.branching_factor, + "namespace" => namespace, + "size" => self.size, + "average log2 error" => self.average_log2_error, + "binary" => true + ); + } +} + +fn measure_rmis(data: &RMITrainingData, + configs: &[(String, u64)]) -> Vec { + let pbar = ProgressBar::new(configs.len() as u64); + + configs.par_iter() + .map(|(models, branch_factor)| { + let mut loc_data = data.soft_copy(); + let res = train::train(&mut loc_data, models, *branch_factor); + pbar.inc(1); + RMIStatistics::from_trained(&res) + }).collect() +} + +pub fn find_pareto_efficient_configs( + data: &RMITrainingData, restrict: usize) + -> Vec{ + let initial_configs = first_phase_configs(); + let first_phase_results = measure_rmis(data, &initial_configs); + + let next_configs = second_phase_configs(&first_phase_results); + let second_phase_results = measure_rmis(data, &next_configs); + + let mut final_front = pareto_front(&second_phase_results); + final_front = narrow_front(&final_front, restrict); + final_front.sort_by( + |a, b| a.average_log2_error.partial_cmp(&b.average_log2_error).unwrap() + ); + + return final_front; +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/lower_bound_correction.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/lower_bound_correction.rs new file mode 100644 index 00000000..dff93fd2 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/lower_bound_correction.rs @@ -0,0 +1,162 @@ +use crate::models::*; + + +fn find_first_below(data: &[Option], idx: usize) -> Option<(usize, T)> { + assert!(idx < data.len()); + if idx == 0 { return None; } + + let mut i = idx - 1; + loop { + if let Some(v) = data[i] { return Some((i, v)); } + if i == 0 { return None; } + i -= 1; + } +} + +fn find_first_above(data: &[Option], idx: usize) -> Option<(usize, T)> { + assert!(idx < data.len()); + if idx == data.len() - 1 { return None; } + + let mut i = idx + 1; + loop { + if let Some(v) = data[i] { return Some((i, v)); } + if i == data.len() - 1 { return None; } + i += 1; + } +} + +// next_for_leaf[i] stores the (key index, key) pairs for the first key in the +// leaf model after leaf i. next_for_leaf[last leaf index] stores the maximum possible key. +fn compute_next_for_leaf(num_leaf_models: u64, + num_keys: usize, + first_key_for_leaf: &[Option<(usize, T)>]) + -> Vec<(usize, T)> { + + let mut next_for_leaf = vec![(0, T::zero_value()) ; num_leaf_models as usize]; + let mut idx: usize = 0; + while idx < num_leaf_models as usize { + match find_first_above(&first_key_for_leaf, idx as usize) { + Some((next_leaf_idx, val)) => { + assert!(next_leaf_idx > idx); + for i in idx..next_leaf_idx { + next_for_leaf[i] = val; + } + idx = next_leaf_idx; + }, + None => { + for i in idx..num_leaf_models as usize { + next_for_leaf[i] = (num_keys, T::max_value()); + } + break; + } + } + } + + return next_for_leaf; +} + +fn compute_prev_for_leaf(num_leaf_models: u64, + last_key_for_leaf: &[Option<(usize, T)>]) + -> Vec<(usize, T)> { + + let mut prev_for_leaf: Vec<(usize, T)> + = vec![(0, T::zero_value()) ; num_leaf_models as usize]; + let mut idx: usize = num_leaf_models as usize - 1; + while idx > 0 { + match find_first_below(&last_key_for_leaf, idx as usize) { + Some((prev_leaf_idx, val)) => { + assert!(prev_leaf_idx < idx); + for i in prev_leaf_idx+1..idx+1 { + prev_for_leaf[i] = val; + } + idx = prev_leaf_idx; + }, + None => { break; } + } + } + + return prev_for_leaf; + +} + + +pub struct LowerBoundCorrection { + first: Vec>, + last: Vec>, + next: Vec<(usize, T)>, + prev: Vec<(usize, T)>, + run_lengths: Vec +} + +impl LowerBoundCorrection { + pub fn new(pred_func: F, num_leaf_models: u64, data: &RMITrainingData) -> LowerBoundCorrection + where F: Fn(T) -> u64 { + + let mut first_key_for_leaf: Vec> + = vec![None ; num_leaf_models as usize]; + let mut last_key_for_leaf: Vec> + = vec![None ; num_leaf_models as usize]; + let mut max_run_length: Vec = vec![0 ; num_leaf_models as usize]; + + let mut last_target = 0; + let mut current_run_length = 0; + let mut current_run_key = data.get_key(0); + for (x, y) in data.iter() { + let leaf_idx = pred_func(x.into()); + let target = u64::min(num_leaf_models - 1, leaf_idx) as usize; + + if target == last_target && x == current_run_key { + current_run_length += 1; + } else if target != last_target || x != current_run_key { + max_run_length[last_target] = u64::max( + max_run_length[last_target], + current_run_length + ); + + current_run_length = 1; + current_run_key = x; + last_target = target; + } + + if first_key_for_leaf[target].is_none() { + first_key_for_leaf[target] = Some((y, x)); + } + last_key_for_leaf[target] = Some((y, x)); + } + + let next_for_leaf = compute_next_for_leaf(num_leaf_models, data.len(), &first_key_for_leaf); + let prev_for_leaf = compute_prev_for_leaf(num_leaf_models, &last_key_for_leaf); + + return LowerBoundCorrection { + first: first_key_for_leaf, + last: last_key_for_leaf, + next: next_for_leaf, + prev: prev_for_leaf, + run_lengths: max_run_length + }; + } + + pub fn first_key(&self, leaf_idx: usize) -> Option { + return self.first[leaf_idx].map(|x| x.1); + } + + pub fn last_key(&self, leaf_idx: usize) -> Option { + return self.last[leaf_idx].map(|x| x.1); + } + + pub fn next(&self, leaf_idx: usize) -> (usize, T) { + return self.next[leaf_idx]; + } + + pub fn next_index(&self, leaf_idx: usize) -> usize { + return self.next[leaf_idx].0; + } + + pub fn prev_key(&self, leaf_idx: usize) -> T { + return self.prev[leaf_idx].1; + } + + pub fn longest_run(&self, leaf_idx: usize) -> u64 { + return self.run_lengths[leaf_idx]; + } +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/mod.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/mod.rs new file mode 100644 index 00000000..c52918fb --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/mod.rs @@ -0,0 +1,184 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +use crate::models::*; +use crate::cache_fix::cache_fix; +use log::*; +use std::time::SystemTime; + +mod two_layer; +//mod multi_layer; +mod lower_bound_correction; + +pub struct TrainedRMI { + pub num_rmi_rows: usize, + pub num_data_rows: usize, + pub model_avg_error: f64, + pub model_avg_l2_error: f64, + pub model_avg_log2_error: f64, + pub model_max_error: u64, + pub model_max_error_idx: usize, + pub model_max_log2_error: f64, + pub last_layer_max_l1s: Vec, + pub rmi: Vec>>, + pub models: String, + pub branching_factor: u64, + pub cache_fix: Option<(usize, Vec<(u64, usize)>)>, + pub build_time: u128 +} + +fn train_model(model_type: &str, + data: &RMITrainingData) -> Box { + let model: Box = match model_type { + "linear" => Box::new(LinearModel::new(data)), + "robust_linear" => Box::new(RobustLinearModel::new(data)), + "linear_spline" => Box::new(LinearSplineModel::new(data)), + "cubic" => Box::new(CubicSplineModel::new(data)), + "loglinear" => Box::new(LogLinearModel::new(data)), + "normal" => Box::new(NormalModel::new(data)), + "lognormal" => Box::new(LogNormalModel::new(data)), + "radix" => Box::new(RadixModel::new(data)), + "radix8" => Box::new(RadixTable::new(data, 8)), + "radix18" => Box::new(RadixTable::new(data, 18)), + "radix22" => Box::new(RadixTable::new(data, 22)), + "radix26" => Box::new(RadixTable::new(data, 26)), + "radix28" => Box::new(RadixTable::new(data, 28)), + "bradix" => Box::new(BalancedRadixModel::new(data)), + "histogram" => Box::new(EquidepthHistogramModel::new(data)), + _ => panic!("Unknown model type: {}", model_type), + }; + + return model; +} + +fn validate(model_spec: &[String]) { + let num_layers = model_spec.len(); + let empty_container: RMITrainingData = RMITrainingData::empty(); + + for (idx, model) in model_spec.iter().enumerate() { + let restriction = train_model(model, &empty_container).restriction(); + + match restriction { + ModelRestriction::None => {} + ModelRestriction::MustBeTop => { + assert_eq!( + idx, 0, + "if used, model type {} must be the root model", + model + ); + } + ModelRestriction::MustBeBottom => { + assert_eq!( + idx, + num_layers - 1, + "if used, model type {} must be the bottommost model", + model + ); + } + } + } +} + +/*fn test_rmi_input(test_key: u64, data: &RMITrainingData, rmi: &TrainedRMI) { + let correct = data.lower_bound(test_key); + println!("Predicting {}", test_key); + let (guess, err) = rmi.test_predict(test_key); + println!("Model prediction for lookup {}: {} with error {}", + test_key, guess, err); + + println!("({}, {}), {}", + guess - err, + guess + err, + correct); +}*/ + +pub fn train(data: &RMITrainingData, + model_spec: &str, branch_factor: u64) -> TrainedRMI { + + let start_time = SystemTime::now(); + let (model_list, last_model): (Vec, String) = { + let mut all_models: Vec = model_spec.split(',').map(String::from).collect(); + validate(&all_models); + let last = all_models.pop().unwrap(); + (all_models, last) + }; + + if model_list.len() == 1 { + let mut res = two_layer::train_two_layer(&mut data.soft_copy(), &model_list[0], + &last_model, branch_factor); + let build_time = SystemTime::now() + .duration_since(start_time) + .map(|d| d.as_nanos()) + .unwrap_or(std::u128::MAX); + res.build_time = build_time; + + return res; + } + + // it is not a simple, two layer rmi + //return multi_layer::train_multi_layer(data, &model_list, last_model, branch_factor); + panic!(); // TODO +} + +pub fn train_for_size(data: &RMITrainingData, + max_size: usize) -> TrainedRMI { + + let start_time = SystemTime::now(); + let pareto = crate::find_pareto_efficient_configs(data, 1000); + // go down the front until we find something small enough + + let config = pareto.into_iter() + .filter(|x| x.size < max_size as u64) + .next() + .expect(format!( + "Could not find any configurations smaller than {}", max_size).as_str()); + + let models = config.models; + let bf = config.branching_factor; + + info!("Found RMI config {} {} with size {} and average log2 {}", + models, bf, config.size, config.average_log2_error); + let mut res = train(data, models.as_str(), bf); + + let build_time = SystemTime::now() + .duration_since(start_time) + .map(|d| d.as_nanos()) + .unwrap_or(std::u128::MAX); + res.build_time = build_time; + return res; +} + +pub fn train_bounded(data: &RMITrainingData, + model_spec: &str, + branch_factor: u64, + line_size: usize) -> TrainedRMI { + let start_time = SystemTime::now(); + // first, transform our data into error-bounded spline points + let spline = cache_fix(data, line_size); + std::mem::drop(data); + + // reindex the spline points so we can build an RMI on top + let reindexed_splines: Vec<(u64, usize)> = spline.iter() + .enumerate() + .map(|(idx, (key, _old_offset))| (*key, idx)) + .collect(); + + // construct new training data from our spline points + let mut new_data = RMITrainingData::new(Box::new(reindexed_splines)); + + let mut res = crate::train(&mut new_data, model_spec, branch_factor); + res.cache_fix = Some((line_size, spline)); + res.num_data_rows = data.len(); + + let build_time = SystemTime::now() + .duration_since(start_time) + .map(|d| d.as_nanos()) + .unwrap_or(std::u128::MAX); + res.build_time = build_time; + return res; +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/multi_layer.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/multi_layer.rs new file mode 100644 index 00000000..01296cf6 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/multi_layer.rs @@ -0,0 +1,118 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + +use crate::models::*; +use crate::train::{train_model, TrainedRMI}; +use log::*; + +pub fn train_multi_layer(data: &mut RMITrainingData, + model_list: &[String], + last_model: String, + branch_factor: u64) -> TrainedRMI { + + let mut rmi: Vec>> = Vec::new(); + let mut data_partitions = vec![data.clone().into_data()]; + let num_rows = data_partitions[0].len(); + + let mut current_model_count = 1; + for (_layer_idx, model_type) in model_list.iter().enumerate() { + info!("Training {} model layer", model_type); + // data_partition contains all of our data partitioned into groups + // based on the previous RMI layer's output + let next_layer_size = current_model_count * branch_factor; + let mut next_layer_data = + vec![Vec::with_capacity(num_rows / next_layer_size as usize); next_layer_size as usize]; + let mut models: Vec> = Vec::with_capacity(next_layer_size as usize); + + for model_data in data_partitions.into_iter() { + let mut md_container = RMITrainingData::new(&model_data); + + // not at the last layer -- rescale + md_container.set_scale(next_layer_size as f64 / num_rows as f64); + let model = train_model(model_type.as_str(), &md_container); + + // rescale back for next layer + md_container.set_scale(1.0); + + for (x, y) in md_container.iter_int_int() { + let model_pred = model.predict_to_int(x.into()); + assert!(model.needs_bounds_check() || model_pred < next_layer_size); + let target = u64::min(next_layer_size - 1, model_pred) as usize; + next_layer_data[target].push((x, y)); + } + + models.push(model); + } + + data_partitions = next_layer_data + .into_iter() + .map(ModelData::IntKeyToIntPos) + .collect(); + + current_model_count *= branch_factor; + rmi.push(models); + } + + info!("Training last level {} model", last_model); + let mut last_layer = Vec::new(); + let mut last_layer_max_l1s: Vec = Vec::new(); + let mut model_avg_error: f64 = 0.0; + let mut model_avg_l2_error: f64 = 0.0; + let mut model_avg_log2_error: f64 = 0.0; + let mut model_max_log2_error: f64 = 0.0; + let mut model_max_error = 0; + let mut model_max_error_idx = 0; + + let mut n = 1; + for (midx, model_data) in data_partitions.into_iter().enumerate() { + let md_container = RMITrainingData::new(&model_data); + let last_model = train_model(last_model.as_str(), &md_container); + let mut max_error = 0; + + for (idx, (x, y)) in md_container.iter_int_int().enumerate() { + let pred = last_model.predict_to_int(x.into()); + let err = u64::max(y, pred) - u64::min(y, pred); + + if let Some(bound) = last_model.error_bound() { + if err > bound { + warn!("Precision issue: model reports max bound of {}, \ + but an error of {} was observed on input {} at index {}. Prediction: {} Actual: {}", + bound, err, x, idx, pred, y); + } + } + + max_error = u64::max(max_error, err); + model_avg_error += ((max_error as f64) - model_avg_error) / (n as f64); + model_avg_l2_error += ((max_error as f64).powf(2.0) - model_avg_l2_error) / (n as f64); + let log2_error = ((2 * max_error + 2) as f64).log2(); + model_avg_log2_error += (log2_error - model_avg_log2_error) / (n as f64); + model_max_log2_error = f64::max(model_max_log2_error, log2_error); + n += 1; + } + if max_error > model_max_error { + model_max_error = max_error; + model_max_error_idx = midx; + } + + last_layer.push(last_model); + last_layer_max_l1s.push(max_error); + } + rmi.push(last_layer); + + return TrainedRMI { + model_avg_error, + model_avg_l2_error, + model_avg_log2_error, + model_max_error, + model_max_error_idx, + model_max_log2_error, + last_layer_max_l1s, + rmi, + models: format!("{},{}", model_list.join(","), last_model), + branching_factor: branch_factor + }; +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/two_layer.rs b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/two_layer.rs new file mode 100644 index 00000000..91685631 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/rmi_lib/src/train/two_layer.rs @@ -0,0 +1,306 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + +use crate::models::TrainingKey; +use crate::models::*; +use crate::train::{validate, train_model, TrainedRMI}; +use crate::train::lower_bound_correction::LowerBoundCorrection; +use log::*; + +fn error_between(v1: u64, v2: u64, max_pred: u64) -> u64 { + let pred1 = u64::min(v1, max_pred); + let pred2 = u64::min(v2, max_pred); + return u64::max(pred1, pred2) - u64::min(pred1, pred2); +} + +fn build_models_from(data: &RMITrainingData, + top_model: &Box, + model_type: &str, + start_idx: usize, end_idx: usize, + first_model_idx: usize, + num_models: usize) -> Vec> { + + assert!(end_idx > start_idx, + "start index was {} but end index was {}", + start_idx, end_idx); + assert!(end_idx <= data.len()); + assert!(start_idx <= data.len()); + + let dummy_md = RMITrainingData::::empty(); + let mut leaf_models: Vec> + = Vec::with_capacity(num_models as usize); + let mut second_layer_data = Vec::with_capacity((end_idx - start_idx) / num_models as usize); + let mut last_target = first_model_idx; + + let bounded_it = data.iter() + .skip(start_idx) + .take(end_idx - start_idx); + + for (x, y) in bounded_it { + let model_pred = top_model.predict_to_int(&x.to_model_input()) as usize; + assert!(top_model.needs_bounds_check() || model_pred < first_model_idx + num_models, + "Top model gave an index of {} which is out of bounds of {}. \ + Subset range: {} to {}", + model_pred, start_idx + num_models, start_idx, end_idx); + let target = usize::min(first_model_idx + num_models - 1, model_pred); + assert!(target >= last_target); + + if target > last_target { + // this is the first datapoint for the next leaf model. + // train the previous leaf model. + + // include the first point of the next leaf node to + // support lower bound searches (not required, but reduces error) + let last_item = second_layer_data.last().copied(); + second_layer_data.push((x, y)); + + let container = RMITrainingData::new(Box::new(second_layer_data)); + let leaf_model = train_model(model_type, &container); + leaf_models.push(leaf_model); + + + // leave empty models for any we skipped. + for _skipped_idx in (last_target+1)..target { + leaf_models.push(train_model(model_type, &dummy_md)); + } + assert_eq!(leaf_models.len() + first_model_idx, target); + + second_layer_data = Vec::new(); + + // include the last item of this leaf in the next leaf + // to support lower bound searches. + if let Some(v) = last_item { + second_layer_data.push(v); + } + + } + + second_layer_data.push((x, y)); + last_target = target; + } + + // train the last remaining model + assert!(! second_layer_data.is_empty()); + let container = RMITrainingData::new(Box::new(second_layer_data)); + let leaf_model = train_model(model_type, &container); + leaf_models.push(leaf_model); + assert!(leaf_models.len() <= num_models); + + // add models at the end with nothing mapped into them + for _skipped_idx in (last_target+1)..(first_model_idx + num_models) as usize { + leaf_models.push(train_model(model_type, &dummy_md)); + } + assert_eq!(num_models as usize, leaf_models.len()); + return leaf_models; +} + +pub fn train_two_layer(md_container: &mut RMITrainingData, + layer1_model: &str, layer2_model: &str, + num_leaf_models: u64) -> TrainedRMI { + validate(&[String::from(layer1_model), String::from(layer2_model)]); + + let num_rows = md_container.len(); + + trace!("Training top-level {} model layer", layer1_model); + md_container.set_scale(num_leaf_models as f64 / num_rows as f64); + let top_model = train_model(layer1_model, &md_container); + + // Check monotonicity if in debug mode + #[cfg(debug_assertions)] + { + let mut last_pred = 0; + for (x, _y) in md_container.iter_model_input() { + let prediction = top_model.predict_to_int(&x); + debug_assert!(prediction >= last_pred, + "Top model {} was non-monotonic on input {:?}", + layer1_model, x); + last_pred = prediction; + } + trace!("Top model was monotonic."); + } + + trace!("Training second-level {} model layer (num models = {})", + layer2_model, num_leaf_models); + md_container.set_scale(1.0); + + // find a prediction boundary near the middle + let midpoint_model = num_leaf_models / 2; + let split_idx = md_container.lower_bound_by(|x| { + let model_idx = top_model.predict_to_int(&x.0.to_model_input()); + let model_target = u64::min(num_leaf_models - 1, model_idx); + return model_target.cmp(&midpoint_model); + }); + + // make sure the split point that we got is valid + if split_idx > 0 && split_idx < md_container.len() { + let key_at = top_model.predict_to_int(&md_container.get_key(split_idx) + .to_model_input()); + let key_pr = top_model.predict_to_int(&md_container.get_key(split_idx - 1) + .to_model_input()); + assert!(key_at > key_pr); + } + + let mut leaf_models = if split_idx >= md_container.len() { + build_models_from(&md_container, &top_model, layer2_model, + 0, md_container.len(), 0, + num_leaf_models as usize) + } else { + let split_idx_target = u64::min(num_leaf_models - 1, + top_model.predict_to_int( + &md_container.get_key(split_idx) + .to_model_input())) + as usize; + + let first_half_models = split_idx_target as usize; + let second_half_models = num_leaf_models as usize - split_idx_target as usize; + + let (mut hf1, mut hf2) + = rayon::join(|| build_models_from(&md_container, &top_model, layer2_model, + 0, split_idx, + 0, + first_half_models), + || build_models_from(&md_container, &top_model, layer2_model, + split_idx + 1, md_container.len(), + split_idx_target, + second_half_models)); + + let mut leaf_models = Vec::new(); + leaf_models.append(&mut hf1); + leaf_models.append(&mut hf2); + leaf_models + }; + + trace!("Computing lower bound stats..."); + let lb_corrections = LowerBoundCorrection::new( + |x| top_model.predict_to_int(&x.to_model_input()), num_leaf_models, md_container + ); + + trace!("Fixing empty models..."); + // replace any empty model with a model that returns the correct constant + // (for LB predictions), if the underlying model supports it. + let mut could_not_replace = false; + for idx in 0..(num_leaf_models as usize)-1 { + assert_eq!(lb_corrections.first_key(idx).is_none(), + lb_corrections.last_key(idx).is_none()); + + if lb_corrections.last_key(idx).is_none() { + // model is empty! + let upper_bound = lb_corrections.next_index(idx); + if !leaf_models[idx].set_to_constant_model(upper_bound as u64) { + could_not_replace = true; + } + } + } + + if could_not_replace { + warn!("Some empty models could not be replaced with constants, \ + negative lookup performance may be poor."); + } + + + trace!("Computing last level errors..."); + // evaluate model, compute last level errors + let mut last_layer_max_l1s = vec![(0, 0) ; num_leaf_models as usize]; + for (x, y) in md_container.iter_model_input() { + let leaf_idx = top_model.predict_to_int(&x); + let target = u64::min(num_leaf_models - 1, leaf_idx) as usize; + + let pred = leaf_models[target].predict_to_int(&x); + let err = error_between(pred, y as u64, md_container.len() as u64); + + let cur_val = last_layer_max_l1s[target]; + last_layer_max_l1s[target] = (cur_val.0 + 1, u64::max(err, cur_val.1)); + } + + // for lower bound searches, we need to make sure that: + // (1) a query for the first key in the next leaf minus one + // includes the key in the next leaf. (upper error) + // (2) a query for the last key in the previous leaf plus one + // includes the first key after the previous leaf (lower error) + // (normally, the first key after the previous leaf is the first + // key in this leaf, but not in the case where this leaf has no keys) + let mut large_corrections = 0; + for leaf_idx in 0..num_leaf_models as usize { + let curr_err = last_layer_max_l1s[leaf_idx].1; + let upper_error = { + let (idx_of_next, key_of_next) = lb_corrections.next(leaf_idx); + let pred = leaf_models[leaf_idx].predict_to_int( + &key_of_next.minus_epsilon().to_model_input() + ); + error_between(pred, idx_of_next as u64 + 1, md_container.len() as u64) + }; + + let lower_error = { + let first_key_before = lb_corrections.prev_key(leaf_idx); + + let prev_idx = if leaf_idx == 0 { 0 } else { leaf_idx - 1 }; + let first_idx = lb_corrections.next_index(prev_idx); + + let pred = leaf_models[leaf_idx].predict_to_int( + &first_key_before.plus_epsilon().to_model_input() + ); + error_between(pred, first_idx as u64, md_container.len() as u64) + }; + + + let new_err = *(&[curr_err, upper_error, lower_error]).iter().max().unwrap() + + lb_corrections.longest_run(leaf_idx); + + let num_items_in_leaf = last_layer_max_l1s[leaf_idx].0; + last_layer_max_l1s[leaf_idx] = (num_items_in_leaf, new_err); + + if new_err - curr_err > 512 && num_items_in_leaf > 100 { + large_corrections += 1; + } + } + + if large_corrections > 1 { + trace!("Of {} models, {} needed large lower bound corrections.", + num_leaf_models, large_corrections); + } + + trace!("Evaluating two-layer RMI..."); + let (m_idx, m_err) = last_layer_max_l1s + .iter().enumerate() + .max_by_key(|(_idx, &x)| x.1).unwrap(); + + let model_max_error = m_err.1; + let model_max_error_idx = m_idx; + + let model_avg_error: f64 = last_layer_max_l1s + .iter().map(|(n, err)| n * err).sum::() as f64 / num_rows as f64; + + let model_avg_l2_error: f64 = last_layer_max_l1s + .iter() + .map(|(n, err)| ((n*err) as f64).powf(2.0) / num_rows as f64).sum::(); + + let model_avg_log2_error: f64 = last_layer_max_l1s + .iter().map(|(n, err)| (*n as f64)*((2*err + 2) as f64).log2()).sum::() / num_rows as f64; + + let model_max_log2_error: f64 = (model_max_error as f64).log2(); + + let final_errors = last_layer_max_l1s.into_iter() + .map(|(_n, err)| err).collect(); + + return TrainedRMI { + num_rmi_rows: md_container.len(), + num_data_rows: md_container.len(), + model_avg_error, + model_avg_l2_error, + model_avg_log2_error, + model_max_error, + model_max_error_idx, + model_max_log2_error, + last_layer_max_l1s: final_errors, + rmi: vec![vec![top_model], leaf_models], + models: format!("{},{}", layer1_model, layer2_model), + branching_factor: num_leaf_models, + cache_fix: None, + build_time: 0 + }; + +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/src/load.rs b/ext/TAL/LISA/ext/build-rmi/RMI/src/load.rs new file mode 100644 index 00000000..9d9bbee6 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/src/load.rs @@ -0,0 +1,157 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + +use memmap::MmapOptions; +use rmi_lib::{RMITrainingData, RMITrainingDataIteratorProvider, KeyType}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::fs::File; +use std::convert::TryInto; + +pub enum DataType { + UINT64, + UINT32, + FLOAT64 +} + +struct SliceAdapterU64 { + data: memmap::Mmap, + length: usize +} + +impl RMITrainingDataIteratorProvider for SliceAdapterU64 { + type InpType = u64; + fn cdf_iter(&self) -> Box + '_> { + Box::new((0..self.length).map(move |i| self.get(i).unwrap())) + } + + fn get(&self, idx: usize) -> Option<(Self::InpType, usize)> { + if idx >= self.length { return None; }; + let mi = u64::from_le_bytes((&self.data[8 + idx * 8..8 + (idx + 1) * 8]) + .try_into().unwrap()); + return Some((mi.into(), idx)); + } + + fn key_type(&self) -> KeyType { + KeyType::U64 + } + + fn len(&self) -> usize { self.length } +} + + +struct SliceAdapterU32 { + data: memmap::Mmap, + length: usize +} + +impl RMITrainingDataIteratorProvider for SliceAdapterU32 { + type InpType = u32; + fn cdf_iter(&self) -> Box + '_> { + Box::new((0..self.length).map(move |i| self.get(i).unwrap())) + } + + fn get(&self, idx: usize) -> Option<(Self::InpType, usize)> { + if idx >= self.length { return None; }; + let mi = (&self.data[8 + idx * 4..8 + (idx + 1) * 4]) + .read_u32::().unwrap().into(); + return Some((mi, idx)); + } + + fn key_type(&self) -> KeyType { + KeyType::U32 + } + + fn len(&self) -> usize { self.length } +} + +struct SliceAdapterF64 { + data: memmap::Mmap, + length: usize +} + +impl RMITrainingDataIteratorProvider for SliceAdapterF64 { + type InpType = f64; + fn cdf_iter(&self) -> Box + '_> { + Box::new((0..self.length).map(move |i| self.get(i).unwrap())) + } + + fn get(&self, idx: usize) -> Option<(Self::InpType, usize)> { + if idx >= self.length { return None; }; + let mi = (&self.data[8 + idx * 8..8 + (idx + 1) * 8]) + .read_f64::().unwrap().into(); + return Some((mi, idx)); + } + + fn key_type(&self) -> KeyType { + KeyType::F64 + } + + fn len(&self) -> usize { self.length } +} + +pub enum RMIMMap { + UINT64(RMITrainingData), + UINT32(RMITrainingData), + FLOAT64(RMITrainingData) +} + +macro_rules! dynamic { + ($funcname: expr, $data: expr $(, $p: expr )*) => { + match $data { + load::RMIMMap::UINT64(mut x) => $funcname(&mut x, $($p),*), + load::RMIMMap::UINT32(mut x) => $funcname(&mut x, $($p),*), + load::RMIMMap::FLOAT64(mut x) => $funcname(&mut x, $($p),*), + } + } +} + + +impl RMIMMap { + pub fn soft_copy(&self) -> RMIMMap { + match self { + RMIMMap::UINT64(x) => RMIMMap::UINT64(x.soft_copy()), + RMIMMap::UINT32(x) => RMIMMap::UINT32(x.soft_copy()), + RMIMMap::FLOAT64(x) => RMIMMap::FLOAT64(x.soft_copy()), + } + } + + pub fn into_u64(self) -> Option> { + match self { + RMIMMap::UINT64(x) => Some(x), + _ => None + } + } +} + + +pub fn load_data(filepath: &str, + dt: DataType) -> (usize, RMIMMap) { + let fd = File::open(filepath).unwrap_or_else(|_| { + panic!("Unable to open data file at {}", filepath) + }); + + let mmap = unsafe { MmapOptions::new().map(&fd).unwrap() }; + let num_items = (&mmap[0..8]).read_u64::().unwrap() as usize; + + let rtd = match dt { + DataType::UINT64 => + RMIMMap::UINT64(RMITrainingData::new(Box::new( + SliceAdapterU64 { data: mmap, length: num_items } + ))), + DataType::UINT32 => + RMIMMap::UINT32(RMITrainingData::new(Box::new( + SliceAdapterU32 { data: mmap, length: num_items } + ))), + DataType::FLOAT64 => + RMIMMap::FLOAT64(RMITrainingData::new(Box::new( + SliceAdapterF64 { data: mmap, length: num_items } + ))) + }; + + return (num_items, rtd); +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/src/main.rs b/ext/TAL/LISA/ext/build-rmi/RMI/src/main.rs new file mode 100644 index 00000000..b6e32ff4 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/src/main.rs @@ -0,0 +1,340 @@ +// < begin copyright > +// Copyright Ryan Marcus 2020 +// +// See root directory of this project for license terms. +// +// < end copyright > + + + +#![allow(clippy::needless_return)] + +#[macro_use] +mod load; + +use load::{load_data, DataType}; +use rmi_lib::{train, train_bounded}; +use rmi_lib::KeyType; +use rmi_lib::optimizer; + +use json::*; +use log::*; +use std::f64; +use std::fs::File; +use std::io::BufWriter; +use std::fs; +use std::path::Path; +use rayon::prelude::*; + +use indicatif::{ProgressBar, ProgressStyle}; +use clap::{App, Arg}; + + +fn main() { + env_logger::init(); + + let matches = App::new("RMI Learner") + .version("0.1") + .author("Ryan Marcus ") + .about("Learns recursive model indexes") + .arg(Arg::with_name("input") + .help("Path to input file containing data") + .index(1).required(true)) + .arg(Arg::with_name("namespace") + .help("Namespace to use in generated code") + .index(2).required(false)) + .arg(Arg::with_name("models") + .help("Comma-separated list of model layers, e.g. linear,linear") + .index(3).required(false)) + .arg(Arg::with_name("branching factor") + .help("Branching factor between each model level") + .index(4).required(false)) + .arg(Arg::with_name("no-code") + .long("no-code") + .help("Skip code generation")) + .arg(Arg::with_name("dump-ll-model-data") + .long("dump-ll-model-data") + .value_name("model_index") + .help("dump the data used to train the last-level model at index")) + .arg(Arg::with_name("dump-ll-errors") + .long("dump-ll-errors") + .help("dump the errors of each last-level model to ll_errors.json")) + .arg(Arg::with_name("stats-file") + .long("stats-file") + .short("s") + .value_name("file") + .help("dump statistics about the learned model into the specified file")) + .arg(Arg::with_name("param-grid") + .long("param-grid") + .value_name("file") + .help("train the RMIs specified in the JSON file and report their errors")) + .arg(Arg::with_name("data-path") + .long("data-path") + .short("d") + .value_name("dir") + .help("exports parameters to files in this directory (default: rmi_data)")) + .arg(Arg::with_name("no-errors") + .long("no-errors") + .help("do not save last-level errors, and modify the RMI function signature")) + .arg(Arg::with_name("threads") + .long("threads") + .short("t") + .value_name("count") + .help("number of threads to use for optimization, default = 4")) + .arg(Arg::with_name("bounded") + .long("bounded") + .value_name("line_size") + .help("construct an error-bounded RMI using the cachefix method for the given line size")) + .arg(Arg::with_name("max-size") + .long("max-size") + .value_name("BYTES") + .help("uses the optimizer fo find an RMI with a size less than specified")) + .arg(Arg::with_name("disable-parallel-training") + .long("disable-parallel-training") + .help("disables training multiple RMIs in parallel")) + .arg(Arg::with_name("zero-build-time") + .long("zero-build-time") + .help("zero out the model build time field")) + .arg(Arg::with_name("optimize") + .long("optimize") + .value_name("file") + .help("Search for Pareto efficient RMI configurations. Specify the name of the output file.")) + .get_matches(); + + // set the max number of threads to 4 by default, otherwise Rayon goes + // crazy on larger machines and allocates too many workers for folds / reduces + let num_threads = matches.value_of("threads") + .map(|x| x.parse::().unwrap()) + .unwrap_or(4); + rayon::ThreadPoolBuilder::new().num_threads(num_threads).build_global().unwrap(); + + let fp = matches.value_of("input").unwrap(); + + + let data_dir = matches.value_of("data-path").unwrap_or("rmi_data"); + + if matches.value_of("namespace").is_some() && matches.value_of("param-grid").is_some() { + panic!("Can only specify one of namespace or param-grid"); + } + + info!("Reading {}...", fp); + + let mut key_type = KeyType::U64; + let (num_rows, data) = if fp.contains("uint64") { + load_data(&fp, DataType::UINT64) + } else if fp.contains("uint32") { + load_data(&fp, DataType::UINT32) + } else if fp.contains("f64") { + key_type = KeyType::F64; + load_data(&fp, DataType::FLOAT64) + } else { + panic!("Data file must contain uint64, uint32, or f64."); + }; + + if matches.is_present("optimize") { + let results = dynamic!(optimizer::find_pareto_efficient_configs, + data, 10); + + optimizer::RMIStatistics::display_table(&results); + + let nmspc_prefix = if matches.value_of("namespace").is_some() { + matches.value_of("namespace").unwrap() + } else { + let path = Path::new(fp); + path.file_name().map(|s| s.to_str()).unwrap_or(Some("rmi")).unwrap() + }; + + let grid_specs: Vec = results.into_iter() + .enumerate() + .map(|(idx, v)| { + let nmspc = format!("{}_{}", nmspc_prefix, idx); + v.to_grid_spec(&nmspc) + }).collect(); + + let grid_specs_json = object!("configs" => grid_specs); + let fp = matches.value_of("optimize").unwrap(); + let f = File::create(fp) + .expect("Could not write optimization results file"); + let mut bw = BufWriter::new(f); + grid_specs_json.write(&mut bw).unwrap(); + return; + } + + // if we aren't optimizing, we should make sure the RMI data directory exists. + if !Path::new(data_dir).exists() { + info!("The RMI data directory specified {} does not exist. Creating it.", + data_dir); + std::fs::create_dir_all(data_dir) + .expect("The RMI data directory did not exist, and it could not be created."); + } + + if let Some(param_grid) = matches.value_of("param-grid").map(|x| x.to_string()) { + let pg = { + let raw_json = fs::read_to_string(param_grid.clone()).unwrap(); + let mut as_json = json::parse(raw_json.as_str()).unwrap(); + as_json["configs"].take() + }; + + let mut to_test = Vec::new(); + if let JsonValue::Array(v) = pg { + for el in v { + let layers = String::from(el["layers"].as_str().unwrap()); + let branching = el["branching factor"].as_u64().unwrap(); + let namespace = match el["namespace"].as_str() { + Some(s) => Some(String::from(s)), + None => None + }; + + to_test.push((layers, branching, namespace)); + } + + trace!("# RMIs to train: {}", to_test.len()); + + let pbar = ProgressBar::new(to_test.len() as u64); + pbar.set_style(ProgressStyle::default_bar() + .template("{pos} / {len} ({msg}) {wide_bar} {eta}")); + + let train_func = + |(models, branch_factor, namespace): &(String, u64, Option)| { + trace!("Training RMI {} with branching factor {}", + models, *branch_factor); + + let loc_data = data.soft_copy(); + let mut trained_model = dynamic!(train, loc_data, models, *branch_factor); + + let size_bs = rmi_lib::rmi_size(&trained_model); + + let result_obj = object! { + "layers" => models.clone(), + "branching factor" => *branch_factor, + "average error" => trained_model.model_avg_error as f64, + "average error %" => trained_model.model_max_error as f64 + / num_rows as f64 * 100.0, + "average l2 error" => trained_model.model_avg_l2_error as f64, + "average log2 error" => trained_model.model_avg_log2_error, + "max error" => trained_model.model_max_error, + "max error %" => trained_model.model_max_error as f64 + / num_rows as f64 * 100.0, + "max log2 error" => trained_model.model_max_log2_error, + "size binary search" => size_bs, + "namespace" => namespace.clone() + }; + + if matches.is_present("zero-build-time") { + trained_model.build_time = 0; + } + + if let Some(nmspc) = namespace { + rmi_lib::output_rmi( + &nmspc, + trained_model, + data_dir, + key_type, + true).unwrap(); + + } + + pbar.inc(1); + return result_obj; + }; + + let results: Vec = + if matches.is_present("disable-parallel-training") { + trace!("Training models sequentially"); + to_test.iter().map(train_func).collect() + } else { + trace!("Training models in parallel"); + to_test.par_iter().map(train_func).collect() + }; + + //let results: Vec = to_test + //.par_iter().map( + pbar.finish(); + + let f = File::create(format!("{}_results", param_grid)).expect("Could not write results file"); + let mut bw = BufWriter::new(f); + let json_results = object! { "results" => results }; + json_results.write(&mut bw).unwrap(); + + } else { + panic!("Configs must have an array as its value"); + } + + } else if matches.value_of("namespace").is_some() { + let namespace = matches.value_of("namespace").unwrap().to_string(); + let mut trained_model = match matches.value_of("max-size") { + None => { + // assume they gave a model spec + let models = matches.value_of("models").unwrap(); + let branch_factor = matches + .value_of("branching factor") + .unwrap() + .parse::() + .unwrap(); + + let trained_model = match matches.value_of("bounded") { + None => dynamic!(train, data, models, branch_factor), + Some(s) => { + let line_size = s.parse::() + .expect("Line size must be a positive integer."); + let d_u64 = data.into_u64() + .expect("Can only construct a bounded RMI on u64 data."); + train_bounded(&d_u64, models, branch_factor, line_size) + } + }; + trained_model + } + Some(max_size_str) => { + let max_size = max_size_str.parse::().unwrap(); + info!("Constructing RMI with size less than {}", max_size); + + let trained_model = dynamic!(rmi_lib::train_for_size, data, max_size); + trained_model + } + }; + + let no_errors = matches.is_present("no-errors"); + info!("Model build time: {} ms", trained_model.build_time / 1_000_000); + + info!( + "Average model error: {} ({}%)", + trained_model.model_avg_error as f64, + trained_model.model_avg_error / num_rows as f64 * 100.0 + ); + info!( + "Average model L2 error: {}", + trained_model.model_avg_l2_error + ); + info!( + "Average model log2 error: {}", + trained_model.model_avg_log2_error + ); + info!( + "Max model log2 error: {}", + trained_model.model_max_log2_error + ); + info!( + "Max model error on model {}: {} ({}%)", + trained_model.model_max_error_idx, + trained_model.model_max_error, + trained_model.model_max_error as f64 / num_rows as f64 * 100.0 + ); + + if !matches.is_present("no-code") { + if matches.is_present("zero-build-time") { + trained_model.build_time = 0; + } + + rmi_lib::output_rmi( + &namespace, + trained_model, + data_dir, + key_type, + !no_errors).unwrap(); + } else { + trace!("Skipping code generation due to CLI flag"); + } + } else { + trace!("Must specify either a name space or a parameter grid."); + } +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/.gitignore b/ext/TAL/LISA/ext/build-rmi/RMI/tests/.gitignore new file mode 100644 index 00000000..e32c90a7 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/.gitignore @@ -0,0 +1,4 @@ +results +wiki_ts_200M_uint64 +osm_cellids_200M_uint64 +rmi diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/Makefile b/ext/TAL/LISA/ext/build-rmi/RMI/tests/Makefile new file mode 100644 index 00000000..76191ef0 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/Makefile @@ -0,0 +1,35 @@ + +ALL_TESTS=$(shell ./find_tests.py) +ALL_RESULTS=$(patsubst %, results/%_result, $(ALL_TESTS)) + +.PHONY: all +all: results $(ALL_RESULTS) + @echo "============== TEST RESULTS ===============" + python3 report.py + +results: + mkdir results + +wiki_ts_200M_uint64: + curl -L https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/JGVF9A/SVN8PI | zstd -d > $@ + +osm_cellids_200M_uint64: + curl -L https://dataverse.harvard.edu/api/access/datafile/:persistentId?persistentId=doi:10.7910/DVN/JGVF9A/8FX9BV | zstd -d > $@ + +rmi: $(shell find ../src/) $(shell find ../rmi_lib/src/) + cd .. && cargo build --release + cp ../target/release/rmi . + + +results/%_result: results rmi wiki_ts_200M_uint64 osm_cellids_200M_uint64 FORCE + @echo "Running test $*" + @cd $* && make -s + @cp $*/result results/$*_result + @echo "Test $* finished." + + +.PHONY: clean FORCE +clean: + rm -rf results rmi + for test in $(ALL_TESTS) ; do make -C $$test clean ; done +FORCE: diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/.gitignore b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/.gitignore new file mode 100644 index 00000000..004a95ac --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/.gitignore @@ -0,0 +1,4 @@ +rmi* +test +stdout +result diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/Makefile b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/Makefile new file mode 100644 index 00000000..48372976 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/Makefile @@ -0,0 +1,16 @@ + +result: test + $(shell ./test > stdout) + echo $(.SHELLSTATUS) > result + cat stdout >> result + +rmi.cpp: ../rmi + ../rmi ../osm_cellids_200M_uint64 rmi cubic,linear 786432 --bounded 8 + +test: main.cpp rmi.cpp + # -lstdc++fs is required for ancient G++s + g++ -std=c++17 -Wall -O3 -ffast-math -march=native main.cpp rmi.cpp -o test -lstdc++fs + +.PHONY: clean +clean: + rm -rf test result rmi* diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/main.cpp b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/main.cpp new file mode 100644 index 00000000..d8d1773d --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_osm/main.cpp @@ -0,0 +1,47 @@ +#include +#include +#include +#include "rmi.h" + +int main() { + // load the data + std::vector data; + std::ifstream in("../osm_cellids_200M_uint64", + std::ios::binary); + + // Read size. + uint64_t size; + in.read(reinterpret_cast(&size), sizeof(uint64_t)); + data.resize(size); + // Read values. + in.read(reinterpret_cast(data.data()), size*sizeof(uint64_t)); + in.close(); + + std::cout << "Data loaded." << std::endl; + + std::cout << "RMI status: " << rmi::load("rmi_data") << std::endl; + + size_t err; + + for (uint64_t key_index = 0; key_index < size; key_index++) { + uint64_t lookup = data[key_index]; + uint64_t true_index = (uint64_t) + std::distance(data.begin(), std::lower_bound(data.begin(), + data.end(), + lookup)); + uint64_t rmi_guess = rmi::lookup(lookup, &err); + + uint64_t diff = (rmi_guess > true_index ? rmi_guess - true_index : true_index - rmi_guess); + + if (diff > 8) { + std::cout << "Search key: " << lookup + << " Key at " << true_index << ": " << data[true_index] + << " RMI guess: " << rmi_guess << " +/- " << err + << " diff: " << diff << std::endl; + exit(-1); + } + } + + rmi::cleanup(); + exit(0); +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/.gitignore b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/.gitignore new file mode 100644 index 00000000..004a95ac --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/.gitignore @@ -0,0 +1,4 @@ +rmi* +test +stdout +result diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/Makefile b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/Makefile new file mode 100644 index 00000000..ce344148 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/Makefile @@ -0,0 +1,16 @@ + +result: test + $(shell ./test > stdout) + echo $(.SHELLSTATUS) > result + cat stdout >> result + +rmi.cpp: ../rmi + ../rmi ../wiki_ts_200M_uint64 rmi linear_spline,linear 4194304 --bounded 8 + +test: main.cpp rmi.cpp + # -lstdc++fs is required for ancient G++s + g++ -std=c++17 -Wall -O3 -ffast-math -march=native main.cpp rmi.cpp -o test -lstdc++fs + +.PHONY: clean +clean: + rm -rf test result rmi* diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/main.cpp b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/main.cpp new file mode 100644 index 00000000..35afc50b --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/cache_fix_wiki/main.cpp @@ -0,0 +1,47 @@ +#include +#include +#include +#include "rmi.h" + +int main() { + // load the data + std::vector data; + std::ifstream in("../wiki_ts_200M_uint64", + std::ios::binary); + + // Read size. + uint64_t size; + in.read(reinterpret_cast(&size), sizeof(uint64_t)); + data.resize(size); + // Read values. + in.read(reinterpret_cast(data.data()), size*sizeof(uint64_t)); + in.close(); + + std::cout << "Data loaded." << std::endl; + + std::cout << "RMI status: " << rmi::load("rmi_data") << std::endl; + + size_t err; + + for (uint64_t key_index = 0; key_index < size; key_index++) { + uint64_t lookup = data[key_index]; + uint64_t true_index = (uint64_t) + std::distance(data.begin(), std::lower_bound(data.begin(), + data.end(), + lookup)); + uint64_t rmi_guess = rmi::lookup(lookup, &err); + + uint64_t diff = (rmi_guess > true_index ? rmi_guess - true_index : true_index - rmi_guess); + + if (diff > 8) { + std::cout << "Search key: " << lookup + << " Key at " << true_index << ": " << data[true_index] + << " RMI guess: " << rmi_guess << " +/- " << err + << " diff: " << diff << std::endl; + exit(-1); + } + } + + rmi::cleanup(); + exit(0); +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/find_tests.py b/ext/TAL/LISA/ext/build-rmi/RMI/tests/find_tests.py new file mode 100755 index 00000000..480867a7 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/find_tests.py @@ -0,0 +1,6 @@ +#! /usr/bin/env python3 + +import glob + +for fn in glob.glob("*/Makefile"): + print(fn.split("/")[0]) diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/.gitignore b/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/.gitignore new file mode 100644 index 00000000..004a95ac --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/.gitignore @@ -0,0 +1,4 @@ +rmi* +test +stdout +result diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/Makefile b/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/Makefile new file mode 100644 index 00000000..774e043e --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/Makefile @@ -0,0 +1,16 @@ + +result: test + $(shell ./test > stdout) + echo $(.SHELLSTATUS) > result + cat stdout >> result + +rmi.cpp: ../rmi + RMI_OPTIMIZER_PROFILE=fast ../rmi ../wiki_ts_200M_uint64 rmi cubic,linear --max-size 50000000 + +test: main.cpp rmi.cpp + # -lstdc++fs is required for ancient G++s + g++ -std=c++17 -Wall -O3 -ffast-math -march=native main.cpp rmi.cpp -o test -lstdc++fs + +.PHONY: clean +clean: + rm -rf test result rmi* diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/main.cpp b/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/main.cpp new file mode 100644 index 00000000..959da88c --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/max_size_wiki/main.cpp @@ -0,0 +1,51 @@ +#include +#include +#include +#include "rmi.h" + +int main() { + // load the data + std::vector data; + std::ifstream in("../wiki_ts_200M_uint64", + std::ios::binary); + + // Read size. + uint64_t size; + in.read(reinterpret_cast(&size), sizeof(uint64_t)); + data.resize(size); + // Read values. + in.read(reinterpret_cast(data.data()), size*sizeof(uint64_t)); + in.close(); + + std::cout << "Data loaded." << std::endl; + + std::cout << "RMI status: " << rmi::load("rmi_data") << std::endl; + + if (rmi::RMI_SIZE > 50000000) { + std::cout << "RMI was larger than 50MB" << std::endl; + exit(-1); + } + + size_t err; + + for (uint64_t key_index = 0; key_index < size; key_index++) { + uint64_t lookup = data[key_index]; + uint64_t true_index = (uint64_t) + std::distance(data.begin(), std::lower_bound(data.begin(), + data.end(), + lookup)); + uint64_t rmi_guess = rmi::lookup(lookup, &err); + + uint64_t diff = (rmi_guess > true_index ? rmi_guess - true_index : true_index - rmi_guess); + if (diff > err) { + std::cout << "Search key: " << lookup + << " Key at " << true_index << ": " << data[true_index] + << " RMI guess: " << rmi_guess << " +/- " << err + << " diff: " << diff << std::endl; + exit(-1); + } + } + + rmi::cleanup(); + exit(0); +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/report.py b/ext/TAL/LISA/ext/build-rmi/RMI/tests/report.py new file mode 100755 index 00000000..5b352912 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/report.py @@ -0,0 +1,34 @@ +#! /usr/bin/env python3 + +import glob +import os + +def all_tests(): + subfolders = [f.path for f in os.scandir(".") if f.is_dir()] + tests = [x[2:] for x in subfolders if x != "./results"] + return tests + + +any_failed = False +tests_remaining = set(all_tests()) +for fn in sorted(glob.glob("results/*_result")): + test_name = fn[8:-7] + with open(fn) as f: + first_line = int(next(f).strip()) + if first_line == 0: + print("PASS", test_name) + else: + print("FAIL", test_name) + any_failed = True + + tests_remaining.remove(fn[8:-7]) + +if tests_remaining: + any_failed = True + + for missing in tests_remaining: + print("MISS", missing) + +if any_failed: + exit(1) +exit(0) diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/.gitignore b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/.gitignore new file mode 100644 index 00000000..004a95ac --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/.gitignore @@ -0,0 +1,4 @@ +rmi* +test +stdout +result diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/Makefile b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/Makefile new file mode 100644 index 00000000..f016fc1a --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/Makefile @@ -0,0 +1,16 @@ + +result: test + $(shell ./test > stdout) + echo $(.SHELLSTATUS) > result + cat stdout >> result + +rmi.cpp: ../rmi + ../rmi ../osm_cellids_200M_uint64 rmi robust_linear,linear 262144 + +test: main.cpp rmi.cpp + # -lstdc++fs is required for ancient G++s + g++ -std=c++17 -Wall -O3 -ffast-math -march=native main.cpp rmi.cpp -o test -lstdc++fs + +.PHONY: clean +clean: + rm -rf test result rmi* diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/main.cpp b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/main.cpp new file mode 100644 index 00000000..06768c38 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_osm/main.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include "rmi.h" + +int main() { + // load the data + std::vector data; + std::ifstream in("../osm_cellids_200M_uint64", + std::ios::binary); + + // Read size. + uint64_t size; + in.read(reinterpret_cast(&size), sizeof(uint64_t)); + data.resize(size); + // Read values. + in.read(reinterpret_cast(data.data()), size*sizeof(uint64_t)); + in.close(); + + std::cout << "Data loaded." << std::endl; + + std::cout << "RMI status: " << rmi::load("rmi_data") << std::endl; + + size_t err; + + for (uint64_t key_index = 0; key_index < size; key_index++) { + uint64_t lookup = data[key_index]; + uint64_t true_index = (uint64_t) + std::distance(data.begin(), std::lower_bound(data.begin(), + data.end(), + lookup)); + uint64_t rmi_guess = rmi::lookup(lookup, &err); + + uint64_t diff = (rmi_guess > true_index ? rmi_guess - true_index : true_index - rmi_guess); + if (diff > err) { + std::cout << "Search key: " << lookup + << " Key at " << true_index << ": " << data[true_index] + << " RMI guess: " << rmi_guess << " +/- " << err + << " diff: " << diff << std::endl; + exit(-1); + } + } + + rmi::cleanup(); + exit(0); +} diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/.gitignore b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/.gitignore new file mode 100644 index 00000000..004a95ac --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/.gitignore @@ -0,0 +1,4 @@ +rmi* +test +stdout +result diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/Makefile b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/Makefile new file mode 100644 index 00000000..f163843b --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/Makefile @@ -0,0 +1,16 @@ + +result: test + $(shell ./test > stdout) + echo $(.SHELLSTATUS) > result + cat stdout >> result + +rmi.cpp: ../rmi + ../rmi ../wiki_ts_200M_uint64 rmi cubic,linear 262144 + +test: main.cpp rmi.cpp + # -lstdc++fs is required for ancient G++s + g++ -std=c++17 -Wall -O3 -ffast-math -march=native main.cpp rmi.cpp -o test -lstdc++fs + +.PHONY: clean +clean: + rm -rf test result rmi* diff --git a/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/main.cpp b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/main.cpp new file mode 100644 index 00000000..264f5499 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/RMI/tests/simple_model_wiki/main.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include "rmi.h" + +int main() { + // load the data + std::vector data; + std::ifstream in("../wiki_ts_200M_uint64", + std::ios::binary); + + // Read size. + uint64_t size; + in.read(reinterpret_cast(&size), sizeof(uint64_t)); + data.resize(size); + // Read values. + in.read(reinterpret_cast(data.data()), size*sizeof(uint64_t)); + in.close(); + + std::cout << "Data loaded." << std::endl; + + std::cout << "RMI status: " << rmi::load("rmi_data") << std::endl; + + size_t err; + + for (uint64_t key_index = 0; key_index < size; key_index++) { + uint64_t lookup = data[key_index]; + uint64_t true_index = (uint64_t) + std::distance(data.begin(), std::lower_bound(data.begin(), + data.end(), + lookup)); + uint64_t rmi_guess = rmi::lookup(lookup, &err); + + uint64_t diff = (rmi_guess > true_index ? rmi_guess - true_index : true_index - rmi_guess); + if (diff > err) { + std::cout << "Search key: " << lookup + << " Key at " << true_index << ": " << data[true_index] + << " RMI guess: " << rmi_guess << " +/- " << err + << " diff: " << diff << std::endl; + exit(-1); + } + } + + rmi::cleanup(); + exit(0); +} diff --git a/ext/TAL/LISA/ext/build-rmi/modify_generated_code.sh b/ext/TAL/LISA/ext/build-rmi/modify_generated_code.sh new file mode 100755 index 00000000..5549cba0 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/modify_generated_code.sh @@ -0,0 +1,8 @@ +set -v +#Prefix +P=$1 +m=$2 +sed -i 's/#include //g' $P.cpp +sed -i 's/std::filesystem::path(dataPath) \/ \"/\"RMI\/rmi_data\//g' $P.cpp +sed -i 's/void cleanup/bool save(char const *filename);\nvoid cleanup/g' $P.h +sed -i "s/void cleanup/bool save(char const *filename)\n{\n int64_t L1_SIZE = $m;\n std::ofstream outstream(filename, std::ofstream::binary);\n outstream.seekp(0);\n outstream.write((char *)\&(L0_PARAMETER0), sizeof(double));\n outstream.write((char *)\&(L0_PARAMETER1), sizeof(double));\n outstream.write((char *)\&(L1_SIZE), sizeof(int64_t));\n outstream.write(L1_PARAMETERS, L1_SIZE * 24);\n outstream.close();\n}\n\nvoid cleanup/g" $P.cpp diff --git a/ext/TAL/LISA/ext/build-rmi/rmi-main.cpp b/ext/TAL/LISA/ext/build-rmi/rmi-main.cpp new file mode 100644 index 00000000..70d7b554 --- /dev/null +++ b/ext/TAL/LISA/ext/build-rmi/rmi-main.cpp @@ -0,0 +1,71 @@ +#include "sorted_doubles_rmi.h" +#include +#include +#include +//#include +#include +#include +#include +#include +#include +#include +#include +#include + + +int main(int argc, char *argv[]) +{ + char *filename = argv[1]; + std::ifstream infile(filename, std::ios::in | std::ios::binary); + if (!infile.good()) + { + printf("%s file not found\n", filename); + exit(0); + } + printf("query file good\n"); + int64_t n = 0; + infile.read((char*)&n, sizeof(int64_t)); + printf("n = %ld\n", n); + double *tmp_data = (double *)malloc(n * sizeof(double)); + if (tmp_data == NULL) + { + printf("tmp_data is NULL\n"); + exit(0); + } + infile.read((char*)tmp_data, n * sizeof(double)); + printf("read query file\n"); + + int64_t i; + sorted_doubles_rmi::load("rmi_data"); + printf("loaded rmi data\n"); + sorted_doubles_rmi::save(argv[2]); + double total_log_err = 0; + double total_log_gap = 0; + int64_t max_err = 0; + int64_t max_gap = 0; + int64_t err_hist[20]; + memset(err_hist, 0, 20*8); + for(i = 0; i < n; i++) + { + size_t err = 0; + size_t pos = sorted_doubles_rmi::lookup(tmp_data[i], &err); + //printf("%d] key = %E, %lu, %lu, %lu\n", i, tmp_data[i], pos, err, labs(pos-i)); + size_t gap = labs(pos-i); + if(gap > err) + printf("%d] key = %E, %lu, %lu, %lu\n", i, tmp_data[i], pos, err, labs(pos-i)); + + int64_t log2_err = log2(1.0 * err); + err_hist[log2_err]++; + if(max_err < err) max_err = err; + if(max_gap < gap) max_gap = gap; + total_log_err += log2(1.0 + err); + total_log_gap += log2(1.0 + gap); + } + printf("avg_log2_err = %lf, avg_log2_gap = %lf\n", total_log_err / n, total_log_gap / n); + printf("max_err = %ld, max_gap = %ld\n", max_err, max_gap); + for(i = 0; i < 20; i++) + { + printf("%ld] log2_err freq = %ld\n", i, err_hist[i]); + } + sorted_doubles_rmi::cleanup(); +} diff --git a/ext/TAL/LISA/scripts/build-lisa-smem-index.sh b/ext/TAL/LISA/scripts/build-lisa-smem-index.sh new file mode 100755 index 00000000..f35038ac --- /dev/null +++ b/ext/TAL/LISA/scripts/build-lisa-smem-index.sh @@ -0,0 +1,12 @@ +make clean +make lisa + +full_path=`readlink -f $1` +K=$2 +num_leaf=$3 +#cd .. + +#build rmi index -rev-comp +./build-index-with-rev-complement-lisa.o ${full_path} $K ${num_leaf} +#build interval tree index +./build-index-with-rev-complement-lisa.o ${full_path} $K ${num_leaf} diff --git a/ext/TAL/LISA/scripts/build-rmi.linear_spline.linear.sh b/ext/TAL/LISA/scripts/build-rmi.linear_spline.linear.sh new file mode 100755 index 00000000..84270460 --- /dev/null +++ b/ext/TAL/LISA/scripts/build-rmi.linear_spline.linear.sh @@ -0,0 +1,24 @@ +#!/usr/bin/sh +set -v +#sorted array +A=$1 +# RMI file +B=$2 +# number of leaf nodes +m=$3 +echo "$A" +echo "$B" +echo "$m" +cd ext/build-rmi/RMI +rm sorted_doubles_rmi.cpp sorted_doubles_rmi.h sorted_doubles_rmi_data.h +time cargo run --release -- $A sorted_doubles_rmi linear_spline,linear $m +cd .. +rm sorted_doubles_rmi.cpp sorted_doubles_rmi.h sorted_doubles_rmi_data.h +cp RMI/sorted_doubles_rmi.cpp . +cp RMI/sorted_doubles_rmi.h . +cp RMI/sorted_doubles_rmi_data.h . +./modify_generated_code.sh sorted_doubles_rmi $m +icpc rmi-main.cpp sorted_doubles_rmi.cpp -o rmi +time ./rmi $A $B > out +grep avg_log2_err out +echo "Built RMI" diff --git a/ext/TAL/LISA/src/chunkEncode.h b/ext/TAL/LISA/src/chunkEncode.h new file mode 100644 index 00000000..7b98a25b --- /dev/null +++ b/ext/TAL/LISA/src/chunkEncode.h @@ -0,0 +1,249 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +void prepareChunkBatchVectorized_v1(Info* qPool, int qPoolSize, uint64_t* str_enc, int64_t* intv_all, int K){ + + int64_t p[21][8]; + + int64_t i = 0; + for(i = 0; (i+8) < qPoolSize; i = i + 8){ + for(int64_t j = 0 ; j < 8; j++){ + Info &q = qPool[i+j]; + int start = q.l - K; + //for(int k = 0; k < 21; k++){ + p[0][j] = q.p[0 + start]; + p[1][j] = q.p[1 + start]; + p[2][j] = q.p[2 + start]; + p[3][j] = q.p[3 + start]; + p[4][j] = q.p[4 + start]; + + p[5][j] = q.p[5 + start]; + p[6][j] = q.p[6 + start]; + p[7][j] = q.p[7 + start]; + p[8][j] = q.p[8 + start]; + p[9][j] = q.p[9 + start]; + + + p[10][j] = q.p[10 + start]; + p[11][j] = q.p[11 + start]; + p[12][j] = q.p[12 + start]; + p[13][j] = q.p[13 + start]; + p[14][j] = q.p[14 + start]; + + + p[15][j] = q.p[15 + start]; + p[16][j] = q.p[16 + start]; + p[17][j] = q.p[17 + start]; + p[18][j] = q.p[18 + start]; + p[19][j] = q.p[19 + start]; + p[20][j] = q.p[20 + start]; + // } + + + intv_all[2 * (i+j)] = q.intv.first; + intv_all[2 * (i+j) + 1] = q.intv.second; + const char *ptr = qPool[i + j + 35].p; int offset = qPool[i + j + 35].l - K; + my_prefetch((const char*)(ptr + offset) , _MM_HINT_T0); + } + + __m512i vAns = _mm512_setzero_si512(); + + int64_t* ptr;// = &p[j][0]; + __m512i vP;// = _mm512_loadu_si512(ptr); + + for(int64_t j = 0 ; j < 21; j++){ + ptr = &p[j][0]; + vAns = _mm512_slli_epi64(vAns, 2); + vP = _mm512_loadu_si512(ptr); + vAns = _mm512_or_epi64(vAns, vP); + } + //Store results of 8 chunks + _mm512_store_epi64(str_enc + i, vAns); + } + + //printf("%lld, %lld\n", i, qPoolSize); + + for( int64_t j = i; j < qPoolSize; j++){ + Info &q = qPool[j]; + uint64_t nxt_ext = 0; + + for(int itr = q.l-K; itr != q.l; itr++) { + nxt_ext = (nxt_ext<<2) | (q.p[itr]); + } + str_enc[j] = nxt_ext; + intv_all[2 * j] = q.intv.first; + intv_all[2 * j + 1] = q.intv.second; + } + + +} +void prepareChunkBatchVectorized(Info* qPool, int qPoolSize, uint64_t* str_enc, int64_t* intv_all, int K){ + + uint64_t offset[24] = {40,38,36,34,32,30,28,26,24,22,20,18,16,14,12,10,8,6,4,2,0,0,0,0}; + uint64_t v_intv[24] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + __m512i vOff1,vOff2,vOff3; + vOff1 = _mm512_loadu_si512(offset); + vOff2 = _mm512_loadu_si512(offset+8); + vOff3 = _mm512_loadu_si512(offset+16); + + for(int64_t j = 0; j < qPoolSize; j++) + { + Info &q = qPool[j]; + intv_all[2 * j] = q.intv.first; + intv_all[2 * j + 1] = q.intv.second; + + int i = q.l - K; +// for( + int itr = 0;// itr < 21; itr++) + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + v_intv[itr++] = q.p[i++]; + + __m512i vA1, vA2, vA3; + vA1 = _mm512_loadu_si512(v_intv); + vA2 = _mm512_loadu_si512(v_intv+8); + vA3 = _mm512_loadu_si512(v_intv+16); + + + __m512i vShift1 = _mm512_sllv_epi64(vA1, vOff1); + __m512i vShift2 = _mm512_sllv_epi64(vA2, vOff2); + __m512i vShift3 = _mm512_sllv_epi64(vA3, vOff3); + + __m512i or1 = _mm512_or_epi64(vShift1, vShift2); + __m512i or2 = _mm512_or_epi64(vShift3, or1); + + uint64_t v_nxt_ext = _mm512_reduce_add_epi64(or2); + + str_enc[j] = v_nxt_ext; + + const char *p = qPool[j + 40].p; int off_set = qPool[j + 40].l - K; + my_prefetch((const char*)(p + off_set) , _MM_HINT_T0); + } +} + +void prepareChunkBatch(Info* qPool, int qPoolSize, uint64_t* str_enc, int64_t* intv_all, int K){ + + for(int64_t j = 0; j < qPoolSize; j++) + { + Info &q = qPool[j]; + uint64_t nxt_ext = 0; +#ifndef NO_DNA_ORD + + for(int i = q.l-K; i != q.l; i++) { + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i]); + } +#else + for(int i = q.l-K; i != q.l; i++) { + nxt_ext = (nxt_ext<<2) | (q.p[i]); + } +#endif + +//Optimization for K=21 with full loop +#if 0 +#ifndef NO_DNA_ORD + + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(q.p[i++]); +#else + nxt_ext = (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + nxt_ext = (nxt_ext<<2) + (q.p[i++]); + + +#endif + +#endif + str_enc[j] = nxt_ext; + + intv_all[2 * j] = q.intv.first; + intv_all[2 * j + 1] = q.intv.second; + const char *p = qPool[j + 40].p; int offset = qPool[j + 40].l - K; + my_prefetch((const char*)(p + offset) , _MM_HINT_T0); + } +} diff --git a/ext/TAL/LISA/src/common.h b/ext/TAL/LISA/src/common.h new file mode 100644 index 00000000..59876691 --- /dev/null +++ b/ext/TAL/LISA/src/common.h @@ -0,0 +1,194 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +#ifndef __rdtsc +#ifdef _rdtsc +#define __rdtsc _rdtsc +#else +#define __rdtsc __builtin_ia32_rdtsc +#endif +#endif + +#ifdef __lg +#undef __lg +#endif + +#ifndef _MM_HINT_NT +#define _MM_HINT_NT _MM_HINT_NTA +#endif + +template +inline constexpr unsigned long __lg(T n) { + return sizeof(uint64_t) * __CHAR_BIT__ - 1 - __builtin_clzll(n); +} + +#define eprintln(...) do{\ + fprintf(stderr,__VA_ARGS__);\ + fprintf(stderr,"\n");\ +}while(0) + +#define error_quit(...) do{\ + eprintln(__VA_ARGS__);\ + exit(1);\ +}while(0) + +const string dna = "ACGT"; +constexpr int dna_ord(const char &a) { +#ifdef NO_DNA_ORD + __builtin_unreachable(); + // assert(0 && "dna_ord is not supported"); +#else + return __lg(a-'A'+2)-1; // "ACGT" -> 0123 +#endif +} + +#ifdef _64BIT + typedef int64_t index_t; +#else + typedef uint32_t index_t; +#endif + + +class SMEM_out { + public: + int id, q_l, q_r; + index_t ref_l, ref_r; + SMEM_out(int _id, int _q_l, int _q_r, index_t _ref_l, index_t _ref_r){ + id = _id; + q_l = _q_l; + q_r = _q_r; + ref_l = _ref_l; + ref_r = _ref_r; + } + +}; + + +class vector_based_output { + public: + int id; + vector> qPos; + vector> refPos; + vector_based_output(int a){ id = a; + + } +}; + + +class Output { + public: + int id; + SMEM_out* smem; + Output(int a){ id = a; + } +}; + + +struct Info { + // TODO use big int? + const char* p; + int l, r; + uint64_t id; + pair intv; + + + + + void set(int a, int b, index_t c, index_t d){ + l = a; r = b; intv = make_pair(c,d); + } + void print(){ +// printf(" %d %d %lld %lld %lld %d %d ", l, r, intv.first, intv.second, intv.second - intv.first, numPrevSuccChk, treeShrinkLength); + } + + + uint64_t get_enc_str(){ + uint64_t nxt_ext = 0; + int i = l - 21; //K; + //TODO: hard coded K + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + nxt_ext = (nxt_ext<<2) | dna_ord(p[i++]); + + return nxt_ext; + } + + void printRead(){ + printf(">\n"); + for(int i = 0; i < r; i++){ + printf("%c",p[i]); + } + printf("\n"); + + + } + +}; + diff --git a/ext/TAL/LISA/src/fmi.h b/ext/TAL/LISA/src/fmi.h new file mode 100644 index 00000000..fc225519 --- /dev/null +++ b/ext/TAL/LISA/src/fmi.h @@ -0,0 +1,235 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +#include "sais.h" +#include +template +class FMI { + public: + FMI(){} + FMI(const string &t, index_t *sa, string _bases, string parent_filename); + FMI(const string &t, index_t t_size, index_t *sa, string _bases, string parent_filename); + struct Interval{ index_t low, high; }; // left-inclusive + Interval backward_extend(Interval intv, char a) const; + ~FMI(); + // private: + static constexpr int INDEX_T_BITS = sizeof(index_t)*__CHAR_BIT__; + string bases; + int ALP_SIZE; + index_t n; + int64_t m; + index_t *cnt; + index_t *occb; + inline index_t occ(int i, index_t j) const { + static constexpr int shift = __lg(INDEX_T_BITS), msk = INDEX_T_BITS-1; + const index_t* d = occb + ((j>>shift)<<3) + ((i-1)<<1); + return d[1] + __builtin_popcountll(((typename make_unsigned::type)d[0])<<(msk&~j)); + } + void load(string filename); + void save(string filename) const; +}; + +template +void FMI::load(string filename) { + ifstream instream(filename.c_str(), ifstream::binary); + instream.seekg(0); + + cnt = new index_t[ALP_SIZE](); + eprintln("MEM-SIZE: %lld", (long long)sizeof(index_t)*ALP_SIZE); + instream.read((char*)cnt, ALP_SIZE*sizeof(cnt[0])); + + +#ifndef HUGE_PAGE + eprintln("FMI - NO -- Huge Page allocation.."); + occb = (index_t*)aligned_alloc(64, (sizeof(occb[0]) * 4 * m + 63) / 64 * 64); +#else + eprintln("FMI - Huge Page allocation.."); + occb = (index_t*) mmap(NULL, (sizeof(occb[0]) * 4 * m + 63) / 64 * 64, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); +#endif + + eprintln("MEM-SIZE: FMI OCCB: %lld",(long long)((sizeof(occb[0]) * 4 * m + 63) / 64 * 64)); + instream.read((char*)occb, 4*m*sizeof(occb[0])); + instream.close(); +} + +template +void FMI::save(string filename) const { + ofstream outstream(filename.c_str(), ofstream::binary); + outstream.seekp(0); + outstream.write((char*)cnt, ALP_SIZE*sizeof(cnt[0])); + outstream.write((char*)occb, 4*m*sizeof(occb[0])); + outstream.close(); +} + +template +FMI::FMI(const string &t, index_t *sa, string _bases, string parent_filename): + bases(_bases), + ALP_SIZE((int)_bases.size()), + n((index_t)t.size()), + m(((n+1+INDEX_T_BITS-1) / INDEX_T_BITS * 2LL)) { + assert(bases.find('$') == string::npos); + assert(bases=="@ACGT"); + assert(is_sorted(bases.begin(), bases.end())); + assert(t[0] == '$' || t.back() == '$'); + + string bin_filename = parent_filename + ".fmi4.compressed.binned.u"; + for(const auto &s:{sizeof(index_t)}) { + bin_filename += string(".") + to_string(s); + } + if(ifstream(bin_filename.c_str()).good()) { + eprintln("Found existing %s!!", (char*)bin_filename.c_str()); + load(bin_filename); + eprintln("Load successful."); + return; + } else { + eprintln("No existing %s. Building...", (char*)bin_filename.c_str()); + } + + vector true_sa; + if(sa == NULL) { + vector _sa(n); + saisxx(t.c_str(), _sa.data(), (int64_t)n); + true_sa.resize(n); + for(index_t i=0;i::type)occb[j-9]); + } + eprintln("%s build done.", (char*)bin_filename.c_str()); + save(bin_filename); + eprintln("save done."); +} + + +template +FMI::FMI(const string &t, index_t t_size, index_t *sa, string _bases, string parent_filename): + bases(_bases), + ALP_SIZE((int)_bases.size()), + n((index_t)t_size), + m(((n+1+INDEX_T_BITS-1) / INDEX_T_BITS * 2LL)) { + assert(bases.find('$') == string::npos); + assert(bases=="@ACGT"); + assert(is_sorted(bases.begin(), bases.end())); +// assert(t[0] == '$' || t.back() == '$'); + + string bin_filename = parent_filename + ".fmi4.compressed.binned.u"; + for(const auto &s:{sizeof(index_t)}) { + bin_filename += string(".") + to_string(s); + } + if(ifstream(bin_filename.c_str()).good()) { + eprintln("Found existing %s!!", (char*)bin_filename.c_str()); + load(bin_filename); + eprintln("Load successful."); + return; + } else { + eprintln("No existing %s. Building...", (char*)bin_filename.c_str()); + } + + vector true_sa; + if(sa == NULL) { + vector _sa(n); + saisxx(t.c_str(), _sa.data(), (int64_t)n); + true_sa.resize(n); + for(index_t i=0;i::type)occb[j-9]); + } + eprintln("%s build done.", (char*)bin_filename.c_str()); + save(bin_filename); + eprintln("save done."); +} + + + +template +typename FMI::Interval FMI::backward_extend(Interval intv, char a) const { +#ifndef NO_DNA_ORD + const int i = __lg(a-'@'+1); // @ACGT -> 01234 +#else + const int i = a+1; // ACGT -> 1234 +#endif + Interval ret = { + cnt[i] + occ(i, intv.low/*-1*/), + cnt[i] + occ(i, intv.high/*-1*/) + }; + return ret; +} + +template +FMI::~FMI(){ + eprintln("FMI memory deallocated\n"); +#ifndef HUGE_PAGE + free(occb); +#else + munmap(occb, (sizeof(occb[0]) * 4 * m + 63) / 64 * 64); +#endif +} diff --git a/ext/TAL/LISA/src/ipbwt_rmi.h b/ext/TAL/LISA/src/ipbwt_rmi.h new file mode 100644 index 00000000..1122a5cd --- /dev/null +++ b/ext/TAL/LISA/src/ipbwt_rmi.h @@ -0,0 +1,1068 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ + + + +#ifdef VECTORIZE +#include +#endif +#include +#include +#include + +#ifdef ENABLE_PREFETCH +enum query_state +{ + GUESS_RMI_ROOT, + GUESS_RMI_LEAF, + LAST_MILE +}; +#endif + +template +class IPBWT_RMI { + public: + IPBWT_RMI(){} + IPBWT_RMI(const string &t, string ref_seq_filename, int K_, int64_t num_rmi_leaf_nodes, index_t *__sa = NULL); + IPBWT_RMI(const string &t, index_t t_size, string ref_seq_filename, int K_, int64_t num_rmi_leaf_nodes, index_t *__sa = NULL); + ~IPBWT_RMI(); + + typedef pair ipbwt_t; + struct ref_pos_t; +#ifdef ENABLE_PREFETCH + struct BatchMetadata; +#endif + int K; + double inv_second_size; + int NUM_POS_BITS = 38; + uint64_t POS_MASK = 0x3fffffffffL; + int NUM_CHUNK_BITS = 42; + uint64_t CHUNK_MASK = 0x3ffffffffffL; + int NUM_IPBWT_BITS = NUM_CHUNK_BITS + NUM_POS_BITS; + uint64_t NUM_IPBWT_BYTES = (NUM_IPBWT_BITS + 7 ) / 8; + __mmask64 mask_ipbwt_load = (1 << NUM_IPBWT_BYTES) - 1; + double L0_PARAMETER0; + double L0_PARAMETER1; + int64_t L1_SIZE; + double *L1_PARAMETERS; + int16_t m0[32]; + int16_t m1[32]; + int32_t m_one_bits[9]; + pair backward_extend_chunk(kenc_t str_enc, pair intv) const; + kenc_t gen_str_enc(const char *a, int len) const; + int64_t bs_ticks; + int64_t vbs_ticks; + int64_t bs_calls; + int64_t vbs_calls; + + // private: + static constexpr int kenc_bits = sizeof(kenc_t)*__CHAR_BIT__; + index_t n, second_size; + + uint8_t *ipbwt_array; +#ifdef ENABLE_PREFETCH + static constexpr int64_t UNROLL = 28; + +#if 0 + inline bool process_partial_query_one_step(BatchMetadataPartial &meta, index_t &p_low, index_t &p_high, bool is_partial); + void backward_extend_chunk_batched_info (Info* str_enc_list, int64_t qs_size, index_t *intv_all, bool is_partial); + void backward_partial_extend_chunk_batched (kenc_t* str_enc_list, int64_t qs_size, index_t *intv_all, bool is_partial); + void backward_partial_extend_chunk_batched_v1 (kenc_t* str_enc_list, int64_t qs_size, index_t *intv_all, bool is_partial); +#endif + inline int64_t get_guess_root_step(double key); + inline int64_t get_guess_leaf_step(double key, int64_t modelIndex, int64_t *err); + inline void last_mile_binary_search_one_step(ipbwt_t ipb_x, int64_t &first, int64_t &m); + inline void last_mile_vectorized_search_final_step(ipbwt_t ipb_x, int64_t &first, int64_t &m); + inline bool process_query_one_step(BatchMetadata &meta, index_t &p_low, index_t &p_high); + void backward_extend_chunk_batched (kenc_t* str_enc_list, int64_t qs_size, index_t *intv_all); + void backward_extend_multi_chunk_batched (Info* str_enc_list, int64_t qs_size, index_t *intv_all); +#endif + + double to_floating_point(pair p) const; + index_t get_guess_from_rmi(ipbwt_t ipb_x, index_t *err) const; + index_t last_mile_from_guess(ipbwt_t ipb_x, index_t guess, index_t err) const; + +inline ipbwt_t ipbwt(uint64_t chunk, uint64_t pos) const +{ + uint64_t second = (pos & POS_MASK) + (chunk << NUM_POS_BITS); + uint64_t first = chunk >> (64 - NUM_POS_BITS); + return {first, second}; +} + +inline ipbwt_t ipbwt(const index_t &i) const +{ + uint8_t *base_addr = ipbwt_array + i * NUM_IPBWT_BYTES; + uint64_t second = ((uint64_t *)base_addr)[0]; + uint64_t first = ((uint16_t *)(base_addr + 8))[0]; + return {first, second}; +} + +inline uint64_t get_ref_pos(ref_pos_t A, index_t i) +{ + uint64_t sa_ms_byte = A.ms_byte[i]; + uint64_t sa_ls_word = A.ls_word[i]; + uint64_t pos = (sa_ms_byte << 32) + sa_ls_word; + return pos; +} + +inline kenc_t get_enc_rev_comp(kenc_t nxt_ext, int K = 21) { + + auto temp = nxt_ext; + temp = ~temp; + kenc_t new_ext = 0, mask = 3; + + for (int i = 0; i < K; i++){ + new_ext = new_ext << 2 | temp & mask; + temp = temp >>2; + } + return new_ext; +} + +inline kenc_t get_enc_suffix(kenc_t nxt_ext, int suffix_length, int K = 21){ + + kenc_t new_ext; + nxt_ext = nxt_ext << ((64 - 2*K) + 2*(K - (suffix_length + 1))); + new_ext = nxt_ext >> (64 - 2*K); + return new_ext; +} + +}; + + +template +struct IPBWT_RMI::ref_pos_t +{ + uint32_t *ls_word; + uint8_t *ms_byte; +}; + +#ifdef ENABLE_PREFETCH +template +struct IPBWT_RMI::BatchMetadata +{ + ipbwt_t ipb_x[2]; + double key[2]; + index_t modelIndex[2]; + index_t first[2]; + index_t m[2]; + int64_t qid; + query_state state; +}; + +#endif + +void load(string filename, vector ptrs, vector sizes) { + ifstream instream(filename.c_str(), ifstream::binary); + instream.seekg(0); + assert(ptrs.size() == sizes.size()); + for(size_t i=0; i ptrs, vector sizes) { + ofstream outstream(filename.c_str(), ofstream::binary); + outstream.seekp(0); + assert(ptrs.size() == sizes.size()); + for(size_t i=0; i bound ? bound : (int64_t)inp); +} + + +template +inline kenc_t IPBWT_RMI::gen_str_enc(const char *a, int len) const { + kenc_t str_enc = 0; + int i; + for(i=0; i +double IPBWT_RMI::to_floating_point(pair p) const { + return p.first + p.second * inv_second_size; +} + + + +template +IPBWT_RMI::IPBWT_RMI(const string &t, index_t t_size, string ref_seq_filename, int K_, int64_t num_rmi_leaf_nodes, index_t *__sa): + n(t_size), K(K_), second_size(n+K_) { + + assert(K <= 21); + eprintln("n = %ld, K = %d", (long)n, K); + eprintln("NUM_IPBWT_BYTES = %ld", NUM_IPBWT_BYTES); + inv_second_size = ((double)1.0)/second_size; + auto build_ipbwt = [&]() { + int64_t startTick, endTick; + + startTick = __rdtsc(); + if(__sa == NULL) { + __sa = (int64_t *)malloc(n * sizeof(int64_t)); + assert(__sa != NULL); + saisxx(t.c_str(), __sa, (int64_t)n); + } + ref_pos_t sa; + // sa.ls_word = (uint32_t *)malloc(n * sizeof(uint32_t)); + // sa.ms_byte = (uint8_t *)malloc(n * sizeof(uint8_t)); + sa.ls_word = (uint32_t *)calloc(n, sizeof(uint32_t)); + sa.ms_byte = (uint8_t *)calloc(n, sizeof(uint8_t)); + assert(sa.ls_word != NULL); + assert(sa.ms_byte != NULL); + for(index_t i = 0; i < n; i++) + { + sa.ls_word[i] = __sa[i] & 0xffffffff; + sa.ms_byte[i] = (__sa[i] >> 32) & 0xff; + } + for(index_t i = 0; i < n; i++) + { + uint64_t sa_ms_byte = sa.ms_byte[i]; + uint64_t sa_ls_word = sa.ls_word[i]; + uint64_t pos = (sa_ms_byte << 32) + sa_ls_word; + if(pos != __sa[i]) + { + eprintln("%ld] pos = %ld, __sa[i] = %ld", (long)i, (long)pos, (long)__sa[i]); + exit(0); + } + } + free(__sa); + endTick = __rdtsc(); + eprintln("SA done in %ld cycles.", endTick - startTick); + + { + startTick = __rdtsc(); + ref_pos_t rnk; + rnk.ls_word = (uint32_t *)malloc(n * sizeof(uint32_t)); + rnk.ms_byte = (uint8_t *)malloc(n * sizeof(uint8_t)); + assert(rnk.ls_word != NULL); + assert(rnk.ms_byte != NULL); + eprintln("rnk allocated, n = %ld", (long)n); +//#pragma omp parallel for + for(index_t i=0; i> 32) & 0xff; + } + endTick = __rdtsc(); + eprintln("RNK done in %ld cycles.", endTick - startTick); + startTick = __rdtsc(); + ipbwt_array = (uint8_t *)aligned_alloc(64, n * NUM_IPBWT_BYTES); + assert(ipbwt_array != NULL); + eprintln("ipbwt_array allocated with size %lu", (unsigned long)n * NUM_IPBWT_BYTES); + assert(K=n) { + p-=n; + pos = K-1-p; // dollar_sign_pos + //((int64_t *)ipbwt_array)[2 * i] = K-1-p; // dollar_sign_pos + } else { + pos = get_ref_pos(rnk, p) + K; + //((int64_t *)ipbwt_array)[2 * i] = get_ref_pos(rnk, p) + K; + } + uint64_t str_enc = gen_str_enc(t.c_str() + get_ref_pos(sa, i), K); + //eprintln("%ld] pos = %lu, str_enc = %lu", i, pos, str_enc); + //((int64_t *)ipbwt_array)[2 * i + 1] = str_enc; + pos = (pos & POS_MASK) + (str_enc << NUM_POS_BITS); + uint16_t chunk = str_enc >> (64 - NUM_POS_BITS); + //eprintln("%ld] pos = %lu, chunk = %u", i, pos, chunk); + //fflush(stderr); + uint64_t *pos_ptr = (uint64_t *)(&(ipbwt_array[i * NUM_IPBWT_BYTES])); + pos_ptr[0] = pos; + uint16_t *chunk_ptr = (uint16_t *)(&(ipbwt_array[i * NUM_IPBWT_BYTES])) + 4; + chunk_ptr[0] = chunk; + } + endTick = __rdtsc(); + eprintln("ipbwt_array done in %ld cycles.", endTick - startTick); + for(index_t i = 1; i < n; i++) + { + if(ipbwt(i - 1) >= ipbwt(i)) + { + //eprintln("%ld] {%lu, %lu} {%lu, %lu}", i, ((int64_t *)ipbwt_array)[2 * (i - 1) + 1], ((int64_t *)ipbwt_array)[2 * (i - 1)], ((int64_t *)ipbwt_array)[2 * i + 1], ((int64_t *)ipbwt_array)[2 * i]); + eprintln("%ld] {%lu, %lu} {%lu, %lu}", (long)i, (unsigned long)ipbwt(i - 1).first, (unsigned long)ipbwt(i - 1). second, (unsigned long)ipbwt(i).first, (unsigned long)ipbwt(i).second); + exit(0); + } + } + free(sa.ls_word); free(sa.ms_byte); + free(rnk.ls_word); free(rnk.ms_byte); + } + }; + + + string ipbwt_filename = ref_seq_filename + ".ipbwt"; + for(const auto &s:{sizeof(index_t), sizeof(kenc_t), (size_t)K, (size_t)NUM_POS_BITS, (size_t)NUM_CHUNK_BITS}) { + ipbwt_filename += string(".") + to_string(s); + } + if(ifstream(ipbwt_filename.c_str()).good()) { + eprintln("Found existing %s!!", (char *)ipbwt_filename.c_str()); + +#ifndef HUGE_PAGE + ipbwt_array = (uint8_t *)aligned_alloc(64, n * NUM_IPBWT_BYTES); +#else + eprintln("Using Huge-Page 2MB"); + ipbwt_array = (uint8_t *) mmap(NULL, n * NUM_IPBWT_BYTES, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); +#endif + eprintln("ipbwt Array size: %lld", (long long)n * NUM_IPBWT_BYTES); + + load(ipbwt_filename, {((char *)ipbwt_array)}, {(n * NUM_IPBWT_BYTES)}); + eprintln("Load successful."); + for(index_t i = 0; i < n; i++) + { + if(i > 0) + { + if(ipbwt(i - 1) >= ipbwt(i)) + { + //eprintln("%ld] {%lu, %lu} {%lu, %lu}", i, ((int64_t *)ipbwt_array)[2 * (i - 1) + 1], ((int64_t *)ipbwt_array)[2 * (i - 1)], ((int64_t *)ipbwt_array)[2 * i + 1], ((int64_t *)ipbwt_array)[2 * i]); + eprintln("%ld] {%lu, %lu} {%lu, %lu}", (long)i, (unsigned long)ipbwt(i - 1).first, (unsigned long)ipbwt(i - 1). second, (unsigned long)ipbwt(i).first, (unsigned long)ipbwt(i).second); + exit(0); + } + assert(ipbwt(i-1) < ipbwt(i)); + } + } + } else { + eprintln("No existing %s. Building...", (char*)ipbwt_filename.c_str()); + build_ipbwt(); + eprintln("%s build done.", (char *)ipbwt_filename.c_str()); + save(ipbwt_filename, {((char *)ipbwt_array)}, {(n * NUM_IPBWT_BYTES)}); + eprintln("%s saved.", (char*)ipbwt_filename.c_str()); + } + eprintln("ipbwt correctness ok."); + + string rmi_filename = ipbwt_filename + "." + to_string(num_rmi_leaf_nodes) + ".rmi_PARAMETERS"; + if(ifstream(rmi_filename.c_str()).good()) { + eprintln("Found existing %s!!", (char*)rmi_filename.c_str()); + vector ptrs;vector sizes; + ptrs.push_back((char *)&(L0_PARAMETER0)); + sizes.push_back(sizeof(double)); + ptrs.push_back((char *)&(L0_PARAMETER1)); + sizes.push_back(sizeof(double)); + ptrs.push_back((char *)&(L1_SIZE)); + sizes.push_back(sizeof(int64_t)); + load(rmi_filename, ptrs, sizes); + +#ifndef HUGE_PAGE + L1_PARAMETERS = (double*) _mm_malloc(L1_SIZE * 3 * sizeof(double), 64); +#else + L1_PARAMETERS = (double*) mmap(NULL, L1_SIZE * 3 * sizeof(double), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); +#endif + + + eprintln("L1 size in Bytes: %lld", (long long)L1_SIZE * 3 * sizeof(double)); + + if (L1_PARAMETERS == NULL) + { + eprintln("L1_PARAMETERS == NULL"); + exit(0);; + } + ptrs.push_back((char *)L1_PARAMETERS); + sizes.push_back(L1_SIZE * 3 * sizeof(double)); + load(rmi_filename, ptrs, sizes); + eprintln("Load successful."); + eprintln("L1_SIZE = %ld", L1_SIZE); + } else { + eprintln("No existing %s. Building train data...", (char*)rmi_filename.c_str()); + double *train_data = (double *)_mm_malloc(n * sizeof(double), 64); + for(index_t i = 0; i < n; i++) + { + ipbwt_t p = ipbwt(i); + uint64_t second = p.second & POS_MASK; + uint64_t first = (p.first << (64 - NUM_POS_BITS)) + (p.second >> NUM_POS_BITS); + train_data[i] = to_floating_point({first, second}); + } + vector ptrs;vector sizes; + ptrs.push_back((char *)&n); + sizes.push_back(sizeof(int64_t)); + ptrs.push_back((char *)train_data); + sizes.push_back(n * sizeof(double)); + string ipbwt_f64_filename = ipbwt_filename + ".f64"; + save(ipbwt_f64_filename, ptrs, sizes); + eprintln("Training data saved at %s. Building RMI using Ryan's code. Run this code again after the RMI is built.", (char*)ipbwt_f64_filename.c_str()); + eprintln("executing ./build-rmi.linear_spline.linear.sh"); + //const char *execv_argv[] = {"./build-rmi.linear_spline.linear.sh", + // ipbwt_f64_filename.c_str(), + // rmi_filename.c_str(), + // to_string(num_rmi_leaf_nodes).c_str()}; + //execv_argv[0] = ipbwt_f64_filename.c_str(); + //execv_argv[1] = rmi_filename.c_str(); + //execv_argv[2] = to_string(num_rmi_leaf_nodes); + int status = execl("./scripts/build-rmi.linear_spline.linear.sh", + "./scripts/build-rmi.linear_spline.linear.sh", + ipbwt_f64_filename.c_str(), + rmi_filename.c_str(), + to_string(num_rmi_leaf_nodes).c_str(), (char *)NULL); + eprintln("ERROR! execl failed with err %d, errno = %d: %s", status, errno, strerror( errno)); + exit(0); + } + + for(int i = 0; i < 4; i++) + { + for(int j = 0; j < 4; j++) + { + m0[i * 4 + j] = i * 5 + j; + m1[i * 4 + j] = 4 + i * 5 + j; + } + } + for(int i = 4; i < 8; i++) + { + for(int j = 0; j < 4; j++) + { + m0[i * 4 + j] = i * 5 + j + 24; + m1[i * 4 + j] = 4 + i * 5 + j + 24; + } + } +#if 0 + for(int i = 0; i < 32; i++) + { + eprintln("%d] %d %d", i, m0[i], m1[i]); + } +#endif + for(int i = 0; i < 9; i++) + { + m_one_bits[i] = (1 << i) - 1; + } + m_one_bits[8] = 0xff; + bs_calls = vbs_calls = 0; + bs_ticks = vbs_ticks = 0; +} + + + + +template +IPBWT_RMI::IPBWT_RMI(const string &t, string ref_seq_filename, int K_, int64_t num_rmi_leaf_nodes, index_t *__sa): + n((index_t)t.size()), K(K_), second_size(n+K_) { + + assert(K <= 21); + eprintln("n = %ld, K = %d", (long)n, K); + eprintln("NUM_IPBWT_BYTES = %ld", NUM_IPBWT_BYTES); + inv_second_size = ((double)1.0)/second_size; + auto build_ipbwt = [&]() { + int64_t startTick, endTick; + + startTick = __rdtsc(); + if(__sa == NULL) { + __sa = (int64_t *)malloc(n * sizeof(int64_t)); + assert(__sa != NULL); + saisxx(t.c_str(), __sa, (int64_t)n); + } + ref_pos_t sa; + //sa.ls_word = (uint32_t *)malloc(n * sizeof(uint32_t)); + //sa.ms_byte = (uint8_t *)malloc(n * sizeof(uint8_t)); + sa.ls_word = (uint32_t *)calloc(n, sizeof(uint32_t)); + sa.ms_byte = (uint8_t *)calloc(n, sizeof(uint8_t)); + assert(sa.ls_word != NULL); + assert(sa.ms_byte != NULL); + for(index_t i = 0; i < n; i++) + { + sa.ls_word[i] = __sa[i] & 0xffffffff; + sa.ms_byte[i] = (__sa[i] >> 32) & 0xff; + } + for(index_t i = 0; i < n; i++) + { + uint64_t sa_ms_byte = sa.ms_byte[i]; + uint64_t sa_ls_word = sa.ls_word[i]; + uint64_t pos = (sa_ms_byte << 32) + sa_ls_word; + if(pos != __sa[i]) + { + eprintln("%ld] pos = %ld, __sa[i] = %ld", (long)i, (long)pos, (long)__sa[i]); + exit(0); + } + } + free(__sa); + endTick = __rdtsc(); + eprintln("SA done in %ld cycles.", endTick - startTick); + + { + startTick = __rdtsc(); + ref_pos_t rnk; + //rnk.ls_word = (uint32_t *)malloc(n * sizeof(uint32_t)); + //rnk.ms_byte = (uint8_t *)malloc(n * sizeof(uint8_t)); + rnk.ls_word = (uint32_t *)calloc(n, sizeof(uint32_t)); + rnk.ms_byte = (uint8_t *)calloc(n, sizeof(uint8_t)); + assert(rnk.ls_word != NULL); + assert(rnk.ms_byte != NULL); + eprintln("rnk allocated, n = %ld", (long)n); +//#pragma omp parallel for + for(index_t i=0; i> 32) & 0xff; + } + endTick = __rdtsc(); + eprintln("RNK done in %ld cycles.", endTick - startTick); + startTick = __rdtsc(); + ipbwt_array = (uint8_t *)aligned_alloc(64, n * NUM_IPBWT_BYTES); + assert(ipbwt_array != NULL); + eprintln("ipbwt_array allocated with size %lu", (unsigned long)n * NUM_IPBWT_BYTES); + assert(K=n) { + p-=n; + pos = K-1-p; // dollar_sign_pos + //((int64_t *)ipbwt_array)[2 * i] = K-1-p; // dollar_sign_pos + } else { + pos = get_ref_pos(rnk, p) + K; + //((int64_t *)ipbwt_array)[2 * i] = get_ref_pos(rnk, p) + K; + } + uint64_t str_enc = gen_str_enc(t.c_str() + get_ref_pos(sa, i), K); + //eprintln("%ld] pos = %lu, str_enc = %lu", i, pos, str_enc); + //((int64_t *)ipbwt_array)[2 * i + 1] = str_enc; + pos = (pos & POS_MASK) + (str_enc << NUM_POS_BITS); + uint16_t chunk = str_enc >> (64 - NUM_POS_BITS); + //eprintln("%ld] pos = %lu, chunk = %u", i, pos, chunk); + //fflush(stderr); + uint64_t *pos_ptr = (uint64_t *)(&(ipbwt_array[i * NUM_IPBWT_BYTES])); + pos_ptr[0] = pos; + uint16_t *chunk_ptr = (uint16_t *)(&(ipbwt_array[i * NUM_IPBWT_BYTES])) + 4; + chunk_ptr[0] = chunk; + } + endTick = __rdtsc(); + eprintln("ipbwt_array done in %ld cycles.", endTick - startTick); + for(index_t i = 1; i < n; i++) + { + if(ipbwt(i - 1) >= ipbwt(i)) + { + //eprintln("%ld] {%lu, %lu} {%lu, %lu}", i, ((int64_t *)ipbwt_array)[2 * (i - 1) + 1], ((int64_t *)ipbwt_array)[2 * (i - 1)], ((int64_t *)ipbwt_array)[2 * i + 1], ((int64_t *)ipbwt_array)[2 * i]); + eprintln("%ld] {%lu, %lu} {%lu, %lu}", (long)i, (unsigned long)ipbwt(i - 1).first, (unsigned long)ipbwt(i - 1). second, (unsigned long)ipbwt(i).first, (unsigned long)ipbwt(i).second); + exit(0); + } + } + free(sa.ls_word); free(sa.ms_byte); + free(rnk.ls_word); free(rnk.ms_byte); + } + }; + + + string ipbwt_filename = ref_seq_filename + ".ipbwt"; + for(const auto &s:{sizeof(index_t), sizeof(kenc_t), (size_t)K, (size_t)NUM_POS_BITS, (size_t)NUM_CHUNK_BITS}) { + ipbwt_filename += string(".") + to_string(s); + } + if(ifstream(ipbwt_filename.c_str()).good()) { + eprintln("Found existing %s!!", (char*)ipbwt_filename.c_str()); + ipbwt_array = (uint8_t *)aligned_alloc(64, n * NUM_IPBWT_BYTES); + load(ipbwt_filename, {((char *)ipbwt_array)}, {(n * NUM_IPBWT_BYTES)}); + eprintln("Load successful."); + for(index_t i = 0; i < n; i++) + { + if(i > 0) + { + if(ipbwt(i - 1) >= ipbwt(i)) + { + //eprintln("%ld] {%lu, %lu} {%lu, %lu}", i, ((int64_t *)ipbwt_array)[2 * (i - 1) + 1], ((int64_t *)ipbwt_array)[2 * (i - 1)], ((int64_t *)ipbwt_array)[2 * i + 1], ((int64_t *)ipbwt_array)[2 * i]); + eprintln("%ld] {%lu, %lu} {%lu, %lu}", (long)i, (unsigned long)ipbwt(i - 1).first, (unsigned long)ipbwt(i - 1). second, (unsigned long)ipbwt(i).first, (unsigned long)ipbwt(i).second); + exit(0); + } + assert(ipbwt(i-1) < ipbwt(i)); + } + } + } else { + eprintln("No existing %s. Building...", (char*)ipbwt_filename.c_str()); + build_ipbwt(); + eprintln("%s build done.", (char*)ipbwt_filename.c_str()); + save(ipbwt_filename, {((char *)ipbwt_array)}, {(n * NUM_IPBWT_BYTES)}); + eprintln("%s saved.", (char*)ipbwt_filename.c_str()); + } + eprintln("ipbwt correctness ok."); + + string rmi_filename = ipbwt_filename + "." + to_string(num_rmi_leaf_nodes) + ".rmi_PARAMETERS"; + if(ifstream(rmi_filename.c_str()).good()) { + eprintln("Found existing %s!!", (char*)rmi_filename.c_str()); + vector ptrs;vector sizes; + ptrs.push_back((char *)&(L0_PARAMETER0)); + sizes.push_back(sizeof(double)); + ptrs.push_back((char *)&(L0_PARAMETER1)); + sizes.push_back(sizeof(double)); + ptrs.push_back((char *)&(L1_SIZE)); + sizes.push_back(sizeof(int64_t)); + load(rmi_filename, ptrs, sizes); + L1_PARAMETERS = (double*) _mm_malloc(L1_SIZE * 3 * sizeof(double), 64); + if (L1_PARAMETERS == NULL) + { + eprintln("L1_PARAMETERS == NULL"); + exit(0);; + } + ptrs.push_back((char *)L1_PARAMETERS); + sizes.push_back(L1_SIZE * 3 * sizeof(double)); + load(rmi_filename, ptrs, sizes); + eprintln("Load successful."); + eprintln("L1_SIZE = %ld", L1_SIZE); + } else { + eprintln("No existing %s. Building train data...", (char*)rmi_filename.c_str()); + double *train_data = (double *)_mm_malloc(n * sizeof(double), 64); + for(index_t i = 0; i < n; i++) + { + ipbwt_t p = ipbwt(i); + uint64_t second = p.second & POS_MASK; + uint64_t first = (p.first << (64 - NUM_POS_BITS)) + (p.second >> NUM_POS_BITS); + train_data[i] = to_floating_point({first, second}); + } + vector ptrs;vector sizes; + ptrs.push_back((char *)&n); + sizes.push_back(sizeof(int64_t)); + ptrs.push_back((char *)train_data); + sizes.push_back(n * sizeof(double)); + string ipbwt_f64_filename = ipbwt_filename + ".f64"; + save(ipbwt_f64_filename, ptrs, sizes); + eprintln("Training data saved at %s. Building RMI using Ryan's code. Run this code again after the RMI is built.", (char*)ipbwt_f64_filename.c_str()); + eprintln("executing ./build-rmi.linear_spline.linear.sh"); + //const char *execv_argv[] = {"./build-rmi.linear_spline.linear.sh", + // ipbwt_f64_filename.c_str(), + // rmi_filename.c_str(), + // to_string(num_rmi_leaf_nodes).c_str()}; + //execv_argv[0] = ipbwt_f64_filename.c_str(); + //execv_argv[1] = rmi_filename.c_str(); + //execv_argv[2] = to_string(num_rmi_leaf_nodes); + int status = execl("./scripts/build-rmi.linear_spline.linear.sh", + "./scripts/build-rmi.linear_spline.linear.sh", + ipbwt_f64_filename.c_str(), + rmi_filename.c_str(), + to_string(num_rmi_leaf_nodes).c_str(), (char *)NULL); + eprintln("ERROR! execl failed with err %d, errno = %d: %s", status, errno, strerror( errno)); + exit(0); + } + + for(int i = 0; i < 4; i++) + { + for(int j = 0; j < 4; j++) + { + m0[i * 4 + j] = i * 5 + j; + m1[i * 4 + j] = 4 + i * 5 + j; + } + } + for(int i = 4; i < 8; i++) + { + for(int j = 0; j < 4; j++) + { + m0[i * 4 + j] = i * 5 + j + 24; + m1[i * 4 + j] = 4 + i * 5 + j + 24; + } + } +#if 0 + for(int i = 0; i < 32; i++) + { + eprintln("%d] %d %d", i, m0[i], m1[i]); + } +#endif + for(int i = 0; i < 9; i++) + { + m_one_bits[i] = (1 << i) - 1; + } + m_one_bits[8] = 0xff; + bs_calls = vbs_calls = 0; + bs_ticks = vbs_ticks = 0; +} + +template +IPBWT_RMI::~IPBWT_RMI() +{ +#ifndef HUGE_PAGE + free(ipbwt_array); + _mm_free(L1_PARAMETERS); +#else + munmap(ipbwt_array, n * NUM_IPBWT_BYTES); + munmap(L1_PARAMETERS, L1_SIZE * 3 * sizeof(double)); +#endif + + //eprintln("bs_calls = %ld, vbs_calls = %ld", bs_calls, vbs_calls); + //eprintln("bs_ticks = %ld, vbs_ticks = %ld", bs_ticks, vbs_ticks); +} + +template +inline index_t IPBWT_RMI::get_guess_from_rmi(ipbwt_t ipb_x, index_t *err) const { + + double key = to_floating_point(ipb_x); + int64_t modelIndex; + double fpred; + fpred = std::fma(L0_PARAMETER1, key, L0_PARAMETER0); + modelIndex = FCLAMP(fpred, L1_SIZE - 1.0); + fpred = std::fma(L1_PARAMETERS[modelIndex * 3 + 1], key, L1_PARAMETERS[modelIndex * 3]); + *err = *((uint64_t*) (L1_PARAMETERS + (modelIndex * 3 + 2))); + + //int64_t pos = FCLAMP(fpred, n - 1.0); + //printf("pos = %ld\n", pos); + return FCLAMP(fpred, n - 1.0); +} + +template +inline index_t IPBWT_RMI::last_mile_from_guess + (ipbwt_t ipb_x, index_t guess, index_t err) const { + //index_t p = guess; + index_t low = guess - err; + if(low < 0) low = 0; + index_t high = guess + err + 1; + if(high > n) high = n; + + index_t m = high - low; + index_t first = low; + + while(m > 0) + { + index_t half = m / 2; + index_t middle = first + half; + + if(ipb_x >= ipbwt(middle)) + { + first = middle + 1; + m -= half + 1; + } + else + { + m = half; + } + } + first--; + return first; + + // TODO: if the final pos is not in the err range, look for it +} + +template +inline pair IPBWT_RMI::backward_extend_chunk +(kenc_t str_enc, pair intv) const { + index_t low = intv.first, high = intv.second; + ipbwt_t ipb_x_low = {str_enc, low+K}; + ipbwt_t ipb_x_high = {str_enc, high+K}; + + int64_t err_low = 0, err_high = 0; + auto low_guess = get_guess_from_rmi(ipb_x_low, &err_low); + + auto high_guess = get_guess_from_rmi(ipb_x_high, &err_high); + + low = last_mile_from_guess(ipb_x_low, low_guess, err_low); + high = last_mile_from_guess(ipb_x_high, high_guess, err_high); + + return {low, high}; +} + +template +inline int64_t IPBWT_RMI::get_guess_root_step(double key) +{ + int64_t modelIndex; + double fpred = std::fma(L0_PARAMETER1, key, L0_PARAMETER0); + modelIndex = FCLAMP(fpred, L1_SIZE - 1.0); + return modelIndex; +} + +template +inline int64_t IPBWT_RMI::get_guess_leaf_step(double key, int64_t modelIndex, int64_t *err) +{ + double fpred = std::fma(L1_PARAMETERS[modelIndex * 3 + 1], key, L1_PARAMETERS[modelIndex * 3]); + *err = *((uint64_t*) (L1_PARAMETERS + (modelIndex * 3 + 2))); + + int64_t guess = FCLAMP(fpred, n - 1.0); + return guess; +} + +template +inline void IPBWT_RMI::last_mile_binary_search_one_step(ipbwt_t ipb_x, int64_t &first, int64_t &m) +{ + //bs_calls++; + //int64_t startTick = __rdtsc(); + int64_t half = m >> 1; + int64_t middle = first + half; + ipbwt_t ipb_middle = ipbwt(middle); + int64_t cond = (ipb_x.first > ipb_middle.first) | ((ipb_x.first == ipb_middle.first) & (ipb_x.second > ipb_middle.second)); + //bool cond = ipb_x >= ipb_middle; + int64_t middle_plus_one = middle + 1; + int64_t m_minus_half_plus_one = m - (half + 1); + first = cond * middle_plus_one + (1 - cond) * first; + m = cond * m_minus_half_plus_one + (1 - cond) * half; + //int64_t endTick = __rdtsc(); + //bs_ticks += (endTick - startTick); +} + +template +inline void IPBWT_RMI::last_mile_vectorized_search_final_step(ipbwt_t ipb_x, int64_t &first, int64_t &m) +{ + //vbs_calls++; + __m512i y0 = _mm512_loadu_si512(&(ipbwt_array[NUM_IPBWT_BYTES * first])); + __m512i y1 = _mm512_loadu_si512(&(ipbwt_array[NUM_IPBWT_BYTES * (first)]) + 16); + + __m512i x_first = _mm512_set1_epi64(ipb_x.first); + __m512i x_second = _mm512_set1_epi64(ipb_x.second); + + __m512i idx0 = _mm512_loadu_si512(m0); + __m512i idx1 = _mm512_loadu_si512(m1); + __m512i y_first = _mm512_maskz_permutex2var_epi16(0x11111111, y0, idx1, y1); + __m512i y_second = _mm512_permutex2var_epi16(y0, idx0, y1); + + __mmask8 mask_gt_first = _mm512_cmpgt_epu64_mask(x_first, y_first); + __mmask8 mask_eq_first = _mm512_cmpeq_epu64_mask(x_first, y_first); + __mmask8 mask_gt_second = _mm512_cmpgt_epu64_mask(x_second, y_second); + + + __mmask8 mask_gt = mask_gt_first | (mask_eq_first & mask_gt_second); + int32_t numgt = _mm_countbits_32(mask_gt & m_one_bits[m]); + first = first + numgt; + m = 0; +} + +template +inline bool IPBWT_RMI::process_query_one_step(BatchMetadata &meta, index_t &p_low, index_t &p_high) +{ + if(meta.state == GUESS_RMI_ROOT) + { + meta.modelIndex[0] = get_guess_root_step(meta.key[0]); + meta.modelIndex[1] = get_guess_root_step(meta.key[1]); + meta.state = GUESS_RMI_LEAF; + _mm_prefetch((const char *)(&L1_PARAMETERS[meta.modelIndex[0] * 3]), _MM_HINT_T0); + _mm_prefetch((const char *)(&L1_PARAMETERS[meta.modelIndex[0] * 3 + 2]), _MM_HINT_T0); + _mm_prefetch((const char *)(&L1_PARAMETERS[meta.modelIndex[1] * 3]), _MM_HINT_T0); + _mm_prefetch((const char *)(&L1_PARAMETERS[meta.modelIndex[1] * 3 + 2]), _MM_HINT_T0); + } + else if(meta.state == GUESS_RMI_LEAF) + { + int64_t err[2]; + int64_t guess[2]; + int64_t last[2]; + err[0] = err[1] = 0; + guess[0] = get_guess_leaf_step(meta.key[0], meta.modelIndex[0], &err[0]); + guess[1] = get_guess_leaf_step(meta.key[1], meta.modelIndex[1], &err[1]); + meta.first[0] = guess[0] - err[0]; + if(meta.first[0] < 0) meta.first[0] = 0; + last[0] = guess[0] + err[0] + 1; + if(last[0] > n) last[0] = n; + meta.m[0] = last[0] - meta.first[0]; + meta.first[1] = guess[1] - err[1]; + if(meta.first[1] < 0) meta.first[1] = 0; + last[1] = guess[1] + err[1] + 1; + if(last[1] > n) last[1] = n; + meta.m[1] = last[1] - meta.first[1]; + meta.state = LAST_MILE; + if(meta.m[0] > 8) + { + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[0] + (meta.m[0] >> 1)) * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[0] + (meta.m[0] >> 1)) * NUM_IPBWT_BYTES]) + 8, _MM_HINT_T0); + } + else + { + _mm_prefetch((const char *)(&ipbwt_array[meta.first[0] * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[0] * NUM_IPBWT_BYTES]) + 40, _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[0] * NUM_IPBWT_BYTES]) + 79, _MM_HINT_T0); + } + if(meta.m[1] > 8) + { + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[1] + (meta.m[1] >> 1)) * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[1] + (meta.m[1] >> 1)) * NUM_IPBWT_BYTES]) + 8, _MM_HINT_T0); + } + else + { + _mm_prefetch((const char *)(&ipbwt_array[meta.first[1] * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[1] * NUM_IPBWT_BYTES]) + 40, _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[1] * NUM_IPBWT_BYTES]) + 79, _MM_HINT_T0); + } + } + else + { + if(meta.m[0] > 8) + { + last_mile_binary_search_one_step(meta.ipb_x[0], meta.first[0], meta.m[0]); + if(meta.m[0] > 8) + { + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[0] + (meta.m[0] >> 1)) * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[0] + (meta.m[0] >> 1)) * NUM_IPBWT_BYTES]) + 8, _MM_HINT_T0); + } + else + { + _mm_prefetch((const char *)(&ipbwt_array[meta.first[0] * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[0] * NUM_IPBWT_BYTES]) + 40, _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[0] * NUM_IPBWT_BYTES]) + 79, _MM_HINT_T0); + } + } + else + { + last_mile_vectorized_search_final_step(meta.ipb_x[0], meta.first[0], meta.m[0]); + } + if(meta.m[1] > 8) + { + last_mile_binary_search_one_step(meta.ipb_x[1], meta.first[1], meta.m[1]); + if(meta.m[1] > 8) + { + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[1] + (meta.m[1] >> 1)) * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[(meta.first[1] + (meta.m[1] >> 1)) * NUM_IPBWT_BYTES]) + 8, _MM_HINT_T0); + } + else + { + _mm_prefetch((const char *)(&ipbwt_array[meta.first[1] * NUM_IPBWT_BYTES]), _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[1] * NUM_IPBWT_BYTES]) + 40, _MM_HINT_T0); + _mm_prefetch((const char *)(&ipbwt_array[meta.first[1] * NUM_IPBWT_BYTES]) + 79, _MM_HINT_T0); + } + } + else + { + last_mile_vectorized_search_final_step(meta.ipb_x[1], meta.first[1], meta.m[1]); + } + if(meta.m[0] == 0 && meta.m[1] == 0) + { + p_low = meta.first[0]; + p_high = meta.first[1]; + return 0; + } + } + return 1; +} + + +template +void IPBWT_RMI::backward_extend_chunk_batched + (kenc_t* str_enc_list, int64_t qs_size, index_t *intv_all) { + + //eprintln("backward_extend_chunk_batched"); + BatchMetadata metadata[UNROLL]; + { + int64_t init_batch_size = min(UNROLL, qs_size); + int64_t next_query; + for(next_query = 0; next_query < init_batch_size; next_query++) + { + BatchMetadata m; + m.qid = next_query; + m.state = GUESS_RMI_ROOT; + m.ipb_x[0] = ipbwt(str_enc_list[next_query], intv_all[next_query * 2]+K); + m.ipb_x[1] = ipbwt(str_enc_list[next_query], intv_all[next_query * 2 + 1]+K); + m.key[0] = to_floating_point({str_enc_list[next_query], intv_all[next_query * 2]+K}); + m.key[1] = to_floating_point({str_enc_list[next_query], intv_all[next_query * 2 + 1]+K}); + metadata[next_query] = m; + } + int64_t num_active = next_query; + + while(num_active > 0) + { + for(int64_t i = 0; i < num_active; i++) + { + index_t pos_low, pos_high; + int status = process_query_one_step(metadata[i], pos_low, pos_high); + if(!status) + { + intv_all[2 * metadata[i].qid] = pos_low; + intv_all[2 * metadata[i].qid + 1] = pos_high; + if(next_query < qs_size) + { + // Get next query + BatchMetadata m; + m.qid = next_query; + m.state = GUESS_RMI_ROOT; + m.ipb_x[0] = ipbwt(str_enc_list[next_query], intv_all[next_query * 2]+K); + m.ipb_x[1] = ipbwt(str_enc_list[next_query], intv_all[next_query * 2 + 1]+K); + m.key[0] = to_floating_point({str_enc_list[next_query], intv_all[next_query * 2]+K}); + m.key[1] = to_floating_point({str_enc_list[next_query], intv_all[next_query * 2 + 1]+K}); + metadata[i] = m; + next_query++; + } + else + { + // No more new queries + metadata[i] = metadata[num_active - 1]; + num_active--; + } + } + } + } + } +} + +template +void IPBWT_RMI::backward_extend_multi_chunk_batched + (Info* str_enc_list, int64_t qs_size, index_t *intv_all) { + + //eprintln("backward_extend_chunk_batched"); + BatchMetadata metadata[UNROLL]; + { + int64_t init_batch_size = min(UNROLL, qs_size); + int64_t next_query; + for(next_query = 0; next_query < init_batch_size; next_query++) + { + BatchMetadata m; + m.qid = next_query; + m.state = GUESS_RMI_ROOT; + uint64_t enc_str = str_enc_list[next_query].get_enc_str(); + m.ipb_x[0] = ipbwt(enc_str, intv_all[next_query * 2]+K); + m.ipb_x[1] = ipbwt(enc_str, intv_all[next_query * 2 + 1]+K); + m.key[0] = to_floating_point({enc_str, intv_all[next_query * 2]+K}); + m.key[1] = to_floating_point({enc_str, intv_all[next_query * 2 + 1]+K}); + metadata[next_query] = m; + } + int64_t num_active = next_query; + + while(num_active > 0) + { + for(int64_t i = 0; i < num_active; i++) + { + index_t pos_low, pos_high; + int status = process_query_one_step(metadata[i], pos_low, pos_high); + if(!status) + { + intv_all[2 * metadata[i].qid] = pos_low; + intv_all[2 * metadata[i].qid + 1] = pos_high; + if(next_query < qs_size) + { + // Get next query + BatchMetadata m; + m.qid = next_query; + m.state = GUESS_RMI_ROOT; + uint64_t enc_str = str_enc_list[next_query].get_enc_str(); + m.ipb_x[0] = ipbwt(enc_str, intv_all[next_query * 2]+K); + m.ipb_x[1] = ipbwt(enc_str, intv_all[next_query * 2 + 1]+K); + m.key[0] = to_floating_point({enc_str, intv_all[next_query * 2]+K}); + m.key[1] = to_floating_point({enc_str, intv_all[next_query * 2 + 1]+K}); + metadata[i] = m; + next_query++; + } + else + { + // No more new queries + metadata[i] = metadata[num_active - 1]; + num_active--; + } + } + } + } + } +} diff --git a/ext/TAL/LISA/src/qbwt-ipbwt-rmi.h b/ext/TAL/LISA/src/qbwt-ipbwt-rmi.h new file mode 100644 index 00000000..323ad996 --- /dev/null +++ b/ext/TAL/LISA/src/qbwt-ipbwt-rmi.h @@ -0,0 +1,465 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +#include "fmi.h" +#include "sais.h" +#include "ipbwt_rmi.h" +template +class QBWT_HYBRID { + public: + QBWT_HYBRID(string t, index_t t_size, string ref_seq_filename, int K, int64_t num_rmi_leaf_nodes); + ~QBWT_HYBRID(); + pair* all_SMEMs(const char* p, const int p_len, pair* ans_ptr, const int min_seed_length) const; + pair* print_all_SMEMs(const char* p, const int p_len, pair* ans_ptr, const int min_seed_length, const int &shift) const; + const index_t n; + typedef typename FMI::Interval Interval; + const Interval init_intv; + void forward_step(const char* p, Interval &intv, int &l, int &r) const; + void backward_step(const char* p, Interval &intv, int &l, int &r) const; + + // private: + FMI *fmi; + typedef float linreg_t; + typedef uint64_t kenc_t; + IPBWT_RMI *rmi; + + struct LcpInfo { + uint16_t s_width: 12; + uint8_t bw_ext_msk: 4; + }; + static constexpr uint16_t WID_MAX = (1<<12)-1; + LcpInfo *lcpi; + vector> b_width; + uint8_t *lcpp1; + static constexpr uint8_t LCPP1_MAX = (1<<8)-1; + vector> large_lcpp1; + inline int64_t get_lcp(index_t i) const { + return (lcpp1[i] == LCPP1_MAX ? + (int64_t)lower_bound(large_lcpp1.begin(), large_lcpp1.end(), i, + [&](pair p, index_t q){return p.first < q;})->second: + (int64_t)lcpp1[i]) - 1; + } + pair forward_shrink_phase(Interval intv, char a) const; + pair advance_chunk(kenc_t first, pair intv) const; + void load(string filename); + void save(string filename) const; +}; + + +template +pair QBWT_HYBRID::advance_chunk(kenc_t first, pair intv) const { + return rmi->backward_extend_chunk(first, {intv.first, intv.second}); +} +template +void QBWT_HYBRID::load(string filename) { + ifstream instream(filename.c_str(), ifstream::binary); + instream.seekg(0); + +//#ifndef HUGE_PAGE + lcpp1 = new uint8_t[n+1](); +//#else +// lcpp1 = (uint8_t*) mmap(NULL, sizeof(lcpp1[0]) * (n+1), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); +//#endif + + + eprintln("MEM-SIZE: lcp1 %lld", (long long)sizeof(lcpp1[0])*(n+1)); + instream.read((char*)lcpp1, (n+1)*sizeof(lcpp1[0])); + +#ifndef HUGE_PAGE + lcpi = new LcpInfo[n+1](); +#else + lcpi = (LcpInfo*) mmap(NULL, sizeof(lcpi[0])*(n+1), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); +#endif + eprintln("MEM-SIZE: lcpi %lld", (long long)sizeof(lcpi[0])*(n+1)); + instream.read((char*)lcpi, (n+1)*sizeof(lcpi[0])); + + int64_t siz; + instream.read((char*)&siz, sizeof(siz)); + b_width.resize(siz); + eprintln("MEM-SIZE: b_width %lld", (long long)sizeof(b_width[0])*siz); + +#define LOADV(v) do{\ + for(auto itl=v.begin(), itr=itl; itl::type>::type tmp[Z];\ + int len = itr-itl;\ + instream.read((char*)tmp, len*sizeof(tmp[0]));\ + for(int i=0; i +void QBWT_HYBRID::save(string filename) const { + ofstream outstream(filename.c_str(), ofstream::binary); + outstream.seekp(0); + + outstream.write((char*)lcpp1, (n+1)*sizeof(lcpp1[0])); + + outstream.write((char*)lcpi, (n+1)*sizeof(lcpi[0])); + + int64_t siz = (int64_t)b_width.size(); + outstream.write((char*)&siz, sizeof(siz)); +#define SAVEV(v) do{\ + for(auto itl=v.begin(), itr=itl; itl::type>::type tmp[Z];\ + int len = itr-itl;\ + for(int i=0; i +pair::Interval, index_t> QBWT_HYBRID::forward_shrink_phase(Interval intv, char a) const { + + index_t e[2] = {intv.low, intv.high}; + LcpInfo info[2] = {lcpi[e[0]], lcpi[e[1]]}; +#define GET_WIDTH(i) (info[i].s_width == WID_MAX ? \ + (index_t)lower_bound(b_width.begin(), b_width.end(), e[i], \ + [&](pair p, index_t q){return p.first < q;})->second: \ + (index_t)info[i].s_width) + index_t siz[2] = {GET_WIDTH(0), GET_WIDTH(1)}; + +#ifndef NO_DNA_ORD + const uint8_t msk = 1< e[big] ? e[big] + siz[small] : e[big] - siz[small]);\ + info[small] = lcpi[e[small]];\ + siz[small] = GET_WIDTH(small);\ + }while(0) + + for(small = siz[1] < siz[0]; !CAN_BW_EXTEND(small); small = siz[1] < siz[0]) { + FORWARD_SHRINK; + } + + if(e[small] > e[big]) { + return {{e[big], e[big] + siz[small]}, get_lcp(e[small])}; + } else { + return {{e[big]-siz[small], e[big]}, get_lcp(e[small])}; + } +#undef FORWARD_SHRINK +#undef big +#undef CAN_BW_EXTEND +#undef GET_WIDTH +} + +template +inline void QBWT_HYBRID::\ + forward_step(const char *p, Interval &intv, int &l, int &r) const +{ + index_t siz; + tie(intv, siz) = forward_shrink_phase(intv, p[l-1]); + r = l + (int)siz; +} + +template +inline void QBWT_HYBRID::\ + backward_step(const char *p, Interval &intv, int &l, int &r) const +{ + for(int i=l-1; i>=0; i--) { + auto next = fmi->backward_extend(intv, p[i]); + if(next.low >= next.high) break; + else l=i, intv=next; + } +} + +template +pair* QBWT_HYBRID::all_SMEMs(const char* p, const int p_len, pair* ans_ptr, const int min_seed_length) const { + Interval intv = init_intv; + auto end = ans_ptr; + int l = p_len, r = p_len; // [l,r) + backward_step(p, intv, l, r); + if(r-l>=min_seed_length) { + *(end++) = make_pair(l, r); + } + + while(l != 0) { + forward_step(p, intv, l, r); + backward_step(p, intv, l, r); + if(r-l>=min_seed_length) { + *(end++) = make_pair(l, r); + } + } + + reverse(ans_ptr, end); + return end; +} + +template +pair* QBWT_HYBRID::print_all_SMEMs(const char* p, const int p_len, pair* ans_ptr, const int min_seed_length, const int &shift) const { + Interval intv = init_intv; + vector vs; + auto end = ans_ptr; + int l = p_len, r = p_len; // [l,r) + backward_step(p, intv, l, r); + if(r-l>=min_seed_length) { + *(end++) = make_pair(l, r); + vs.push_back(intv); + } + + while(l != 0) { + forward_step(p, intv, l, r); + backward_step(p, intv, l, r); + if(r-l>=min_seed_length) { + *(end++) = make_pair(l, r); + vs.push_back(intv); + } + } + + reverse(ans_ptr, end); + reverse(vs.begin(), vs.end()); + for(int i=0; i<(int)vs.size(); i++) { + printf("+ %ld %ld %lld %lld\n", + (long)ans_ptr[i].first+shift, (long)ans_ptr[i].second+shift,(long long) 1LL*vs[i].low, (long long)1LL*vs[i].high); + } + return end; +} + + +template +QBWT_HYBRID::QBWT_HYBRID(string t, index_t t_size, string ref_seq_filename, int K, int64_t num_rmi_leaf_nodes): +#ifdef REV_COMP +#ifndef BWA_MEM_BUG + n(2*(index_t)t_size+2), +#else + n(2*(index_t)t_size+1), +#endif +#else + n(t_size+1), +#endif + init_intv({0, n}) { + +#ifdef REV_COMP + string bin_filename = ref_seq_filename + ".qbwt4.walg.rev_comp"; +#else + string bin_filename = ref_seq_filename + ".qbwt4.walg"; +#endif + string rmi_filename = bin_filename; + for(const auto &s:{sizeof(index_t)}) { + bin_filename += string(".") + to_string(s); + } + if(ifstream(bin_filename.c_str()).good()) { + eprintln("Found existing %s!!", (char*)bin_filename.c_str()); + load(bin_filename); + fmi = new FMI(t.c_str(), n, NULL, "@"+dna, bin_filename); + rmi = new IPBWT_RMI(t, n, rmi_filename, K, num_rmi_leaf_nodes, NULL); + eprintln("Load successful."); + eprintln("large lcp size = %lu", large_lcpp1.size()); + eprintln("large lcp space usage = %.6fN", (double)(large_lcpp1.size()*sizeof(large_lcpp1[0])*1.0/n)); + eprintln("large width size = %lu", b_width.size()); + eprintln("large width space usage = %.6fN", (double)(b_width.size()*sizeof(b_width[0])*1.0/n)); + return; + } else { + eprintln("No existing %s. Building...", (char*)bin_filename.c_str()); + } + + + assert(t.find('@') == string::npos && t.find('$') == string::npos); + +#ifdef REV_COMP +#ifndef BWA_MEM_BUG + t.push_back('@'); +#else + eprintln("No char placed between ref seq and reverse complement, to replicate BWA-MEM bug."); +#endif + // appending reverse complement + for(int64_t i=(index_t)t.size()-1-(t.back()=='@');i>=0;i--) { +#ifndef NO_DNA_ORD + t.push_back(dna[3-dna_ord(t[i])]); +#else + t.push_back(dna[3-(__lg(t[i]-'A'+2)-1)]); +#endif + } +#endif + t.push_back('$'); + assert(n == (index_t)t.size()); + + + + + + vector sa(n); + if(numeric_limits::max() > n && numeric_limits::min() < 0) { + saisxx(t.c_str(), sa.data(), (index_t)n); + } else { + vector _sa(n); + saisxx(t.c_str(), _sa.data(), (int64_t)n); + for(index_t i=0;i(t.c_str(), sa.data(), "@"+dna, bin_filename); // do not directly call load/save! + rmi = new IPBWT_RMI(t, rmi_filename, K, num_rmi_leaf_nodes, sa.data()); + + // build rnk = sa^(-1) + + // build lcp + { + lcpp1 = new uint8_t[n+1](); + large_lcpp1.clear(); + auto write_lcp = [&](index_t i, int64_t result) { + if(result+1 >= LCPP1_MAX) { + large_lcpp1.push_back({i, (index_t)(result+1)}); + lcpp1[i] = LCPP1_MAX; + } else { + lcpp1[i] = (uint8_t)(result+1); + } + }; + { + vector rnk(n); + for(index_t i=0; i dec_lcp_order(n-1); { + int64_t max_lcp = LCPP1_MAX; + for(auto p: large_lcpp1) max_lcp = max(max_lcp, (int64_t)p.second); + vector cnt(max_lcp+1, 0); + for(index_t i=1; i<=n-1; i++) cnt[get_lcp(i)]++; + for(index_t j=1; j<(index_t)cnt.size(); j++) { + cnt[j] += cnt[j-1]; + } + for(int64_t i=n-1; i>=1; i--) { + dec_lcp_order[--cnt[get_lcp(i)]] = i; + } + reverse(dec_lcp_order.begin(), dec_lcp_order.end()); + assert(is_sorted(dec_lcp_order.begin(), dec_lcp_order.end(), [&](index_t i, index_t j){return make_pair(get_lcp(i), i) > make_pair(get_lcp(j),j);})); + eprintln("dec lcp ordering ok."); + } + + // il, ir for O(1) interval-merge algorithm + vector il(n); + iota(il.begin(), il.end(), ((index_t)0)); + vector ir=il; + + lcpi = new LcpInfo[n+1](); + + lcpi[0].s_width = lcpi[n].s_width = WID_MAX; + b_width.push_back({0,n+1}); + b_width.push_back({n,n+1}); + + + for(auto itl=dec_lcp_order.begin(), itr=itl; itl!=dec_lcp_order.end(); itl=itr) { + for(itr++; itr!=dec_lcp_order.end() && + get_lcp(*itr)==get_lcp(*itl) && + il[(*prev(itr))-1] == *itr; itr++); + + if(itr-itl>1) reverse(itl+1, itr); + + for(auto it=itl;it!=itr;it++) { + const index_t i = *it; + const index_t l = il[i-1]; + const index_t r = ir[i]; + il[r]=l; + ir[l]=r; + auto w = r-l+1; + if(w>=WID_MAX) { + b_width.push_back({i, w}); + lcpi[i].s_width = WID_MAX; + } else { + lcpi[i].s_width = (uint16_t)w; + } + if(it+1==itr) { + uint8_t my_mask = 0; + for(int j=0;j<(int)dna.size();j++) { + auto intv = fmi->backward_extend({l, r+1}, dna[j]); + if(intv.low < intv.high) my_mask |= 1< +QBWT_HYBRID::~QBWT_HYBRID(){ + eprintln("qbwt rmi deallocated"); + delete fmi; + delete rmi; + + delete lcpp1; + +#ifndef HUGE_PAGE + delete lcpi; +#else + munmap(lcpi, sizeof(lcpi[0])*(n+1)); +#endif + +} + diff --git a/ext/TAL/LISA/src/qbwt-rmi-batched.h b/ext/TAL/LISA/src/qbwt-rmi-batched.h new file mode 100644 index 00000000..24f1014d --- /dev/null +++ b/ext/TAL/LISA/src/qbwt-rmi-batched.h @@ -0,0 +1,364 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +#ifdef VTUNE_ANALYSIS +#include +#endif +#include "common.h" +#include +#include "read.h" +#include +#include +#include + +#ifdef ENABLE_PREFETCH +#define my_prefetch(a, b) _mm_prefetch(a, b) +#else +#define my_prefetch(a, b) +#endif +#include "qbwt-ipbwt-rmi.h" +#include "chunkEncode.h" + +class threadData { + + public: + + Info *chunk_pool; + int chunk_cnt ; + uint64_t *str_enc; + int64_t *intv_all; + + int64_t numSMEMs; + Info *fmi_pool; + int fmi_cnt; + + Info *tree_pool; + int tree_cnt; + index_t **s_siz; + QBWT_HYBRID::LcpInfo **s_info; + uint8_t *s_msk; + threadData(int64_t pool_size){ + + numSMEMs = 0; + chunk_pool = (Info*) aligned_alloc(64, sizeof(Info)*pool_size); + chunk_cnt = 0; + str_enc = (uint64_t *)aligned_alloc(64, pool_size * sizeof(uint64_t)); + intv_all = (int64_t *)aligned_alloc(64, pool_size * 2 * sizeof(int64_t)); + fmi_pool = (Info*) aligned_alloc(64, sizeof(Info)*pool_size); + fmi_cnt = 0; + + tree_pool = (Info*) aligned_alloc(64, sizeof(Info)*pool_size); + tree_cnt = 0; + index_t* s_siz_one_d = (index_t*) aligned_alloc(64, sizeof(index_t) * 2 * pool_size); + s_siz = (index_t**) aligned_alloc(64, sizeof(index_t*) * pool_size); + for(int64_t i = 0; i < 2*pool_size; i = i + 2) + s_siz[i/2] = &s_siz_one_d[i]; + + + QBWT_HYBRID::LcpInfo* s_info_one_d = (QBWT_HYBRID::LcpInfo*) aligned_alloc(64, sizeof(QBWT_HYBRID::LcpInfo) * 2 * pool_size); + + s_info = (QBWT_HYBRID::LcpInfo**) aligned_alloc(64, sizeof(QBWT_HYBRID::LcpInfo*)*pool_size); + for(int64_t i = 0; i < 2*pool_size; i = i + 2) + s_info[i/2] = &s_info_one_d[i]; + + s_msk = (uint8_t*) aligned_alloc(64, sizeof(uint8_t) * pool_size); + + } + + +}; + + +#define S_SWP_END do{\ + int c; \ + if(cnt > pref_dist)\ + c = --cnt;\ + else\ + c = --shrink_batch_size;\ + q = tree_pool[c]; \ + siz[0] = s_siz[c][0]; siz[1] = s_siz[c][1]; \ + info[0] = s_info[c][0]; info[1] = s_info[c][1]; \ + msk = s_msk[c]; \ +}while(0) + +#define S_RUN do{\ + bool small = siz[1] < siz[0]; \ + if(info[small].bw_ext_msk & msk) { \ + if(small) { \ + q.r = q.l + qbwt.get_lcp(q.intv.second); \ + q.intv.second = q.intv.first + siz[1]; \ + } else { \ + q.r = q.l + qbwt.get_lcp(q.intv.first); \ + q.intv.first = q.intv.second - siz[0]; \ + } \ + fmi_pool[fmi_cnt++] = q;\ + S_SWP_END;\ + } else { \ + if(small) { \ + q.intv.second = q.intv.first + siz[1]; \ + info[1] = qbwt.lcpi[q.intv.second]; \ + siz[1] = GET_WIDTH(info[1].s_width, q.intv.second); \ + } else { \ + q.intv.first = q.intv.second - siz[0]; \ + info[0] = qbwt.lcpi[q.intv.first]; \ + siz[0] = GET_WIDTH(info[0].s_width, q.intv.first); \ + } \ + }\ +}while(0) + + +#define S_PREFETCH do{\ + bool small = siz[1] < siz[0]; \ + my_prefetch((const char*)(qbwt.lcpp1 + (small ? q.intv.second : q.intv.first)), _MM_HINT_T0); \ + my_prefetch((const char*)(qbwt.lcpi + (small ? \ + q.intv.first + siz[1] : q.intv.second - siz[0])), _MM_HINT_T0); \ +}while(0) + +#define GET_WIDTH(A, B) (A == qbwt.WID_MAX ? \ + (index_t)lower_bound(qbwt.b_width.begin(), qbwt.b_width.end(), B, \ + [&](pair p, index_t qq){return p.first < qq;})->second: \ + (index_t)A) + + +#define S_LOAD(i) \ + Info &q = tree_pool[i]; \ + index_t* siz = s_siz[i]; \ + QBWT_HYBRID::LcpInfo* info = s_info[i]; \ + uint8_t &msk = s_msk[i] + +inline void s_pb(QBWT_HYBRID &qbwt, Info &_q, int cnt, threadData &td) { + + Info &q = td.tree_pool[cnt]; + index_t* siz = td.s_siz[cnt]; + QBWT_HYBRID::LcpInfo* info = td.s_info[cnt]; + uint8_t &msk = td.s_msk[cnt]; + q = _q; + info[0] = qbwt.lcpi[q.intv.first]; info[1] = qbwt.lcpi[q.intv.second]; + siz[0] = GET_WIDTH(info[0].s_width, q.intv.first); siz[1] = GET_WIDTH(info[1].s_width, q.intv.second); +#ifndef NO_DNA_ORD + msk = 1< &qbwt, int cnt, threadData &td){ + + Info* tree_pool = td.tree_pool; + index_t** s_siz = td.s_siz; + QBWT_HYBRID::LcpInfo** s_info = td.s_info; + uint8_t* s_msk = td.s_msk; + + Info* fmi_pool = td.fmi_pool; + int &fmi_cnt = td.fmi_cnt; + + int pref_dist = 50; + int shrink_batch_size = pref_dist = min(pref_dist, cnt); + pref_dist = shrink_batch_size; + + while(shrink_batch_size > 0) { + for(int i = 0; i < shrink_batch_size; i++){ + S_LOAD(i); + S_RUN; + S_PREFETCH; + } + } +} + + + +void fmi_extend_batched( QBWT_HYBRID &qbwt, int cnt, Info* q_batch, threadData &td, Output* output, int min_seed_len){ + + Info* tree_pool = td.tree_pool; + int &tree_cnt = td.tree_cnt; + + int pref_dist = 30; + int fmi_batch_size = pref_dist = min(pref_dist, cnt); + pref_dist = fmi_batch_size; + + Info pf_batch[fmi_batch_size]; + + auto cnt1 = fmi_batch_size; + + // prepare first batch + for(int i = 0; i < fmi_batch_size; i++){ + Info &q = q_batch[i]; + static constexpr int INDEX_T_BITS = sizeof(index_t)*__CHAR_BIT__; + static constexpr int shift = __lg(INDEX_T_BITS); + auto ls = ((q.intv.first>>shift)<<3), hs = ((q.intv.second>>shift)<<3); + my_prefetch((const char*)(qbwt.fmi->occb + ls), _MM_HINT_T0); + my_prefetch((const char*)(qbwt.fmi->occb + hs), _MM_HINT_T0); + my_prefetch((const char*)(q.p + q.l - 1) , _MM_HINT_T0); + } + + while(fmi_batch_size > 0) { + + for(int i = 0; i < fmi_batch_size; i++){ + Info &q = q_batch[i]; + //process one step + int it = q.l -1; + + if(it >=0){ + auto next = qbwt.fmi->backward_extend({q.intv.first, q.intv.second}, q.p[it]); + if(next.low >= next.high) { + + tree_pool[tree_cnt++] = q; //State change + + if(q.r - q.l >= min_seed_len){ +#ifdef OUTPUT +/* output[q.id].qPos.push_back({q.l, q.r}); + output[q.id].refPos.push_back(q.intv); + output[q.id].smem.push_back(SMEM_out(q.id, q.l, q.r, q.intv.first, q.intv.second)); +*/ + output->smem[td.numSMEMs++] = SMEM_out(q.id, q.l, q.r, q.intv.first, q.intv.second); +#endif + } + if(cnt1< cnt) //More queries to be processed? + q = q_batch[cnt1++]; //direction + + else + q = q_batch[--fmi_batch_size]; + my_prefetch((const char*)(q.p + q.l - 1) , _MM_HINT_T0); + + } + else { + q.l = it, q.intv={next.low, next.high}; //fmi-continue + } + } + else{ + //query finished + if(q.r - q.l >= min_seed_len){ +#ifdef OUTPUT +// output[q.id].qPos.push_back({q.l, q.r}); +// output[q.id].refPos.push_back(q.intv); +// output[q.id].smem.push_back(SMEM_out(q.id, q.l, q.r, q.intv.first, q.intv.second)); + + output->smem[td.numSMEMs++] = SMEM_out(q.id, q.l, q.r, q.intv.first, q.intv.second); +#endif + } + if(cnt1 < cnt) //More queries to be processed? + q = q_batch[cnt1++]; + else + q = q_batch[--fmi_batch_size]; + my_prefetch((const char*)(q.p + q.l - 1) , _MM_HINT_T0); + } + static constexpr int INDEX_T_BITS = sizeof(index_t)*__CHAR_BIT__; + static constexpr int shift = __lg(INDEX_T_BITS); + auto ls = ((q.intv.first>>shift)<<3), hs = ((q.intv.second>>shift)<<3); + my_prefetch((const char*)(qbwt.fmi->occb + ls + 4), _MM_HINT_T0); + my_prefetch((const char*)(qbwt.fmi->occb + hs + 4), _MM_HINT_T0); + } + } +} + +void smem_rmi_batched(Info *qs, int64_t qs_size, int64_t batch_size, QBWT_HYBRID &qbwt, threadData &td, Output* output, int min_seed_len){ + Info *chunk_pool = td.chunk_pool; + int &chunk_cnt = td.chunk_cnt; + + uint64_t *str_enc = td.str_enc; + int64_t *intv_all = td.intv_all; + + Info* fmi_pool = td.fmi_pool; + int &fmi_cnt = td.fmi_cnt; + + Info* tree_pool = td.tree_pool; + int &tree_cnt = td.tree_cnt; + + int K = qbwt.rmi->K; + + + int64_t next_q = 0; + while(next_q < qs_size || (chunk_cnt + fmi_cnt ) > 0){ + while(next_q < qs_size && chunk_cnt < batch_size && fmi_cnt < batch_size){ + if(qs[next_q].r >= K){ + chunk_pool[chunk_cnt++] = qs[next_q++]; + } + else + fmi_pool[fmi_cnt++] = qs[next_q++]; + } + + // process chunk batch + if(next_q >= qs_size || !(chunk_cnt < batch_size)){ + // prepareChunkBatchVectorized_v1(chunk_pool, chunk_cnt, str_enc, intv_all); + prepareChunkBatch(chunk_pool, chunk_cnt, str_enc, intv_all, K); + + qbwt.rmi->backward_extend_chunk_batched(&str_enc[0], chunk_cnt, intv_all); + auto cnt = chunk_cnt; + chunk_cnt = 0; + for(int64_t j = 0; j < cnt; j++) { + Info &q = chunk_pool[j]; + auto next_intv = make_pair(intv_all[2*j],intv_all[2*j + 1]); + + // next state: chunk batching + if(next_intv.first < next_intv.second){ + q.intv = next_intv; + q.l -= K; + + if(q.l >= K)// heuristics: && !(q.intv.second - q.intv.first > 1 && q.intv.second - q.intv.first < 100)) + chunk_pool[chunk_cnt++] = q; + else if(q.l > 0){ + fmi_pool[fmi_cnt++] = q; + } + else if(q.r - q.l >= min_seed_len){ + +#ifdef OUTPUT + // output[q.id].qPos.push_back({q.l, q.r}); + // output[q.id].refPos.push_back(q.intv); + // output[q.id].smem.push_back(SMEM_out(q.id, q.l, q.r, q.intv.first, q.intv.second)); + output->smem[td.numSMEMs++] = SMEM_out(q.id, q.l, q.r, q.intv.first, q.intv.second); +#endif + } + } + else { + //Next state: fmi procssing + fmi_pool[fmi_cnt++] = q; + } + + } + } + + // fmi processing + if(next_q >= qs_size || !(fmi_cnt < batch_size)){ + auto cnt = fmi_cnt; + fmi_cnt = 0; + + fmi_extend_batched(qbwt, cnt, &fmi_pool[0], td, output, min_seed_len); + + cnt = tree_cnt; + tree_cnt = 0; + for(int i = 0; i < cnt; i++) { + auto &q = tree_pool[i]; + s_pb(qbwt, q, i, td); + my_prefetch((const char*)(qbwt.lcpi + tree_pool[i+50].intv.first) , _MM_HINT_T0); + my_prefetch((const char*)(qbwt.lcpi + tree_pool[i+50].intv.second) , _MM_HINT_T0); + my_prefetch((const char*)(tree_pool[i + 50].p + tree_pool[i + 50].l - 1) , _MM_HINT_T0); + } + tree_shrink_batched(qbwt, cnt, td); + } + } + + +} diff --git a/ext/TAL/LISA/src/read.h b/ext/TAL/LISA/src/read.h new file mode 100644 index 00000000..8f5e5ab3 --- /dev/null +++ b/ext/TAL/LISA/src/read.h @@ -0,0 +1,178 @@ +/************************************************************************************* +MIT License + +Copyright (c) 2020 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Saurabh Kalikar ; Sanchit Misra +*****************************************************************************************/ +#include +#include +#include "kseq.h" +#include "utils.h" +//#include "bwa.h" +KSEQ_INIT(gzFile, gzread) + + +int64_t pac_seq_len(const char *fn_pac) +{ + FILE *fp; + int64_t pac_len; + uint8_t c; + fp = xopen(fn_pac, "rb"); + err_fseek(fp, -1, SEEK_END); + pac_len = err_ftell(fp); + err_fread_noeof(&c, 1, 1, fp); + err_fclose(fp); + return (pac_len - 1) * 4 + (int)c; +} + +void read_seq_lisa(string filename, string &reference_seq){ + uint8_t *buf2; + int64_t i, pac_size, seq_len; + FILE *fp; + filename += ".pac"; + const char *fn_pac = filename.c_str(); + + // initialization + seq_len = pac_seq_len(fn_pac); + assert(seq_len > 0); + assert(seq_len <= 0x7fffffffffL); + fp = xopen(fn_pac, "rb"); + + // prepare sequence + pac_size = (seq_len>>2) + ((seq_len&3) == 0? 0 : 1); + buf2 = (uint8_t*)calloc(pac_size, 1); + assert(buf2 != NULL); + err_fread_noeof(buf2, 1, pac_size, fp); + err_fclose(fp); + for (i = 0; i < seq_len; ++i) { + int nt = buf2[i>>2] >> ((3 - (i&3)) << 1) & 3; + switch(nt) + { + case 0: + reference_seq += "A"; + break; + case 1: + reference_seq += "C"; + break; + case 2: + reference_seq += "G"; + break; + case 3: + reference_seq += "T"; + break; + default: + fprintf(stderr, "ERROR! Value of nt is not in 0,1,2,3!"); + exit(EXIT_FAILURE); + } + } + free(buf2); +} + +string read_seq(string filename) { + static char in[123465]; + FILE* fp = fopen(filename.c_str(),"r"); + assert(fp != NULL); + fscanf(fp,"%123464[^\n]",in); + assert(in[0] == '>'); + + string result = ""; + while(fscanf(fp,"%123464s",in)==1) { + result += in; + } + static_assert(sizeof(result.size()) * __CHAR_BIT__ == 64, "you are not using 64-bit system?!"); + eprintln("Read ref seq length = %lu", result.size()); + + for(char &c:result) if(!strchr(dna.c_str(),c)) { + c = dna[rand()%4]; + // error_quit("Error: ref file not in upper-case ACGT format"); + } + fclose(fp); + return result; +} +pair read_query(string filename) { + //rem: static char in[123465]; + gzFile fp = gzopen(filename.c_str(), "r"); + kseq_t *ks = kseq_init(fp); + + string ret; + int num_queries = 0; + int max_query_len = 0; + int64_t m, n; + //rem: char *seq; + m = n = 0; + while (kseq_read(ks) >= 0) + { + num_queries++; + string cur = string(ks->seq.s); + + max_query_len = max(max_query_len, (int)cur.size()); + //for(char &c:cur) if(!strchr(dna.c_str(),c)) { + // c = '.'; + // // error_quit("Error: query set file not in upper-case ACGT format"); + //} + ret += cur; + } + gzclose(fp); + eprintln("Detected num of queries = %d, max_query_len = %d", num_queries, max_query_len); + + //Fix: Memory leak + kseq_destroy(ks); + return {ret, max_query_len}; +} + + + +pair read_query_separated_with_dot(string filename) { + //rem: static char in[123465]; + gzFile fp = gzopen(filename.c_str(), "r"); + kseq_t *ks = kseq_init(fp); + + string ret; + int num_queries = 0; + int max_query_len = 0; + int64_t m, n; + //rem: char *seq; + m = n = 0; + while (kseq_read(ks) >= 0) + { + num_queries++; + string cur = string(ks->seq.s); + max_query_len = max(max_query_len, (int)cur.size()); + for(char &c:cur) if(!strchr(dna.c_str(),c)) { + c = '.'; + // error_quit("Error: query set file not in upper-case ACGT format"); + } + ret += cur + ";"; + } + gzclose(fp); + eprintln("Detected num of queries = %d, max_query_len = %d", num_queries, max_query_len); + //Fix: Memory leak + kseq_destroy(ks); + return {ret, max_query_len}; +} + + + + + + + diff --git a/ext/TAL/Makefile b/ext/TAL/Makefile new file mode 100644 index 00000000..a7ca1273 --- /dev/null +++ b/ext/TAL/Makefile @@ -0,0 +1,137 @@ +##/************************************************************************************* +## The MIT License +## +## Copyright (C) 2020 Intel Labs. +## +## Permission is hereby granted, free of charge, to any person obtaining a copy +## of this software and associated documentation files (the "Software"), to deal +## in the Software without restriction, including without limitation the rights +## to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +## copies of the Software, and to permit persons to whom the Software is +## furnished to do so, subject to the following conditions: +## +## The above copyright notice and this permission notice shall be included in all +## copies or substantial portions of the Software. +## +## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +## SOFTWARE. +## +##Contacts: Sanchit Misra ; Vasimuddin Md +## Saurabh Kalikar +##*****************************************************************************************/ + +BUILD_INDEX_FOR_ONLY= build-index-forward-only +BUILD_INDEX_WITH_RC= build-index-with-rev-complement +BENCH_SMEM= bench-smem +BENCH_FIXED_LEN_E2E= bench-fixed-len-e2e-match +#CXX= icpc +ifeq ($(CXX), icpc) + CC= icc +else ifeq ($(CXX), g++) + CC=gcc +endif + +ARCH_FLAGS= -msse4.1 +#ARCH_FLAGS= -mavx512bw +MEM_FLAGS= -DSAIS=1 +CPPFLAGS= -DENABLE_PREFETCH $(MEM_FLAGS) -DKSW=1 +INCLUDES= -Iext -Iext/safestringlib/include -Isrc/FMI/ +LIBS= -fopenmp -lm -lz -L. -ltal -Lext/safestringlib/ -lsafestring +OBJS= ext/utils.o \ + ext/kstring.o ext/bntseq.o \ + src/FMI/FMI_search.o ext/bseq.o +TAL_LIB= libtal.a +SAFE_STR_LIB= ext/safestringlib/libsafestring.a + +ifeq ($(arch),sse) + ARCH_FLAGS=-msse4.1 +else ifeq ($(arch),avx2) + ifeq ($(CXX), icpc) + ARCH_FLAGS=-march=core-avx2 #-xCORE-AVX2 + else + ARCH_FLAGS=-mavx2 + endif +else ifeq ($(arch),avx512) + ifeq ($(CXX), icpc) + ARCH_FLAGS=-xCORE-AVX512 + else + ARCH_FLAGS=-mavx512bw + endif +else ifeq ($(arch),native) + ARCH_FLAGS=-march=native +else ifneq ($(arch),) +## To provide a different architecture flag like -march=core-avx2. + ARCH_FLAGS=$(arch) +endif + +#CXXFLAGS= -g -O3 -fpermissive $(ARCH_FLAGS) #-Wall ##-xSSE2 +CXXFLAGS= -g -O3 -fopenmp $(ARCH_FLAGS) + +#.PHONY:all clean depend +.PHONY:all clean depend +#.PHONY:all clean +.SUFFIXES:.cpp .o + +.cpp.o: + $(CXX) -c $(CXXFLAGS) $(CPPFLAGS) $(INCLUDES) $< -o $@ + +all:$(BUILD_INDEX_FOR_ONLY) $(BUILD_INDEX_WITH_RC) $(BENCH_SMEM) $(BENCH_FIXED_LEN_E2E) + +$(BUILD_INDEX_FOR_ONLY):$(TAL_LIB) $(SAFE_STR_LIB) benchmarks/build-index-forward-only.o + $(CXX) $(CXXFLAGS) benchmarks/build-index-forward-only.o $(LIBS) -o $@ + +$(BUILD_INDEX_WITH_RC):$(TAL_LIB) $(SAFE_STR_LIB) benchmarks/build-index-with-rev-complement.o + $(CXX) $(CXXFLAGS) benchmarks/build-index-with-rev-complement.o $(LIBS) -o $@ + +$(BENCH_SMEM):$(TAL_LIB) $(SAFE_STR_LIB) benchmarks/bench-smem.o + $(CXX) $(CXXFLAGS) benchmarks/bench-smem.o $(LIBS) -o $@ + +$(BENCH_FIXED_LEN_E2E):$(TAL_LIB) $(SAFE_STR_LIB) benchmarks/bench-fixed-len-e2e-match.o + $(CXX) $(CXXFLAGS) benchmarks/bench-fixed-len-e2e-match.o $(LIBS) -o $@ + +$(TAL_LIB):$(OBJS) + ar rcs $(TAL_LIB) $(OBJS) + +$(SAFE_STR_LIB): + cd ext/safestringlib/ && make clean && make CC=$(CC) directories libsafestring.a + +clean: + rm -fr src/*.o ext/*.o benchmarks/*.o $(TAL_LIB) $(BUILD_INDEX_FOR_ONLY) $(BUILD_INDEX_WITH_RC) $(BENCH_SMEM) $(BENCH_FIXED_LEN_E2E) + cd ext/safestringlib && make CC=$(CC) clean + +depend: + (LC_ALL=C; export LC_ALL; makedepend -Y -- $(CXXFLAGS) $(CPPFLAGS) -I. -- src/*.cpp) + +KEY_TYPE=-DUINT64 +ifeq ($(key_type),f64) + KEY_TYPE=-DF64 +else ifeq ($(key_type),uint64) + KEY_TYPE=-DUINT64 +endif + +#LISA hash +lisa_hash: ./benchmarks/build-lisa-hash-index.cpp ./src/LISA-hash/lisa_hash.h + $(CXX) ./benchmarks/build-lisa-hash-index.cpp -o build-lisa-hash-index -Ofast -DVECTORIZE ${KEY_TYPE} -I./src/LISA-hash -march=native + +#Dynamic programming based chaining benchmark +dp_chain: ./benchmarks/bench-dp-chaining.cpp ./src/dynamic-programming/parallel_chaining_32_bit.h + $(CXX) ./benchmarks/bench-dp-chaining.cpp -I src/dynamic-programming/ -Ofast -o bench-dp-chaining -march=native -DDEBUG + + +# DO NOT DELETE + +src/FMI/FMI_search.o: src/FMI/FMI_search.h ext/bntseq.h +src/FMI/FMI_search.o: ext/utils.h ext/sais.h +ext/bntseq.o: ext/bntseq.h ext/utils.h ext/kseq.h +#ext/bwa.o: ext/bntseq.h ext/bwa.h ext/utils.h +#ext/bwa.o: ext/kstring.h ext/kseq.h +ext/kstring.o: ext/kstring.h +ext/utils.o: ext/utils.h ext/kseq.h +ext/bseq.o: ext/bseq.h ext/utils.h ext/kseq.h +benchmarks/bench-smem.o: src/FMI/FMI_search.h ext/bntseq.h ext/utils.h +benchmarks/bench-smem.o: ext/sais.h diff --git a/ext/TAL/README.md b/ext/TAL/README.md new file mode 100644 index 00000000..12ffa64c --- /dev/null +++ b/ext/TAL/README.md @@ -0,0 +1,22 @@ +# Trans-Omics Acceleration Library + +## Introduction +The Trans-Omics Acceleration Library (TAL) contains key modules for Trans-Omics analysis that are optimized for Xeon. + +The contributors to TAL are Sanchit Misra, Vasimuddin Md, Saurabh Kalikar, Narendra Chaudhary and Kanak Mahadik. + +## Getting Started +```sh +git clone --recursive +make +``` + +## Publications + +Vasimuddin Md, Sanchit Misra, Heng Li, Srinivas Aluru. + Efficient Architecture-Aware Acceleration of BWA-MEM for Multicore Systems. + IEEE Parallel and Distributed Processing Symposium (IPDPS), 2019. + +Sanchit Misra, Tony Pan, Kanak Mahadik, George Powley, Priya N Vaidya, Md Vasimuddin, Srinivas Aluru. + Performance extraction and suitability analysis of multi- and many-core architectures for next generation sequencing secondary analysis. + International Conference on Parallel Architectures and Compilation Techniques (PACT), 2018. diff --git a/ext/TAL/applications/ATAC-Seq/AtacWorks_cpu_optimization_patch.patch b/ext/TAL/applications/ATAC-Seq/AtacWorks_cpu_optimization_patch.patch new file mode 100644 index 00000000..2c64d6bd --- /dev/null +++ b/ext/TAL/applications/ATAC-Seq/AtacWorks_cpu_optimization_patch.patch @@ -0,0 +1,935 @@ +diff --git a/Dockerfile b/Dockerfile +index 4f1ce56..79f5ab4 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -31,3 +31,4 @@ RUN git clone --recursive https://github.com/clara-genomics/AtacWorks.git + + # Install AtacWorks requirements + RUN pip3 install -r AtacWorks/requirements-base.txt && pip3 install -r AtacWorks/requirements-macs2.txt ++RUN pip install . +diff --git a/claragenomics/dl4atac/dataset.py b/claragenomics/dl4atac/dataset.py +index 92b00a9..11b2586 100755 +--- a/claragenomics/dl4atac/dataset.py ++++ b/claragenomics/dl4atac/dataset.py +@@ -86,6 +86,7 @@ class DatasetTrain(DatasetBase): + idx: Index for which the batch is to be returned. + + """ ++ + if self._h5_gen is None: + self._h5_gen = self._get_generator() + next(self._h5_gen) +@@ -104,6 +105,7 @@ class DatasetTrain(DatasetBase): + for i, filename in enumerate(self.files): + # print('loading H5Py file %s' % filename) + hf = h5py.File(filename, 'r') ++ + # Read noisy data and labels + for key in hf.keys(): + hrecs[key].append(hf[key]) +@@ -123,7 +125,7 @@ class DatasetTrain(DatasetBase): + rec['input'], + hrecs[layer_key][file_id][local_idx])) + rec['input'] = np.swapaxes(rec['input'], 0, 1) +- yield rec ++ idx = yield rec # Inportant (Change) + + + class DatasetInfer(DatasetBase): +diff --git a/claragenomics/dl4atac/evaluate.py b/claragenomics/dl4atac/evaluate.py +index d4f5b35..27962fb 100755 +--- a/claragenomics/dl4atac/evaluate.py ++++ b/claragenomics/dl4atac/evaluate.py +@@ -52,7 +52,6 @@ def evaluate(*, rank, gpu, task, model, val_loader, metrics_reg, + print('Eval for %d batches' % num_batches) + with torch.no_grad(): + for i, batch in enumerate(val_loader): +- # idxes = batch['idx'] + x = batch['input'] + y_reg = batch['label_reg'] + y_cla = batch['label_cla'] +@@ -68,7 +67,9 @@ def evaluate(*, rank, gpu, task, model, val_loader, metrics_reg, + x = x.unsqueeze(1) # (N, 1, L) + else: + x = np.swapaxes(x, 1, 2) +- x = x.cuda(gpu, non_blocking=True) ++ # x = x.cuda(gpu, non_blocking=True) ++ if gpu >= 0: # (Change) ++ x = x.cuda(gpu, non_blocking=True) + + # transform coverage track if required + if transform == 'log': +@@ -135,8 +136,6 @@ def evaluate(*, rank, gpu, task, model, val_loader, metrics_reg, + del y_cla_list + del pred_cla_list + +- # gather_start = time.time() +- # gather the results across all devices + if distributed: + if task == 'both' or task == 'regression': + ys_reg = gather_tensor(ys_reg, world_size=world_size, +@@ -148,10 +147,7 @@ def evaluate(*, rank, gpu, task, model, val_loader, metrics_reg, + rank=rank) + preds_cla = gather_tensor( + preds_cla, world_size=world_size, rank=rank) +- # myprint("Gathering takes {}s".format(time.time()-gather_start), +- # rank=rank) + +- # now with the results of whole dataset, compute metrics on device 0 + if rank == 0: + if task == 'both' or task == 'classification': + for metric in metrics_cla: +diff --git a/claragenomics/dl4atac/infer.py b/claragenomics/dl4atac/infer.py +index 88d3b7b..11dc8af 100644 +--- a/claragenomics/dl4atac/infer.py ++++ b/claragenomics/dl4atac/infer.py +@@ -60,7 +60,10 @@ def infer(*, rank, gpu, task, model, infer_loader, print_freq, + x = x.unsqueeze(1) # (N, 1, L) + else: + x = np.swapaxes(x, 1, 2) +- x = x.cuda(gpu, non_blocking=True) ++ ++ if gpu >= 0: ++ x = x.cuda(gpu, non_blocking=True) ++ + count += x.shape[0] + + # transform coverage track if required +diff --git a/claragenomics/dl4atac/layers.py b/claragenomics/dl4atac/layers.py +index fb914bf..82a34ec 100755 +--- a/claragenomics/dl4atac/layers.py ++++ b/claragenomics/dl4atac/layers.py +@@ -13,6 +13,10 @@ import torch + import torch.nn as nn + import torch.nn.functional as F + ++try: ++ from Conv1dOpti_ext import Conv1dOpti, ReLU_bf16 # (Change) ++except ImportError as e: ++ print(e) + + class ZeroSamePad1d(nn.Module): + """Apply SAME zero padding to input.""" +@@ -121,7 +125,10 @@ class ConvAct1d(nn.Module): + + self.padding_layer = ZeroSamePad1d( + interval_size, kernel_size, stride, dilation) +- self.conv_layer = nn.Conv1d( ++ # self.conv_layer = nn.Conv1d( ++ # in_channels, out_channels, kernel_size, stride, padding=0, ++ # dilation=dilation, bias=bias) ++ self.conv_layer = Conv1dOpti( # (Change) + in_channels, out_channels, kernel_size, stride, padding=0, + dilation=dilation, bias=bias) + self.bn_layer = nn.BatchNorm1d(out_channels) if bn else None +@@ -138,8 +145,15 @@ class ConvAct1d(nn.Module): + x = self.conv_layer(x) + if self.bn_layer: + x = self.bn_layer(x) +- if self.act_layer: +- x = self.act_layer(x) ++ if self.act_layer: # (Change) ++ if x.dtype != torch.float32: ++ x = x.to(torch.float32) ++ x = self.act_layer(x) ++ x = x.to(torch.bfloat16) ++ x = ReLU_bf16.apply(x) ++ # pass ++ else: ++ x = self.act_layer(x) + return x + + +@@ -200,8 +214,14 @@ class ResBlock(nn.Module): + x = self.conv_act2(x) + x = self.conv_act3(x) + x = x + self.conv_input(input) +- x = self.activation(x) +- ++ if x.dtype != torch.float32: # (Change) ++ x = x.to(torch.float32) ++ x = self.activation(x) ++ x = x.to(torch.bfloat16) ++ x = ReLU_bf16.apply(x) ++ # pass ++ else: ++ x = self.activation(x) + return x + + +diff --git a/claragenomics/dl4atac/losses.py b/claragenomics/dl4atac/losses.py +index 2da4ef6..1fd4437 100755 +--- a/claragenomics/dl4atac/losses.py ++++ b/claragenomics/dl4atac/losses.py +@@ -133,8 +133,11 @@ class MultiLoss(object): + + """ + # TODO: error checking for self.device +- if self.device: +- loss_func = loss_func.cuda(self.device) ++ if self.device >= 0: # (Change) ++ if self.device: ++ loss_func = loss_func.cuda(self.device) ++ # else: ++ # loss_func = loss_func.to('cpu') + return loss_func + + def single_output_loss(self, pred, label): +diff --git a/claragenomics/dl4atac/models/model_utils.py b/claragenomics/dl4atac/models/model_utils.py +index 5e89016..db2d2ba 100644 +--- a/claragenomics/dl4atac/models/model_utils.py ++++ b/claragenomics/dl4atac/models/model_utils.py +@@ -158,11 +158,11 @@ def build_model(rank, interval_size, resume, + dilation_class=model_args.dil_cla, + in_channels=model_args.in_channels) + +- elif model == 'linear': ++ elif model_args.model == 'linear': + model = DenoisingLinear( + interval_size=interval_size, field=model_args.field) + +- elif model == 'logistic': ++ elif model_args.model == 'logistic': + model = DenoisingLogistic( + interval_size=interval_size, field=model_args.field) + +@@ -171,11 +171,16 @@ def build_model(rank, interval_size, resume, + if resume or infer or evaluate: + model = load_model(model, weights_path, rank) + +- model = model.cuda(gpu) +- ++ # model = model.cuda(gpu) ++ if gpu >= 0: # (Change) ++ model = model.cuda(gpu) ++ + if distributed: + _logger.info('Compiling model in DistributedDataParallel') +- model = DistributedDataParallel(model, device_ids=[gpu]) ++ if gpu < 0: ++ model = DistributedDataParallel(model) ++ else: ++ model = DistributedDataParallel(model, device_ids=[gpu]) + elif gpu > 1: + _logger.info('Compiling model in DataParallel') + model = nn.DataParallel( +diff --git a/claragenomics/dl4atac/models/models.py b/claragenomics/dl4atac/models/models.py +index 6a5aa60..b05080a 100644 +--- a/claragenomics/dl4atac/models/models.py ++++ b/claragenomics/dl4atac/models/models.py +@@ -51,6 +51,7 @@ class DenoisingResNet(nn.Module): + blocks in the classification part of the model + + """ ++ + self.interval_size = interval_size + super(DenoisingResNet, self).__init__() + +@@ -100,12 +101,18 @@ class DenoisingResNet(nn.Module): + """ + for res_block in self.res_blocks: + x = res_block(x) ++ + x = self.regressor(x) +- out_reg = x.squeeze(1) ++ if x.dtype != torch.float32: # (Change for Bfloat16) ++ out_reg = x.squeeze(1).to(torch.float32) ++ else: ++ out_reg = x.squeeze(1) ++ + for res_block in self.res_blocks_class: + x = res_block(x) +- out_cla = torch.sigmoid(self.classifier(x).squeeze(1)) + ++ out_cla = torch.sigmoid(self.classifier(x).squeeze(1)) ++ + return out_reg, out_cla + + +diff --git a/claragenomics/dl4atac/train.py b/claragenomics/dl4atac/train.py +index 42f01d8..d30416e 100755 +--- a/claragenomics/dl4atac/train.py ++++ b/claragenomics/dl4atac/train.py +@@ -14,10 +14,23 @@ import time + from claragenomics.dl4atac.utils import myprint, progbar, equal_width_formatter + + import torch ++# import extend_profiler + import torch.distributed as dist +- ++# print(extend_profiler.__file__) + import numpy as np + ++# def print_op_timings(prof, use_gpu, prefix): ++# sorted_fe = sorted(prof.function_events, key=lambda event: [event.cpu_interval.start, -event.cpu_interval.end],) ++# start_time = sorted_fe[0].cpu_interval.start if len(sorted_fe) > 0 else 0 ++ ++# with open("%s.OpTimings_cpu_bf16_2.txt" % prefix, "w") as f: ++# for i, fe in enumerate(sorted_fe): ++# fe_name = getattr(fe, 'nested_key', fe.name) ++# cstr = "" ++# if use_gpu: ++# for kinfo in fe.kernels: cstr += " %10.3f %10.3f %8.3f %8.3f " % ((kinfo.interval.start - start_time) /1000.0, (kinfo.interval.end - start_time)/1000.0, (kinfo.interval.start - fe.cpu_interval.start) /1000.0, kinfo.interval.elapsed_us()/1000.0) ++# #for kinfo in fe.kernels: cstr += " %12.4f %12.4f %12.4f " % ((kinfo.interval.start - fe.cpu_interval.start) /1000.0, (kinfo.interval.end - fe.cpu_interval.start)/1000.0, kinfo.interval.elapsed_us()/1000.0) ++# print("%-3d %-6d %6d %12.4f %12.4f %12.4f %2s %s %-40s %s" % (0, i, fe.id, (fe.cpu_interval.start - start_time)/1000.0, (fe.cpu_interval.end - start_time)/1000.0, fe.cpu_interval.elapsed_us()/1000.0, fe.thread, cstr, fe_name.replace(' ', '_'), fe.input_shapes), file=f) + + def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + epoch, epochs, clip_grad, print_freq, distributed, world_size, +@@ -43,6 +56,7 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + + """ + num_batches = len(train_loader) ++ + epoch_formatter = "Epoch " + \ + equal_width_formatter(total=epochs).format(epoch) + start = time.time() +@@ -50,6 +64,12 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + backward_time = 0. + print_time = 0. + ++ enable_profiling = False ++ if gpu >= 0: ++ use_cuda = True ++ else: ++ use_cuda = False ++ + model.train() + + print( +@@ -57,6 +77,7 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + + # Loop training data + for i, batch in enumerate(train_loader): ++ + x = batch['input'] + y_reg = batch['label_reg'] + y_cla = batch['label_cla'] +@@ -66,16 +87,18 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + x = x.unsqueeze(1) # (N, 1, L) + else: + x = np.swapaxes(x, 1, 2) +- x = x.cuda(gpu, non_blocking=True) +- +- if task == 'regression': +- y = y_reg.cuda(gpu, non_blocking=True) +- elif task == 'classification': +- y = y_cla.cuda(gpu, non_blocking=True) +- elif task == 'both': +- y_reg = y_reg.cuda(gpu, non_blocking=True) +- y_cla = y_cla.cuda(gpu, non_blocking=True) + ++ if gpu >= 0: # (Change) ++ x = x.cuda(gpu, non_blocking=True) ++ if task == 'regression': ++ y = y_reg.cuda(gpu, non_blocking=True) ++ elif task == 'classification': ++ y = y_cla.cuda(gpu, non_blocking=True) ++ elif task == 'both': ++ y_reg = y_reg.cuda(gpu, non_blocking=True) ++ y_cla = y_cla.cuda(gpu, non_blocking=True) ++ ++ # with torch.autograd.profiler.profile(enable_profiling, use_cuda, True) as prof: + # transform tracks if required + if transform == 'log': + x = torch.log(x + 1) +@@ -87,7 +110,7 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + # Model forward pass + t = time.time() + pred = model(x) +- ++ + # Remove padding + if pad is not None: + center = range(pad, x.shape[2] - pad) +@@ -119,7 +142,9 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + # one gradient descent step + optimizer.zero_grad() + t = time.time() ++ + total_loss_value.backward() ++ + if clip_grad > 0: + torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad) + optimizer.step() +@@ -137,11 +162,30 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + if rank == 0: + post_bar_msg = " | ".join( + [k + ':{:8.3f}'.format(v.cpu().item()) for k, v in +- losses_values.items()]) ++ losses_values.items()]) + progbar(curr=i, total=num_batches, progbar_len=20, + pre_bar_msg=epoch_formatter, post_bar_msg=post_bar_msg) + print_time += time.time() - t + ++ # if enable_profiling: ++ # if use_cuda == True: ++ # with open("profiler_out/Atacwork_cuda_new.prof", "w") as prof_f: ++ # prof_f.write(prof.key_averages().table(sort_by="cuda_time_total")) ++ # with open("profiler_out/Atacwork_cuda_new.nested.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages().table(sort_by="cuda_time_total")) ++ # with open("profiler_out/Atacwork_cuda_new.toplevel.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages(only_top_level=True).table(sort_by="cuda_time_total")) ++ # print_op_timings(prof, use_cuda, "profiler_out/") ++ # else: ++ # with open("profiler_out/Atacwork_cpu_bf16_2.prof", "w") as prof_f: ++ # prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) ++ # with open("profiler_out/Atacwork_cpu_bf16_2.nested.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages().table(sort_by="cpu_time_total")) ++ # with open("profiler_out/Atacwork_cpu_bf16_2.toplevel.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages(only_top_level=True).table(sort_by="cpu_time_total")) ++ # print_op_timings(prof, use_cuda, "profiler_out/") ++ ++ + myprint( + epoch_formatter + " Time Taken: {:7.3f}s".format(time.time() - start), + color='yellow', rank=rank) +@@ -154,3 +198,5 @@ def train(*, rank, gpu, task, model, train_loader, loss_func, optimizer, pad, + 'time: %.3f\tRemain (data) time: %.3f' % (total_time, forward_time, + backward_time, print_time, + remainder_time)) ++ ++ +\ No newline at end of file +diff --git a/claragenomics/dl4atac/utils.py b/claragenomics/dl4atac/utils.py +index fc822d8..4aae253 100755 +--- a/claragenomics/dl4atac/utils.py ++++ b/claragenomics/dl4atac/utils.py +@@ -78,7 +78,11 @@ def make_experiment_dir(label, out_home, timestamp=True, + "{}_{}".format(label, "latest")) + if (os.path.islink(latest_symlink)): + os.remove(latest_symlink) +- os.symlink(os.path.abspath(exp_path), latest_symlink) ++ # os.symlink(os.path.abspath(exp_path), latest_symlink) ++ try: ++ os.symlink(os.path.abspath(exp_path), latest_symlink) # (Change) ++ except: ++ pass + return exp_path + + +diff --git a/configs/infer_config.yaml b/configs/infer_config.yaml +index e0421cd..80868ad 100644 +--- a/configs/infer_config.yaml ++++ b/configs/infer_config.yaml +@@ -14,12 +14,12 @@ label: 'inference_output' + task: 'both' + print_freq: 50 + bs: 512 +-num_workers: 4 ++num_workers: 1 + weights_path: "None" +-gpu: 0 ++gpu: -1 + distributed: False +-dist-url: 'tcp://127.0.0.1:4321' +-dist-backend: 'gloo' ++dist-url: "env://" # # 'tcp://127.0.0.1:4321' ++dist-backend: 'ccl' # 'gloo' + debug: False + + # Data processing args +diff --git a/configs/model_structure.yaml b/configs/model_structure.yaml +index e296b3b..e587cf4 100644 +--- a/configs/model_structure.yaml ++++ b/configs/model_structure.yaml +@@ -18,7 +18,7 @@ dil: 8 # dilation for regression blocks in resnet + width: 51 # kernel size for regression blocks in resnet + nfilt: 15 # number of filters for regression blocks in resnet + nblocks_cla: 2 # number of classification blocks for resnet +-dil_cla: 8 # dilation for classification blocks in resnet ++dil_cla: 8 # dilation for classification blocks in resnet + width_cla: 51 # kernel size for classification blocks in resnet + nfilt_cla: 15 # number of filters for classification blocks in resnet + field: None # receptive field, only defined for linear/logistic regression +diff --git a/configs/train_config.yaml b/configs/train_config.yaml +index 27bbcf3..79252aa 100644 +--- a/configs/train_config.yaml ++++ b/configs/train_config.yaml +@@ -14,13 +14,13 @@ out_home: './' + label: 'trained_models' + task: 'both' + print_freq: 50 +-bs: 64 +-num_workers: 4 ++bs: 54 #64 original ++num_workers: 1 #4 original + weights_path: "None" +-gpu: 0 ++gpu: -1 + distributed: False +-dist-url: 'tcp://127.0.0.1:4321' +-dist-backend: 'gloo' ++dist-url: "env://" # 'tcp://127.0.0.1:4321' ++dist-backend: 'ccl' # 'gloo' + debug: False + + # Data processing args +@@ -35,8 +35,8 @@ save_freq: 1 + + #Learning args + clip_grad: 0 +-lr: 0.0002 +-epochs: 25 ++lr: 0.0002 # 0.0002 original ++epochs: 25 #25 original + mse_weight: 0.0005 + pearson_weight: 1 + poisson_weight: 0 +diff --git a/requirements-base.txt b/requirements-base.txt +index 549dc41..9081f05 100644 +--- a/requirements-base.txt ++++ b/requirements-base.txt +@@ -19,8 +19,11 @@ pyyaml==5.3 + scikit-learn==0.21.3 + scipy==1.3.2 + termcolor==1.1.0 +-torch==1.2.0 +-torchvision==0.3.0 ++# torch==1.2.0 ++# torchvision==0.3.0 + scikit-learn==0.21.3 + scipy==1.3.2 + configargparse==0.15.2 ++pytest==4.4.1 ++psutil==5.7.0 ++# torch-ccl==1.0 +\ No newline at end of file +diff --git a/scripts/main.py b/scripts/main.py +index 9f8e4b3..20d82b1 100755 +--- a/scripts/main.py ++++ b/scripts/main.py +@@ -43,6 +43,12 @@ import torch.multiprocessing as mp + + from worker import eval_worker, infer_worker, train_worker + ++try: ++ import torch_ccl # (Change) CCL import ++except ImportError as e: ++ print(e) ++ torch_ccl = False ++ + warnings.filterwarnings("ignore") + + # Set up logging +@@ -173,17 +179,13 @@ def writer(infer, intervals_file, exp_dir, result_fname, + out_base_path = os.path.join(exp_dir, prefix + "_" + result_fname) + if task == "regression": + channels = [0] +- outfiles = [os.path.join(out_base_path + ".track.bedGraph")] +- rounding = [reg_rounding] + elif task == "classification": + channels = [1] +- outfiles = [os.path.join(out_base_path + ".peaks.bedGraph")] +- rounding = [cla_rounding] + elif task == "both": + channels = [0, 1] +- outfiles = [os.path.join(out_base_path + ".track.bedGraph"), +- os.path.join(out_base_path + ".peaks.bedGraph")] +- rounding = [reg_rounding, cla_rounding] ++ outfiles = [os.path.join(out_base_path + ".track.bedGraph"), ++ os.path.join(out_base_path + ".peaks.bedGraph")] ++ rounding = [reg_rounding, cla_rounding] + + # Temp dir used to save temp files during multiprocessing. + temp_dir = tempfile.mkdtemp() +@@ -314,7 +316,12 @@ def main(): + + # check gpu + # TODO: add cpu support +- if not torch.cuda.is_available(): ++ # if not torch.cuda.is_available(): ++ # raise Exception("No GPU available. Check your machine configuration.") ++ ++ if args.gpu < 0: # (Change) ++ print("Running on CPU: ", args.gpu) ++ elif not torch.cuda.is_available(): + raise Exception("No GPU available. Check your machine configuration.") + + # all output will be written in the exp_dir folder +@@ -337,7 +344,7 @@ def main(): + with h5py.File(args.files_train[0], 'r') as f: + args.interval_size = f['input'].shape[1] + args.batch_size = 1 +- ++ + ngpus_per_node = torch.cuda.device_count() + # WAR: gloo distributed doesn't work if world size is 1. + # This is fixed in newer torch version - +@@ -346,7 +353,8 @@ def main(): + + config_dir = os.path.join(args.exp_dir, "configs") + if not os.path.exists(config_dir): +- os.mkdir(config_dir) ++ # os.mkdir(config_dir) ++ os.makedirs(config_dir, exist_ok=True) + if args.distributed: + _logger.info('Distributing to %s GPUS' % str(ngpus_per_node)) + args.world_size = ngpus_per_node +@@ -355,7 +363,10 @@ def main(): + else: + assert_device_available(args.gpu) + _logger.info('Running on GPU: %s' % str(args.gpu)) +- args.world_size = 1 ++ # args.world_size = 1 #(Change) ++ args.world_size = int(os.environ.get("PMI_SIZE",1)) ++ if args.world_size > 1: ++ args.distributed = True + train_worker(args.gpu, ngpus_per_node, args, timers=Timers) + + # infer & eval +@@ -409,7 +420,8 @@ def main(): + write_proc = mp.Process(target=writer, kwargs=keyword_args) + write_proc.start() + ############################################################# +- ++ ++ + ngpus_per_node = torch.cuda.device_count() + # WAR: gloo distributed doesn't work if world size is 1. + # This is fixed in newer torch version - +@@ -425,6 +437,9 @@ def main(): + else: + assert_device_available(args.gpu) + args.world_size = 1 ++ # args.world_size = int(os.environ.get("PMI_SIZE",1)) ++ # if args.world_size > 1: ++ # args.distributed = True + worker(args.gpu, ngpus_per_node, args, res_queue) + + # finish off writing +diff --git a/scripts/worker.py b/scripts/worker.py +index 564afe1..a00ab47 100644 +--- a/scripts/worker.py ++++ b/scripts/worker.py +@@ -35,6 +35,14 @@ import torch + import torch.distributed as dist + + from torch.optim import Adam ++import os ++ ++# import extend_profiler ++# import psutil ++try: ++ import psutil # (Change) ++except ImportError as e: ++ print(e) + + warnings.filterwarnings("ignore") + +@@ -48,6 +56,28 @@ _handler.setFormatter(log_formatter) + _logger.setLevel(logging.INFO) + _logger.addHandler(_handler) + ++def worker_init_fn(worker_id): # (Change) ++ cpu_aff = psutil.Process().cpu_affinity() ++ cpu_aff_new = [cpu_aff[0] - worker_id -1] ++ try: ++ psutil.Process().cpu_affinity(cpu_aff_new) ++ print("Worker {} with pid {} called, new affinity = {}".format(worker_id, os.getpid(), psutil.Process().cpu_affinity())) ++ except: ++ print("Unable to set worker affinity {} for worker {}".format(cpu_aff_new, worker_id)) ++ ++# def print_op_timings(prof, use_gpu, prefix): ++# sorted_fe = sorted(prof.function_events, key=lambda event: [event.cpu_interval.start, -event.cpu_interval.end],) ++# start_time = sorted_fe[0].cpu_interval.start if len(sorted_fe) > 0 else 0 ++ ++# with open("%s.OpTimings_cpu_evaluation_multi.txt" % prefix, "w") as f: ++# for i, fe in enumerate(sorted_fe): ++# fe_name = getattr(fe, 'nested_key', fe.name) ++# cstr = "" ++# if use_gpu: ++# for kinfo in fe.kernels: cstr += " %10.3f %10.3f %8.3f %8.3f " % ((kinfo.interval.start - start_time) /1000.0, (kinfo.interval.end - start_time)/1000.0, (kinfo.interval.start - fe.cpu_interval.start) /1000.0, kinfo.interval.elapsed_us()/1000.0) ++# #for kinfo in fe.kernels: cstr += " %12.4f %12.4f %12.4f " % ((kinfo.interval.start - fe.cpu_interval.start) /1000.0, (kinfo.interval.end - fe.cpu_interval.start)/1000.0, kinfo.interval.elapsed_us()/1000.0) ++# print("%-3d %-6d %6d %12.4f %12.4f %12.4f %2s %s %-40s %s" % (0, i, fe.id, (fe.cpu_interval.start - start_time)/1000.0, (fe.cpu_interval.end - start_time)/1000.0, fe.cpu_interval.elapsed_us()/1000.0, fe.thread, cstr, fe_name.replace(' ', '_'), fe.input_shapes), file=f) ++ + + def get_losses(task, mse_weight, pearson_weight, gpu, poisson_weight): + """Return loss function. +@@ -135,7 +165,13 @@ def get_model(args, gpu, rank): + model_params : model parameters + + """ +- torch.cuda.set_device(gpu) ++ #torch.cuda.set_device(gpu) ++ ++ if gpu >= 0: # (Change) ++ torch.cuda.set_device(gpu) ++ # else: ++ # device = torch.device('cpu') ++ + _logger.debug('Rank %s' % str(rank)) + + if args.distributed: +@@ -167,9 +203,15 @@ def train_worker(gpu, ngpu_per_node, args, timers=None): + """ + # fix random seed so models have the same starting weights + torch.manual_seed(42) ++ + +- rank = gpu if args.distributed else 0 +- ++ # rank = gpu if args.distributed else 0 ++ if gpu < 0: #(Change) ++ # rank = 0 ++ rank = int(os.environ.get("PMI_RANK",0)) ++ else: ++ rank = gpu if args.distributed else 0 ++ + model, model_params = get_model(args, gpu, rank) + + optimizer = Adam(model.parameters(), lr=args.lr) +@@ -183,12 +225,22 @@ def train_worker(gpu, ngpu_per_node, args, timers=None): + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler( + train_dataset) +- train_loader = torch.utils.data.DataLoader( +- train_dataset, batch_size=args.bs, shuffle=(train_sampler is None), +- # collate_fn=custom_collate_train, +- num_workers=args.num_workers, pin_memory=True, sampler=train_sampler, +- drop_last=False +- ) ++ ++ if gpu < 0: # (Change) ++ train_loader = torch.utils.data.DataLoader( ++ train_dataset, batch_size=args.bs, shuffle=(train_sampler is None), ++ # collate_fn=custom_collate_train, ++ num_workers=args.num_workers, pin_memory=False, sampler=train_sampler, ++ drop_last=False, worker_init_fn=worker_init_fn ++ ) ++ else: ++ train_loader = torch.utils.data.DataLoader( ++ train_dataset, batch_size=args.bs, shuffle=(train_sampler is None), ++ # collate_fn=custom_collate_train, ++ num_workers=args.num_workers, pin_memory=True, sampler=train_sampler, ++ drop_last=False ++ ) ++ + + # TODO: need DatasetVal? Not for now + val_dataset = DatasetTrain(files=args.val_files, layers=args.layers) +@@ -196,29 +248,42 @@ def train_worker(gpu, ngpu_per_node, args, timers=None): + if args.distributed: + val_sampler = torch.utils.data.distributed.DistributedSampler( + val_dataset) +- val_loader = torch.utils.data.DataLoader( +- val_dataset, batch_size=args.bs, shuffle=False, +- # collate_fn=custom_collate_train, +- num_workers=args.num_workers, pin_memory=True, sampler=val_sampler, +- drop_last=False +- # drop_last=True # need to drop irregular batch for distributed +- # evaluation due to limitation of dist.all_gather +- ) ++ ++ if gpu < 0: #(Change) ++ val_loader = torch.utils.data.DataLoader( ++ val_dataset, batch_size=args.bs, shuffle=False, ++ # collate_fn=custom_collate_train, ++ num_workers=args.num_workers, pin_memory=False, sampler=val_sampler, ++ drop_last=False, worker_init_fn=worker_init_fn ++ # drop_last=True # need to drop irregular batch for distributed ++ # evaluation due to limitation of dist.all_gather ++ ) ++ else: ++ val_loader = torch.utils.data.DataLoader( ++ val_dataset, batch_size=args.bs, shuffle=False, ++ # collate_fn=custom_collate_train, ++ num_workers=args.num_workers, pin_memory=True, sampler=val_sampler, ++ drop_last=False ++ # drop_last=True # need to drop irregular batch for distributed ++ # evaluation due to limitation of dist.all_gather ++ ) + + loss_func = get_losses(args.task, args.mse_weight, + args.pearson_weight, gpu, args.poisson_weight) + + current_best = None ++ + for epoch in range(args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) ++ + train(rank=rank, gpu=gpu, task=args.task, model=model, +- train_loader=train_loader, +- loss_func=loss_func, optimizer=optimizer, epoch=epoch, +- epochs=args.epochs, clip_grad=args.clip_grad, +- print_freq=args.print_freq, pad=args.pad, +- distributed=args.distributed, world_size=args.world_size, +- transform=args.transform) ++ train_loader=train_loader, ++ loss_func=loss_func, optimizer=optimizer, epoch=epoch, ++ epochs=args.epochs, clip_grad=args.clip_grad, ++ print_freq=args.print_freq, pad=args.pad, ++ distributed=args.distributed, world_size=args.world_size, ++ transform=args.transform) + + if epoch % args.eval_freq == 0: + # either create new objects or call reset on each metric obj +@@ -227,12 +292,36 @@ def train_worker(gpu, ngpu_per_node, args, timers=None): + + # best_metric is the metric used to compare results + # across different evaluation runs. It's modified in place. ++ # enable_eval_profiling = False ++ # if gpu >= 0: ++ # use_cuda = True ++ # else: ++ # use_cuda = False ++ # with torch.autograd.profiler.profile(enable_eval_profiling, use_cuda, True) as prof: + evaluate(rank=rank, gpu=gpu, task=args.task, +- model=model, val_loader=val_loader, +- metrics_reg=metrics_reg, metrics_cla=metrics_cla, +- world_size=args.world_size, distributed=args.distributed, +- best_metric=best_metric, pad=args.pad, +- print_freq=args.print_freq, transform=args.transform) ++ model=model, val_loader=val_loader, ++ metrics_reg=metrics_reg, metrics_cla=metrics_cla, ++ world_size=args.world_size, distributed=args.distributed, ++ best_metric=best_metric, pad=args.pad, ++ print_freq=args.print_freq, transform=args.transform) ++ ++ # if enable_eval_profiling: ++ # if use_cuda == True: ++ # with open("profiler_out/Atacwork_cuda_evaluation.prof", "w") as prof_f: ++ # prof_f.write(prof.key_averages().table(sort_by="cuda_time_total")) ++ # with open("profiler_out/Atacwork_cuda_evaluation.nested.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages().table(sort_by="cuda_time_total")) ++ # with open("profiler_out/Atacwork_cuda_evaluation.toplevel.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages(only_top_level=True).table(sort_by="cuda_time_total")) ++ # print_op_timings(prof, use_cuda, "profiler_out/") ++ # else: ++ # with open("profiler_out/Atacwork_cpu_evaluation_multi.prof", "w") as prof_f: ++ # prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) ++ # with open("profiler_out/Atacwork_cpu_evaluation_multi.nested.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages().table(sort_by="cpu_time_total")) ++ # with open("profiler_out/Atacwork_cpu_evaluation_multi.toplevel.prof", "w") as prof_f: ++ # prof_f.write(prof.nested_key_averages(only_top_level=True).table(sort_by="cpu_time_total")) ++ # print_op_timings(prof, use_cuda, "profiler_out/") + + if rank == 0: + new_best = best_metric.better_than(current_best) +@@ -263,7 +352,10 @@ def infer_worker(gpu, ngpu_per_node, args, res_queue=None): + res_queue : Inference queue. + + """ +- rank = gpu if args.distributed else 0 ++ if gpu < 0: # (Change) ++ rank = 0 ++ else: ++ rank = gpu if args.distributed else 0 + + model, _ = get_model(args, gpu, rank) + +@@ -274,11 +366,18 @@ def infer_worker(gpu, ngpu_per_node, args, res_queue=None): + infer_sampler = torch.utils.data.distributed.DistributedSampler( + infer_dataset, shuffle=False) + +- infer_loader = torch.utils.data.DataLoader( +- infer_dataset, batch_size=args.bs, shuffle=False, +- num_workers=args.num_workers, pin_memory=True, sampler=infer_sampler, +- drop_last=False +- ) ++ if gpu < 0: # (Change) ++ infer_loader = torch.utils.data.DataLoader( ++ infer_dataset, batch_size=args.bs, shuffle=False, ++ num_workers=args.num_workers, pin_memory=False, sampler=infer_sampler, ++ drop_last=False, worker_init_fn=worker_init_fn ++ ) ++ else: ++ infer_loader = torch.utils.data.DataLoader( ++ infer_dataset, batch_size=args.bs, shuffle=False, ++ num_workers=args.num_workers, pin_memory=True, sampler=infer_sampler, ++ drop_last=False ++ ) + + infer(rank=rank, gpu=gpu, task=args.task, model=model, + infer_loader=infer_loader, +@@ -296,7 +395,10 @@ def eval_worker(gpu, ngpu_per_node, args, res_queue=None): + res_queue : Evaluate queue. + + """ +- rank = gpu if args.distributed else 0 ++ if gpu < 0: # (Change) ++ rank=0 ++ else: ++ rank = gpu if args.distributed else 0 + + model, _ = get_model(args, gpu, rank) + +@@ -307,11 +409,18 @@ def eval_worker(gpu, ngpu_per_node, args, res_queue=None): + eval_sampler = torch.utils.data.distributed.DistributedSampler( + eval_dataset) + +- eval_loader = torch.utils.data.DataLoader( +- eval_dataset, batch_size=args.bs, shuffle=False, +- num_workers=args.num_workers, pin_memory=True, sampler=eval_sampler, +- drop_last=False +- ) ++ if gpu < 0: # (Change) ++ eval_loader = torch.utils.data.DataLoader( ++ eval_dataset, batch_size=args.bs, shuffle=False, ++ num_workers=args.num_workers, pin_memory=False, sampler=eval_sampler, ++ drop_last=False, worker_init_fn=worker_init_fn ++ ) ++ else: ++ eval_loader = torch.utils.data.DataLoader( ++ eval_dataset, batch_size=args.bs, shuffle=False, ++ num_workers=args.num_workers, pin_memory=True, sampler=eval_sampler, ++ drop_last=False, ++ ) + + metrics_reg, metrics_cla, best_metric = get_metrics( + args.task, args.threshold, args.best_metric_choice) +diff --git a/tests/end-to-end/train.sh b/tests/end-to-end/train.sh +index 953ee04..75a210c 100755 +--- a/tests/end-to-end/train.sh ++++ b/tests/end-to-end/train.sh +@@ -24,3 +24,16 @@ python $root_dir/main.py train\ + --width 50 --width_cla 50 --dil_cla 10 --pad 0 + # Training is not deterministic, so we are not comparing results. + check_status $? "Training run not succesful!" ++ ++echo "" ++echo "Test classification mode of training" ++echo "" ++python $root_dir/main.py train \ ++ --files_train $out_dir/train_data.h5 \ ++ --val_files $out_dir/val_data.h5 \ ++ --model logistic --field 8401 \ ++ --out_home $out_dir --label logistic \ ++ --task classification --bs 4 \ ++ --epochs 1 --pad 5000 ++# Training is not deterministic, so we are not comparing results. ++check_status $? "Training run not succesful!" +diff --git a/tutorials/tutorial1.md b/tutorials/tutorial1.md +index b53f3ae..97dcea8 100644 +--- a/tutorials/tutorial1.md ++++ b/tutorials/tutorial1.md +@@ -164,6 +164,8 @@ See [Tutorial 2](tutorial2.md) for step-by-step instructions on how to apply thi + + To change any of the parameters for the deep learning model, you can edit the appropriate parameters in `configs/train_config.yaml` or `configs/model_structure.yaml` and run the command in step 7 above. Type `python $atacworks/scripts/main.py train --help` for an explanation of the parameters. + ++Note: `train_config.yaml` is set up to use multiple GPUs. If you are using a single GPU, edit `train_config.yaml` to change the line `gpu: "None"` to read `gpu: 0`. ++ + ## References + (1) Lal, A., Chiang, Z.D., Yakovenko, N., Duarte, F.M., Israeli, J. and Buenrostro, J.D., 2019. AtacWorks: A deep convolutional neural network toolkit for epigenomics. BioRxiv, p.829481. (https://www.biorxiv.org/content/10.1101/829481v1) + +diff --git a/tutorials/tutorial2.md b/tutorials/tutorial2.md +index a733e86..cedbb19 100644 +--- a/tutorials/tutorial2.md ++++ b/tutorials/tutorial2.md +@@ -87,6 +87,7 @@ python $atacworks/scripts/main.py infer \ + --config configs/infer_config.yaml \ + --config_mparams configs/model_structure.yaml \ + ``` ++Note: `infer_config.yaml` is set up to use multiple GPUs. If you are using a single GPU, edit `infer_config.yaml` to change the line `gpu: "None"` to read `gpu: 0`. + + The inference results will be saved in the folder `output_latest`. This folder will contain four files: + 1. `NK_inferred.track.bedGraph` +@@ -96,7 +97,7 @@ The inference results will be saved in the folder `output_latest`. This folder w + + `NK_inferred.track.bedGraph` and `NK_inferred.track.bw` contain the denoised ATAC-seq track. `NK_inferred.peaks.bedGraph` and `NK_inferred.peaks.bw` contain the positions in the genome that are designated as peaks (the model predicts that the probability of these positions being part of a peak is at least 0.5) + +-To change any of the parameters for inference with the deep learning model, you can edit the parameters in `configs/infer_config.yaml` or `configs/model_structure.yaml` and run the commands in step 7-8 above. ++To change any of the parameters for inference with the deep learning model, you can edit the parameters in `configs/infer_config.yaml` or `configs/model_structure.yaml` and run the command above. + + Type `python $atacworks/scripts/main.py infer --help` for an explanation of the parameters. + diff --git a/ext/TAL/applications/ATAC-Seq/Batchfile_CPU.slurm b/ext/TAL/applications/ATAC-Seq/Batchfile_CPU.slurm new file mode 100644 index 00000000..ee50c4a8 --- /dev/null +++ b/ext/TAL/applications/ATAC-Seq/Batchfile_CPU.slurm @@ -0,0 +1,34 @@ +#!/bin/bash + +#SBATCH -o /home/test_output.%j.out +#SBATCH -D /home/AtacWorks/ +#SBATCH -J AtacworkJob_CPU +#SBATCH --nodes=8 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=64 +##SBATCH --time=8:00:00 + +hostname +export MASTER_PORT=29500 +export LD_LIBRARY_PATH=/home/libxsmm/lib/ + +source /opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin/mpivars.sh release_mt + +source /home/anaconda3/etc/profile.d/conda.sh +conda activate Atac +atacworks=/home/AtacWorks/ + +export KMP_AFFINITY=compact,1,0,granularity=fine + +export OMP_NUM_THREADS=30 # number of cores - 2 (N - 2) + +export LD_PRELOAD=/home/libtcmalloc.so +export LD_PRELOAD=/home/libjemalloc.so + +mpiexec.hydra -np 16 -l -genv I_MPI_PIN_DOMAIN=[0x00000000FFFFFFFC,0xFFFFFFFC00000000] -genv CCL_WORKER_AFFINITY=0,32 \ + -genv CCL_WORKER_COUNT=1 -genv OMP_NUM_THREADS=30 \ + python $atacworks/scripts/main.py train \ + --config configs/train_config.yaml \ + --config_mparams configs/model_structure.yaml \ + --files_train $atacworks/Mono.50.2400.train.h5 \ + --val_files $atacworks/Mono.50.2400.val.h5 diff --git a/ext/TAL/applications/ATAC-Seq/README.md b/ext/TAL/applications/ATAC-Seq/README.md new file mode 100644 index 00000000..62ecf753 --- /dev/null +++ b/ext/TAL/applications/ATAC-Seq/README.md @@ -0,0 +1,172 @@ +# Instructions for ubuntu + +## Update +```bash +apt-get update +``` + +## Install software +```bash +apt-get install -y git +apt-get install -y libcurl4-openssl-dev +apt-get install -y hdf5-tools +apt-get install -y rsync +apt-get install -y make +apt-get install -y gcc +apt-get install -y libblas-dev +apt-get install -y python3.7 python3-pip +ln -nsf /usr/bin/python3.7 /usr/bin/python +``` +## Anaconda Environment +```bash +conda create --name Atac python=3.7 +conda activate Atac +``` + +## Clone the libxsmm repository and set library path +```bash +cd /home/ +git clone https://github.com/libxsmm/libxsmm.git +cd /home/libxsmm +git checkout b3da2b1bed9d27f9d6bae91a683f8cf76fe299b5 +make -j # Use AVX=2 for AVX2 and AVX=3 for AVX512 +cd /home/ +export LD_LIBRARY_PATH=/home/libxsmm/lib/ +``` + +## Clone atacworks repo +```bash +git clone --branch v0.2.0 https://github.com/clara-parabricks/AtacWorks.git +``` + +## Clone the OpenOmics version +```bash +git clone https://github.com/IntelLabs/Trans-Omics-Acceleration-Library.git +``` + +## Apply patch +```bash +cd /home/AtacWorks/ +git apply /home/Trans-Omics-Acceleration-Library/applications/ATAC-Seq/AtacWorks_cpu_optimization_patch.patch +``` + +## Install python packages +```bash +python3.7 -m pip install -r requirements-base.txt +python3.7 -m pip install torch torchvision torchaudio +python3.7 -m pip install -r requirements-macs2.txt +``` + +## (Optional) Install torch-ccl +```bash +# Install torch-ccl +# git clone --branch v1.1.0 https://github.com/intel/torch-ccl.git && cd torch-ccl +# git submodule sync +# git submodule update --init --recursive +# python3.7 setup.py install +``` + +## Install 1D convolution module +```bash +cd /home/libxsmm/samples/deeplearning/conv1dopti_layer/Conv1dOpti-extension/ +python setup.py install +``` + +## Install AtacWorks folder ans set path +```bash +cd /home/AtacWorks/ +python3.7 -m pip install . +atacworks=/home/AtacWorks/ +``` + +## Download data to train +```bash +wget https://atacworks-paper.s3.us-east-2.amazonaws.com/dsc_atac_blood_cell_denoising_experiments/50_cells/train_data/noisy_data/dsc.1.Mono.50.cutsites.smoothed.200.bw +wget https://atacworks-paper.s3.us-east-2.amazonaws.com/dsc_atac_blood_cell_denoising_experiments/50_cells/train_data/clean_data/dsc.Mono.2400.cutsites.smoothed.200.bw +wget https://atacworks-paper.s3.us-east-2.amazonaws.com/dsc_atac_blood_cell_denoising_experiments/50_cells/train_data/clean_data/dsc.Mono.2400.cutsites.smoothed.200.3.narrowPeak +``` + +## Download file conversion binaries and set path +```bash +rsync -aP rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bedGraphToBigWig /home/ +rsync -aP rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bigWigToBedGraph /home/ +export PATH="$PATH:/home/" >> /home/.bashrc # set the path for bedGraphToBigWig binaries +``` + +## Data preprocessing + +```python +python $atacworks/scripts/peak2bw.py \ + --input dsc.Mono.2400.cutsites.smoothed.200.3.narrowPeak \ + --sizes $atacworks/data/reference/hg19.chrom.sizes \ + --out_dir ./ \ + --skip 1 + +python $atacworks/scripts/get_intervals.py \ + --sizes $atacworks/data/reference/hg19.auto.sizes \ + --intervalsize 50000 \ + --out_dir ./ \ + --val chr20 \ + --holdout chr10 + +python $atacworks/scripts/bw2h5.py \ + --noisybw dsc.1.Mono.50.cutsites.smoothed.200.bw \ + --cleanbw dsc.Mono.2400.cutsites.smoothed.200.bw \ + --cleanpeakbw dsc.Mono.2400.cutsites.smoothed.200.3.narrowPeak.bw \ + --intervals training_intervals.bed \ + --out_dir ./ \ + --prefix Mono.50.2400.train \ + --pad 5000 \ + --nonzero + +python $atacworks/scripts/bw2h5.py \ + --noisybw dsc.1.Mono.50.cutsites.smoothed.200.bw \ + --cleanbw dsc.Mono.2400.cutsites.smoothed.200.bw \ + --cleanpeakbw dsc.Mono.2400.cutsites.smoothed.200.3.narrowPeak.bw \ + --intervals val_intervals.bed \ + --out_dir ./ \ + --prefix Mono.50.2400.val \ + --pad 5000 +``` + +## Set affinity and threads +```bash +export KMP_AFFINITY=compact,1,0,granularity=fine +export LD_PRELOAD=/home/libtcmalloc.so # Copy these files in the /home folder first +export LD_PRELOAD=/home/libjemalloc.so +export OMP_NUM_THREADS=31 # (Available cores (N) - 1) +``` + +## Training run (Single Socket) +```python +# In numactl command, "-C 1-31" is for running on cores 1 to 31. +# General case for an N core machine is "-C 1-(N-1)". +# Keep batch size in config/train_config.yaml to a multiple of (N-1) for optimum performance + +numactl --membind 0 -C 1-31 python $atacworks/scripts/main.py train \ + --config configs/train_config.yaml \ + --config_mparams configs/model_structure.yaml \ + --files_train $atacworks/Mono.50.2400.train.h5 \ + --val_files $atacworks/Mono.50.2400.val.h5 +``` + +Option - Another option to use on machines without NUMA --- "taskset -c 1-31 python ..." + +## Training run (Multiple Sockets/Nodes) + +``` bash +export OMP_NUM_THREADS=30 # (Available cores (N) - 2) +# 1. change line 23 in configs/train_config.yaml with the following +# dist-backend: 'gloo' +# 2. change line 22 in configs/train_config.yaml with the following +# dist-backend: 'gloo' +# 3. Keep batch size (bs) in config/train_config.yaml to a multiple of (N-2) for optimum performance. +# Batch size gets multiplied by number of socket. Hence, if bs=30, no. of sockets = 16 than batch size = 30*16 = 480 +# 4. Comment line the following line (79,80) in AtacWorks/claragenomics/dl4atac/utils.py and reinstall AtacWorks using "pip install ." command. +# if (os.path.islink(latest_symlink)): +# os.remove(latest_symlink) +# 5. Run the following Slurm batch script that uses MPI commands. + +sbatch Batchfile_CPU.slurm + +``` diff --git a/ext/TAL/applications/ATAC-Seq/libjemalloc.so b/ext/TAL/applications/ATAC-Seq/libjemalloc.so new file mode 100755 index 00000000..169896e8 Binary files /dev/null and b/ext/TAL/applications/ATAC-Seq/libjemalloc.so differ diff --git a/ext/TAL/applications/ATAC-Seq/libtcmalloc.so b/ext/TAL/applications/ATAC-Seq/libtcmalloc.so new file mode 100755 index 00000000..b1c685e4 Binary files /dev/null and b/ext/TAL/applications/ATAC-Seq/libtcmalloc.so differ diff --git a/ext/TAL/applications/UMAP_fast/LICENSE b/ext/TAL/applications/UMAP_fast/LICENSE new file mode 100644 index 00000000..860ad047 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/README.md b/ext/TAL/applications/UMAP_fast/README.md new file mode 100644 index 00000000..953af8e5 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/README.md @@ -0,0 +1,38 @@ + +## Create an Anaconda environment +```bash +conda create --name umap_env python=3.8.0 +conda activate umap_env +``` + +## Load library path for libmkl_rt.so +```bash +export LD_LIBRARY_PATH=~/anaconda3/umap_env/lib/ # If loading from anaconda3 +``` + +## Install scanpy +```bash +pip install scanpy==1.8.1 +``` + +## Install pybind11 +```bash +pip install pybind11 +``` + +## Install umap_extend and umap +```bash + +pip uninstall umap-learn +cd umap_extend +python setup.py install + + +cd ../umap +python setup.py install +``` + +## Run Test +``` +python test_umap_fast.py +``` \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/requirements.txt b/ext/TAL/applications/UMAP_fast/requirements.txt new file mode 100644 index 00000000..7c740f19 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/requirements.txt @@ -0,0 +1,9 @@ +python>=3.8.0 +numpy>=1.17 +scipy>=1.3.1 +scikit-learn>=0.22 +numba>=0.51.2 +pynndescent>=0.5 +tbb>=2019.0 +pybind11>=2.8.0 +scanpy>=1.8.1 diff --git a/ext/TAL/applications/UMAP_fast/test_umap_fast.py b/ext/TAL/applications/UMAP_fast/test_umap_fast.py new file mode 100644 index 00000000..bbfc5f61 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/test_umap_fast.py @@ -0,0 +1,56 @@ +""" +MIT License + +Copyright (c) 2022 Intel Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Authors: Narendra Chaudhary ; Sanchit Misra +""" + + + +""" +This example needs scanpy package to run. Please install using pip install scanpy==1.8.1 and uninstall the existing umap-learn package. +""" + +import numpy as np +import mkl +import scanpy as sc +import os +import time + +# UMAP +umap_min_dist = 0.3 +umap_spread = 1.0 + +sc.settings.n_jobs = 56 # Set it to number of cpus on a CPU socket + +os.environ["OMP_NUM_THREADS"] = str(sc.settings.n_jobs) +mkl.set_num_threads(sc.settings.n_jobs) + +adata = sc.read('before_umap.h5ad') +print(adata.shape) +umap_time = time.time() +sc.tl.umap(adata, min_dist=umap_min_dist, spread=umap_spread) +print("UMAP time : %s" % (time.time() - umap_time)) + +sc.pl.umap(adata, color=["Stmn2_raw"], color_map="Blues", vmax=1, vmin=-0.05, save="_Stmn2_raw.png") +sc.pl.umap(adata, color=["Hes1_raw"], color_map="Blues", vmax=1, vmin=-0.05, save="_Hes1_raw.png") + diff --git a/ext/TAL/applications/UMAP_fast/umap/.travis.yml b/ext/TAL/applications/UMAP_fast/umap/.travis.yml new file mode 100644 index 00000000..ccfb3f27 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/.travis.yml @@ -0,0 +1,35 @@ +language: python + +cache: + apt: true + # We use three different cache directory + # to work around a Travis bug with multi-platform cache + directories: + - $HOME/.cache/pip + - $HOME/download +env: + global: + # Directory where tests are run from + - TEST_DIR=/tmp/test_dir/ + - MODULE=umap + +matrix: + include: + - python: 3.6 + os: linux + - env: DISTRIB="conda" PYTHON_VERSION="3.7" NUMPY_VERSION="1.17" SCIPY_VERSION="1.3.1" + os: linux + - env: DISTRIB="conda" PYTHON_VERSION="3.8" NUMPY_VERSION="1.20.0" SCIPY_VERSION="1.6.0" + os: linux + - env: DISTRIB="conda" PYTHON_VERSION="3.8" COVERAGE="true" NUMPY_VERSION="1.20.0" SCIPY_VERSION="1.6.0" + os: linux +# - env: DISTRIB="conda" PYTHON_VERSION="3.7" NUMBA_VERSION="0.51.2" +# os: osx +# language: generic +# - env: DISTRIB="conda" PYTHON_VERSION="3.8" NUMBA_VERSION="0.51.2" +# os: osx +# language: generic + +install: source ci_scripts/install.sh +script: travis_wait 90 bash ci_scripts/test.sh +after_success: source ci_scripts/success.sh diff --git a/ext/TAL/applications/UMAP_fast/umap/CODE_OF_CONDUCT.md b/ext/TAL/applications/UMAP_fast/umap/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..7f9a36b5 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at leland.mcinnes@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/ext/TAL/applications/UMAP_fast/umap/CONTRIBUTING.md b/ext/TAL/applications/UMAP_fast/umap/CONTRIBUTING.md new file mode 100644 index 00000000..b54deba4 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Contributing + +Contributions of all kinds are welcome. In particular pull requests are appreciated. +The authors will endeavour to help walk you through any issues in the pull request +discussion, so please feel free to open a pull request even if you are new to such things. + +## Issues + +The easiest contribution to make is to [file an issue](https://github.com/lmcinnes/umap/issues/new). +It is beneficial if you check the [FAQ](https://umap-learn.readthedocs.io/en/latest/faq.html), +and do a cursory search of [existing issues](https://github.com/lmcinnes/umap/issues?utf8=%E2%9C%93&q=is%3Aissue). +It is also helpful, but not necessary, if you can provide clear instruction for +how to reproduce a problem. If you have resolved an issue yourself please consider +contributing to the FAQ to add your problem, and its resolution, so others can +benefit from your work. + +## Documentation + +Contributing to documentation is the easiest way to get started. Providing simple +clear or helpful documentation for new users is critical. Anything that *you* as +a new user found hard to understand, or difficult to work out, are excellent places +to begin. Contributions to more detailed and descriptive error messages is +especially appreciated. To contribute to the documentation please +[fork the project](https://github.com/lmcinnes/umap/issues#fork-destination-box) +into your own repository, make changes there, and then submit a pull request. + +## Code + +Code contributions are always welcome, from simple bug fixes, to new features. To +contribute code please +[fork the project](https://github.com/lmcinnes/umap/issues#fork-destination-box) +into your own repository, make changes there, and then submit a pull request. If +you are fixing a known issue please add the issue number to the PR message. If you +are fixing a new issue feel free to file an issue and then reference it in the PR. +You can [browse open issues](https://github.com/lmcinnes/umap/issues), +or consult the [project roadmap](https://github.com/lmcinnes/umap/issues/15), for potential code +contributions. Fixes for issues tagged with 'help wanted' are especially appreciated. + +### Code formatting + +If possible, install the [black code formatter](https://github.com/python/black) (e.g. +`pip install black`) and run it before submitting a pull request. This helps maintain consistency +across the code, but also there is a check in the Travis-CI continuous integration system which +will show up as a failure in the pull request if `black` detects that it hasn't been run. + +Formatting is as simple as running: + +```bash +black . +``` + +in the root of the project. diff --git a/ext/TAL/applications/UMAP_fast/umap/LICENSE.txt b/ext/TAL/applications/UMAP_fast/umap/LICENSE.txt new file mode 100644 index 00000000..5e0d0b7a --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/LICENSE.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2017, Leland McInnes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ext/TAL/applications/UMAP_fast/umap/Makefile b/ext/TAL/applications/UMAP_fast/umap/Makefile new file mode 100644 index 00000000..99426d39 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/Makefile @@ -0,0 +1,11 @@ +# make gh-pages in repo base directory to automatically build and deploy documents to github + +gh-pages: + echo "Make gh-pages" + cd doc; make html + git checkout gh-pages + rm -rf _sources _static _modules _downloads _images auto_examples + mv -fv doc/_build/html/* . + rm -rf doc + git add -A + git commit -m "Generated gh-pages for `git log master -1 --pretty=short --abbrev-commit`" && git push origin gh-pages ; git checkout master diff --git a/ext/TAL/applications/UMAP_fast/umap/README.rst b/ext/TAL/applications/UMAP_fast/umap/README.rst new file mode 100644 index 00000000..0fef242d --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/README.rst @@ -0,0 +1,519 @@ +.. -*- mode: rst -*- + +.. image:: doc/logo_large.png + :width: 600 + :alt: UMAP logo + :align: center + +|pypi_version|_ |pypi_downloads|_ + +|conda_version|_ |conda_downloads|_ + +|License|_ |build_status|_ |Coverage|_ + +|Docs|_ |joss_paper|_ + +.. |pypi_version| image:: https://img.shields.io/pypi/v/umap-learn.svg +.. _pypi_version: https://pypi.python.org/pypi/umap-learn/ + +.. |pypi_downloads| image:: https://pepy.tech/badge/umap-learn/month +.. _pypi_downloads: https://pepy.tech/project/umap-learn + +.. |conda_version| image:: https://anaconda.org/conda-forge/umap-learn/badges/version.svg +.. _conda_version: https://anaconda.org/conda-forge/umap-learn + +.. |conda_downloads| image:: https://anaconda.org/conda-forge/umap-learn/badges/downloads.svg +.. _conda_downloads: https://anaconda.org/conda-forge/umap-learn + +.. |License| image:: https://img.shields.io/pypi/l/umap-learn.svg +.. _License: https://github.com/lmcinnes/umap/blob/master/LICENSE.txt + +.. |build_status| image:: https://dev.azure.com/lelandmcinnes/UMAP%20project%20builds/_apis/build/status/lmcinnes.umap?branchName=master +.. _build_status: https://dev.azure.com/lelandmcinnes/UMAP%20project%20builds/_build/latest?definitionId=2&branchName=master + +.. |Coverage| image:: https://coveralls.io/repos/github/lmcinnes/umap/badge.svg +.. _Coverage: https://coveralls.io/github/lmcinnes/umap + +.. |Docs| image:: https://readthedocs.org/projects/umap-learn/badge/?version=latest +.. _Docs: https://umap-learn.readthedocs.io/en/latest/?badge=latest + +.. |joss_paper| image:: http://joss.theoj.org/papers/10.21105/joss.00861/status.svg +.. _joss_paper: https://doi.org/10.21105/joss.00861 + +==== +UMAP +==== + +Uniform Manifold Approximation and Projection (UMAP) is a dimension reduction +technique that can be used for visualisation similarly to t-SNE, but also for +general non-linear dimension reduction. The algorithm is founded on three +assumptions about the data: + +1. The data is uniformly distributed on a Riemannian manifold; +2. The Riemannian metric is locally constant (or can be approximated as such); +3. The manifold is locally connected. + +From these assumptions it is possible to model the manifold with a fuzzy +topological structure. The embedding is found by searching for a low dimensional +projection of the data that has the closest possible equivalent fuzzy +topological structure. + +The details for the underlying mathematics can be found in +`our paper on ArXiv `_: + +McInnes, L, Healy, J, *UMAP: Uniform Manifold Approximation and Projection +for Dimension Reduction*, ArXiv e-prints 1802.03426, 2018 + +The important thing is that you don't need to worry about that—you can use +UMAP right now for dimension reduction and visualisation as easily as a drop +in replacement for scikit-learn's t-SNE. + +Documentation is `available via Read the Docs `_. + +**New: this package now also provides support for densMAP.** The densMAP algorithm augments UMAP +to preserve local density information in addition to the topological structure of the data. +Details of this method are described in the following `paper `_: + +Narayan, A, Berger, B, Cho, H, *Density-Preserving Data Visualization Unveils +Dynamic Patterns of Single-Cell Transcriptomic Variability*, bioRxiv, 2020 + +---------- +Installing +---------- + +UMAP depends upon ``scikit-learn``, and thus ``scikit-learn``'s dependencies +such as ``numpy`` and ``scipy``. UMAP adds a requirement for ``numba`` for +performance reasons. The original version used Cython, but the improved code +clarity, simplicity and performance of Numba made the transition necessary. + +Requirements: + +* Python 3.6 or greater +* numpy +* scipy +* scikit-learn +* numba + +Recommended packages: + +* `pynndescent `_ +* For plotting + * matplotlib + * datashader + * holoviews +* for Parametric UMAP + * tensorflow > 2.0.0 + + +Installing pynndescent can significantly increase performance, and in later versions +it will become a hard dependency. + +**Install Options** + +Conda install, via the excellent work of the conda-forge team: + +.. code:: bash + + conda install -c conda-forge umap-learn + +The conda-forge packages are available for Linux, OS X, and Windows 64 bit. + +PyPI install, presuming you have numba and sklearn and all its requirements +(numpy and scipy) installed: + +.. code:: bash + + pip install umap-learn + +If you wish to use the plotting functionality you can use + +.. code:: bash + + pip install umap-learn[plot] + +to install all the plotting dependencies. + +If you wish to use Parametric UMAP, you need to install Tensorflow, which can be +installed either using the instructions at https://www.tensorflow.org/install +(reccomended) or using + +.. code:: bash + + pip install umap-learn[parametric_umap] + +for a CPU-only version of Tensorflow. + +If pip is having difficulties pulling the dependencies then we'd suggest installing +the dependencies manually using anaconda followed by pulling umap from pip: + +.. code:: bash + + conda install numpy scipy + conda install scikit-learn + conda install numba + pip install umap-learn + +For a manual install get this package: + +.. code:: bash + + wget https://github.com/lmcinnes/umap/archive/master.zip + unzip master.zip + rm master.zip + cd umap-master + +Install the requirements + +.. code:: bash + + sudo pip install -r requirements.txt + +or + +.. code:: bash + + conda install scikit-learn numba + +Install the package + +.. code:: bash + + python setup.py install + +--------------- +How to use UMAP +--------------- + +The umap package inherits from sklearn classes, and thus drops in neatly +next to other sklearn transformers with an identical calling API. + +.. code:: python + + import umap + from sklearn.datasets import load_digits + + digits = load_digits() + + embedding = umap.UMAP().fit_transform(digits.data) + +There are a number of parameters that can be set for the UMAP class; the +major ones are as follows: + + - ``n_neighbors``: This determines the number of neighboring points used in + local approximations of manifold structure. Larger values will result in + more global structure being preserved at the loss of detailed local + structure. In general this parameter should often be in the range 5 to + 50, with a choice of 10 to 15 being a sensible default. + + - ``min_dist``: This controls how tightly the embedding is allowed compress + points together. Larger values ensure embedded points are more evenly + distributed, while smaller values allow the algorithm to optimise more + accurately with regard to local structure. Sensible values are in the + range 0.001 to 0.5, with 0.1 being a reasonable default. + + - ``metric``: This determines the choice of metric used to measure distance + in the input space. A wide variety of metrics are already coded, and a user + defined function can be passed as long as it has been JITd by numba. + +An example of making use of these options: + +.. code:: python + + import umap + from sklearn.datasets import load_digits + + digits = load_digits() + + embedding = umap.UMAP(n_neighbors=5, + min_dist=0.3, + metric='correlation').fit_transform(digits.data) + +UMAP also supports fitting to sparse matrix data. For more details +please see `the UMAP documentation `_ + +---------------- +Benefits of UMAP +---------------- + +UMAP has a few signficant wins in its current incarnation. + +First of all UMAP is *fast*. It can handle large datasets and high +dimensional data without too much difficulty, scaling beyond what most t-SNE +packages can manage. This includes very high dimensional sparse datasets. UMAP +has successfully been used directly on data with over a million dimensions. + +Second, UMAP scales well in embedding dimension—it isn't just for +visualisation! You can use UMAP as a general purpose dimension reduction +technique as a preliminary step to other machine learning tasks. With a +little care it partners well with the `hdbscan +`_ clustering library (for +more details please see `Using UMAP for Clustering +`_). + +Third, UMAP often performs better at preserving some aspects of global structure +of the data than most implementations of t-SNE. This means that it can often +provide a better "big picture" view of your data as well as preserving local neighbor +relations. + +Fourth, UMAP supports a wide variety of distance functions, including +non-metric distance functions such as *cosine distance* and *correlation +distance*. You can finally embed word vectors properly using cosine distance! + +Fifth, UMAP supports adding new points to an existing embedding via +the standard sklearn ``transform`` method. This means that UMAP can be +used as a preprocessing transformer in sklearn pipelines. + +Sixth, UMAP supports supervised and semi-supervised dimension reduction. +This means that if you have label information that you wish to use as +extra information for dimension reduction (even if it is just partial +labelling) you can do that—as simply as providing it as the ``y`` +parameter in the fit method. + +Seventh, UMAP supports a variety of additional experimental features including: an +"inverse transform" that can approximate a high dimensional sample that would map to +a given position in the embedding space; the ability to embed into non-euclidean +spaces including hyperbolic embeddings, and embeddings with uncertainty; very +preliminary support for embedding dataframes also exists. + +Finally, UMAP has solid theoretical foundations in manifold learning +(see `our paper on ArXiv `_). +This both justifies the approach and allows for further +extensions that will soon be added to the library. + +------------------------ +Performance and Examples +------------------------ + +UMAP is very efficient at embedding large high dimensional datasets. In +particular it scales well with both input dimension and embedding dimension. +For the best possible performance we recommend installing the nearest neighbor +computation library `pynndescent `_ . +UMAP will work without it, but if installed it will run faster, particularly on +multicore machines. + +For a problem such as the 784-dimensional MNIST digits dataset with +70000 data samples, UMAP can complete the embedding in under a minute (as +compared with around 45 minutes for scikit-learn's t-SNE implementation). +Despite this runtime efficiency, UMAP still produces high quality embeddings. + +The obligatory MNIST digits dataset, embedded in 42 +seconds (with pynndescent installed and after numba jit warmup) +using a 3.1 GHz Intel Core i7 processor (n_neighbors=10, min_dist=0.001): + +.. image:: images/umap_example_mnist1.png + :alt: UMAP embedding of MNIST digits + +The MNIST digits dataset is fairly straightforward, however. A better test is +the more recent "Fashion MNIST" dataset of images of fashion items (again +70000 data sample in 784 dimensions). UMAP +produced this embedding in 49 seconds (n_neighbors=5, min_dist=0.1): + +.. image:: images/umap_example_fashion_mnist1.png + :alt: UMAP embedding of "Fashion MNIST" + +The UCI shuttle dataset (43500 sample in 8 dimensions) embeds well under +*correlation* distance in 44 seconds (note the longer time +required for correlation distance computations): + +.. image:: images/umap_example_shuttle.png + :alt: UMAP embedding the UCI Shuttle dataset + +The following is a densMAP visualization of the MNIST digits dataset with 784 features +based on the same parameters as above (n_neighbors=10, min_dist=0.001). densMAP reveals +that the cluster corresponding to digit 1 is noticeably denser, suggesting that +there are fewer degrees of freedom in the images of 1 compared to other digits. + +.. image:: images/densmap_example_mnist.png + :alt: densMAP embedding of the MNIST dataset + +-------- +Plotting +-------- + +UMAP includes a subpackage ``umap.plot`` for plotting the results of UMAP embeddings. +This package needs to be imported separately since it has extra requirements +(matplotlib, datashader and holoviews). It allows for fast and simple plotting and +attempts to make sensible decisions to avoid overplotting and other pitfalls. An +example of use: + +.. code:: python + + import umap + import umap.plot + from sklearn.datasets import load_digits + + digits = load_digits() + + mapper = umap.UMAP().fit(digits.data) + umap.plot.points(mapper, labels=digits.target) + +The plotting package offers basic plots, as well as interactive plots with hover +tools and various diagnostic plotting options. See the documentation for more details. + +--------------- +Parametric UMAP +--------------- + +Parametric UMAP provides support for training a neural network to learn a UMAP based +transformation of data. This can be used to support faster inference of new unseen +data, more robust inverse transforms, autoencoder versions of UMAP and +semi-supervised classification (particularly for data well separated by UMAP and very +limited amounts of labelled data). See the +`documentation of Parametric UMAP `_ +or the +`example notebooks `_ +for more. + + +------- +densMAP +------- + +The densMAP algorithm augments UMAP to additionally preserve local density information +in addition to the topological structure captured by UMAP. One can easily run densMAP +using the umap package by setting the ``densmap`` input flag: + +.. code:: python + + embedding = umap.UMAP(densmap=True).fit_transform(data) + +This functionality is built upon the densMAP `implementation `_ provided by the developers +of densMAP, who also contributed to integrating densMAP into the umap package. + +densMAP inherits all of the parameters of UMAP. The following is a list of additional +parameters that can be set for densMAP: + + - ``dens_frac``: This determines the fraction of epochs (a value between 0 and 1) that will include the density-preservation term in the optimization objective. This parameter is set to 0.3 by default. Note that densMAP switches density optimization on after an initial phase of optimizing the embedding using UMAP. + + - ``dens_lambda``: This determines the weight of the density-preservation objective. Higher values prioritize density preservation, and lower values (closer to zero) prioritize the UMAP objective. Setting this parameter to zero reduces the algorithm to UMAP. Default value is 2.0. + + - ``dens_var_shift``: Regularization term added to the variance of local densities in the embedding for numerical stability. We recommend setting this parameter to 0.1, which consistently works well in many settings. + + - ``output_dens``: When this flag is True, the call to ``fit_transform`` returns, in addition to the embedding, the local radii (inverse measure of local density defined in the `densMAP paper `_) for the original dataset and for the embedding. The output is a tuple ``(embedding, radii_original, radii_embedding)``. Note that the radii are log-transformed. If False, only the embedding is returned. This flag can also be used with UMAP to explore the local densities of UMAP embeddings. By default this flag is False. + +For densMAP we recommend larger values of ``n_neighbors`` (e.g. 30) for reliable estimation of local density. + +An example of making use of these options (based on a subsample of the mnist_784 dataset): + +.. code:: python + + import umap + from sklearn.datasets import fetch_openml + from sklearn.utils import resample + + digits = fetch_openml(name='mnist_784') + subsample, subsample_labels = resample(digits.data, digits.target, n_samples=7000, + stratify=digits.target, random_state=1) + + embedding, r_orig, r_emb = umap.UMAP(densmap=True, dens_lambda=2.0, n_neighbors=30, + output_dens=True).fit_transform(subsample) + +See `the documentation `_ for more details. + +---------------- +Help and Support +---------------- + +Documentation is at `Read the Docs `_. +The documentation `includes a FAQ `_ that +may answer your questions. If you still have questions then please +`open an issue `_ +and I will try to provide any help and guidance that I can. + +-------- +Citation +-------- + +If you make use of this software for your work we would appreciate it if you +would cite the paper from the Journal of Open Source Software: + +.. code:: bibtex + + @article{mcinnes2018umap-software, + title={UMAP: Uniform Manifold Approximation and Projection}, + author={McInnes, Leland and Healy, John and Saul, Nathaniel and Grossberger, Lukas}, + journal={The Journal of Open Source Software}, + volume={3}, + number={29}, + pages={861}, + year={2018} + } + +If you would like to cite this algorithm in your work the ArXiv paper is the +current reference: + +.. code:: bibtex + + @article{2018arXivUMAP, + author = {{McInnes}, L. and {Healy}, J. and {Melville}, J.}, + title = "{UMAP: Uniform Manifold Approximation + and Projection for Dimension Reduction}", + journal = {ArXiv e-prints}, + archivePrefix = "arXiv", + eprint = {1802.03426}, + primaryClass = "stat.ML", + keywords = {Statistics - Machine Learning, + Computer Science - Computational Geometry, + Computer Science - Learning}, + year = 2018, + month = feb, + } + +Additionally, if you use the densMAP algorithm in your work please cite the following reference: + +.. code:: bibtex + + @article {NBC2020, + author = {Narayan, Ashwin and Berger, Bonnie and Cho, Hyunghoon}, + title = {Density-Preserving Data Visualization Unveils Dynamic Patterns of Single-Cell Transcriptomic Variability}, + journal = {bioRxiv}, + year = {2020}, + doi = {10.1101/2020.05.12.077776}, + publisher = {Cold Spring Harbor Laboratory}, + URL = {https://www.biorxiv.org/content/early/2020/05/14/2020.05.12.077776}, + eprint = {https://www.biorxiv.org/content/early/2020/05/14/2020.05.12.077776.full.pdf}, + } + +If you use the Parametric UMAP algorithm in your work please cite the following reference: + +.. code:: bibtex + + @article {NBC2020, + author = {Sainburg, Tim and McInnes, Leland and Gentner, Timothy Q.}, + title = {Parametric UMAP: learning embeddings with deep neural networks for representation and semi-supervised learning}, + journal = {ArXiv e-prints}, + archivePrefix = "arXiv", + eprint = {2009.12981}, + primaryClass = "stat.ML", + keywords = {Statistics - Machine Learning, + Computer Science - Computational Geometry, + Computer Science - Learning}, + year = 2020, + } + + +------- +License +------- + +The umap package is 3-clause BSD licensed. + +We would like to note that the umap package makes heavy use of +NumFOCUS sponsored projects, and would not be possible without +their support of those projects, so please `consider contributing to NumFOCUS `_. + +------------ +Contributing +------------ + +Contributions are more than welcome! There are lots of opportunities +for potential projects, so please get in touch if you would like to +help out. Everything from code to notebooks to +examples and documentation are all *equally valuable* so please don't feel +you can't contribute. To contribute please +`fork the project `_ +make your changes and +submit a pull request. We will do our best to work through any issues with +you and get your code merged into the main branch. + + diff --git a/ext/TAL/applications/UMAP_fast/umap/appveyor.yml b/ext/TAL/applications/UMAP_fast/umap/appveyor.yml new file mode 100644 index 00000000..2012ea59 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/appveyor.yml @@ -0,0 +1,28 @@ +build: "off" + +environment: + matrix: + - PYTHON_VERSION: "3.6" + MINICONDA: C:\Miniconda36-x64 + - PYTHON_VERSION: "3.7" + MINICONDA: C:\Miniconda3-x64 + - PYTHON_VERSION: "3.8" + MINICONDA: C:\Miniconda3-x64 + +init: + - "ECHO %PYTHON_VERSION% %MINICONDA%" + +install: + - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a + - "conda create -q -n test-environment python=%PYTHON_VERSION% numpy scipy scikit-learn numba pandas bokeh holoviews datashader scikit-image pytest" + - activate test-environment + - pip install "tensorflow>=2.1" + - pip install pytest-benchmark + - pip install -e . + +test_script: + - pytest --show-capture=no -v --disable-warnings + \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/azure-pipelines.yml b/ext/TAL/applications/UMAP_fast/umap/azure-pipelines.yml new file mode 100644 index 00000000..a42f03ec --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/azure-pipelines.yml @@ -0,0 +1,178 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +jobs: + - job: Linux + pool: + vmImage: 'ubuntu-latest' + strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + Python38: + python.version: '3.8' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + + - script: | + pip install -e . + pip install .[plot] + pip install .[parametric_umap] + displayName: 'Install package' + + - script: | + pip install pytest pytest-benchmark + pytest --show-capture=no -v --disable-warnings --junitxml=pytest.xml + displayName: 'Run tests' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: 'pytest.xml' + testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)' + condition: succeededOrFailed() + + - job: MacOS + pool: + vmImage: 'macOS-latest' + strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + Python38: + python.version: '3.8' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + + - script: | + pip install -e . + pip install .[plot] + pip install .[parametric_umap] + displayName: 'Install package' + + - script: | + pip install pytest pytest-benchmark + pytest --show-capture=no -v --disable-warnings --junitxml=pytest.xml + displayName: 'Run tests' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: 'pytest.xml' + testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)' + condition: succeededOrFailed() + + - job: Windows + pool: + vmImage: 'windows-latest' + strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + Python38: + python.version: '3.8' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + + - script: | + pip install -e . + pip install .[plot] + pip install .[parametric_umap] + displayName: 'Install package' + + - script: | + pip install pytest pytest-benchmark + pytest --show-capture=no -v --disable-warnings --junitxml=pytest.xml + displayName: 'Run tests' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: 'pytest.xml' + testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)' + condition: succeededOrFailed() + + - job: Coverage + pool: + vmImage: 'ubuntu-latest' + strategy: + matrix: + Python38: + python.version: '3.8' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + + - script: | + pip install -e . + pip install .[plot] + pip install .[parametric_umap] + pip install pytest pytest-benchmark + pip install pytest-cov + pip install coveralls + displayName: 'Install package' + + - script: | + export NUMBA_DISABLE_JIT=1 + pytest umap/tests --show-capture=no -v --disable-warnings --junitxml=pytest.xml --cov=umap/ --cov-report=xml --cov-report=html + displayName: 'Run tests' + + - script: | + export COVERALLS_REPO_TOKEN=$(COVERALLS_REPO_TOKEN) + coveralls + displayName: 'Publish to coveralls' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: 'coverage.xml' + testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)' + condition: succeededOrFailed() + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' diff --git a/ext/TAL/applications/UMAP_fast/umap/ci_scripts/install.sh b/ext/TAL/applications/UMAP_fast/umap/ci_scripts/install.sh new file mode 100644 index 00000000..0097e360 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/ci_scripts/install.sh @@ -0,0 +1,86 @@ +if [[ "$DISTRIB" == "conda" ]]; then + + # Deactivate the travis-provided virtual environment and setup a + # conda-based environment instead + if [ $TRAVIS_OS_NAME = 'linux' ]; then + # Only Linux has a virtual environment activated; Mac does not. + deactivate + fi + + # Use the miniconda installer for faster download / install of conda + # itself + pushd . + cd + mkdir -p download + cd download + echo "Cached in $HOME/download :" + ls -l + echo +# For now, ignoring the cached file. +# if [[ ! -f miniconda.sh ]] +# then + if [ $TRAVIS_OS_NAME = 'osx' ]; then + # MacOS URL found here: https://docs.conda.io/en/latest/miniconda.html + wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh \ + -O miniconda.sh + else + wget \ + http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + -O miniconda.sh + fi +# fi + chmod +x miniconda.sh && ./miniconda.sh -b -p $HOME/miniconda + cd .. + export PATH=$HOME/miniconda/bin:$HOME/miniconda3/bin:$PATH + conda update --yes conda + popd + + # Configure the conda environment and put it in the path using the + # provided versions +# conda create -n testenv --yes python=$PYTHON_VERSION pip \ +# numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION numba=$NUMBA_VERSION scikit-learn \ +# pytest "tensorflow-mkl>=2.2.0" + if [ $TRAVIS_OS_NAME = 'osx' ]; then + conda create -q -n testenv --yes python=$PYTHON_VERSION numpy scipy scikit-learn \ + numba pytest pandas +# pip install bokeh +# pip install datashader +# pip install holoviews + conda install --yes "tensorflow>=2.0.0" + else + conda create -q -n testenv --yes python=$PYTHON_VERSION numpy scipy scikit-learn \ + numba pandas bokeh holoviews datashader scikit-image pytest pytest-benchmark \ + "tensorflow-mkl>=2.2.0" + fi + + source activate testenv + + # black requires Python 3.x; don't try to install for Python 2.7 test + if [[ "$PYTHON_VERSION" != "2.7" ]]; then + pip install black + pip install pynndescent + fi + + if [[ "$COVERAGE" == "true" ]]; then + pip install coverage coveralls + pip install pytest-cov pytest-benchmark # pytest coverage plugin + fi + + python --version + python -c "import numpy; print('numpy %s' % numpy.__version__)" + python -c "import scipy; print('scipy %s' % scipy.__version__)" + python -c "import numba; print('numba %s' % numba.__version__)" + python -c "import sklearn; print('scikit-learn %s' % sklearn.__version__)" + python setup.py develop +else + pip install pynndescent # test with optional pynndescent dependency + pip install pandas + pip install bokeh + pip install datashader + pip install matplotlib + pip install holoviews + pip install scikit-image + pip install "tensorflow>=2.2.0" + pip install -e . +fi diff --git a/ext/TAL/applications/UMAP_fast/umap/ci_scripts/success.sh b/ext/TAL/applications/UMAP_fast/umap/ci_scripts/success.sh new file mode 100644 index 00000000..8327fc8f --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/ci_scripts/success.sh @@ -0,0 +1,13 @@ +set -e + +if [[ "$COVERAGE" == "true" ]]; then +# # Need to run coveralls from a git checkout, so we copy .coverage +# # from TEST_DIR where nosetests has been run +# cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR +# cd $TRAVIS_BUILD_DIR + # Ignore coveralls failures as the coveralls server is not + # very reliable but we don't want travis to report a failure + # in the github UI just because the coverage report failed to + # be published. + coveralls || echo "Coveralls upload failed" +fi diff --git a/ext/TAL/applications/UMAP_fast/umap/ci_scripts/test.sh b/ext/TAL/applications/UMAP_fast/umap/ci_scripts/test.sh new file mode 100644 index 00000000..63ccc510 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/ci_scripts/test.sh @@ -0,0 +1,12 @@ +set -e + +#if [[ "$COVERAGE" == "true" ]]; then +# black --check $MODULE +#fi + +if [[ "$COVERAGE" == "true" ]]; then + export NUMBA_DISABLE_JIT=1 + pytest --cov=umap/ --cov-report=xml --cov-report=html --show-capture=no -v --disable-warnings +else + pytest --show-capture=no -v --disable-warnings +fi diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/.gitignore b/ext/TAL/applications/UMAP_fast/umap/doc/.gitignore new file mode 100644 index 00000000..6655421e --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/.gitignore @@ -0,0 +1,7 @@ +venv +umap +setup.py +paper.md +paper.bib +LICENSE.txt +CODE_OF_CONDUCT.md diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/Makefile b/ext/TAL/applications/UMAP_fast/umap/doc/Makefile new file mode 100644 index 00000000..9981f907 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = umap +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/_static/.gitkeep b/ext/TAL/applications/UMAP_fast/umap/doc/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_basic_usage.rst b/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_basic_usage.rst new file mode 100644 index 00000000..c5893237 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_basic_usage.rst @@ -0,0 +1,496 @@ +How to use AlignedUMAP +====================== + +It may happen that it would be beneficial to have different UMAP +embeddings aligned with each other. There are several ways to go about +doing this. One simple approach is to simply embed each dataset with +UMAP independently and then solve for a `Procrustes +transformation `__ +on shared points. An alternative approach is to embed the first dataset +and then construct an initial embedding for the second dataset based on +locations of shared points in the first embedding and then go from +there. A third approach, which will provide better alignments in +general, is to optimize both embeddings at the same time with some form +of constraint as to how far shared points can take different locations +in different embeddings *during* the optimization. This last option is +possible, but is not easily tractable to implement yourself (unlike the +first two options). To remedy this issue it has been implemented as a +separate model class in ``umap-learn`` called ``AlignedUMAP``. The +resulting class is quite flexible, but here we will walk through simple +usage on some basic (and somewhat contrived) data just to demonstrate +how to get it running on data. + +.. code:: python3 + + import numpy as np + import sklearn.datasets + import umap + import umap.plot + import umap.utils as utils + import umap.aligned_umap + import matplotlib.pyplot as plt + +For our demonstration we’ll just use the pendigits dataset from sklearn. + +.. code:: python3 + + digits = sklearn.datasets.load_digits() + +To make a sequence of datasets with some shared points between each +different dataset we’ll first sort the data so we have some vaguely +sensible progression. In this case we’ll sort by the total amount of +“ink” in the handwritten digit. This isn’t meant to be meaningful, it is +merely meant to provide something useful to slicing into overlapping +chunks that we will want to embed separately and yet keep aligned. + +.. code:: python3 + + ordered_digits = digits.data[np.argsort(digits.data.sum(axis=1))] + ordered_target = digits.target[np.argsort(digits.data.sum(axis=1))] + plt.matshow(ordered_digits[-1].reshape((8,8))) + + +.. image:: images/aligned_umap_basic_usage_5_1.png + + +We can then divide up the dataset into slices of 400 samples, moving +along in chunks of 150 to ensure that there are overlaps between +consecutive slices. This will give us a list of ten different datasets +that we can embed, with the goal being to ensure that the positions of +points in the embeddings are relatively consistent. + +.. code:: python3 + + slices = [ordered_digits[150 * i:min(ordered_digits.shape[0], 150 * i + 400)] for i in range(10)] + +To ensure that consistency ``AlignedUMAP`` will need more information +than *just* the datasets – we also need some information about how the +datasets relate to one another. These take the form of dictionaries that +relate the indices of one dataset to the indices of another. Currently +``AlignedUMAP`` only supports sequences of datasets with relations +between each consecutive pair in the sequence. To construct the +relations for this dataset we note that the last 250 samples of one +dataset are going to be the same samples as the first 250 samples of the +next dataset – this makes it easy to construct the dictionary: it is +mapping + +:: + + 150 --> 0 + 151 --> 1 + ... + 398 --> 248 + 399 --> 249 + +which we can construct easily using a dictionary comprehension. We will +have the same relation between each consecutive pair, so to make the +list of relations between pairs we can just duplicate the constructed +relation the requisite number of times. + +.. code:: python3 + + relation_dict = {i+150:i for i in range(400-150)} + relation_dicts = [relation_dict.copy() for i in range(len(slices) - 1)] + +Note that while in this case the relation defines a map between +identical samples in different datasets it can be much more general – +see the politics example later for a case where the relation is +constructed from external information (representatives names and +states). + +Now that we have both a list of data slices and a list of relations +between the consecutive pairs we can use the ``AlignedUMAP`` class to +generate a list of embeddings. The ``AlignedUMAP`` class takes most of +the parameters that UMAP accepts. The major difference is that the fit +method requires a *list* of datasets, and a keyword argument +``relations`` that specifies the relation dictionaries between +consecutive pairs of datasets. Other than that things are essentially +push-button. + +.. code:: python3 + + %%time + aligned_mapper = umap.AlignedUMAP().fit(slices, relations=relation_dicts) + + +.. parsed-literal:: + + CPU times: user 57.4 s, sys: 8.43 s, total: 1min 5s + Wall time: 57.4 s + + +You will note that this took a non-trivial amount of time to run, +despite being on the relatively small pendigits dataset. This is because +we are completing 10 different UMAP embeddings at once, so on average we +are taking about five seconds per embedding, which is more reasonable – +the alignment does have overhead cost however. + +The next step is to look at the results. To ensure that the plots we +produce have a consistent x and y axis we’ll use a small function to +compute a set of axis bounds for plotting. + +.. code:: python3 + + def axis_bounds(embedding): + left, right = embedding.T[0].min(), embedding.T[0].max() + bottom, top = embedding.T[1].min(), embedding.T[1].max() + adj_h, adj_v = (right - left) * 0.1, (top - bottom) * 0.1 + return [left - adj_h, right + adj_h, bottom - adj_v, top + adj_v] + +Now it is just a matter of plotting the results in ten different scatter +plots. We can do this most easily with matplotlib directly, setting up a +grid of plots. Note that the progression proceeds by row then column, so +read the progression as if you were reading a page of text (across, then +down). + +.. code:: python3 + + fig, axs = plt.subplots(5,2, figsize=(10, 20)) + ax_bound = axis_bounds(np.vstack(aligned_mapper.embeddings_)) + for i, ax in enumerate(axs.flatten()): + current_target = ordered_target[150 * i:min(ordered_target.shape[0], 150 * i + 400)] + ax.scatter(*aligned_mapper.embeddings_[i].T, s=2, c=current_target, cmap="Spectral") + ax.axis(ax_bound) + ax.set(xticks=[], yticks=[]) + plt.tight_layout() + + + +.. image:: images/aligned_umap_basic_usage_15_0.png + + +So despite being different embeddings on different datasets, the +clusters keep their general alignment – the top left plot and bottom +right plot have the same rough positions for specific digit clusters. We +can also, to a degree, see how the structure changes over the course of +the different slices. Thus we are keeping the various embeddings +aligned, but allowing the changes dictated by the differing structures +of each different slice of data. + +Online updating of aligned embeddings +------------------------------------- + +It may be the case that we have incoming temporal data and would like to +have embeddings of time-windows that, ideally, align with the embeddings +of prior time-windows. As long as we overlap the time-windows we use to +allow for relations between time windows then this is possible – except +that the previous code required all the time-windows to be input *at +once* for fitting. We would instead like to train an initial model and +then update it as we go. This is possible via the ``update`` method +which we’ll demonstrate below. + +First we need to fit a base ``AlignedUMAP`` model; we’ll use the first +two slices and the first relation dict to do so. + +.. code:: python3 + + %%time + updating_mapper = umap.AlignedUMAP().fit(slices[:2], relations=relation_dicts[:1]) + + +.. parsed-literal:: + + CPU times: user 9.32 s, sys: 1.47 s, total: 10.8 s + Wall time: 9.17 s + + +Note that this is fairly quick, since we are only fitting two slices. +Given the trained model the update method requires a new slice of data +to add, along with a relation dictionary (passed in with the +``relations`` keyword argument as with ``fit``). This will append a new +embedding to the ``embeddings_`` attribute of the model for the new +data, aligned with what has been seen so far. + +.. code:: python3 + + for i in range(2,len(slices)): + %time updating_mapper.update(slices[i], relations={v:k for k,v in relation_dicts[i-1].items()}) + + +.. parsed-literal:: + + CPU times: user 7.78 s, sys: 1.15 s, total: 8.93 s + Wall time: 7.92 s + CPU times: user 6.64 s, sys: 1.17 s, total: 7.81 s + Wall time: 6.6 s + CPU times: user 6.94 s, sys: 1.17 s, total: 8.11 s + Wall time: 6.81 s + CPU times: user 6.45 s, sys: 1.51 s, total: 7.96 s + Wall time: 6.45 s + CPU times: user 7.44 s, sys: 1.32 s, total: 8.76 s + Wall time: 7.16 s + CPU times: user 7.68 s, sys: 1.73 s, total: 9.41 s + Wall time: 7.59 s + CPU times: user 7.88 s, sys: 1.65 s, total: 9.54 s + Wall time: 7.39 s + CPU times: user 7.82 s, sys: 1.98 s, total: 9.8 s + Wall time: 7.7 s + + +Note that each new slice takes a relatively short period of time, as we +might hope. The downside of this, as you can imagine, is that we have no +“forward” relations – the windows over slices only look backward. This +means the results are less good, but we are trading that for the ability +to quickly and easily update as we go. + +We can look at how we did using essentially the same code as before. + +.. code:: python3 + + fig, axs = plt.subplots(5,2, figsize=(10, 20)) + ax_bound = axis_bounds(np.vstack(updating_mapper.embeddings_)) + for i, ax in enumerate(axs.flatten()): + current_target = ordered_target[150 * i:min(ordered_target.shape[0], 150 * i + 400)] + ax.scatter(*updating_mapper.embeddings_[i].T, s=2, c=current_target, cmap="Spectral") + ax.axis(ax_bound) + ax.set(xticks=[], yticks=[]) + plt.tight_layout() + + + +.. image:: images/aligned_umap_basic_usage_22_0.png + + +Wee see that the alignment is indeed working, so new slices remain +comparable with previously trained slices. As noted the overall +alignments and progression is not as nice as the previous version, but +it does have the significant benefit of allowing an update as you go +approach. + +Note that right now this model keeps all the previous data, so it will +only really work in a batch streaming approach where occasionally a +fresh model is trained, dropping some of the historical data before +continuing with updates. + +Aligning varying parameters +--------------------------- + +It is possible to align UMAP embedding that vary in the parameters used +instead of the data. To demonstrate how this can work we’ll continue to +use the pendigits dataset, but instead of slicing the data as we did +before, we’ll use the full dataset. That means that our relations +between datasets are simply constant relations. We can construct those +ahead of time: + +.. code:: python3 + + constant_dict = {i:i for i in range(digits.data.shape[0])} + constant_relations = [constant_dict for i in range(9)] + +To run AlignedUMAP over a range of parameters you simply need to pass in +a *list* of the sequence of parameters you wish to use. You can do this +for several different parameters – just ensure that all the lists are +the same length! In this case we’ll try looking at how the embeddings +change if we change ``n_neighbors`` and ``min_dist``. This means that +when we create the AlignedUMAP object we pass a list, instead of a +single value, to each of those parameters. To make the visualization a +little more interesting we’ll also vary some of the alignment parameters +(there are only two of major consequence). Specifically we’ll adjust the +``alignment_window_size``, which controls how far forward and backward +across the datasets we look when doing alignment, and the +``alignment_regularisation`` which controls how heavily we weight the +alignment aspect versus the UMAP layout. Larger values of +``alignment_regularisation`` will work harder to keep points aligned +across embeddings (at the cost of the embedding quality at each slice), +while smaller values will allow the optimisation to focus more on the +individual embeddings and put less emphasis on aligning the embeddings +with each other. + +Given a model we can then fit it. As before we need to hand it a list of +datasets, and a list of relations. Since we are using the same data each +time (and varying the parameters) we can just repeat the full pendigits +dataset. Note that the number of datasets needs to match the number of +parameter values being used. The same goes for the number of relations +(one less than the number of parameter values). + +.. code:: python3 + + neighbors_mapper = umap.AlignedUMAP( + n_neighbors=[3,4,5,7,11,16,22,29,37,45,54], + min_dist=[0.01,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45], + alignment_window_size=2, + alignment_regularisation=1e-3, + ).fit( + [digits.data for i in range(10)], relations=constant_relations + ) + + +As before we can look at the results by plotting each of the embeddings. + +.. code:: python3 + + fig, axs = plt.subplots(5,2, figsize=(10, 20)) + ax_bound = axis_bounds(np.vstack(neighbors_mapper.embeddings_)) + for i, ax in enumerate(axs.flatten()): + ax.scatter(*neighbors_mapper.embeddings_[i].T, s=2, c=digits.target, cmap="Spectral") + ax.axis(ax_bound) + ax.set(xticks=[], yticks=[]) + plt.tight_layout() + + + +.. image:: images/aligned_umap_basic_usage_29_1.png + + +To get a better feel for the evolution of the embedding over the change +in parameter values we can plot the data in three dimensions, with the +third dimension being the parameter value chosen. To better show how +data points in the embedding *move* with respect to the changing +parameters we can plot them not as points, but as *curves* connecting +the same point in each sequential embedding. For three dimensional plots +like this we’ll make use of the `plotly `__ plotting +library. + +.. code:: python3 + + import plotly.graph_objects as go + import plotly.express as px + import pandas as pd + +The first thing we’ll have to do is wrangle the data into a suitable +format for plotly. That’s the reason we loaded up pandas as well – +plotly likes dataframes. This involves stacking all the embeddings +together, and then assigning an extra ``z`` value according to which +embedding we are in. For the purposes of visualization we’ll just have a +linear scale from 0 to 1 of the appropriate length for the z +coordinates. + +.. code:: python3 + + n_embeddings = len(neighbors_mapper.embeddings_) + es = neighbors_mapper.embeddings_ + embedding_df = pd.DataFrame(np.vstack(es), columns=('x', 'y')) + embedding_df['z'] = np.repeat(np.linspace(0, 1.0, n_embeddings), es[0].shape[0]) + embedding_df['id'] = np.tile(np.arange(es[0].shape[0]), n_embeddings) + embedding_df['digit'] = np.tile(digits.target, n_embeddings) + +The next thing we can do to improve the visualization is to smooth out +the curves rather than leaving them as piecewise linear lines. To to +this we can use the ``scipy.interpolate`` functionality to create smooth +cubic splines that pass through all the points of the curve we wish to +create. + +.. code:: python3 + + import scipy.interpolate + +The interpolate module has a function ``interp1d`` that generates a +(vector of) smooth function given a one dimensional set of datapoints +that it needs to pass through. We can generate separate functions for +the x and y coordinates for each pendigit sample, allowing us to +generate smooth curves in three dimensions. + +.. code:: python3 + + fx = scipy.interpolate.interp1d( + embedding_df.z[embedding_df.id == 0], embedding_df.x.values.reshape(n_embeddings, digits.data.shape[0]).T, kind="cubic" + ) + fy = scipy.interpolate.interp1d( + embedding_df.z[embedding_df.id == 0], embedding_df.y.values.reshape(n_embeddings, digits.data.shape[0]).T, kind="cubic" + ) + z = np.linspace(0, 1.0, 100) + +With that in hand it is just a matter of plotting all the curves. In +plotly parlance each curve is a “trace” and we generate each one +separately (along with a suitable colour given by the digit the sample +represents). We then add all the traces to a figure, and display the +figure. + +.. code:: python3 + + palette = px.colors.diverging.Spectral + interpolated_traces = [fx(z), fy(z)] + traces = [ + go.Scatter3d( + x=interpolated_traces[0][i], + y=interpolated_traces[1][i], + z=z*3.0, + mode="lines", + line=dict( + color=palette[digits.target[i]], + width=3.0 + ), + opacity=1.0, + ) + for i in range(digits.data.shape[0]) + ] + fig = go.Figure(data=traces) + fig.update_layout( + width=800, + height=700, + autosize=False, + showlegend=False, + ) + fig.show() + + +.. image:: images/aligned_umap_pendigits_3d_1.png + + +Since it is tricky to get the interactive plotly figure embedded in +documentation we have a static image here, but if you run this yourself +you will have a fully interactive view of the data. + +Alternatively, we can visualize the third dimension as an evolution of the +embeddings through time by rendering each z-slice as a frame in an animated +GIF. To do this, we'll first need to import some notebook display tools and +matplotlib's `animation `_ +module. + +.. code:: python3 + + from IPython.display import display, Image, HTML + from matplotlib import animation + + +Next, we'll create a new figure, initialize a blank scatter plot, then use +``FuncAnimation`` to update the point positions (called "offsets") one frame at +a time. + +.. code:: python3 + + fig = plt.figure(figsize=(4, 4), dpi=150) + ax = fig.add_subplot(1, 1, 1) + + scat = ax.scatter([], [], s=2) + scat.set_array(digits.target) + scat.set_cmap('Spectral') + text = ax.text(ax_bound[0] + 0.5, ax_bound[2] + 0.5, '') + ax.axis(ax_bound) + ax.set(xticks=[], yticks=[]) + plt.tight_layout() + + offsets = np.array(interpolated_traces).T + num_frames = offsets.shape[0] + + def animate(i): + scat.set_offsets(offsets[i]) + text.set_text(f'Frame {i}') + return scat + + anim = animation.FuncAnimation( + fig, + init_func=None, + func=animate, + frames=num_frames, + interval=40) + + +Then we can save the animation as a GIF and close our animation. Depending on +your machine, you may need to change which writer the save method uses. + +.. code:: python3 + + anim.save("aligned_umap_pendigits_anim.gif", writer="pillow") + plt.close(anim._fig) + + +Finally, we can read in our rendered GIF and display it in the notebook. + +.. code:: python3 + + with open("aligned_umap_pendigits_anim.gif", "rb") as f: + display(Image(f.read())) + + +.. image:: images/aligned_umap_pendigits_anim.gif diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_plotly_plot.html b/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_plotly_plot.html new file mode 100644 index 00000000..1d7bc63a --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_plotly_plot.html @@ -0,0 +1,7 @@ + + + +
+
+ + \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_politics_demo.rst b/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_politics_demo.rst new file mode 100644 index 00000000..d87ffabe --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/aligned_umap_politics_demo.rst @@ -0,0 +1,1088 @@ +AlignedUMAP for Time Varying Data +================================= + +It is not uncommon to have datasets that can be partitioned into +segments, often with respect to time, where we want to understand not +only the structure of each segment, but how that structure changes over +the different segments. An example of this is the relative political +leanings of the US congress over time. In determining the relative +political leanings we can look at the representatives voting record on +roll call votes, with the presumption that representatives with similar +political principles will have similar voting records. We can, of +course, look at such data for any given congress, but since +representatives are commonly re-elected we can also consider how their +relative position in congress changes with time – an ideal use case for +AlignedUMAP. + +First we’ll need a selection of libraries. Aside from UMAP we will need +to do a little bit of data wrangling; for that we’ll need pandas, and +also for matching up names of representatives we’ll make use of the +library ``fuzzywuzzy`` which provides easy to use fuzzy string matching. + +.. code:: python3 + + import umap + import umap.utils as utils + import umap.aligned_umap + import sklearn.decomposition + + import pandas as pd + import numpy as np + + import matplotlib.pyplot as plt + import seaborn as sns + + from fuzzywuzzy import fuzz, process + import re + + +.. code:: python3 + + sns.set(style="darkgrid", color_codes=True) + +Next we’ll need to voting records for the representatives, along with +the associated metadata from the roll call vote records. You can obtain +the data https://clerk.house.gov; a notebook demonstrating how to pull +down the data and parse it into the csv files used here is available +`here `__. + +Processing Congressional Voting Records +--------------------------------------- + +The voting records provide a row for each representative with a -1 for +“No”, 0 for “Present” or “Not Voting”, and 1 for “Aye”. A separate csv +file contains the raw data of all the votes with a row for each +legislators vote on each roll-call item. We really just need some +metadata – which state they represent and the party they represent so we +can decorate the results with this kind of information later. For that +we just need to extra the names, states, and parties for each year. We +can grab those columns and then drop duplicates. A catch: the party is +very occasionally entered incorrectly, and occasionally representatives +switch parties, making duplicated rows. We’ll just take the first entry +of such duplciates for now. + +.. code:: python3 + + votes = [pd.read_csv(f"house_votes/{year}_voting_record.csv", index_col=0).sort_index() + for year in range(1990,2021)] + metadata = [pd.read_csv( + f"house_votes/{year}_full.csv", + index_col=0 + )[["legislator", "state", "party"]].drop_duplicates(["legislator", "state"]).sort_values('legislator') + for year in range(1990,2021)] + +Let’s take a look at the voting record for a single year to see what +sort of data we are looking at: + +.. code:: python3 + + votes[5] + + + + +.. raw:: html + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
104-1st-1104-1st-10104-1st-100104-1st-101104-1st-102104-1st-103104-1st-104104-1st-105104-1st-106104-1st-107...104-1st-90104-1st-91104-1st-92104-1st-93104-1st-94104-1st-95104-1st-96104-1st-97104-1st-98104-1st-99
legislator
Abercrombie0.01.0-1.0-1.0-1.0-1.01.01.0-1.01.0...-1.0-1.01.0-1.01.0-1.0-1.01.01.01.0
Ackerman0.01.0-1.01.0-1.0-1.01.01.0-1.01.0...1.0-1.0-1.01.01.0-1.0-1.01.01.01.0
Allard0.01.01.01.0-1.01.0-1.0-1.01.0-1.0...-1.0-1.0-1.0-1.01.01.01.01.00.0-1.0
Andrews0.01.00.0-1.0-1.01.0-1.00.00.00.0...-1.01.0-1.0-1.01.01.0-1.01.0-1.0-1.0
Archer0.01.01.0-1.0-1.01.0-1.0-1.01.0-1.0...-1.0-1.0-1.0-1.0-1.01.01.01.0-1.00.0
..................................................................
Young (AK)0.01.01.01.0-1.01.0-1.0-1.01.0-1.0...-1.0-1.0-1.0-1.0-1.01.01.01.0-1.0-1.0
Young (FL)0.01.01.0-1.0-1.01.0-1.0-1.01.0-1.0...-1.0-1.0-1.0-1.0-1.01.01.01.0-1.0-1.0
Zeliff0.01.01.0-1.0-1.01.0-1.0-1.01.0-1.0...-1.0-1.0-1.0-1.0-1.01.01.01.0-1.0-1.0
Zimmer0.01.01.0-1.0-1.01.0-1.0-1.01.0-1.0...-1.01.0-1.0-1.0-1.01.01.01.0-1.0-1.0
de la Garza0.01.01.01.0-1.01.01.01.0-1.01.0...0.0-1.0-1.01.01.0-1.01.01.0-1.01.0
+

438 rows × 885 columns

+
+ + + +We can examine the associated metadata for the same year. + +.. code:: python3 + + metadata[5] + + + + +.. raw:: html + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
legislatorstateparty
0AbercrombieHID
1AckermanNYD
2AllardCOR
3AndrewsNJD
4ArcherTXR
............
430Young (AK)AKR
431Young (FL)FLR
432ZeliffNHR
433ZimmerNJR
89de la GarzaTXD
+

438 rows × 3 columns

+
+ + + +You may note that sometimes representatives names list a state in +parenthesis afterwards. This is to provide disambiguation for +representatives that happen to have the last name. This actually +complicates matters for us since the disambiguation is only applied in +those cases where there is a name collision in that sitting of congress. +That means that for several years a representative may have simply their +last name, but then switch to being disambiguated, before potentially +switching back again. This would make it much harder to consistently +treck representatives over their entire career in congress. To fix this +up we’ll simply re-index by a unique representative ID that has their +last name, party, and state all listed over all the voting dataframes. +We’ll need a function to generate those from the metadata, and then +we’ll need to apply it to all the reocrds. Importantly we’ll have to +finesse those situations where representatives are listed twice (under +un-ambiguous and disambiguated names) with some groupby tricks. + +.. code:: python3 + + def unique_legislator(row): + name, state, party = row.legislator, row.state, row.party + # Strip of disambiguating state designators + if re.search(r'(\w+) \([A-Z]{2}\)', name) is not None: + name = name[:-5] + return f"{name} ({party}, {state})" + +.. code:: python3 + + for i, _ in enumerate(votes): + votes[i].index = pd.Index(metadata[i].apply(unique_legislator, axis=1), name="legislator_index") + votes[i] = votes[i].groupby(level=0).sum() + metadata[i].index = pd.Index(metadata[i].apply(unique_legislator, axis=1), name="legislator_index") + metadata[i] = metadata[i].groupby(level=0).first() + +Now that we have the data at least a little wrangled into order there is +the question of ensuring some degree of continuity fore representatives. +To make this a little easier we’ll use voting records over *four year +spans* instead of over single years. Equally importantly we’ll do this +in a sliding window fashion so that we consider the record for 1990-1994 +and then the record for 1991-1995 and so on. By overlapping the windows +in this way we can ensure a little greater continuity of political +stance through the years. To make this happen we just have to merge data +frames in a sliding set of pairs, and then merge the pairs via the same +approach: + +.. code:: python3 + + votes = [ + pd.merge( + v1, v2, how="outer", on="legislator_index" + ).fillna(0.0).sort_index() + for v1, v2 in zip(votes[:-1], votes[1:]) + ] + votes[-1:] + + metadata = [ + pd.concat([m1, m2]).groupby("legislator_index").first().sort_index() + for m1, m2 in zip(metadata[:-1], metadata[1:]) + ] + metadata[-1:] + +That’s the pairs of years; now we merge these pairwise to get sets of +four years worth of votes. + +.. code:: python3 + + votes = [ + pd.merge( + v1, v2, how="outer", on="legislator_index" + ).fillna(0.0).sort_index() + for v1, v2 in zip(votes[:-1], votes[1:]) + ] + votes[-1:] + + metadata = [ + pd.concat([m1, m2]).groupby(level=0).first().sort_index() + for m1, m2 in zip(metadata[:-1], metadata[1:]) + ] + metadata[-1:] + +Applying AlignedUMAP +-------------------- + +To make use of AlignedUMAP we need to generate relations between +consecutive dataset slices. In this case that means we need to have a +relation describing row from one four year slice corresponds to a row +from the following four year slice for the same representative. For +AlignedUMAP to work this should be formatted as a list of dictionaries; +each dictionary gives a mapping from indices of one slice to indices of +the next. Importantly this mapping can be partial – it only has to +relate indices for which there is a match between the two slices. + +The vote dataframes that we are using for slices are already indexed +with unique identifiers for representatives, so to make relations we +simply have to match them up, creating a dictionary of indices from one +to the other. In practice we can do this relatively efficiently by using +pandas to merge dataframes on the pandas indexes of the two vote +dataframes with the data being simply the numeric indices of the rows. +The resulting dictionary is then just the dictionary of pairs given by +the inner join. + +.. code:: python3 + + def make_relation(from_df, to_df): + left = pd.DataFrame(data=np.arange(len(from_df)), index=from_df.index) + right = pd.DataFrame(data=np.arange(len(to_df)), index=to_df.index) + merge = pd.merge(left, right, left_index=True, right_index=True) + return dict(merge.values) + +With a function for relation creation in place we simply need to apply +it to each consecutive pair of vote dataframes. + +.. code:: python3 + + relations = [make_relation(x,y) for x, y in zip(votes[:-1], votes[1:])] + +If you are still unsure of what these relations are it might be +beneficial to look at a few of the dictionaries, along with the +corresponding pairs of vote dataframes. Here is (part of) the first +relation dictionary: + +.. code:: python3 + + relations[0] + + + + +.. parsed-literal:: + + {0: 0, + 1: 1, + 3: 2, + 4: 3, + 5: 4, + 6: 5, + 7: 6, + 8: 7, + 9: 8, + 10: 9, + 11: 10, + 12: 11, + 13: 12, + 14: 13, + 15: 14, + ... + 475: 547, + 476: 549, + 477: 550, + 478: 552, + 479: 553, + 480: 554, + 481: 555, + 482: 556, + 483: 557, + 484: 559} + + + +Now we are finally in a position to run AlignedUMAP. Most of the +standard UMAP parameters are available for use, including choosing a +metric and a number of neighbors. Here we will also make use of the +extra AlignedUMAP parameters ``alignment_regularisation`` and +``alignment_window_size``. The first is a value that weights how +important retaining alignment is. Typically the value is much smaller +than this (the default is 0.01), but given the relatively high +volatility in voting records we are going to increase it here. The +second parameter, ``alignment_window_size`` determines how far out on +either side AlignedUMAP will look when aligning embeddings – even though +the relations are specified only between consecutive slices it will +chain them together to construct relations reaching further. In this +case we’ll have it look as far out as 5 slices either side. + +.. code:: python3 + + %%time + aligned_mapper = umap.aligned_umap.AlignedUMAP( + metric="cosine", + n_neighbors=20, + alignment_regularisation=0.1, + alignment_window_size=5, + n_epochs=200, + random_state=42, + ).fit(votes, relations=relations) + embeddings = aligned_mapper.embeddings_ + + +.. parsed-literal:: + + CPU times: user 6min 7s, sys: 30.6 s, total: 6min 37s + Wall time: 5min 57s + + +Visualizing the Results +----------------------- + +Now we need to plot the data somehow. To make the visualization +interesting it would be beneficial to have some colour variation – +ideally showing a different view of the relative political stance. For +that we want to attempt to get an idea of the position of each candidate +from an alternative source. To do this we can try to extract the vote +margin that the representative won by. The catch here is that while the +election data can be collected and processed, the names don’t match +perfectly as they come from a different source. That means we need to do +our best to get a name match for each candidate. We’ll use fuzzy string +matching restricted to the relevant year and state to try to get a good +match. A notebook providing details for obtaining and processing the +election winners data can be found +`here `__. + +.. code:: python3 + + election_winners = pd.read_csv('election_winners_1976-2018.csv', index_col=0) + election_winners.head() + + + + +.. raw:: html + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
yearstatedistrictwinnerpartywinning_ratio
01976AK0Don Youngrepublican0.289986
01976AL1Jack Edwardsrepublican0.374808
01976AL2William L. \\"Bill\"\" Dickinson"republican0.423953
01976AL3Bill Nicholsdemocrat1.000000
01976AL4Tom Bevilldemocrat0.803825
+
+ + + +Now we need to simply go through the metadata and fill it out with the +extra information we can glean from the election winners data. Since we +can’t do exact name matching (the data for both is somewhat messy when +it comes to text fields like names) we can’t simply perform a join, but +must instead process things year by year and representative by +representative, finding the best string match on name that we can for +the given year and state election. In practice we are undoubtedly going +to get some of these wrong, and if the goal was a rigorous analysis +based on this data a lot more care would need to be taken. Since this is +just a demonstration and we’ll only be using this extra information as a +colour channel in plots we can excuise a few errors here and there from +in-exact data processing. + +.. code:: python3 + + n_name_misses = 0 + for year, df in enumerate(metadata, 1990): + df["partisan_lean"] = 0.5 + df["district"] = np.full(len(df), -1, dtype=np.int8) + for idx, (loc, row) in enumerate(df.iterrows()): + name, state, party = row.legislator, row.state, row.party + # Strip of disambiguating state designators + if re.search(r'(\w+) \([A-Z]{2}\)', name) is not None: + name = name[:-5] + # Get a party designator matching the election_winners data + party = "republican" if party == "R" else "democrat" + # Restrict to the right state and time-frame + state_election_winners = election_winners[(election_winners.state == state) + & (election_winners.year <= year + 4) + & (election_winners.year >= year - 4)] + # Try to match a name; and fail "gracefully" + try: + matched_name = process.extractOne( + name, + state_election_winners.winner.tolist(), + scorer=fuzz.partial_token_sort_ratio, + score_cutoff=50, + ) + except: + matched_name = None + + # If we got a unique match, get the election data + if matched_name is not None: + winner = state_election_winners[state_election_winners.winner == matched_name[0]] + else: + winner = [] + + # We either have none, one, or *several* match elections. Take a best guess. + if len(winner) < 1: + df.loc[loc, ["partisan_lean"]] = 0.25 if party == "republican" else 0.75 + n_name_misses += 1 + elif len(winner) > 1: + df.iloc[idx, 4] = int(winner.district.values[-1]) + df.iloc[idx, 3] = float(winner.winning_ratio.values[-1]) + else: + df.iloc[idx, 4] = int(winner.district.values) + df.iloc[idx, 3] = float(winner.winning_ratio.values[0]) + + print(f"Failed to match a name {n_name_misses} times") + + +.. parsed-literal:: + + Failed to match a name 100 times + + +Now that we have the relative partisan leanings based on district +election margins we can color the plot. We can obviously label the plot +with the representatives names. The last remaining catch (when using +matplotlib for the plotting) is the get the plot bounds (since we will +be placing text markers directly into the plot, and thus not +autogenerating bounds). This is a simple enough matter of computing some +bounds as an adjustment a little outside the data limits. + +.. code:: python3 + + def axis_bounds(embedding): + left = embedding.T[0].min() + right = embedding.T[0].max() + bottom = embedding.T[1].min() + top = embedding.T[1].max() + width = right - left + height = top - bottom + adj_h = width * 0.1 + adj_v = height * 0.05 + return [left - adj_h, right + adj_h, bottom - adj_v, top + adj_v] + +Now for the plot. Let’s pick a random time slice (you are welcome to try +others) and draw the representatives names in their embedded locations +for that slice, coloured by their relative election victory margin. + +.. code:: python3 + + fig, ax = plt.subplots(figsize=(12,12)) + e = 5 + ax.axis(axis_bounds(embeddings[e])) + ax.set_aspect('equal') + for i in range(embeddings[e].shape[0]): + ax.text(embeddings[e][i, 0], + embeddings[e][i, 1], + metadata[e].index.values[i], + color=plt.cm.RdBu(np.float32(metadata[e]["partisan_lean"].values[i])), + fontsize=8, + horizontalalignment='center', + verticalalignment='center', + ) + + + + +.. image:: images/aligned_umap_politics_demo_31_0.png + + +This gives a good idea of the layout in a single time slices, and by +plotting different time slices we can get some idea of how things have +evolved. We can go further, however, by plotting a representative as +curve through time as their relative political position in congress +changes. For that we will need a 3D plot – we need both the UMAP x and y +coordinates, as well as a third coordinate giving the year. I found this +easiest to do in plotly, so let’s import that. To make nice smooth +curves through time we will also import the ``scipy.interpolate`` module +which will let is interpolate a smooth curve from the discrete positions +that a representatives appears in over time. + +.. code:: python3 + + import plotly.graph_objects as go + import scipy.interpolate + +Wrangling the data into shape for this is the next step; first let’s get +everything in a single dataframe that we can extract relevant data from +on an as-needed basis. + +.. code:: python3 + + df = pd.DataFrame(np.vstack(embeddings), columns=('x', 'y')) + df['z'] = np.concatenate([[year] * len(embeddings[i]) for i, year in enumerate(range(1990, 2021))]) + df['representative_id'] = np.concatenate([v.index for v in votes]) + df['partisan_lean'] = np.concatenate([m["partisan_lean"].values for m in metadata]) + +Next we’ll need that interpolation of the curve for a given +representative. We’ll write a function to handle that as there is a +little bit of case-based logic that makes it non-trivial. We are going +to get handed year data and want to interpolate the UMAP x and y +coordinates for a single representative. + +The first major catch is that many representatives don’t have a single +contiguous block of years for which they were in congress: they were +elected for several years, missed re-election, and then came back to +congress several years later (possibly in another district). Each such +block of contiguous years needs to be a separate path, and we shouldn’t +connect them. We therefore need some logic to find the contiguous blocks +and generate smooth paths for each of them. + +Another catch is that some representatives have only been in office for +a year or two (special elections and so forth) and we can’t do a cubic +spline interpolation for that; we can devolve to linear interpolation or +quadratic splines for those cases, so simply add the point itself for +the odd single year cases. + +With those issues in hand we can then simply use the scipy ``interp1d`` +function to generate smooth curves through the points. + +.. code:: python3 + + INTERP_KIND = {2:"linear", 3:"quadratic", 4:"cubic"} + + def interpolate_paths(z, x, y, c, rep_id): + consecutive_year_blocks = np.where(np.diff(z) != 1)[0] + 1 + z_blocks = np.split(z, consecutive_year_blocks) + x_blocks = np.split(x, consecutive_year_blocks) + y_blocks = np.split(y, consecutive_year_blocks) + c_blocks = np.split(c, consecutive_year_blocks) + + paths = [] + + for block_idx, zs in enumerate(z_blocks): + + text = f"{rep_id} -- partisan_lean: {np.mean(c_blocks[block_idx]):.2f}" + + if len(zs) > 1: + kind = INTERP_KIND.get(len(zs), "cubic") + else: + paths.append( + (zs, x_blocks[block_idx], y_blocks[block_idx], c_blocks[block_idx], text) + ) + continue + + z = np.linspace(np.min(zs), np.max(zs), 100) + x = scipy.interpolate.interp1d(zs, x_blocks[block_idx], kind=kind)(z) + y = scipy.interpolate.interp1d(zs, y_blocks[block_idx], kind=kind)(z) + c = scipy.interpolate.interp1d(zs, c_blocks[block_idx], kind="linear")(z) + + paths.append((z, x, y, c, text)) + + return paths + +And now we can use plotly to draw the resulting curves. For plotly we +use the ``Scatter3D`` method, which supports a “lines” mode that can +draw curves in 3D space. We can colour the curves by the partisan lean +score we derived from the election data – in fact the colour can vary +through the trace as the election margins vary. Since this is a plotly +plot it is interactive, so you can rotate it around and view it from all +angles. + +Unfortunately the interactive plotly plot does not embed into the documentation +well, so we present here a static image. If you run this yourself, however, +you will get the interactive version. + +.. code:: python3 + + traces = [] + for rep in df.representative_id.unique(): + z = df.z[df.representative_id == rep].values + x = df.x[df.representative_id == rep].values + y = df.y[df.representative_id == rep].values + c = df.partisan_lean[df.representative_id == rep] + + for z, x, y, c, text in interpolate_paths(z, x, y, c, rep): + trace = go.Scatter3d( + x=x, y=z, z=y, + mode="lines", + hovertext=text, + hoverinfo="text", + line=dict( + color=c, + cmin=0.0, + cmid=0.5, + cmax=1.0, + cauto=False, + colorscale="RdBu", + colorbar=dict(), + width=2.5, + ), + opacity=1.0, + ) + traces.append(trace) + + fig = go.Figure(data=traces) + fig.update_layout( + width=800, + height=600, + scene=dict( + aspectratio = dict( x=0.5, y=1.25, z=0.5 ), + yaxis_title="Year", + xaxis_title="UMAP-X", + zaxis_title="UMAP-Y", + ), + scene_camera=dict(eye=dict( x=0.5, y=0.8, z=0.75 )), + autosize=False, + showlegend=False, + ) + fig_widget = go.FigureWidget(fig) + fig_widget + + +.. image:: images/aligned_umap_politics_demo_spaghetti.png + +.. + .. raw:: html + :file: aligned_umap_plotly_plot.html + +This concludes our exploration for now. + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/api.rst b/ext/TAL/applications/UMAP_fast/umap/doc/api.rst new file mode 100644 index 00000000..3e27b4a1 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/api.rst @@ -0,0 +1,19 @@ +UMAP API Guide +============== + +UMAP has only a single class :class:`UMAP`. + +UMAP +---- + +.. autoclass:: umap.umap_.UMAP + :members: + +A number of internal functions can also be accessed separately for more fine tuned work. + +Useful Functions +---------------- + +.. automodule:: umap.umap_ + :members: + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/basic_usage.rst b/ext/TAL/applications/UMAP_fast/umap/doc/basic_usage.rst new file mode 100644 index 00000000..08a3db5f --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/basic_usage.rst @@ -0,0 +1,609 @@ +How to Use UMAP +=============== + +UMAP is a general purpose manifold learning and dimension reduction +algorithm. It is designed to be compatible with +`scikit-learn `__, making use +of the same API and able to be added to sklearn pipelines. If you are +already familiar with sklearn you should be able to use UMAP as a drop +in replacement for t-SNE and other dimension reduction classes. If you +are not so familiar with sklearn this tutorial will step you through the +basics of using UMAP to transform and visualise data. + +First we'll need to import a bunch of useful tools. We will need numpy +obviously, but we'll use some of the datasets available in sklearn, as +well as the ``train_test_split`` function to divide up data. Finally +we'll need some plotting tools (matplotlib and seaborn) to help us +visualise the results of UMAP, and pandas to make that a little easier. + +.. code:: python3 + + import numpy as np + from sklearn.datasets import load_digits + from sklearn.model_selection import train_test_split + from sklearn.preprocessing import StandardScaler + import matplotlib.pyplot as plt + import seaborn as sns + import pandas as pd + %matplotlib inline + +.. code:: python3 + + sns.set(style='white', context='notebook', rc={'figure.figsize':(14,10)}) + +Penguin data +------------ + +.. image:: https://github.com/allisonhorst/palmerpenguins/raw/master/man/figures/lter_penguins.png + :width: 300px + :align: center + :alt: Penguins + +The next step is to get some data to work with. To ease us into things +we'll start with the `penguin +dataset `__. It isn't very +representative of what real data would look like, but it is small both +in number of points and number of features, and will let us get an idea +of what the dimension reduction is doing. + +.. code:: python3 + + penguins = pd.read_csv("https://github.com/allisonhorst/palmerpenguins/raw/5b5891f01b52ae26ad8cb9755ec93672f49328a8/data/penguins_size.csv") + penguins.head() + + + + +.. raw:: html + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
species_shortislandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
0AdelieTorgersen39.118.7181.03750.0MALE
1AdelieTorgersen39.517.4186.03800.0FEMALE
2AdelieTorgersen40.318.0195.03250.0FEMALE
3AdelieTorgersenNaNNaNNaNNaNNaN
4AdelieTorgersen36.719.3193.03450.0FEMALE
+
+ + + +Since this is for demonstration purposes we will get rid of the NAs in +the data; in a real world setting one would wish to take more care with +proper handling of missing data. + +.. code:: python3 + + penguins = penguins.dropna() + penguins.species_short.value_counts() + + + + +.. parsed-literal:: + + Adelie 146 + Gentoo 120 + Chinstrap 68 + Name: species_short, dtype: int64 + + +.. image:: https://github.com/allisonhorst/palmerpenguins/raw/master/man/figures/culmen_depth.png + :width: 300px + :align: center + :alt: Diagram of culmen measurements on a penguin + +See the `github repostiory `__ +for more details about the dataset itself. It consists of measurements +of bill (culmen) and flippers and weights of three species of penguins, +along with some other metadata about the penguins. In total we have 334 +different penguins measured. Visualizing this data is a little bit +tricky since we can't plot in 4 dimensions easily. Fortunately four is +not that large a number, so we can just to a pairwise feature +scatterplot matrix to get an ideas of what is going on. Seaborn makes +this easy. + +.. code:: python3 + + sns.pairplot(penguins, hue='species_short') + + + +.. image:: images/basic_usage_8_1.png + + +This gives us some idea of what the data looks like by giving as all the +2D views of the data. Four dimensions is low enough that we can (sort +of) reconstruct what the full dimensional data looks like in our heads. +Now that we sort of know what we are looking at, the question is what +can a dimension reduction technique like UMAP do for us? By reducing the +dimension in a way that preserves as much of the structure of the data +as possible we can get a visualisable representation of the data +allowing us to "see" the data and its structure and begin to get some +intuition about the data itself. + +To use UMAP for this task we need to first construct a UMAP object that +will do the job for us. That is as simple as instantiating the class. So +let's import the umap library and do that. + +.. code:: python3 + + import umap + +.. code:: python3 + + reducer = umap.UMAP() + +Before we can do any work with the data it will help to clean up it a +little. We won't need NAs, we just want the measurement columns, and +since the measurements are on entirely different scales it will be +helpful to convert each feature into z-scores (number of standard +deviations from the mean) for comparability. + +.. code:: python3 + + penguin_data = penguins[ + [ + "culmen_length_mm", + "culmen_depth_mm", + "flipper_length_mm", + "body_mass_g", + ] + ].values + scaled_penguin_data = StandardScaler().fit_transform(penguin_data) + +Now we need to train our reducer, letting it learn about the manifold. +For this UMAP follows the sklearn API and has a method ``fit`` which we +pass the data we want the model to learn from. Since, at the end of the +day, we are going to want to reduced representation of the data we will +use, instead, the ``fit_transform`` method which first calls ``fit`` and +then returns the transformed data as a numpy array. + +.. code:: python3 + + embedding = reducer.fit_transform(scaled_penguin_data) + embedding.shape + + + + +.. parsed-literal:: + + (334, 2) + + + +The result is an array with 334 samples, but only two feature columns +(instead of the four we started with). This is because, by default, UMAP +reduces down to 2D. Each row of the array is a 2-dimensional +representation of the corresponding penguin. Thus we can plot the +``embedding`` as a standard scatterplot and color by the target array +(since it applies to the transformed data which is in the same order as +the original). + +.. code:: python3 + + plt.scatter( + embedding[:, 0], + embedding[:, 1], + c=[sns.color_palette()[x] for x in penguins.species_short.map({"Adelie":0, "Chinstrap":1, "Gentoo":2})]) + plt.gca().set_aspect('equal', 'datalim') + plt.title('UMAP projection of the Penguin dataset', fontsize=24) + + + +.. image:: images/basic_usage_17_1.png + + +This does a useful job of capturing the structure of the data, and as +can be seen from the matrix of scatterplots this is relatively accurate. +Of course we learned at least this much just from that matrix of +scatterplots -- which we could do since we only had four different +dimensions to analyse. If we had data with a larger number of dimensions +the scatterplot matrix would quickly become unwieldy to plot, and far +harder to interpret. So moving on from the Penguin dataset, let's consider +the digits dataset. + +Digits data +----------- + +First we will load the dataset from sklearn. + +.. code:: python3 + + digits = load_digits() + print(digits.DESCR) + + +.. parsed-literal:: + + .. _digits_dataset: + + Optical recognition of handwritten digits dataset + -------------------------------------------------- + + **Data Set Characteristics:** + + :Number of Instances: 5620 + :Number of Attributes: 64 + :Attribute Information: 8x8 image of integer pixels in the range 0..16. + :Missing Attribute Values: None + :Creator: E. Alpaydin (alpaydin '@' boun.edu.tr) + :Date: July; 1998 + + This is a copy of the test set of the UCI ML hand-written digits datasets + https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits + + The data set contains images of hand-written digits: 10 classes where + each class refers to a digit. + + Preprocessing programs made available by NIST were used to extract + normalized bitmaps of handwritten digits from a preprinted form. From a + total of 43 people, 30 contributed to the training set and different 13 + to the test set. 32x32 bitmaps are divided into nonoverlapping blocks of + 4x4 and the number of on pixels are counted in each block. This generates + an input matrix of 8x8 where each element is an integer in the range + 0..16. This reduces dimensionality and gives invariance to small + distortions. + + For info on NIST preprocessing routines, see M. D. Garris, J. L. Blue, G. + T. Candela, D. L. Dimmick, J. Geist, P. J. Grother, S. A. Janet, and C. + L. Wilson, NIST Form-Based Handprint Recognition System, NISTIR 5469, + 1994. + + .. topic:: References + + - C. Kaynak (1995) Methods of Combining Multiple Classifiers and Their + Applications to Handwritten Digit Recognition, MSc Thesis, Institute of + Graduate Studies in Science and Engineering, Bogazici University. + - E. Alpaydin, C. Kaynak (1998) Cascading Classifiers, Kybernetika. + - Ken Tang and Ponnuthurai N. Suganthan and Xi Yao and A. Kai Qin. + Linear dimensionalityreduction using relevance weighted LDA. School of + Electrical and Electronic Engineering Nanyang Technological University. + 2005. + - Claudio Gentile. A New Approximate Maximal Margin Classification + Algorithm. NIPS. 2000. + + +We can plot a number of the images to get an idea of what we are looking +at. This just involves matplotlib building a grid of axes and then +looping through them plotting an image into each one in turn. + +.. code:: python3 + + fig, ax_array = plt.subplots(20, 20) + axes = ax_array.flatten() + for i, ax in enumerate(axes): + ax.imshow(digits.images[i], cmap='gray_r') + plt.setp(axes, xticks=[], yticks=[], frame_on=False) + plt.tight_layout(h_pad=0.5, w_pad=0.01) + + + +.. image:: images/basic_usage_22_0.png + + +As you can see these are quite low resolution images -- for the most +part they are recognisable as digits, but there are a number of cases +that are sufficiently blurred as to be questionable even for a human to +guess at. The zeros do stand out as the easiest to pick out as notably +different and clearly zeros. Beyond that things get a little harder: +some of the squashed thing eights look awfully like ones, some of the +threes start to look a little like crossed sevens when drawn badly, and +so on. + +Each image can be unfolded into a 64 element long vector of grayscale +values. It is these 64 dimensional vectors that we wish to analyse: how +much of the digits structure can we discern? At least in principle 64 +dimensions is overkill for this task, and we would reasonably expect +that there should be some smaller number of "latent" features that would +be sufficient to describe the data reasonably well. We can try a +scatterplot matrix -- in this case just of the first 10 dimensions so +that it is at least plottable, but as you can quickly see that approach +is not going to be sufficient for this data. + +.. code:: python3 + + digits_df = pd.DataFrame(digits.data[:,1:11]) + digits_df['digit'] = pd.Series(digits.target).map(lambda x: 'Digit {}'.format(x)) + sns.pairplot(digits_df, hue='digit', palette='Spectral') + + +.. image:: images/basic_usage_24_2.png + + +In contrast we can try using UMAP again. It works exactly as before: +construct a model, train the model, and then look at the transformed +data. To demonstrate more of UMAP we'll go about it differently this +time and simply use the ``fit`` method rather than the ``fit_transform`` +approach we used for Penguins. + +.. code:: python3 + + reducer = umap.UMAP(random_state=42) + reducer.fit(digits.data) + + +.. parsed-literal:: + + UMAP(a=None, angular_rp_forest=False, b=None, + force_approximation_algorithm=False, init='spectral', learning_rate=1.0, + local_connectivity=1.0, low_memory=False, metric='euclidean', + metric_kwds=None, min_dist=0.1, n_components=2, n_epochs=None, + n_neighbors=15, negative_sample_rate=5, output_metric='euclidean', + output_metric_kwds=None, random_state=42, repulsion_strength=1.0, + set_op_mix_ratio=1.0, spread=1.0, target_metric='categorical', + target_metric_kwds=None, target_n_neighbors=-1, target_weight=0.5, + transform_queue_size=4.0, transform_seed=42, unique=False, verbose=False) + + + +Now, instead of returning an embedding we simply get back the reducer +object, now having trained on the dataset we passed it. To access the +resulting transform we can either look at the ``embedding_`` attribute +of the reducer object, or call transform on the original data. + +.. code:: python3 + + embedding = reducer.transform(digits.data) + # Verify that the result of calling transform is + # idenitical to accessing the embedding_ attribute + assert(np.all(embedding == reducer.embedding_)) + embedding.shape + + + + +.. parsed-literal:: + + (1797, 2) + + + +We now have a dataset with 1797 rows (one for each hand-written digit +sample), but only 2 columns. As with the Penguins example we can now plot +the resulting embedding, coloring the data points by the class that +they belong to (i.e. the digit they represent). + +.. code:: python3 + + plt.scatter(embedding[:, 0], embedding[:, 1], c=digits.target, cmap='Spectral', s=5) + plt.gca().set_aspect('equal', 'datalim') + plt.colorbar(boundaries=np.arange(11)-0.5).set_ticks(np.arange(10)) + plt.title('UMAP projection of the Digits dataset', fontsize=24); + +.. image:: images/basic_usage_30_1.png + + +We see that UMAP has successfully captured the digit classes. There are +also some interesting effects as some digit classes blend into one +another (see the eights, ones, and sevens, with some nines in between), +and also cases where digits are pushed away as clearly distinct (the +zeros on the right, the fours at the top, and a small subcluster of ones +at the bottom come to mind). To get a better idea of why UMAP chose to +do this it is helpful to see the actual digits involve. One can do this +using `bokeh `__ and mouseover +tooltips of the images. + +First we'll need to encode all the images for inclusion in a dataframe. + +.. code:: python3 + + from io import BytesIO + from PIL import Image + import base64 + +.. code:: python3 + + def embeddable_image(data): + img_data = 255 - 15 * data.astype(np.uint8) + image = Image.fromarray(img_data, mode='L').resize((64, 64), Image.BICUBIC) + buffer = BytesIO() + image.save(buffer, format='png') + for_encoding = buffer.getvalue() + return 'data:image/png;base64,' + base64.b64encode(for_encoding).decode() + +Next we need to load up bokeh and the various tools from it that will be +needed to generate a suitable interactive plot. + +.. code:: python3 + + from bokeh.plotting import figure, show, output_notebook + from bokeh.models import HoverTool, ColumnDataSource, CategoricalColorMapper + from bokeh.palettes import Spectral10 + + output_notebook() + + + +.. raw:: html + + +
+ + Loading BokehJS ... +
+ + + + +Finally we generate the plot itself with a custom hover tooltip that +embeds the image of the digit in question in it, along with the digit +class that the digit is actually from (this can be useful for digits +that are hard even for humans to classify correctly). + +.. code:: python3 + + digits_df = pd.DataFrame(embedding, columns=('x', 'y')) + digits_df['digit'] = [str(x) for x in digits.target] + digits_df['image'] = list(map(embeddable_image, digits.images)) + + datasource = ColumnDataSource(digits_df) + color_mapping = CategoricalColorMapper(factors=[str(9 - x) for x in digits.target_names], + palette=Spectral10) + + plot_figure = figure( + title='UMAP projection of the Digits dataset', + plot_width=600, + plot_height=600, + tools=('pan, wheel_zoom, reset') + ) + + plot_figure.add_tools(HoverTool(tooltips=""" +
+
+ +
+
+ Digit: + @digit +
+
+ """)) + + plot_figure.circle( + 'x', + 'y', + source=datasource, + color=dict(field='digit', transform=color_mapping), + line_alpha=0.6, + fill_alpha=0.6, + size=4 + ) + show(plot_figure) + + + +.. raw:: html + :file: basic_usage_bokeh_example.html + +As can be seen, the nines that blend between the ones and the sevens are +odd looking nines (that aren't very rounded) and do, indeed, interpolate +surprisingly well between ones with hats and crossed sevens. In contrast +the small disjoint cluster of ones at the bottom of the plot is made up +of ones with feet (a horizontal line at the base of the one) which are, +indeed, quite distinct from the general mass of ones. + +This concludes our introduction to basic UMAP usage -- hopefully this +has given you the tools to get started for yourself. Further tutorials, +covering UMAP parameters and more advanced usage are also available when +you wish to dive deeper. + +-------------- + +.. raw:: html + +

+ +Penguin data information + +.. raw:: html + +

+ +Peguin data are from: + +**Gorman KB, Williams TD, Fraser WR** (2014) Ecological Sexual +Dimorphism and Environmental Variability within a Community of Antarctic +Penguins (Genus *Pygoscelis*). PLoS ONE 9(3): e90081. +doi:10.1371/journal.pone.0090081 + +See the full paper +`HERE `__. + +.. raw:: html + +

+ +Original data access and use + +.. raw:: html + +

+ +From Gorman et al.: “Data reported here are publicly available within +the PAL-LTER data system (datasets #219, 220, and 221): +http://oceaninformatics.ucsd.edu/datazoo/data/pallter/datasets. These +data are additionally archived within the United States (US) LTER +Network’s Information System Data Portal: https://portal.lternet.edu/. +Individuals interested in using these data are therefore expected to +follow the US LTER Network’s Data Access Policy, Requirements and Use +Agreement: https://lternet.edu/data-access-policy/.” + +Anyone interested in publishing the data should contact `Dr. Kristen +Gorman `__ +about analysis and working together on any final products. + +Penguin images by Alison Horst. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/basic_usage_bokeh_example.html b/ext/TAL/applications/UMAP_fast/umap/doc/basic_usage_bokeh_example.html new file mode 100644 index 00000000..8c278888 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/basic_usage_bokeh_example.html @@ -0,0 +1,85 @@ + + + + + + + + + + + Bokeh Plot + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/benchmarking.rst b/ext/TAL/applications/UMAP_fast/umap/doc/benchmarking.rst new file mode 100644 index 00000000..5ed3b088 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/benchmarking.rst @@ -0,0 +1,239 @@ + +Performance Comparison of Dimension Reduction Implementations +============================================================= + +Different dimension reduction techniques can have quite different +computational complexity. Beyond the algorithm itself there is also the +question of how exactly it is implemented. These two factors can have a +significant role in how long it actually takes to run a given dimension +reduction. Furthermore the nature of the data you are trying to reduce +can also matter -- mostly the involves the dimensionality of the +original data. Here we will take a brief look at the performance +characterstics of a number of dimension reduction implementations. + +To start let's get the basic tools we'll need loaded up -- numpy and +pandas obviously, but also tools to get and resample the data, and the +time module so we can perform some basic benchmarking. + +.. code:: ipython3 + + import numpy as np + import pandas as pd + from sklearn.datasets import fetch_openml + from sklearn.utils import resample + import time + +Next we'll need the actual dimension reduction implementations. For the +purposes of this explanation we'll mostly stick with +`scikit-learn `__, but for the sake of +comparison we'll also include the +`MulticoreTSNE `__ +implementation of t-SNE, which has significantly better performance than +the current scikit-learn t-SNE. + +.. code:: ipython3 + + from sklearn.manifold import TSNE, LocallyLinearEmbedding, Isomap, MDS, SpectralEmbedding + from sklearn.decomposition import PCA + from MulticoreTSNE import MulticoreTSNE + from umap import UMAP + +Next we'll need out plotting tools, and, of course, some data to work +with. For this performance comparison we'll default to the now standard +benchmark of manifold learning: the MNIST digits dataset. We can use +scikit-learn's ``fetch_mldata`` to grab it for us. + +.. code:: ipython3 + + import matplotlib.pyplot as plt + import seaborn as sns + %matplotlib inline + +.. code:: ipython3 + + sns.set(context='notebook', + rc={'figure.figsize':(12,10)}, + palette=sns.color_palette('tab10', 10)) + +.. code:: ipython3 + + mnist = fetch_openml('Fashion-MNIST', version=1) + +Now it is time to start looking at performance. To start with let's look +at how performance scales with increasing dataset size. + +Performance scaling by dataset size +----------------------------------- + +As the size of a dataset increases the runtime of a given dimension +reduction algorithm will increase at varying rates. If you ever want to +run your algorithm on larger datasets you will care not just about the +comparative runtime on a single small dataset, but how the performance +scales out as you move to larger datasets. We can similate this by +subsampling from MNIST digits (via scikit-learn's convenient +``resample`` utility) and looking at the runtime for varying sized +subsamples. Since there is some randomness involved here (both in the +subsample selection, and in some of the algorithms which have stochastic +aspects) we will want to run a few examples for each dataset size. We +can easily package all of this up in a simple function that will return +a convenient pandas dataframe of dataset sizes and runtimes given an +algorithm. + +.. code:: ipython3 + + def data_size_scaling(algorithm, data, sizes=[100, 200, 400, 800, 1600], n_runs=5): + result = [] + for size in sizes: + for run in range(n_runs): + subsample = resample(data, n_samples=size) + start_time = time.time() + algorithm.fit(subsample) + elapsed_time = time.time() - start_time + del subsample + result.append((size, elapsed_time)) + return pd.DataFrame(result, columns=('dataset size', 'runtime (s)')) + +Now we just want to run this for each of the various dimension reduction +implementations so we can look at the results. Since we don't know how +long these runs might take we'll start off with a very small set of +samples, scaling up to only 1600 samples. + +.. code:: ipython3 + + all_algorithms = [ + PCA(), + UMAP(), + MulticoreTSNE(), + LocallyLinearEmbedding(), + SpectralEmbedding(), + Isomap(), + TSNE(), + MDS(), + ] + performance_data = {} + for algorithm in all_algorithms: + alg_name = str(algorithm) + if 'MulticoreTSNE' in alg_name: + alg_name = 'MulticoreTSNE' + else: + alg_name = alg_name.split('(')[0] + + performance_data[alg_name] = data_size_scaling(algorithm, mnist.data, n_runs=3) + + + +Now let's plot the results so we can see what is going on. We'll use +seaborn's regression plot to interpolate the effective scaling. + +.. code:: ipython3 + + for alg_name, perf_data in performance_data.items(): + sns.regplot('dataset size', 'runtime (s)', perf_data, order=2, label=alg_name) + plt.legend(); + + + +.. image:: images/performance_14_1.png + + +We can see straight away that there are some outliers here. The +scikit-learn t-SNE is clearly much slower than most of the other +algorithms. It does not have the scaling properties of MDS however; for +larger dataset sizes MDS is going to quickly become completely +unmanageable. At the same time MulticoreTSNE demonstrates that t-SNE can +run fairly efficiently. It is hard to tell much about the other +implementations other than the fact that PCA is far and away the fastest +option. To see more we'll have to look at runtimes on larger dataset +sizes. Both MDS and scikit-learn's t-SNE are going to take too long to +run so let's restrict ourselves to the fastest performing +implementations and see what happens as we extend out to larger dataset +sizes. + +.. code:: ipython3 + + fast_algorithms = [ + PCA(), + UMAP(), + MulticoreTSNE(), + LocallyLinearEmbedding(), + SpectralEmbedding(), + Isomap(), + ] + fast_performance_data = {} + for algorithm in fast_algorithms: + alg_name = str(algorithm) + if 'MulticoreTSNE' in alg_name: + alg_name = 'MulticoreTSNE' + else: + alg_name = alg_name.split('(')[0] + + fast_performance_data[alg_name] = data_size_scaling(algorithm, mnist.data, + sizes=[800, 1600, 3200, 6400, 12800], n_runs=3) + + +.. code:: ipython3 + + for alg_name, perf_data in fast_performance_data.items(): + sns.regplot('dataset size', 'runtime (s)', perf_data, order=2, label=alg_name) + plt.legend(); + + + +.. image:: images/performance_17_1.png + + +At this point we begin to see some significant differentiation among the +different implementations. In the earlier plot MulticoreTSNE looked to +be slower than some of the other algorithms, but as we scale out to +larger datasets we see that its relative scaling performance is far +superior to the scikit-learn implementations of Isomap, spectral +embedding, and locally linear embedding. + +It is probably worth extending out further -- up to the full MNIST +digits dataset. To manage to do that in any reasonable amount of time +we'll have to restrict out attention to an even smaller subset of +implementations. We will pare things down to just MulticoreTSNE, PCA and +UMAP. + +.. code:: ipython3 + + very_fast_algorithms = [ + PCA(), + UMAP(), + MulticoreTSNE(), + ] + vfast_performance_data = {} + for algorithm in very_fast_algorithms: + alg_name = str(algorithm) + if 'MulticoreTSNE' in alg_name: + alg_name = 'MulticoreTSNE' + else: + alg_name = alg_name.split('(')[0] + + vfast_performance_data[alg_name] = data_size_scaling(algorithm, mnist.data, + sizes=[3200, 6400, 12800, 25600, 51200, 70000], n_runs=2) + + +.. code:: ipython3 + + for alg_name, perf_data in vfast_performance_data.items(): + sns.regplot('dataset size', 'runtime (s)', perf_data, order=2, label=alg_name) + plt.legend(); + + + + +.. image:: images/performance_20_1.png + + +Here we see UMAP's advantages over t-SNE really coming to the forefront. +While UMAP is clearly slower than PCA, its scaling performance is +dramatically better than MulticoreTSNE, and for even larger datasets the +difference is only going to grow. + +This concludes our look at scaling by dataset size. The short summary is +that PCA is far and away the fastest option, but you are potentially +giving up a lot for that speed. UMAP, while not competitive with PCA, is +clearly the next best option in terms of performance among the +implementations explored here. Given the quality of results that UMAP +can provide we feel it is clearly a good option for dimension reduction. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/bokeh_digits_plot.py b/ext/TAL/applications/UMAP_fast/umap/doc/bokeh_digits_plot.py new file mode 100644 index 00000000..eea675dd --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/bokeh_digits_plot.py @@ -0,0 +1,73 @@ +import numpy as np +from sklearn.datasets import load_digits +import pandas as pd + +digits = load_digits() + +import umap + +reducer = umap.UMAP(random_state=42) +embedding = reducer.fit_transform(digits.data) + +from io import BytesIO +from PIL import Image +import base64 + + +def embeddable_image(data): + img_data = 255 - 15 * data.astype(np.uint8) + image = Image.fromarray(img_data, mode="L").resize((64, 64), Image.BICUBIC) + buffer = BytesIO() + image.save(buffer, format="png") + for_encoding = buffer.getvalue() + return "data:image/png;base64," + base64.b64encode(for_encoding).decode() + + +from bokeh.plotting import figure, show, output_file +from bokeh.models import HoverTool, ColumnDataSource, CategoricalColorMapper +from bokeh.palettes import Spectral10 + +output_file("basic_usage_bokeh_example.html") + +digits_df = pd.DataFrame(embedding, columns=("x", "y")) +digits_df["digit"] = [str(x) for x in digits.target] +digits_df["image"] = list(map(embeddable_image, digits.images)) + +datasource = ColumnDataSource(digits_df) +color_mapping = CategoricalColorMapper( + factors=[str(9 - x) for x in digits.target_names], palette=Spectral10 +) + +plot_figure = figure( + title="UMAP projection of the Digits dataset", + plot_width=600, + plot_height=600, + tools=("pan, wheel_zoom, reset"), +) + +plot_figure.add_tools( + HoverTool( + tooltips=""" +
+
+ +
+
+ Digit: + @digit +
+
+""" + ) +) + +plot_figure.circle( + "x", + "y", + source=datasource, + color=dict(field="digit", transform=color_mapping), + line_alpha=0.6, + fill_alpha=0.6, + size=4, +) +show(plot_figure) diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/clustering.rst b/ext/TAL/applications/UMAP_fast/umap/doc/clustering.rst new file mode 100644 index 00000000..53688f1b --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/clustering.rst @@ -0,0 +1,403 @@ +Using UMAP for Clustering +========================= + +UMAP can be used as an effective preprocessing step to boost the +performance of density based clustering. This is somewhat controversial, +and should be attempted with care. For a good discussion of some of the +issues involved in this, please see the various answers `in this +stackoverflow +thread `__ +on clustering the results of t-SNE. Many of the points of concern raised +there are salient for clustering the results of UMAP. The most notable +is that UMAP, like t-SNE, does not completely preserve density. UMAP, +like t-SNE, can also create false tears in clusters, resulting in a +finer clustering than is necessarily present in +the data. Despite these concerns there are still valid reasons to use +UMAP as a preprocessing step for clustering. As with any clustering +approach one will want to do some exploration and evaluation of the +clusters that come out to try to validate them if possible. + +With all of that said, let's work through an example to demonstrate the +difficulties that can face clustering approaches and how UMAP can +provide a powerful tool to help overcome them. + +First we'll need a selection of libraries loaded up. Obviously we'll +need data, and we can use sklearn's ``fetch_openml`` to get it. We'll +also need the usual tools of numpy, and plotting. Next we'll need umap, +and some clustering options. Finally, since we'll be working with +labeled data, we can make use of strong cluster evaluation metrics +`Adjusted Rand +Index `__ +and `Adjusted Mutual +Information `__. + +.. code:: python3 + + from sklearn.datasets import fetch_openml + from sklearn.decomposition import PCA + import numpy as np + import matplotlib.pyplot as plt + %matplotlib inline + + # Dimension reduction and clustering libraries + import umap + import hdbscan + import sklearn.cluster as cluster + from sklearn.metrics import adjusted_rand_score, adjusted_mutual_info_score + +Now let's set up the plotting and grab the data we'll be using -- in +this case the MNIST handwritten digits dataset. MNIST consists of 28x28 +pixel grayscale images of handwritten digits (0 through 9). These can be +unraveled such that each digit is described by a 784 dimensional vector +(the gray scale value of each pixel in the image). Ideally we would like +the clustering to recover the digit structure. + +.. code:: python3 + + mnist = fetch_openml('mnist_784', version=1) + mnist.target = mnist.target.astype(int) + +For visualization purposes we can reduce the data to 2-dimensions using +UMAP. When we cluster the data in high dimensions we can visualize the +result of that clustering. First, however, we'll view the data colored +by the digit that each data point represents -- we'll use a different +color for each digit. This will help frame what follows. + +.. code:: python3 + + standard_embedding = umap.UMAP(random_state=42).fit_transform(mnist.data) + plt.scatter(standard_embedding[:, 0], standard_embedding[:, 1], c=mnist.target.astype(int), s=0.1, cmap='Spectral'); + +.. image:: images/clustering_6_1.png + + +Traditional clustering +~~~~~~~~~~~~~~~~~~~~~~ + +Now we would like to cluster the data. As a first attempt let's try the +traditional approach: K-Means. In this case we can solve one of the hard +problems for K-Means clustering -- choosing the right k value, giving +the number of clusters we are looking for. In this case we know the +answer is exactly 10. We will use sklearns K-Means implementation +looking for 10 clusters in the original 784 dimensional data. + +.. code:: python3 + + kmeans_labels = cluster.KMeans(n_clusters=10).fit_predict(mnist.data) + +And how did the clustering do? We can look at the results by coloring +out UMAP embedded data by cluster membership. + +.. code:: python3 + + plt.scatter(standard_embedding[:, 0], standard_embedding[:, 1], c=kmeans_labels, s=0.1, cmap='Spectral'); + + +.. image:: images/clustering_10_1.png + + +This is not really the result we were looking for (though it does expose +interesting properties of how K-Means chooses clusters in high +dimensional space, and how UMAP unwraps manifolds by finding manifold +boundaries). While K-Means gets some cases correct, such as the two clusters +on the right side which are mostly correct, most of the rest of the data looks +somewhat arbitrarily carved up among the remaining clusters. We can put +this impression to the test by evaluating the adjusted Rand score and +adjusted mutual information for this clustering as compared with the +true labels. + +.. code:: python3 + + ( + adjusted_rand_score(mnist.target, kmeans_labels), + adjusted_mutual_info_score(mnist.target, kmeans_labels) + ) + + + + +.. parsed-literal:: + + (0.36675295135972552, 0.49614118437750965) + + + +As might be expected, we have not done a particularly good job -- both +scores take values in the range 0 to 1, with 0 representing a bad +(essentially random) clustering and 1 representing perfectly recovering +the true labels. K-Means definitely was not random, but it was also +quite a long way from perfectly recovering the true labels. Part of the +problem is the way K-Means works, based on centroids with an assumption +of largely spherical clusters -- this is responsible for some of the +sharp divides that K-Means puts across digit classes. We can potentially +improve on this by using a smarter density based algorithm. In this case +we've chosen to try HDBSCAN, which we believe to be among the most +advanced density based techniques. For the sake of performance we'll +reduce the dimensionality of the data down to 50 dimensions via PCA +(this recovers most of the variance), since HDBSCAN scales somewhat +poorly with the dimensionality of the data it will work on. + +.. code:: python3 + + lowd_mnist = PCA(n_components=50).fit_transform(mnist.data) + hdbscan_labels = hdbscan.HDBSCAN(min_samples=10, min_cluster_size=500).fit_predict(lowd_mnist) + +We can now inspect the results. Before we do, however, it should be +noted that one of the features of HDBSCAN is that it can refuse to +cluster some points and classify them as "noise". To visualize this +aspect we will color points that were classified as noise gray, and then +color the remaining points according to the cluster membership. + +.. code:: python3 + + clustered = (hdbscan_labels >= 0) + plt.scatter(standard_embedding[~clustered, 0], + standard_embedding[~clustered, 1], + color=(0.5, 0.5, 0.5), + s=0.1, + alpha=0.5) + plt.scatter(standard_embedding[clustered, 0], + standard_embedding[clustered, 1], + c=hdbscan_labels[clustered], + s=0.1, + cmap='Spectral'); + + + +.. image:: images/clustering_16_1.png + + +This looks somewhat underwhelming. It meets HDBSCAN's approach of "not +being wrong" by simply refusing to classify the majority of the data. +The result is a clustering that almost certainly fails to recover all +the labels. We can verify this by looking at the clustering validation +scores. + +.. code:: python3 + + ( + adjusted_rand_score(mnist.target, hdbscan_labels), + adjusted_mutual_info_score(mnist.target, hdbscan_labels) + ) + + + + +.. parsed-literal:: + + (0.053830107882840102, 0.19756104096566332) + + + +These scores are far worse than K-Means! Partially this is due to the +fact that these scores assume that the noise points are simply an extra +cluster. We can instead only look at the subset of the data that HDBSCAN +was actually confident enough to assign to clusters -- a simple +sub-selection will let us recompute the scores for only that data. + +.. code:: python3 + + clustered = (hdbscan_labels >= 0) + ( + adjusted_rand_score(mnist.target[clustered], hdbscan_labels[clustered]), + adjusted_mutual_info_score(mnist.target[clustered], hdbscan_labels[clustered]) + ) + + + + +.. parsed-literal:: + + (0.99843407988303912, 0.99405521087764015) + + + +And here we see that where HDBSCAN was willing to cluster it got things +almost entirely correct. This is what it was designed to do -- be right +for what it can, and defer on anything that it couldn't have sufficient +confidence in. Of course the catch here is that it deferred clustering a +lot of the data. How much of the data did HDBSCAN actually assign to +clusters? We can compute that easily enough. + +.. code:: python3 + + np.sum(clustered) / mnist.data.shape[0] + + + + +.. parsed-literal:: + + 0.17081428571428572 + + + +It seems that less than 18% of the data was clustered. While HDBSCAN did +a great job on the data it could cluster it did a poor job of actually +managing to cluster the data. The problem here is that, as a density +based clustering algorithm, HDBSCAN tends to suffer from the curse of +dimensionality: high dimensional data requires more observed samples to +produce much density. If we could reduce the dimensionality of the data +more we would make the density more evident and make it far easier for +HDBSCAN to cluster the data. The problem is that trying to use PCA to do +this is going to become problematic. While reducing the 50 dimensions +still explained a lot of the variance of the data, reducing further is +going to quickly do a lot worse. This is due to the linear nature of +PCA. What we need is strong manifold learning, and this is where UMAP +can come into play. + +UMAP enhanced clustering +~~~~~~~~~~~~~~~~~~~~~~~~ + +Our goal is to make use of UMAP to perform non-linear manifold aware +dimension reduction so we can get the dataset down to a number of +dimensions small enough for a density based clustering algorithm to make +progress. One advantage of UMAP for this is that it doesn't require you +to reduce to only two dimensions -- you can reduce to 10 dimensions +instead since the goal is to cluster, not visualize, and the performance +cost with UMAP is minimal. As it happens MNIST is such a simple dataset +that we really can push it all the way down to only two dimensions, but +in general you should explore different embedding dimension options. + +The next thing to be aware of is that when using UMAP for dimension +reduction you will want to select different parameters than if you were +using it for visualization. First of all we will want a larger +``n_neighbors`` value -- small values will focus more on very local +structure and are more prone to producing fine grained cluster structure +that may be more a result of patterns of noise in the data than actual +clusters. In this case we'll double it from the default 15 up to 30. +Second it is beneficial to set ``min_dist`` to a very low value. Since +we actually want to pack points together densely (density is what we +want after all) a low value will help, as well as making cleaner +separations between clusters. In this case we will simply set +``min_dist`` to be 0. + +.. code:: python3 + + clusterable_embedding = umap.UMAP( + n_neighbors=30, + min_dist=0.0, + n_components=2, + random_state=42, + ).fit_transform(mnist.data) + +We can visualize the results of this so see how it compares with more +visualization attuned parameters: + +.. code:: python3 + + plt.scatter(clusterable_embedding[:, 0], clusterable_embedding[:, 1], + c=mnist.target, s=0.1, cmap='Spectral'); + + +.. image:: images/clustering_27_1.png + + +As you can see we still have the general global structure, but we are +packing points together more tightly within clusters, and consequently +we can see larger gaps between the clusters. Ultimately this embedding +was for clustering purposes only, and we will go back to the original +embedding for visualization purposes from here on out. + +The next step is to cluster this data. We'll use HDBSCAN again, with the +same parameter setting as before. + +.. code:: python3 + + labels = hdbscan.HDBSCAN( + min_samples=10, + min_cluster_size=500, + ).fit_predict(clusterable_embedding) + +And now we can visualize the results, just as before. + +.. code:: python3 + + clustered = (labels >= 0) + plt.scatter(standard_embedding[~clustered, 0], + standard_embedding[~clustered, 1], + color=(0.5, 0.5, 0.5), + s=0.1, + alpha=0.5) + plt.scatter(standard_embedding[clustered, 0], + standard_embedding[clustered, 1], + c=labels[clustered], + s=0.1, + cmap='Spectral'); + + +.. image:: images/clustering_31_1.png + + +We can see that we have done a much better job of finding clusters +rather than merely assigning the majority of data as noise. This is +because we no longer have to try to cope with the relative lack +of density in 50 dimensional space and now HDBSCAN can more cleanly +discern the clusters. + +We can also make a quantitative assessment by using the clustering +quality measures as before. + +.. code:: python3 + + adjusted_rand_score(mnist.target, labels), adjusted_mutual_info_score(mnist.target, labels) + + + + +.. parsed-literal:: + + (0.9239306564265013, 0.90302671641133736) + + + +Where before HDBSCAN performed very poorly, we now have scores of 0.9 or +better. This is because we actually clustered far more of the data. As +before we can also look at how the clustering did on just the data that +HDBSCAN was confident in clustering. + +.. code:: python3 + + clustered = (labels >= 0) + ( + adjusted_rand_score(mnist.target[clustered], labels[clustered]), + adjusted_mutual_info_score(mnist.target[clustered], labels[clustered]) + ) + + + + +.. parsed-literal:: + + (0.93240371696811541, 0.91912906363537572) + + + +This is a little worse than the original HDBSCAN, but it is unsurprising +that you are going to be wrong more often if you make more predictions. +The question is how much more of the data is HDBSCAN actually +clustering? Previously we were clustering only 17% of the data. + +.. code:: python3 + + np.sum(clustered) / mnist.data.shape[0] + + + + +.. parsed-literal:: + + 0.99164285714285716 + + + +Now we are clustering over 99% of the data! And our results in terms of +adjusted Rand score and adjusted mutual information are in line with the +current state of the art techniques using convolutional autoencoder +techniques. That's not bad for an approach that is simply viewing the +data as arbitrary 784 dimensional vectors. + +Hopefully this has outlined how UMAP can be beneficial for clustering. +As with all things care must be taken, but clearly UMAP can provide +significantly better clustering results when used judiciously. + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/composing_models.rst b/ext/TAL/applications/UMAP_fast/umap/doc/composing_models.rst new file mode 100644 index 00000000..f72b2dd8 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/composing_models.rst @@ -0,0 +1,640 @@ +Combining multiple UMAP models +============================== + +It is possible to combine together multiple UMAP models, assuming that +they are operating on the same underlying data. To get an idea of how +this works recall that UMAP uses an intermediate fuzzy topological +representation (see :ref:`how_umap_works`). Given different views of the +same underlying data this will generate different fuzzy topological +representations. We can apply intersections or unions to these +representations to get a new composite fuzzy topological representation +which we can then embed into low dimensional space in the standard UMAP +way. The key is that, to be able to sensibly intersect or union these +representations, there must be one-to-one correspondences between the +data samples from the two different views. + +To get an idea of how this might work it is useful to see it in +practice. Let’s load some libraries and get started. + +.. code:: python3 + + import sklearn.datasets + from sklearn.preprocessing import RobustScaler + import seaborn as sns + import pandas as pd + import numpy as np + import umap + import umap.plot + +MNIST digits example +-------------------- + +To begin with let’s use a relatively familiar dataset – the MNIST digits +dataset that we’ve used in other sections of this tutorial. The data is +(grayscale) 28x28 pixel images of handwritten digits (0 through 9); in +total there are 70,000 such images, and each image is unrolled into a +784 element vector. + +.. code:: python3 + + mnist = sklearn.datasets.fetch_openml("mnist_784") + +To ensure we have an idea of what this dataset looks like through the +lens of UMAP we can run UMAP on the full dataset. + +.. code:: python3 + + mapper = umap.UMAP(random_state=42).fit(mnist.data) + +.. code:: python3 + + umap.plot.points(mapper, labels=mnist.target, width=500, height=500) + +.. image:: images/composing_models_6_1.png + + +To make the problem more interesting let’s carve the dataset in two – not +into two sets of 35,000 samples, but instead carve each image in half. +That is, we’ll end up with 70,000 samples each of which is the top half +of the image of the handwritten digit, and another 70,000 samples each +of which is the bottom half of the image of the handwritten digit. + +.. code:: python3 + + top = mnist.data[:, :28 * 14] + bottom = mnist.data[:, 28 * 14:] + +This is a little artificial, but it provides us with an example dataset +where we have two distinct views of the data which we can still well +understand. In practice this situation would be more likely to arise +when there are two different data collection processes sampling from the +same underlying population. In our case we could simply glue the data +back together (hstack the numpy arrays for example), but potentially +this isn’t feasible as the different data views may have different +scales or modalities. So, despite the fact that we could glue things +back together in this case, we will proceed as if we can’t – as may be +the case for many real world problems. + +Let’s first look at what UMAP does individually on each dataset. We’ll +start with the top halves of the digits: + +.. code:: python3 + + top_mapper = umap.UMAP(random_state=42).fit(top) + +.. code:: python3 + + umap.plot.points(top_mapper, labels=mnist.target, width=500, height=500) + +.. image:: images/composing_models_11_1.png + + +While UMAP still manages to mostly separate the different digit classes +we can see the results are quite different from UMAP on the full +standard MNIST dataset. The twos and threes are blurred together (as we +would expect given that we don’t have the bottom half of the image wich +would let us tell them apart); The twos and threes are also in a large +grouping that pulls together all of the eights, sevens and nines (again, +what we would expect given only the top half of the digit), while the +fives and sixes are somewhat distinct, but clearly are similar to each +other. It is only the ones, fours and zeros that are very clearly +discernible. + +Now let’s see what sorts of results we get with the bottom halves of the +digits: + +.. code:: python3 + + bot_mapper = umap.UMAP(random_state=42).fit(bottom) + +.. code:: python3 + + umap.plot.points(bot_mapper, labels=mnist.target, width=500, height=500) + +.. image:: images/composing_models_14_1.png + + +This is clearly a very different view of the data. Now it is the fours +and nines that blur together (presumably many of the nines are drawn +with straight rather than curved stems), with sevens nearby. The twos +and the threes are very distinct from each other, but the threes and the +fives are combined (as one might expect given that the bottom halves +*should* look similar). Zeros and sixes are distinct, but close to each +other. Ones, eights and twos are the most distinctive digits in this +view. + +So, assuming we can’t just glue the raw data together and stick a +reasonable metric on it, what can we do? We can perform intersections or +unions on the fuzzy topological representations. There is also some work +to be done re-asserting UMAP’s theoretical assumptions (local +connectivity, approximately uniform distributions). Fortunately UMAP +makes this relatively easy as long as you have a copy of fitted UMAP +models on hand (which we do in this case). To intersect two models +simply use the ``*`` operator; to union them use the ``+`` operator. +Note that this will actually take some time since we need to compute the +2D embedding of the combined model. + +.. code:: python3 + + intersection_mapper = top_mapper * bot_mapper + union_mapper = top_mapper + bot_mapper + +With that complete we can visualize the results. First let’s look at the +intersection: + +.. code:: python3 + + umap.plot.points(intersection_mapper, labels=mnist.target, width=500, height=500) + + + +.. image:: images/composing_models_18_1.png + + +As you can see, while this isn’t as good as a UMAP plot for the full +MNIST dataset it has recovered the individual digits quite well. The +worst of the remaining overlap is between the threes and fives in the +center, which is it still struggling to fully distinguish. But note, +also, that we have recovered more of the overall structure than either +of the two different individual views, with the layout of different +digit classes more closely resembling that of the UMAP run on the full +dataset. + +Now let’s look at the union. + +.. code:: python3 + + umap.plot.points(union_mapper, labels=mnist.target, width=500, height=500) + +.. image:: images/composing_models_20_1.png + + +Given that UMAP is agnostic to rotation or reflection of the final +layout, this is essentially the same result as the intersection since it +is almost the reflection of it in the y-axis. This sort of result +(intersection and union being similar) is not always the case (in fact +it is not that common), but since the underlying structure of the digits +dataset is so clear we find that either way of piecing it together from +the two half datasets manage to find the same core underlying structure. + +If you are willing to try something a little more experimental there is +also a third option using the ``-`` operator which effectively +intersects with the fuzzy set complement (and is thus not commutative, +just as ``-`` implies). The goal here is to try to provide a sense of +what the data looks like when we contrast it against a second view. + +.. code:: python3 + + contrast_mapper = top_mapper - bot_mapper + +.. code:: python3 + + umap.plot.points(contrast_mapper, labels=mnist.target, width=500, height=500) + +.. image:: images/composing_models_23_1.png + + +In this case the result is not overly dissimilar from the embedding of +just the top half, so the contrast has perhaps not shown is as much as +we might have hoped. + +Diamonds dataset example +------------------------ + +Now let’s try the same approach on a different dataset where the option +of just running UMAP on the full dataset is not available. For this +we’ll use the diamonds dataset. In this dataset each row represents a +different diamond and provides details on the weight (carat), cut, +color, clarity, size (depth, table, x, y, z) and price of the given +diamond. How these different factors interplay is somewhat complicated. + +.. code:: python3 + + diamonds = sns.load_dataset('diamonds') + diamonds.head() + + + + +.. raw:: html + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
caratcutcolorclaritydepthtablepricexyz
00.23IdealESI261.555.03263.953.982.43
10.21PremiumESI159.861.03263.893.842.31
20.23GoodEVS156.965.03274.054.072.31
30.29PremiumIVS262.458.03344.204.232.63
40.31GoodJSI263.358.03354.344.352.75
+
+ + + +For our purposes let’s take “price” as a “target” variable (as is often +the case when the dataset is used in machine learning contexts). What we +would like to do is provide a UMAP embedding of the data using the +remaining features. This is tricky since we can’t exactly use a +euclidean metric over the whole thing. What we can do, however, is split +the data into two distinct types: the purely numeric features relating +to size and weight, and the categorical features of color, cut and +clarity. Let’s pull each of those feature sets out so we can work with +them independently. + +.. code:: python3 + + numeric = diamonds[["carat", "table", "x", "y", "z"]].copy() + ordinal = diamonds[["cut", "color", "clarity"]].copy() + +Now we have a new problem: the numeric features are not at all on the +same scales, so any sort of standard distance metric across them will be +dominated by those features with the largest ranges. We can correct for +that by performing feature scaling. To do that we’ll make use of +sklearn’s ``RobustScaler`` which uses robust statistics (such as the +median and interquartile range) to center and rescale the data feature +by feature. If we look at the results on the first five rows we see that +the different features are all now reasonably comparable, and it is +reasonable to apply something like euclidean distance across them. + +.. code:: python3 + + scaled_numeric = RobustScaler().fit_transform(numeric) + scaled_numeric[:5] + + + + +.. parsed-literal:: + + array([[-0.734375 , -0.66666667, -0.95628415, -0.95054945, -0.97345133], + [-0.765625 , 1.33333333, -0.98907104, -1.02747253, -1.07964602], + [-0.734375 , 2.66666667, -0.90163934, -0.9010989 , -1.07964602], + [-0.640625 , 0.33333333, -0.81967213, -0.81318681, -0.79646018], + [-0.609375 , 0.33333333, -0.7431694 , -0.74725275, -0.69026549]]) + + + +What is the best way to handle the categorical features? If they are +purely categorical it would make sense to one-hot encode the categories +and use “dice” distance between them. A downside of that is that, with +so few categories, it is a very coarse metric which will fail to provide +much differentiation. For the diamonds dataset, however, the categories +come with a strict order: Ideal cut is better than Premium cut, which is +better than Very Good cut and so on. Color grades work similarly, and +there is a distinct grading scheme for clarity as well. We can use an +ordinal encoding on these categories. Now, while the *ranges* of values +may vary, the differences between them are all comparable – a difference +of 1 for each grade level. That means we don’t need to rescale this data +after the ordinal coding. + +.. code:: python3 + + ordinal["cut"] = ordinal.cut.map({"Fair":0, "Good":1, "Very Good":2, "Premium":3, "Ideal":4}) + ordinal["color"] = ordinal.color.map({"D":0, "E":1, "F":2, "G":3, "H":4, "I":5, "J":6}) + ordinal["clarity"] = ordinal.clarity.map({"I1":0, "SI2":1, "SI1":2, "VS2":3, "VS1":4, "VVS2":5, "VVS1":6, "IF":7}) + +.. code:: python3 + + ordinal + + + + +.. raw:: html + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
cutcolorclarity
0411
1312
2114
3353
4161
............
53935402
53936102
53937202
53938341
53939401
+

53940 rows × 3 columns

+
+ + + +As noted we can use euclidean as a sensible distance on the rescaled +numeric data. On the other hand since the different ordinal categories +are entirelty independent of each other, and we have a strict ordinal +codin, the socalled “manhattan” metric makes more sense here – it is +simply the sum of the absolute differences in each category. As before +we can now train UMAP models on each dataset – this time, however, since +the datasets are different we need different metrics and even different +values of ``n_neighbors``. + +.. code:: python3 + + numeric_mapper = umap.UMAP(n_neighbors=15, random_state=42).fit(scaled_numeric) + ordinal_mapper = umap.UMAP(metric="manhattan", n_neighbors=150, random_state=42).fit(ordinal.values) + +We can look at the results of each of these independent views of the +dataset reduced to 2D using UMAP. Let’s first look at the numeric data +on size and weight of the diamonds. We can colour by the price to get +some idea of how the dataset fits together. + +.. code:: python3 + + umap.plot.points(numeric_mapper, values=diamonds["price"], cmap="viridis") + +.. image:: images/composing_models_36_1.png + + +We see that while the data generally correlates somewhat with the price +of the diamonds there are distinctly different threads in the data, +presumably corresponding to different styles of cut, and how that +results in different sizing of diamonds in the various dimensions, +depending on the weight. + +In contrast we ca look at the ordinal data. In this case we’ll colour it +by the different categories as well as by price. + +.. code:: python3 + + fig, ax = umap.plot.plt.subplots(2, 2, figsize=(12,12)) + umap.plot.points(ordinal_mapper, labels=diamonds["color"], ax=ax[0,0]) + umap.plot.points(ordinal_mapper, labels=diamonds["clarity"], ax=ax[0,1]) + umap.plot.points(ordinal_mapper, labels=diamonds["cut"], ax=ax[1,0]) + umap.plot.points(ordinal_mapper, values=diamonds["price"], cmap="viridis", ax=ax[1,1]) + +.. image:: images/composing_models_38_1.png + + +As you can see this is a markedly different result! The ordinal data has +a relatively coarse metric, since the different categories can only take +on a small range of discrete values. This means that, with respect to +the trio of color, cut, and clarity, diamonds are largely either almost +identical, or quite distinct. The result is very tight groupings which +have very high density. You can see a gradient of color from left to +right in the plot; colouring by cut or clarity show different +stratifications. The combination of these very distinct statifications +results in this highly clustered embedding. It is exactly for this +reason that we need such a high ``n_neighbors`` value: the very local +structure of the data is merely clusters of identical categories; we +need to see wider to learn more structure. + +Given these radically different views of the data, what do we get if we +try to integrate them together? As before we can use the intersection +and union operators to simply combine the models. As noted before this +is a somewhat time-consuming operation as a new 2D representation for +the combined models needs to be optimized. + +.. code:: python3 + + intersection_mapper = numeric_mapper * ordinal_mapper + union_mapper = numeric_mapper + ordinal_mapper + +Let’s start by looking at the intersection; here we are only really +decreasing connectivity since edges are assigned the probability of +existing in *both* data views (before re-asserting local connectivity +and uniform distribution assumptions). + +.. code:: python3 + + umap.plot.points(intersection_mapper, values=diamonds["price"], cmap="viridis") + +.. image:: images/composing_models_42_1.png + + +What we get most closely represents the numeric data view. Why is this? +Because the categorical data view has points either connected with +certainty (because they are, or are nearly, identical) or very loosely. +The points connected with near certainty are very dense clusters – +almost points in the plot – and mostly what we are doing with the +intersection is breaking up those clusters with the more fine-grained +and variable connectivity provided by the numerical data. At th esame +time we have shifted the result significantly from the numerical data +view on its own; the categorical information has made each cluster more +uniform (rather than being a gradient) in its price. + +Given this result, what would you expect of the union? + +.. code:: python3 + + umap.plot.points(union_mapper, labels=diamonds["color"]) + +.. image:: images/composing_models_44_1.png + + +What we get in practice looks a lot more like the categorical view of +the data. This time we are only *increasing* the connectivity (prior to +re-asserting local connectivity and uniform distribution assumptions); +thus we retain most of the structure of the high-connectivity +categorical view. Note, however, that we have created more connected and +coherent clusters in the center of the plot, showing a range of diamond +colors, and the introduction of the numerical size and weight +information has induced a rearrangement of the individual clusters +around the fringes. + +We can go a step further and experiment with the contrast composition +method. + +.. code:: python3 + + contrast_mapper = numeric_mapper - ordinal_mapper + +.. code:: python3 + + umap.plot.points(contrast_mapper, values=diamonds["price"], cmap="viridis") + +.. image:: images/composing_models_47_1.png + + +Here we see that we’ve retained a lot of the structure of the numeric +data view, but have refined and broken it down further into clear +clusters with price gradients running through each of them. + +To further demonstrate the power of this approach we can go a step +further and intersect a higher ``n_neighbors`` based embedding of the +numeric data view with our existing union of numeric and categorical +data – providing a model that is a composition of three simpler models. + +.. code:: python3 + + intersect_union_mapper = umap.UMAP(random_state=42, n_neighbors=60).fit(numeric) * union_mapper + +.. code:: python3 + + umap.plot.points(intersect_union_mapper, values=diamonds["price"], cmap="viridis") + +.. image:: images/composing_models_50_1.png + +Here the greater global structure from the larger ``n_neighbors`` value +glues together longer strands and we get an interesting result out. In +this case it is not necessarily particularly informative, but it is +included as a demonstration that even composed models can be composed +with each other, stacking together potentially many different views. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/conf.py b/ext/TAL/applications/UMAP_fast/umap/doc/conf.py new file mode 100644 index 00000000..35610a7d --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/conf.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# umap documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 8 10:09:40 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +import sys + +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", + # 'bokeh.sphinxext.bokeh_plot', + "sphinx_gallery.gen_gallery", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "umap" +copyright = "2018, Leland McInnes" +author = "Leland McInnes" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = "0.5" +# The full version, including alpha/beta/rc tags. +release = "0.5" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'alabaster' +html_theme = "sphinx_rtd_theme" + + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + "navigation_depth": 3, + "logo_only": True, +} + +html_logo = "logo.png" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +# html_sidebars = { +# '**': [ +# 'relations.html', # needs 'show_related': True theme option to display +# 'searchbox.html', +# ] +# } + +html_sidebars = { + "**": [ + "globaltoc.html", + "relations.html", # needs 'show_related': True theme option to display + "searchbox.html", + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "umapdoc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "umap.tex", "umap Documentation", "Leland McInnes", "manual"), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "umap", "umap Documentation", [author], 1)] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "umap", + "umap Documentation", + author, + "umap", + "One line description of project.", + "Miscellaneous", + ), +] + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("https://docs.python.org/{.major}".format(sys.version_info), None), + "numpy": ("https://docs.scipy.org/doc/numpy/", None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), + "matplotlib": ("https://matplotlib.org/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), + "sklearn": ("http://scikit-learn.org/stable/", None), + "bokeh": ("http://bokeh.pydata.org/en/latest/", None), +} + +# -- Options for sphinx-gallery --------------------------------------------- + +sphinx_gallery_conf = { + # path to your examples scripts + "examples_dirs": "../examples", + "ignore_pattern": r"(.*torus.*|inverse_transform.*)\.py", + # path where to save gallery generated examples + "gallery_dirs": "auto_examples", + "plot_gallery": False, # Turn off running the examples for now + "reference_url": { + "umap": None, + "python": "https://docs.python.org/{.major}".format(sys.version_info), + "numpy": "https://docs.scipy.org/doc/numpy/", + "scipy": "https://docs.scipy.org/doc/scipy/reference", + "matplotlib": "https://matplotlib.org/", + "pandas": "https://pandas.pydata.org/pandas-docs/stable/", + "sklearn": "http://scikit-learn.org/stable/", + "bokeh": "http://bokeh.pydata.org/en/latest/", + }, +} + + +def setup(app): + app.add_javascript("https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js") + app.add_javascript("https://cdn.plot.ly/plotly-latest.min.js") \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/densmap_demo.rst b/ext/TAL/applications/UMAP_fast/umap/doc/densmap_demo.rst new file mode 100644 index 00000000..59a9b0fc --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/densmap_demo.rst @@ -0,0 +1,384 @@ +Better Preserving Local Density with DensMAP +============================================ + +A notable assumption in UMAP is that the data is uniformly distributed +on some manifold and that it is ultimately this manifold that we would +like to present. This is highly effective for many use cases, but it can +be the case that one would like to preserve more information about the +relative local density of data. A recent paper presented a technique +called +`DensMAP `__ +the computes estimates of the local density and uses those estimates as +a regularizer in the optimization of the low dimensional representation. +The details are well explained in `the +paper `__ +and we encourage those curious about the details to read it. The result +is a low dimensional representation that preserves information about the +relative local density of the data. To see what this means in practice +let’s load some modules and try it out on some familiar data. + +.. code:: python3 + + import sklearn.datasets + import umap + import umap.plot + +For test data we will make use of the now familiar (see earlier tutorial +sections) MNIST and Fashion-MNIST datasets. MNIST is a collection of +70,000 gray-scale images of hand-written digits. Fashion-MNIST is a +collection of 70,000 gray-scale images of fashion items. + +.. code:: python3 + + mnist = sklearn.datasets.fetch_openml("mnist_784") + fmnist = sklearn.datasets.fetch_openml("Fashion-MNIST") + +Before we try out DensMAP let’s run standard UMAP so we have a baseline +to compare to. We’ll start with MNIST digits. + +.. code:: python3 + + %%time + mapper = umap.UMAP(random_state=42).fit(mnist.data) + + +.. parsed-literal:: + + CPU times: user 2min, sys: 15 s, total: 2min 15s + Wall time: 1min 43s + + +.. code:: python3 + + umap.plot.points(mapper, labels=mnist.target, width=500, height=500) + +.. image:: images/densmap_demo_6_1.png + + +Now let’s try running DensMAP instead. In practice this is as easy as +adding the parameter ``densmap=True`` to the UMAP constructor – this +will cause UMAP use use DensMAP regularization with the default DensMAP +parameters. + +.. code:: python3 + + %%time + dens_mapper = umap.UMAP(densmap=True, random_state=42).fit(mnist.data) + + +.. parsed-literal:: + + CPU times: user 3min 42s, sys: 12.9 s, total: 3min 55s + Wall time: 2min 20s + + +Note that this is a little slower than standard UMAP – there is a little +more work to be done. It is worth noting, however, that the DensMAP +overhead is relatively constant, so the difference in runtime won’t +increase much as you scale out DensMAP to larger datasets. + +Now let’s see what sort of results we get: + +.. code:: python3 + + umap.plot.points(dens_mapper, labels=mnist.target, width=500, height=500) + +.. image:: images/densmap_demo_10_1.png + + +This is a significantly different result – although notably the same +groupings of digits and overall structure have resulted. The most +striking aspects are that the ones cluster has be compressed into a very +narrow and dense stripe, while other digit clusters, most notably the +zeros and the twos have expanded out to fill more space in the plot. +This is due to the fact that in the high dimensional space the ones are +indeed more densely packed together, with largely only variation along +one dimension (the angle with which the stroke of the one is drawn). In +contrast a digit like the zero has a lot more variation (rounder, +narrower, taller, shorter, sloping one way or another); this results in +less local density in high dimensional space, and this lack of local +density has been preserved by DensMAP. + +Let’s now look at the Fashion-MNIST dataset; as before we’ll start by +reminding ourselves what the default UMAP results look like: + +.. code:: python3 + + %%time + mapper = umap.UMAP(random_state=42).fit(fmnist.data) + + +.. parsed-literal:: + + CPU times: user 1min 6s, sys: 8.66 s, total: 1min 15s + Wall time: 49.8 s + + +.. code:: python3 + + umap.plot.points(mapper, labels=fmnist.target, width=500, height=500) + +.. image:: images/densmap_demo_13_1.png + + +Now let’s try running DensMAP. As before that is as simple as setting +the ``densmap=True`` flag. + +.. code:: python3 + + %%time + dens_mapper = umap.UMAP(densmap=True, random_state=42).fit(fmnist.data) + + +.. parsed-literal:: + + CPU times: user 3min 48s, sys: 8.07 s, total: 3min 56s + Wall time: 2min 21s + + +.. code:: python3 + + umap.plot.points(dens_mapper, labels=fmnist.target, width=500, height=500) + +.. image:: images/densmap_demo_16_1.png + + +Again we see that DensMAP provides a plot similar to UMAP broadly, but +with striking differences. Here we get to see that the cluster of bags +(label 8 in blue) is actually quite sparse, while the cluster of pants +(label 1 in red) is actually quite dense with little variation compared +to other categories. We even see information internal to clusters. +Consider the cluster of boots (label 9 in violet): at the top end it is +quite dense, but it fades out into a much sparse region. + +So far we have used DensMAP with default parameters, but the +implementation provides several parameters for adjusting exactly how the +local density regularisation is handled. We encourage readers to consult +the paper for the details of the many parameters available. For general +use the main parameter of interest is called ``dens_lambda`` and it +controls how strongly the local density regularisation acts. Larger +values of ``dens_lambda`` with make preserving the local density a +priority over the the standard UMAP objective, while smaller values lean +more towards classical UMAP. The default value is 2.0. Let’s play with +it a little so we can see the effects of varying it. To start we’ll use +a higher ``dens_lambda`` of 5.0: + +.. code:: python3 + + %%time + dens_mapper = umap.UMAP(densmap=True, dens_lambda=5.0, random_state=42).fit(fmnist.data) + + +.. parsed-literal:: + + CPU times: user 3min 47s, sys: 5.04 s, total: 3min 52s + Wall time: 2min 18s + + +.. code:: python3 + + umap.plot.points(dens_mapper, labels=fmnist.target, width=500, height=500) + +.. image:: images/densmap_demo_19_1.png + + +This looks kind of like what we had before, but blurrier. And also … +smaller? The plot bounds are set by the data, so the fact that it is +smaller represents the fact that there are some points right out to the +edges of the plot. These are likely points that are in locally very +sparse regions of the high dimensional space and are thus pushed well +away from everything else. We can see this better if we use raw +matplotlib and a scatter plot with larger point size: + +.. code:: python3 + + fig, ax = umap.plot.plt.subplots(figsize=(7,7)) + ax.scatter(*dens_mapper.embedding_.T, c=fmnist.target.astype('int8'), cmap="Spectral", s=1) + +.. image:: images/densmap_demo_21_1.png + + +Aside from seeing the issues with overplotting we can see that there +are, in fact, quite a few points that create a very soft halo of of +sparse points around the fringes. + +Now let’s try going the other way and reduce ``dens_lambda`` to a small +value, so that in principle we can recover something quite close to the +default UMAP plot, with just a hint of local density information +encoded. + +.. code:: python3 + + %%time + dens_mapper = umap.UMAP(densmap=True, dens_lambda=0.1, random_state=42).fit(fmnist.data) + + +.. parsed-literal:: + + CPU times: user 3min 47s, sys: 3.78 s, total: 3min 51s + Wall time: 2min 16s + + +.. code:: python3 + + umap.plot.points(dens_mapper, labels=fmnist.target, width=500, height=500) + +.. image:: images/densmap_demo_24_1.png + + +And indeed, this looks very much like the original plot, but the bags +(label 8 in blue) are slightly more diffused, and the pants (label 1 in +red) are a little denser. This is very much the default UMAP with just a +tweak to better reflect some notion of local density. + +Supervised DensMAP on the Galaxy10SDSS dataset +---------------------------------------------- + +The `Galaxy10SDSS dataset `__ +is a crowd sourced human labelled dataset of galaxy images, +which have been separated in to ten classes. DensMAP can +learn an embedding that partially separates the data. To +keep runtime small, DensMAP is applied to a subset of the +data. + +.. code:: python3 + + import numpy as np + import h5py + import matplotlib.pyplot as plt + import umap + import os + import math + import requests + + if not os.path.isfile("Galaxy10.h5"): + url = "http://astro.utoronto.ca/~bovy/Galaxy10/Galaxy10.h5" + r = requests.get(url, allow_redirects=True) + open("Galaxy10.h5", "wb").write(r.content) + + # To get the images and labels from file + with h5py.File("Galaxy10.h5", "r") as F: + images = np.array(F["images"]) + labels = np.array(F["ans"]) + + X_train = np.empty([math.floor(len(labels) / 100), 14283], dtype=np.float64) + y_train = np.empty([math.floor(len(labels) / 100)], dtype=np.float64) + X_test = X_train + y_test = y_train + # Get a subset of the data + for i in range(math.floor(len(labels) / 100)): + X_train[i, :] = np.array(np.ndarray.flatten(images[i, :, :, :]), dtype=np.float64) + y_train[i] = labels[i] + X_test[i, :] = np.array( + np.ndarray.flatten(images[i + math.floor(len(labels) / 100), :, :, :]), + dtype=np.float64, + ) + y_test[i] = labels[i + math.floor(len(labels) / 100)] + + # Plot distribution + classes, frequency = np.unique(y_train, return_counts=True) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.bar(classes, frequency) + plt.xlabel("Class") + plt.ylabel("Frequency") + plt.title("Data Subset") + plt.savefig("galaxy10_subset.svg") + + + +.. image:: images/galaxy10_subset.svg + + +The figure shows that the selected subset of the data set is +unbalanced, but the entire dataset is also unbalanced, so +this experiment will still use this subset. The next step is +to examine the output of the standard DensMAP algorithm. + +.. code:: python3 + + reducer = umap.UMAP( + densmap=True, n_components=2, random_state=42, verbose=False + ) + reducer.fit(X_train) + + galaxy10_densmap = reducer.transform(X_train) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.scatter( + galaxy10_densmap[:, 0], + galaxy10_densmap[:, 1], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, + ) + plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) + plt.savefig("galaxy10_2D_densmap.svg") + + + +.. image:: images/galaxy10_2D_densmap.svg + + +The standard DensMAP algorithm does not separate the galaxies +according to their type. Supervised DensMAP can do better. + +.. code:: python3 + + reducer = umap.UMAP( + densmap=True, n_components=2, random_state=42, verbose=False + ) + reducer.fit(X_train, y_train) + + galaxy10_densmap_supervised = reducer.transform(X_train) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.scatter( + galaxy10_densmap_supervised[:, 0], + galaxy10_densmap_supervised[:, 1], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, + ) + plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) + plt.savefig("galaxy10_2D_densmap_supervised.svg") + + + +.. image:: images/galaxy10_2D_densmap_supervised.svg + + +Supervised DensMAP does indeed do better. There is a litle overlap +between some of the classes, but the original dataset +also has some ambiguities in the classification. The best +check of this method is to project the testing data onto the +learned embedding. + +.. code:: python3 + + galaxy10_densmap_supervised_prediction = reducer.transform(X_test) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.scatter( + galaxy10_densmap_supervised_prediction[:, 0], + galaxy10_densmap_supervised_prediction[:, 1], + c=y_test, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_test, + ) + plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) + plt.savefig("galaxy10_2D_densmap_supervised_prediction.svg") + + + +.. image:: images/galaxy10_2D_densmap_supervised_prediction.svg + + +This shows that the learned embedding can be used on new data +sets, and so this method may be helpful for examining images +of galaxies. Try out this method on the full 200 Mb dataset +as well as the newer 2.54 Gb +`Galaxy 10 DECals dataset `__ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/doc_requirements.txt b/ext/TAL/applications/UMAP_fast/umap/doc/doc_requirements.txt new file mode 100644 index 00000000..4ceb3df5 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/doc_requirements.txt @@ -0,0 +1,8 @@ +numpy>=1.13 +scipy>=0.19 +scikit-learn>=0.19 +numba>=0.37 +bokeh>=0.13 +datashader>=0.6 +seaborn>=0.8 +sphinx-gallery \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/document_embedding.rst b/ext/TAL/applications/UMAP_fast/umap/doc/document_embedding.rst new file mode 100644 index 00000000..9decf783 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/document_embedding.rst @@ -0,0 +1,329 @@ +Document embedding using UMAP +============================= + +This is a tutorial of using UMAP to embed text (but this can be extended +to any collection of tokens). We are going to use the `20 newsgroups +dataset `__ which is a collection +of forum posts labelled by topic. We are going to embed these documents +and see that similar documents (i.e. posts in the same subforum) will +end up close together. You can use this embedding for other downstream +tasks, such as visualizing your corpus, or run a clustering algorithm +(e.g. HDBSCAN). We will use a bag of words model and use UMAP on the +count vectors as well as the TF-IDF vectors. + + +To start with let's load the relevant libraries. **This requires UMAP version >= 0.4.0.** + +.. code:: python3 + + import pandas as pd + import umap + import umap.plot + + # Used to get the data + from sklearn.datasets import fetch_20newsgroups + from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer + + # Some plotting libraries + import matplotlib.pyplot as plt + %matplotlib notebook + from bokeh.plotting import show, save, output_notebook, output_file + from bokeh.resources import INLINE + output_notebook(resources=INLINE) + +Next let's download and explore the 20 newsgroups dataset. + +.. code:: python3 + + %%time + dataset = fetch_20newsgroups(subset='all', + shuffle=True, random_state=42) + + +.. parsed-literal:: + + CPU times: user 280 ms, sys: 52 ms, total: 332 ms + Wall time: 460 ms + +Let's see the size of the corpus: + +.. code:: python3 + + print(f'{len(dataset.data)} documents') + print(f'{len(dataset.target_names)} categories') + + +.. parsed-literal:: + + 18846 documents + 20 categories + + +Here are the categories of documents. As you can see many are related to +one another (e.g. ‘comp.sys.ibm.pc.hardware’ and +‘comp.sys.mac.hardware’) but they are not all correlated (e.g. ‘sci.med’ +and ‘rec.sport.baseball’). + +.. code:: python3 + + dataset.target_names + + + + +.. parsed-literal:: + + ['alt.atheism', + 'comp.graphics', + 'comp.os.ms-windows.misc', + 'comp.sys.ibm.pc.hardware', + 'comp.sys.mac.hardware', + 'comp.windows.x', + 'misc.forsale', + 'rec.autos', + 'rec.motorcycles', + 'rec.sport.baseball', + 'rec.sport.hockey', + 'sci.crypt', + 'sci.electronics', + 'sci.med', + 'sci.space', + 'soc.religion.christian', + 'talk.politics.guns', + 'talk.politics.mideast', + 'talk.politics.misc', + 'talk.religion.misc'] + + + +Let’s look at a couple of sample documents: + +.. code:: python3 + + for idx, document in enumerate(dataset.data[:3]): + category = dataset.target_names[dataset.target[idx]] + + print(f'Category: {category}') + print('---------------------------') + # Print the first 500 characters of the post + print(document[:500]) + print('---------------------------') + + +.. parsed-literal:: + + Category: rec.sport.hockey + --------------------------- + From: Mamatha Devineni Ratnam + Subject: Pens fans reactions + Organization: Post Office, Carnegie Mellon, Pittsburgh, PA + Lines: 12 + NNTP-Posting-Host: po4.andrew.cmu.edu + + + + I am sure some bashers of Pens fans are pretty confused about the lack + of any kind of posts about the recent Pens massacre of the Devils. Actually, + I am bit puzzled too and a bit relieved. However, I am going to put an end + to non-PIttsburghers' relief with a bit of praise for the Pens. Man, they + are killin + --------------------------- + Category: comp.sys.ibm.pc.hardware + --------------------------- + From: mblawson@midway.ecn.uoknor.edu (Matthew B Lawson) + Subject: Which high-performance VLB video card? + Summary: Seek recommendations for VLB video card + Nntp-Posting-Host: midway.ecn.uoknor.edu + Organization: Engineering Computer Network, University of Oklahoma, Norman, OK, USA + Keywords: orchid, stealth, vlb + Lines: 21 + + My brother is in the market for a high-performance video card that supports + VESA local bus with 1-2MB RAM. Does anyone have suggestions/ideas on: + + - Diamond Stealth Pro Local + --------------------------- + Category: talk.politics.mideast + --------------------------- + From: hilmi-er@dsv.su.se (Hilmi Eren) + Subject: Re: ARMENIA SAYS IT COULD SHOOT DOWN TURKISH PLANES (Henrik) + Lines: 95 + Nntp-Posting-Host: viktoria.dsv.su.se + Reply-To: hilmi-er@dsv.su.se (Hilmi Eren) + Organization: Dept. of Computer and Systems Sciences, Stockholm University + + + + + |>The student of "regional killings" alias Davidian (not the Davidian religios sect) writes: + + + |>Greater Armenia would stretch from Karabakh, to the Black Sea, to the + |>Mediterranean, so if you use the term "Greater Armenia + --------------------------- + + +Now we will create a dataframe with the target labels to be used in plotting. This will allow us to see the newsgroup +when we hover over the plotted points (if using interactive plotting). This will help us evaluate (by eye) how good the embedding looks. + +.. code:: python3 + + category_labels = [dataset.target_names[x] for x in dataset.target] + hover_df = pd.DataFrame(category_labels, columns=['category']) + +Using raw counts +---------------- + +Next, we are going to use a bag-of-words approach (i.e. word order doesn’t +matter) and construct a word document matrix. In this matrix the rows +will correspond to a document (i.e. post) and each column will +correspond to a particular word. The values will be the counts of how +many times a given word appeared in a particular document. + +We will use sklearns CountVectorizer function to do this for us along +with a couple other preprocessing steps: + +1) Split the text into tokens (i.e. words) by splitting on whitespace + +2) Remove english stopwords (the, and, etc) + +3) Remove all words which occur less than 5 times in the entire corpus + (via the min_df parameter) + +.. code:: python3 + + vectorizer = CountVectorizer(min_df=5, stop_words='english') + word_doc_matrix = vectorizer.fit_transform(dataset.data) + +This gives us a 18846x34880 matrix where there are 18846 documents (same +as above) and 34880 unique tokens. This matrix is sparse since most +words do not appear in most documents. + +.. code:: python3 + + word_doc_matrix + +.. parsed-literal:: + + <18846x34880 sparse matrix of type '' + with 1939023 stored elements in Compressed Sparse Row format> + + + +Now we are going to do dimension reduction using UMAP to reduce the matrix +from 34880 dimensions to 2 dimensions (since n_components=2). We need a +distance metric and will use `Hellinger +distance `__ which +measures the similarity between two probability distributions. Each +document has a set of counts generated by a `multinomial +distribution `__ +where we can use Hellinger distance to measure the similarity of these +distributions. + +.. code:: python3 + + %%time + embedding = umap.UMAP(n_components=2, metric='hellinger').fit(word_doc_matrix) + + +.. parsed-literal:: + + CPU times: user 2min 24s, sys: 1.18 s, total: 2min 25s + Wall time: 2min 3s + + +Now we have an embedding of 18846x2. + +.. code:: python3 + + embedding.embedding_.shape + + +.. parsed-literal:: + + (18846, 2) + + +Let’s plot the embedding. If you are running this in a notebook, you should use the +interactive plotting method as it lets you hover over your points and see what +category they belong to. + +.. code:: python3 + + # For interactive plotting use + # f = umap.plot.interactive(embedding, labels=dataset.target, hover_data=hover_df, point_size=1) + # show(f) + f = umap.plot.points(embedding, labels=hover_df['category']) + +.. image:: images/20newsgroups_hellinger_counts.png + +As you can see this does reasonably well. There is some separation and +groups that you would expect to be similar (such as ‘rec.sport.baseball’ +and ‘rec.sport.hockey’) are close together. The big clump in the middle +corresponds to a lot of extremely similar newsgroups like +‘comp.sys.ibm.pc.hardware’ and ‘comp.sys.mac.hardware’. + +Using TF-IDF +------------ + +We will now do the same pipeline with the only change being the use of +`TF-IDF `__ weighting. +TF-IDF gives less weight to words that appear frequently across a large +number of documents since they are more popular in general. It asserts +a higher weight to words that appear frequently in a smaller subset of +documents since they are probably important words for those documents. + +To do the TF-IDF weighting we will use sklearns TfidfVectorizer with the +same parameters as CountVectorizer above. + +.. code:: python3 + + tfidf_vectorizer = TfidfVectorizer(min_df=5, stop_words='english') + tfidf_word_doc_matrix = tfidf_vectorizer.fit_transform(dataset.data) + +We get a matrix of the same size as before: + +.. code:: python3 + + tfidf_word_doc_matrix + +.. parsed-literal:: + + <18846x34880 sparse matrix of type '' + with 1939023 stored elements in Compressed Sparse Row format> + +Again we use Hellinger distance and UMAP to embed the documents + +.. code:: python3 + + %%time + tfidf_embedding = umap.UMAP(metric='hellinger').fit(tfidf_word_doc_matrix) + + +.. parsed-literal:: + + CPU times: user 2min 19s, sys: 1.27 s, total: 2min 20s + Wall time: 1min 57s + + +.. code:: python3 + + # For interactive plotting use + # fig = umap.plot.interactive(tfidf_embedding, labels=dataset.target, hover_data=hover_df, point_size=1) + # show(fig) + fig = umap.plot.points(tfidf_embedding, labels=hover_df['category']) + +.. image:: images/20newsgroups_hellinger_tfidf.png + +The results look fairly similar to before but this can be a useful trick +to have in your toolbox. + +Potential applications +---------------------- + +Now that we have an embedding, what can we do with it? + +- Explore/visualize your corpus to identify topics/trends +- Cluster the embedding to find groups of related documents +- Look for nearest neighbours to find related documents +- Look for anomalous documents \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/embedding_space.rst b/ext/TAL/applications/UMAP_fast/umap/doc/embedding_space.rst new file mode 100644 index 00000000..abd7b0c2 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/embedding_space.rst @@ -0,0 +1,579 @@ +Embedding to non-Euclidean spaces +================================= + +By default UMAP embeds data into Euclidean space. For 2D visualization +that means that data is embedded into a 2D plane suitable for a +scatterplot. In practice, however, there aren't really any major +constraints that prevent the algorithm from working with other more +interesting embedding spaces. In this tutorial we'll look at how to get +UMAP to embed into other spaces, how to embed into your own custom +space, and why this sort of approach might be useful. + +To start we'll load the usual selection of libraries. In this case we +will not be using the ``umap.plot`` functionality, but working with +matplotlib directly since we'll be generating some custom visualizations +for some of the more unique embedding spaces. + +.. code:: python3 + + import numpy as np + import numba + import sklearn.datasets + import matplotlib.pyplot as plt + import seaborn as sns + from mpl_toolkits.mplot3d import Axes3D + import umap + %matplotlib inline + +.. code:: python3 + + sns.set(style='white', rc={'figure.figsize':(10,10)}) + +As a test dataset we'll use the PenDigits dataset from sklearn -- +embedding into exotic spaces can be considerably more computationally +taxing, so a simple relatively small dataset is going to be useful. + +.. code:: python3 + + digits = sklearn.datasets.load_digits() + +Plane embeddings +---------------- + +Plain old plane embeddings are simple enough -- it is the default for +UMAP. Here we'll run through the example again, just to ensure you are +familiar with how this works, and what the result of a UMAP embedding of +the PenDigits dataset looks like in the simple case of embedding in the +plane. + +.. code:: python3 + + plane_mapper = umap.UMAP(random_state=42).fit(digits.data) + +.. code:: python3 + + plt.scatter(plane_mapper.embedding_.T[0], plane_mapper.embedding_.T[1], c=digits.target, cmap='Spectral') + +.. image:: images/embedding_space_7_1.png + + +Spherical embeddings +-------------------- + +What if we wanted to embed data onto a sphere rather than a plane? This +might make sense, for example, if we have reason to expect some sort of +periodic behaviour or other reasons to expect that no point can be +infinitely far from any other. To make UMAP embed onto a sphere we need +to make use of the ``output_metric`` parameter, which specifies what +metric to use for the **output** space. By default UMAP uses a Euclidean +``output_metric`` (and even has a special faster code-path for this +case), but you can pass in other metrics. Among the metrics UMAP +supports is the Haversine metric, used for measuring distances on a +sphere, given in latitude and longitude (in radians). If we set the +``output_metric`` to ``"haversine"`` then UMAP will use that to measure +distance in the embedding space. + +.. code:: python3 + + sphere_mapper = umap.UMAP(output_metric='haversine', random_state=42).fit(digits.data) + +The result is the pendigits data embedded with respect to haversine +distance on a sphere. The catch is that if we visualize this naively +then we will get nonsense. + +.. code:: python3 + + plt.scatter(sphere_mapper.embedding_.T[0], sphere_mapper.embedding_.T[1], c=digits.target, cmap='Spectral') + + +.. image:: images/embedding_space_11_1.png + + +What has gone astray is that under the embedding distance metric a point +at :math:`(0, \pi)` is distance zero from a point at :math:`(0, 3\pi)` +since that will wrap all the way around the equator. You'll note that +the scales on the x and y axes of the above plot go well outside the +ranges :math:`(-\pi, \pi)` and :math:`(0, 2\pi)`, so this isn't the +right representation of the data. We can, however, use straightforward +formulas to map this data onto a sphere embedded in 3d-space. + +.. code:: python3 + + x = np.sin(sphere_mapper.embedding_[:, 0]) * np.cos(sphere_mapper.embedding_[:, 1]) + y = np.sin(sphere_mapper.embedding_[:, 0]) * np.sin(sphere_mapper.embedding_[:, 1]) + z = np.cos(sphere_mapper.embedding_[:, 0]) + +Now ``x``, ``y``, and ``z`` give 3d coordinates for each embedding point +that lies on the surface of a sphere. We can visualize this using +matplotlib's 3d plotting capabilities, and see that we have in fact +induced a quite reasonable embedding of the data onto the surface of a +sphere. + +.. code:: python3 + + fig = plt.figure() + ax = fig.add_subplot(111, projection='3d') + ax.scatter(x, y, z, c=digits.target, cmap='Spectral') + + +.. image:: images/embedding_space_15_1.png + + +If you prefer a 2d plot we can convert these into lat/long coordinates +in the appropriate ranges and get the equivalent of a map projection of +the sphere data. + +.. code:: python3 + + x = np.arctan2(x, y) + y = -np.arccos(z) + +.. code:: python3 + + plt.scatter(x, y, c=digits.target.astype(np.int32), cmap='Spectral') + + +.. image:: images/embedding_space_18_1.png + + +Embedding on a Custom Metric Space +---------------------------------- + +What if you have some other custom notion of a metric space that you +would like to embed data into? In the same way that UMAP can support +custom written distance metrics for the input data (as long as they can +be compiled with numba), the ``output_metric`` parameter can accept +custom distance functions. One catch is that, to support gradient +descent optimization, the distance function needs to return both the +distance, and a vector for the gradient of the distance. This latter +point may require a little bit of calculus on the users part. A second +catch is that it is highly beneficial to parameterize the embedding +space in a way that has no coordinate constraints -- otherwise the +gradient descent may step a point outside the embedding space, resulting +in bad things happening. This is why, for example, the sphere example +simply has points wrap around rather than constraining coordinates to be +in the appropriate ranges. + +Let's work through an example where we construct a distance metric and +gradient for a different sort of space: a +`torus `__. A torus is essentially +just the outer surface of a donut. We can parameterize the torus in +terms of x, y coordinates with the caveat that we can `"wrap around" +(similar to the +sphere) `__. In such a +model distances are mostly just euclidean distances, we just have to +check for which is the shorter direction -- across or wrapping around -- +and ensure we account for the equivalence of wrapping around several +times. We can write a simple function to calculate that. + +.. code:: python3 + + @numba.njit(fastmath=True) + def torus_euclidean_grad(x, y, torus_dimensions=(2*np.pi,2*np.pi)): + """Standard euclidean distance. + + ..math:: + D(x, y) = \sqrt{\sum_i (x_i - y_i)^2} + """ + distance_sqr = 0.0 + g = np.zeros_like(x) + for i in range(x.shape[0]): + a = abs(x[i] - y[i]) + if 2*a < torus_dimensions[i]: + distance_sqr += a ** 2 + g[i] = (x[i] - y[i]) + else: + distance_sqr += (torus_dimensions[i]-a) ** 2 + g[i] = (x[i] - y[i]) * (a - torus_dimensions[i]) / a + distance = np.sqrt(distance_sqr) + return distance, g/(1e-6 + distance) + +Note that the gradient just derives from the standard euclidean +gradient, we just have to check the direction according to the way we've +wrapped around to compute the distance. We can now plug that function +directly in to the ``output_metric`` parameter and end up embedding data +on a torus. + +.. code:: python3 + + torus_mapper = umap.UMAP(output_metric=torus_euclidean_grad, random_state=42).fit(digits.data) + +As with the sphere case, a naive visualisation will look strange, due +the the wrapping around and equivalence of looping several times. But, +also just like the torus, we can construct a suitable visualization by +computing the 3d coordinates for the points using a little bit of +straightforward geometry (yes, I still had to look it up to check). + +.. code:: python3 + + R = 3 # Size of the doughnut circle + r = 1 # Size of the doughnut cross-section + + x = (R + r * np.cos(torus_mapper.embedding_[:, 0])) * np.cos(torus_mapper.embedding_[:, 1]) + y = (R + r * np.cos(torus_mapper.embedding_[:, 0])) * np.sin(torus_mapper.embedding_[:, 1]) + z = r * np.sin(torus_mapper.embedding_[:, 0]) + +Now we can visualize the result using matplotlib and see that, indeed, +the data has been suitably embedded onto a torus. + +.. code:: python3 + + fig = plt.figure() + ax = fig.add_subplot(111, projection='3d') + ax.scatter(x, y, z, c=digits.target, cmap='Spectral') + ax.set_zlim3d(-3, 3) + ax.view_init(35, 70) + + + +.. image:: images/embedding_space_26_0.png + + +And as with the torus we can do a little geometry and unwrap the torus +into a flat plane with the appropriate bounds. + +.. code:: python3 + + u = np.arctan2(x,y) + v = np.arctan2(np.sqrt(x**2 + y**2) - R, z) + +.. code:: python3 + + plt.scatter(u, v, c=digits.target, cmap='Spectral') + + + +.. image:: images/embedding_space_29_1.png + + +A Practical Example +------------------- + +While the examples given so far may have some use (because some data +does have suitable periodic or looping structures that we expect will be +better represented in a sphere or a torus), most data doesn't really +fall in the realm of something that a user can, apriori, expect to lie +on an exotic manifold. Are there more practical uses for the ability to +embed in other spaces? It turns out that there are. One interesting +example to consider is the space formed by 2d-Gaussian distributions. We +can measure the distance between two Gaussians (parameterized by a 2d +vector for the mean, and 2x2 matrix giving the covariance) by the +negative log of the inner product between the PDFs (since this has a +nice closed form solution, and is reasonably computable). That gives us +a metric space to embed into where samples are represented not as points +in 2d, but as Gaussian distributions in 2d, encoding some uncertainty in +how each sample in the high dimensional space is to be embedded. + +Of course we still have the issues of parameterizations that are +suitable for SGD -- requiring that the covariance matrix be symmetric +and positive definite is challenging. Instead we can parameterize the +covariance in terms of a width, height and angle, and recover the +covariance matrix from these if required. That gives us a total of 5 +components to embed into (two for the mean, 3 for parameters describing +the covariance). We can simply do this since the appropriate metric is +defined already. Note that we have to specifically pass +``n_components=5`` since we need to explicitly embed into a 5 +dimensional space to support all the covariance parameters associated to +2d Gaussians. + +.. code:: python3 + + gaussian_mapper = umap.UMAP(output_metric='gaussian_energy', + n_components=5, + random_state=42).fit(digits.data) + +Since we have embedded the data into a 5 dimensional space visualization +is not as trivial as it was earlier. We can get a start on visualizing +the results by looking at just the means, which are the 2d locations of +the modes of the Gaussians. A traditional scatter plot will suffice for +this. + +.. code:: python3 + + plt.scatter(gaussian_mapper.embedding_.T[0], gaussian_mapper.embedding_.T[1], c=digits.target, cmap='Spectral') + + +.. image:: images/embedding_space_33_1.png + + +We see that we have gotten a result similar to a standard embedding into +euclidean space, but with less clear clustering, and more points between +clusters. To get a clearer idea of what is going on it will be necessary +to devise a means to display some of the extra information contained in +the extra 3 dimensions providing covariance data. To do this it will be +helpful to be able to draw ellipses corresponding to super-level sets of +the PDF of the 2d Gaussian. We can start on this by writing a simple +function to draw ellipses on a plot accoriding to a position, a width, a +height, and an angle (since this is the format the embedding computed +the data). + +.. code:: python3 + + from matplotlib.patches import Ellipse + + def draw_simple_ellipse(position, width, height, angle, + ax=None, from_size=0.1, to_size=0.5, n_ellipses=3, + alpha=0.1, color=None, + **kwargs): + ax = ax or plt.gca() + angle = (angle / np.pi) * 180 + width, height = np.sqrt(width), np.sqrt(height) + # Draw the Ellipse + for nsig in np.linspace(from_size, to_size, n_ellipses): + ax.add_patch(Ellipse(position, nsig * width, nsig * height, + angle, alpha=alpha, lw=0, color=color, **kwargs)) + +Now we can plot the data by providing a scatterplot of the centers (as +before), but overlaying that over a super-level-set ellipses of the +associated Gaussians. The obvious catch is that this will induce a lot +of over-plotting, but it will at least provide a way to start +understanding the embedding we have produced. + +.. code:: python3 + + fig = plt.figure(figsize=(10,10)) + ax = fig.add_subplot(111) + colors = plt.get_cmap('Spectral')(np.linspace(0, 1, 10)) + for i in range(gaussian_mapper.embedding_.shape[0]): + pos = gaussian_mapper.embedding_[i, :2] + draw_simple_ellipse(pos, gaussian_mapper.embedding_[i, 2], + gaussian_mapper.embedding_[i, 3], + gaussian_mapper.embedding_[i, 4], + ax, color=colors[digits.target[i]], + from_size=0.2, to_size=1.0, alpha=0.05) + ax.scatter(gaussian_mapper.embedding_.T[0], + gaussian_mapper.embedding_.T[1], + c=digits.target, cmap='Spectral', s=3) + +.. image:: images/embedding_space_37_1.png + + +Now we can see that the covariance structure for the points can vary +greatly, both in absolute size, and in shape. We note that many of the +points falling between clusters have much larger variances, in a sense +representing the greater uncertainty of the location of the embedding. +It is also worth noting that the shape of the ellipses can vary +significantly -- there are several very stretched ellipses, quite +distinct from many of the very round ellipses; in a sense this +represents where the uncertainty falls more along a single line for +example. + +While this plot highlights some of the covariance structure in the +outlying points, in practice the overplotting here obscures a lot of the +more interesting structure in the clusters themselves. We can try to see +this structure better by plotting only a single ellipse per point and +using a lower alpha channel value for the ellipses, making them more +translucent. + +.. code:: python3 + + fig = plt.figure(figsize=(10,10)) + ax = fig.add_subplot(111) + for i in range(gaussian_mapper.embedding_.shape[0]): + pos = gaussian_mapper.embedding_[i, :2] + draw_simple_ellipse(pos, gaussian_mapper.embedding_[i, 2], + gaussian_mapper.embedding_[i, 3], + gaussian_mapper.embedding_[i, 4], + ax, n_ellipses=1, + color=colors[digits.target[i]], + from_size=1.0, to_size=1.0, alpha=0.01) + ax.scatter(gaussian_mapper.embedding_.T[0], + gaussian_mapper.embedding_.T[1], + c=digits.target, cmap='Spectral', s=3) + + +.. image:: images/embedding_space_39_1.png + + +This lets us see the variation of density of clusters with respect to +the covariance structure -- some clusters have consistently very tight +covariance, while others are more spread out (and hence have, in a sense, +greater associated uncertainty. Of course we still have a degree of +overplotting even here, and it will become increasingly difficult to +tune alpha channels to make things visible. Instead what we would want +is an actual density plot, showing the the density of the sum over all +of these Gaussians. + +To do this we'll need to define some functions, whose execution will be +accelerated using numba: the evaluation of the density of a 2d Gaussian +at a given point; an evaluation of the density of a given point summing +over a set of several Gaussians; and a function to generate the density +for each point in some grid (summing only over nearby Gaussians to make +this naive approach more computable). + +.. code:: python3 + + from sklearn.neighbors import KDTree + + @numba.njit(fastmath=True) + def eval_gaussian(x, pos=np.array([0, 0]), cov=np.eye(2, dtype=np.float32)): + det = cov[0,0] * cov[1,1] - cov[0,1] * cov[1,0] + if det > 1e-16: + cov_inv = np.array([[cov[1,1], -cov[0,1]], [-cov[1,0], cov[0,0]]]) * 1.0 / det + diff = x - pos + m_dist = cov_inv[0,0] * diff[0]**2 - \ + (cov_inv[0,1] + cov_inv[1,0]) * diff[0] * diff[1] + \ + cov_inv[1,1] * diff[1]**2 + return (np.exp(-0.5 * m_dist)) / (2 * np.pi * np.sqrt(np.abs(det))) + else: + return 0.0 + + @numba.njit(fastmath=True) + def eval_density_at_point(x, embedding): + result = 0.0 + for i in range(embedding.shape[0]): + pos = embedding[i, :2] + t = embedding[i, 4] + U = np.array([[np.cos(t), np.sin(t)], [np.sin(t), -np.cos(t)]]) + cov = U @ np.diag(embedding[i, 2:4]) @ U + result += eval_gaussian(x, pos=pos, cov=cov) + return result + + def create_density_plot(X, Y, embedding): + Z = np.zeros_like(X) + tree = KDTree(embedding[:, :2]) + for i in range(X.shape[0]): + for j in range(X.shape[1]): + nearby_points = embedding[tree.query_radius([[X[i,j],Y[i,j]]], r=2)[0]] + Z[i, j] = eval_density_at_point(np.array([X[i,j],Y[i,j]]), nearby_points) + return Z / Z.sum() + +Now we simply need an appropriate grid of points. We can use the plot +bounds seen above, and a grid size selected for the sake of +computability. The numpy ``meshgrid`` function can supply the actual +grid. + +.. code:: python3 + + X, Y = np.meshgrid(np.linspace(-7, 9, 300), np.linspace(-8, 8, 300)) + +Now we can use the function defined above to compute the density at each +point in the grid, given the Gaussians produced by the embedding. + +.. code:: python3 + + Z = create_density_plot(X, Y, gaussian_mapper.embedding_) + +Now we can view the result as a density plot using ``imshow``. + +.. code:: python3 + + plt.imshow(Z, origin='lower', cmap='Reds', extent=(-7, 9, -8, 8), vmax=0.0005) + plt.colorbar() + + +.. image:: images/embedding_space_47_1.png + + +Here we see the finer structure within the various clusters, including +some of the interesting linear structures, demonstrating that this +Gaussian uncertainty based embedding has captured quite detailed and +useful information about the inter-relationships among the PenDigits +dataset. + +Bonus: Embedding in Hyperbolic space +------------------------------------ + +As a bonus example let's look at embedding data into hyperbolic space. +The most popular model for this for visualization is `Poincare's disk +model `__. An +example of a regular tiling of hyperbolic space in Poincare's disk model +is shown below; you may note it is similar to famous images by M.C. +Escher. + +.. image:: images/Hyperbolic_tiling.png + :height: 400 px + :width: 400 px + + +Ideally we would be able to embed directly into this Poincare disk +model, but in practice this proves to be very difficult. The issue is +that the disk has a "line at infinity" in a circle of radius one +bounding the disk. Outside of that circle things are not well defined. +As you may recall from the discussion of embedding onto spheres and +toruses it is best if we can have a parameterisation of the embedding +space that it is hard to move out of. The Poincare disk model is almost +the opposite of this -- as soon as we move outside the unit circle we +have moved off the manifold and further updates will be badly defined. +We therefore instead need a different parameterisation of hyperbolic +space that is less constrained. One option is the Poincare half-plane +model, but this, again, has a boundary that it is easy to move beyond. +The simplest option is the `hyperboloid +model `__. Under this +model we can simply move in x and y coordinates, and solve for the +corresponding z coordinate when we need to compute distances. This model +has been implemented under the distance metric ``"hyperboloid"`` so we +can simply use it out-of-the-box. + +.. code:: python3 + + hyperbolic_mapper = umap.UMAP(output_metric='hyperboloid', + random_state=42).fit(digits.data) + +A straightforward visualization option is to simply view the x and y +coordinates we have arrived at: + +.. code:: python3 + + plt.scatter(hyperbolic_mapper.embedding_.T[0], + hyperbolic_mapper.embedding_.T[1], + c=digits.target, cmap='Spectral') + + +.. image:: images/embedding_space_52_1.png + + +We can also solve for the z coordinate and view the data lying on a +hyperboloid in 3d space. + +.. code:: python3 + + x = hyperbolic_mapper.embedding_[:, 0] + y = hyperbolic_mapper.embedding_[:, 1] + z = np.sqrt(1 + np.sum(hyperbolic_mapper.embedding_**2, axis=1)) + +.. code:: python3 + + fig = plt.figure() + ax = fig.add_subplot(111, projection='3d') + ax.scatter(x, y, z, c=digits.target, cmap='Spectral') + ax.view_init(35, 80) + + + +.. image:: images/embedding_space_55_0.png + + +But we can do more -- since we have embedded the data successfully in +hyperbolic space we can map the data into the Poincare disk model. This +is, in fact, a straightforward computation. + +.. code:: python3 + + disk_x = x / (1 + z) + disk_y = y / (1 + z) + +Now we can visualize the data in a Poincare disk model embedding as we +first wanted. For this we simply generate a scatterplot of the data, and +then draw in the bounding circle of the line at infinity. + +.. code:: python3 + + fig = plt.figure() + ax = fig.add_subplot(111) + ax.scatter(disk_x, disk_y, c=digits.target, cmap='Spectral') + boundary = plt.Circle((0,0), 1, fc='none', ec='k') + ax.add_artist(boundary) + ax.axis('off'); + + + +.. image:: images/embedding_space_59_0.png + + +Hopefully this has provided a useful example of how to go about +embedding into non-euclidean spaces. This last example ideally +highlights the limitations of this approach (we really need a suitable +parameterisation), and some potential approaches to get around this: we +can use an alternative parameterisation for the embedding, and then +transform the data into the desired representation. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/exploratory_analysis.rst b/ext/TAL/applications/UMAP_fast/umap/doc/exploratory_analysis.rst new file mode 100644 index 00000000..cd552377 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/exploratory_analysis.rst @@ -0,0 +1,103 @@ +Exploratory Analysis of Interesting Datasets +============================================ + +UMAP is a useful tool for general exploratory analysis of data -- it can provide +a unique lens through which to view data that can highlight structures and +properties hiding in data that are not as apparent when analysed with other techniques. +Below is a selection of uses cases of UMAP being used for interesting explorations +of intriguing datasets -- everything from pure math and outputs of neural networks, +to philosophy articles, and scientific texts. + +Prime factorizations of numbers +------------------------------- +What would happen if we applied UMAP to the integers? First we would need a way +to express an integer in a high dimensional space. That can be done by looking +at the prime factorization of each number. Next you have to take enough numbers +to actually generate an interesting visualization. John Williamson set about doing +exactly this, and the results are fascinating. While they may not actually tell us +anything new about number theory they do highlight interesting structures +in prime factorizations, and demonstrate how UMAP can aid in interesting explorations +of datasets that we might think we know well. It's worth visiting the linked article +below as Dr. Williamson provides a rich and detailed exploration of UMAP as +applied to prime factorizations of integers. + +.. image:: images/umap_primes.png + :width: 400px + +`UMAP on prime factorizations `__ + +Thanks to John Williamson. + +Structure of Recent Philosophy +------------------------------ +Philosophy is an incredibly diverse subject, ranging from social and moral philosophy to +logic and philosophy of math; from analysis of ancient Greek philosophy to modern business +ethics. If we could get an overview of all the philosophy papers published in the last +century what might it look like? Maximilian Noichl provides just such an exploration, +looking at a large sampling of philosophy papers and comparing them according to their +citations. The results are intriguing, and can be explored interactively in the +viewer Maximilian built for it. + +.. image:: images/structure_recent_phil.png + :width: 400px + +`Structure of Recent Philosophy `__ + +Thanks to Maximilian Noichl. + +Language, Context, and Geometry in Neural Networks +-------------------------------------------------- +Among recent developments in natural language processing is the BERT neural network +based technique for analysis of language. Among many things that BERT can do one is +context sensitive embeddings of words -- providing numeric vector representations of words +that are sensitive to the context of how the word is used. Exactly what goes on inside +the neural network to do this is a little mysterious (since the network is very complex +with many many parameters). A tram of researchers from Google set out to explore the +word embedding space generated by BERT, and among the tools used was UMAP. The linked +blog post provides a detailed and inspiring analysis of what BERT's word embeddings +look like, and how the different layers of BERT represent different aspects of language. + +.. image:: images/bert_embedding.png + :width: 400px + +`Language, Context, and Geometry in Neural Networks `__ + +Thanks to Andy Coenen, Emily Reif, Ann Yuan, Been Kim, Adam Pearce, Fernanda Viégas, and Martin Wattenberg. + +Activation Atlas +---------------- +Understanding the image processing capabilities (and deficits!) of modern +convolutional neural networks is a challenge. Certainly these models are capable +of amazing feats in, for example, image classification. They can also be brittle +in unexpected ways, with carefully designed images able to induce otherwise +baffling mis-classifications. To better understand this researchers from +Google and OpenAI built the activation atlas -- analysing the space of activations +of a neural network. Here UMAP provides a means to compress the activation landscape +down to 2 dimensions for visualization. The result was an impressive interactive paper +in the Distill journal, providing rich visualizations and new insights into +the working of convolutional neural networks. + +.. image:: images/activation_atlas.png + :width: 400px + +`The Activation Atlas `__ + +Thanks to Shan Carter, Zan Armstrong, Ludwig Schubert, Ian Johnson, and Chris Olah. + +Open Syllabus Galaxy +-------------------- +Suppose you wanted to explore the space of commonly assigned texts from Open Syllabus? That +gives you over 150,000 texts to consider. Since the texts are open you can actually analyse +the text content involved. With some NLP and neural network wizardry David McClure build +a network of such texts and then used node2vec and UMAP to generate a map of them. The result +is a galaxy of textbooks showing inter-relationships between subjects, similar and related texts, +and generally just a an interesting ladscape of science to be explored. As with some +of the other projects here David made a great interactive viewer allowing for rich exploration +of the results. + +.. image:: images/syllabus_galaxy.png + :width: 400px + +`Open Syllabus Galaxy `__ + +Thanks to David McClure. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/faq.rst b/ext/TAL/applications/UMAP_fast/umap/doc/faq.rst new file mode 100644 index 00000000..be713924 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/faq.rst @@ -0,0 +1,294 @@ +Frequently Asked Questions +========================== + +Compiled here are a set of frequently asked questions, +along with answers. If you don't find your question listed here +then please feel free to add an +`issue on github `_. +More questions are always welcome, and the authors will do +their best to answer. If you feel you have a common question +that isn't answered here then please suggest that the question +(and answer) be added to the FAQ when you file the issue. + +Should I normalise my features? +------------------------------- + +The default answer is yes, but, of course, the real answer is +"it depends". If your features have meaningful relationships +with one another (say, latitude and longitude values) then +normalising per feature is not a good idea. For features that +are essentially independent it does make sense to get all the +features on (relatively) the same scale. The best way to do +this is to use +`pre-processing tools from scikit-learn `_. +All the advice given there applies as sensible preprocessing +for UMAP, and since UMAP is scikit-learn compatible you +can put all of this together into a `scikit-learn pipeline `_. + + +Can I cluster the results of UMAP? +---------------------------------- + +This is hard to answer well, but essentially the answer is +"yes, with care". To start with it matters what clustering +algorithm you are going to use. Since UMAP does not necessarily +produce clean spherical clusters something like K-Means +is a poor choice. I would recommend +`HDBSCAN `_ or +similar. The catch here is that UMAP, with its uniform density +assumption, does not preserve density well. What UMAP will do, +however, is contract connected components of the manifold +together. Providing you have enough data for UMAP to +distinguish that information then you can get *useful* +clustering results out since algorithms like HDBSCAN will +easily pick out the components after applying UMAP. + +UMAP does offer significant improvements over algorithms +like t-SNE for clustering. First, by preserving more +global structure and creating meaningful separation +between connected components of the manifold on which +the data lies, UMAP offers more meaningful clusters. +Second, because it supports arbitrary embedding +dimensions, UMAP allows embedding to larger dimensional +spaces that make it more amenable to clustering. + +The clusters are all squashed together and I can't see internal structure +------------------------------------------------------------------------- + +One of UMAPs goals is to have distance between clusters of points +be meaningful. This means that clusters can end up spread out +with a fair amount of space between them. As a result the +clusters themselves can end up more visually packed together +than in, say, t-SNE. This is intended. A catch, however, is +that many plots (for example matplotlib's scatter plot with +default parameters) tend to show the clusters only as indistinct +blobs with no internal structure. The solution for this is +really a matter of tuning the plot more than anything else. + +If you are using matplotlib consider using the ``s`` parameter +that specifies the glyph size in scatter plots. Depending on +how much data you have reducing this to anything from 5 to +0.001 can have a notable effect. The ``size`` parameter in +bokeh is similarly useful (but does not need to be quite so small). + +More generally the real solution, particular with large datasets, +is to use `datashader `_ for plotting. +Datashader is a plotting library that handles aggregation +of large scale data in scatter plots in a way that can better +show the underlying detail that can otherwise be lost. We +highly recommend investing the time to learn datashader for +UMAP plot particularly for larger datasets. + +I ran out of memory. Help! +-------------------------- + +For some datasets the default options for approximate +nearest neighbor search can result in excessive memory use. +If your dataset is not especially large but you have found +that UMAP runs out of memory when operating on it consider +using the ``low_memory=True`` option, which will switch +to a slower but less memory intensive approach to computing +the approximate nearest neighbors. This may alleviate your +issues. + +UMAP is eating all my cores. Help! +---------------------------------- + +If run without a random seed UMAP will use numba's parallel +implementation to do multithreaded work and use many cores. +By default this will make use of as many cores as are available. +If you are on a shared machine or otherwise don't wish to +use *all* the cores at once you can restrict the number of +threads that numba uses by making use of the environment +variable ``NUMBA_NUM_THREADS``; see the `numba +documentation `__ +for more details. + +Is there GPU or multicore-CPU support? +-------------------------------------- + +There is basic multicore support as of version 0.4. +In the future it is possible that GPU support may +be added. + +There is a UMAP implementation for GPU available in +the NVIDIA RAPIDS cuML library, so if you need GPU +support that is currently the best place to go. + +Can I add a custom loss function? +--------------------------------- + +To allow for fast performance the SGD phase of UMAP has +been hand-coded for the specific needs of UMAP. This makes +custom loss functions a little difficult to handle. Now +that Numba (as of version 0.38) supports passing functions +it is possible that future versions of UMAP may support +such functionality. In the meantime you should definitely +look into `smallvis `_, +a library for t-SNE, LargeVis, UMAP, and related algorithms. +Smallvis only works for small datasets, but provides +much greater flexibility and control. + +Is there support for the R language? +------------------------------------ + +Yes! A number of people have worked hard to make UMAP +available to R users. + +If you want to use the reference +implementation under the hood but want a nice R interface +then we recommend `umap `_, +which wraps the python code with +`reticulate `_. +Another reticulate interface is +`umapr `_, but it +may not be under active development. + +If you want a pure R version then we recommend +`uwot `_ at this time. +`umap `_ also provides +a pure R implementation in addition to its reticulate +wrapper. + +Both umap and uwot are available on CRAN. + +Is there a C/C++ implementation? +-------------------------------- + +Not that we are aware of. For now Numba has done a very +admirable job of providing high performance and the +developers of UMAP have not felt the need to move to +lower level languages. At some point a multithreaded +C++ implementation may be made available, but there are +no time-frames for when that would happen. + +I can't get UMAP to run properly! +--------------------------------- + +There are, inevitably, a number of issues and corner cases +that can cause issues for UMAP. Some know issues that can +cause problems are: + +- UMAP doesn't currently support 32-bit Windows. + This is due to issues with Numba of that platform + and will not likely be resolved soon. Sorry :-( +- If you have pip installed the package ``umap`` + at any time (instead of ``umap-learn``) this can + cause serious issues. You will want to purge/remove + everything umap related in your ``site-packages`` + directory and re-install ``umap-learn``. +- Having any files called ``umap.py`` in the current + directory you will have issues as that will be + loaded instead of the ``umap`` module. + +It is worth checking the +`issues page on github `_ +for potential solutions. If all else fails please add an +`issue on github `_. + +What is the difference between PCA / UMAP / VAEs? +------------------------------------------------- + +This is an example of an embedding for a popular Fashion MNIST dataset. + +.. figure:: images/umap_vae_pca.png + :alt: Comparison of PCA / UMAP / VAE embeddings + + Comparison of PCA / UMAP / VAE embeddings + +Note that FMNIST is mostly a toy dataset (MNIST on steroids). +On such a simplistic case UMAP shows distillation results +(i.e. if we use its embedding in a downstream task like classification) +comparable to VAEs, which are more computationally expensive. + +By definition: + +- PCA is linear transformation, you can apply it + to mostly any kind of data in an unsupervised fashion. + Also it works really fast. For most real world tasks + its embeddings are mostly too simplistic / useless. +- VAE is a kind of encoder-decoder neural network, + trained with KLD loss and BCE (or MSE) loss + to enforce the resulting embedding to be continuous. + VAE is an extension of auto-encoder networks, + which by design should produce embeddings that are + not only relevant to actually encoding the data, but are + also smooth. + +From a more practical standpoint: + +- PCA mostly works for any reasonable dataset on a modern machine. + (up to tens or hundreds of millions of rows); +- VAEs have been shown to work only for toy datasets + and to our knowledge there was no real life useful application to + a real world sized dataset (i.e. ImageNet); +- Applying UMAP to real world tasks usually provides a good starting + point for downstream tasks (data visualization, clustering, classification) + and works reasonably fast; +- Consider a typical pipeline: high-dimensional embedding (300+) + => PCA to reduce to 50 dimensions => UMAP to reduce to 10-20 dimensions + => HDBSCAN for clustering / some plain algorithm for classification; + +Which tool should I use? + +- PCA for very large or high dimensional datasets (or maybe consider finding + a domain specific matrix factorization technique, e.g. topic modelling for texts); +- UMAP for smaller datasets; +- VAEs are mostly experimental; + +Where can I learn more? + +- While PCA is ubiquitous, you may `look `_ + at this example comparing PCA / UMAP / VAEs; + +How UMAP can go wrong +--------------------- + +One way UMAP can go wrong is the introduction of data points that are maximally far apart +from all other points in your data set. In other words, a points nearest neighbour is maximally +far from it. A common example of this could be a point which shares no features in common +with any other point under a Jaccard distance or a point whose nearest neighbour is np.inf from +it under a continuous distance function. In both these cases UMAPs assumption of all points +lying on a connected manifold can lead us astray. From this points perspective all other points +are equally valid nearest neighbours so its k-nearest neighbour query will return a random +selection of neighbours all at this maximal distance. Next we will normalize this distance by +applying our UMAP kernel which says that a point should be maximally similar to it's nearest neighbour. +Since all k-nearest neighbours are identically far apart they will all be considered maximally +similar by our point in question. When we try to embed our data into a low dimensional space +our optimization will attempt to pull all these randomly selected points together. Add a +sufficiently large number of these points and our entire space gets pulled together destroying +any of the structure we had hoped to identify. + +To circumvent this problem we've added a disconnection_distance parameter to UMAP which will cut +any edge with a distance greater than the value passed in. This parameter defaults to ``None``. +When set to ``None`` the disconnection_distance will be set to the maximal value for any of our +supported bounded metrics and otherwise set to np.inf. Removing these edges from the UMAP graph +will disconnect our manifold and cause these points to start where they are initialized and get pushed +away from all other points via the our optimization. + +If a user has a good understanding of their distance metric they can set this value by hand to prevent +data in particularly sparse regions of their space from becoming connected to their manifold. + +If vertices in your graph are disconnected a warning message will be thrown. At that point a user can +make use of the umap.utils.disconnected_vertices() function to identify the disconnected points. +This can be used either for filtering and retraining a new UMAP model or simple to bed used as a +filter for visualization purposes as seen below. + +.. code:: python3 + + umap_model = umap.UMAP().fit(data) + disconnected_points = umap.utils.disconnected_vertices(umap_model) + umap.plot.points(umap_model, subset_points=~disconnected_points) + +Successful use-cases +-------------------- + +UMAP can be / has been successfully applied to the following domains: + +- Single cell data visualization in biology; +- Mapping malware based on behavioural data; +- Pre-processing phrase vectors for clustering; +- Pre-processing image embeddings (Inception) for clustering; + +and many more -- if you have a successful use-case please submit +a pull request adding it to this list! \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/how_umap_works.rst b/ext/TAL/applications/UMAP_fast/umap/doc/how_umap_works.rst new file mode 100644 index 00000000..14705fb8 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/how_umap_works.rst @@ -0,0 +1,559 @@ +.. _how_umap_works: + +How UMAP Works +============== + +UMAP is an algorithm for dimension reduction based on manifold learning +techniques and ideas from topological data analysis. It provides a very +general framework for approaching manifold learning and dimension +reduction, but can also provide specific concrete realizations. This +article will discuss how the algorithm works in practice. There exist +deeper mathematical underpinnings, but for the sake of readability by a +general audience these will merely be referenced and linked. If you are +looking for the mathematical description please see the `UMAP +paper `__. + +To begin making sense of UMAP we will need a little bit of mathematical +background from algebraic topology and topological data analysis. This +will provide a basic algorithm that works well in theory, but +unfortunately not so well in practice. The next step will be to make use +of some basic Riemannian geometry to bring real world data a little +closer to the underlying assumptions of the topological data analysis +algorithm. Unfortunately this will introduce new complications, which +will be resolved through a combination of deep math (details of which +will be elided) and fuzzy logic. We can then put the pieces back +together again, and combine them with a new approach to finding a low +dimensional representation more fitting to the new data structures at +hand. Putting this all together we arrive at the basic UMAP algorithm. + +Topological Data Analysis and Simplicial Complexes +-------------------------------------------------- + +Simplicial complexes are a means to construct topological spaces out of +simple combinatorial components. This allows one to reduce the +complexities of dealing with the continuous geometry of topological +spaces to the task of relatively simple combinatorics and counting. This +method of taming geometry and topology will be fundamental to our +approach to topological data analysis in general, and dimension +reduction in particular. + +The first step is to provide some simple combinatorial building blocks +called `*simplices* `__. +Geometrically a simplex is a very simple way to build an +:math:`k`-dimensional object. A :math:`k` dimensional simplex is called +a :math:`k`-simplex, and it is formed by taking the convex hull of +:math:`k+1` independent points. Thus a 0-simplex is a point, a 1-simplex +is a line segment (between two zero simplices), a 2-simplex is a +triangle (with three 1-simplices as "faces"), and a 3-simplex is a +tetrahedron (with four 2-simplices as "faces"). Such a simple +construction allows for easy generalization to arbitrary dimensions. + +.. figure:: images/simplices.png + :alt: Low dimensional simplices + + Low dimensional simplices + + + +This has a very simple combinatorial underlying structure, and +ultimately one can regard a :math:`k`-simplex as an arbitrary set of +:math:`k+1` objects with faces (and faces of faces etc.) given by +appropriately sized subsets -- one can always provide a "geometric +realization" of this abstract set description by constructing the +corresponding geometric simplex. + +Simplices can provide building blocks, but to construct interesting +topological spaces we need to be able to glue together such building +blocks. This can be done by constructing a `*simplicial +complex* `__. +Ostensibly a simplicial complex is a set of simplices glued together +along faces. More explicitly a simplicial complex :math:`\mathcal{K}` is +a set of simplices such that any face of any simplex in +:math:`\mathcal{K}` is also in :math:`\mathcal{K}` (ensuring all faces +exist), and the intersection of any two simplices in :math:`\mathcal{K}` +is a face of both simplices. A large class of topological spaces can be +constructed in this way -- just gluing together simplices of various +dimensions along their faces. A little further abstraction will get to +`*simplicial sets* `__ +which are purely combinatorial, have a nice category theoretic +presentation, and can generate a much broader class of topological +spaces, but that will take us too far afield for this article. The +intuition of simplicial complexes will be enough to illustrate the +relevant ideas and motivation. + +How does one apply these theoretical tools from topology to finite sets +of data points? To start we'll look at how one might construct a +simplicial complex from a topological space. The tool we will consider is +the construction of a `Čech +complex `__ given an +open cover of a topological space. That's a lot of verbiage if you +haven't done much topology, but we can break it down fairly easily for +our use case. An open cover is essentially just a family of sets whose +union is the whole space, and a Čech complex is a combinatorial way to +covert that into a simplicial complex. It works fairly simply: let each +set in the cover be a 0-simplex; create a 1-simplex between two such +sets if they have a non-empty intersection; create a 2-simplex between +three such sets if the triple intersection of all three is non-empty; +and so on. Now, that doesn't sound very advanced -- just looking at +intersections of sets. The key is that the background topological theory +actually provides guarantees about how well this simple process can +produce something that represents the topological space itself in a +meaningful way (the `Nerve +theorem `__ is the relevant +result for those interested). Obviously the quality of the cover is +important, and finer covers provide more accuracy, but the reality is +that despite its simplicity the process captures much of the topology. + +Next we need to understand how to apply that process to a finite set of +data samples. If we assume that the data samples are drawn from some +underlying topological space then to learn about the topology of that +space we need to generate an open cover of it. If our data actually lie +in a metric space (i.e. we can measure distance between points) then one +way to approximate an open cover is to simply create balls of some fixed +radius about each data point. Since we only have finite samples, and not +the topological space itself, we cannot be sure it is truly an open +cover, but it might be as good an approximation as we could +reasonably expect. This approach also has the advantage that the Čech +complex associated to the cover will have a 0-simplex for each data +point. + +To demonstrate the process let's consider a test dataset like this + +.. figure:: images/how_umap_works_raw_data.png + :alt: Test data set of a noisy sine wave + + Test data set of a noisy sine wave + + + +If we fix a radius we can then picture the open sets of our cover as +circles (since we are in a nice visualizable two dimensional case). The +result is something like this + +.. figure:: images/how_umap_works_open_cover.png + :alt: A basic open cover of the test data + + A basic open cover of the test data + + + +We can then depict the the simplicial complex of 0-, 1-, and 2-simplices +as points, lines, and triangles + +.. figure:: images/how_umap_works_basic_graph.png + :alt: A simplicial complex built from the test data + + A simplicial complex built from the test data + + + +It is harder to easily depict the higher dimensional simplices, but you +can imagine how they would fit in. There are two things to note here: +first, the simplicial complex does a reasonable job of starting to +capture the fundamental topology of the dataset; second, most of the +work is really done by the 0- and 1-simplices, which are easier to deal +with computationally (it is just a graph, in the nodes and edges sense). +The second observation motivates the `Vietoris-Rips +complex `__, +which is similar to the Čech complex but is entirely determined by the +0- and 1-simplices. Vietoris-Rips complexes are much easier to work with +computationally, especially for large datasets, and are one of the major +tools of topological data analysis. + +If we take this approach to get a topological representation then we can +build a dimension reduction algorithm by finding a low dimensional +representation of the data that has a similar topological +representation. If we only care about the 0- and 1-simplices then the +topological representation is just a graph, and finding a low +dimensional representation can be described as a `graph layout +problem <>`__. If one wants to use, for example, spectral methods for +graph layout then we arrive at algorithms like `Laplacian +eigenmaps <>`__ and `Diffusion maps <>`__. Force directed layouts are +also an option, and provide algorithms closer to `MDS <>`__ or `Sammon +mapping <>`__ in flavour. + +I would not blame those who have read this far to wonder why we took +such an abstract roundabout road to simply building a neighborhood-graph +on the data and then laying out that graph. There are a couple of +reasons. The first reason is that the topological approach, while +abstract, provides sound theoretical justification for what we are +doing. While building a neighborhood-graph and laying it out in lower +dimensional space makes heuristic sense and is computationally tractable, +it doesn't provide the same underlying motivation of capturing the +underlying topological structure of the data faithfully -- for that we +need to appeal to the powerful topological machinery I've hinted lies in +the background. The second reason is that it is this more abstract +topological approach that will allow us to generalize the approach and +get around some of the difficulties of the sorts of algorithms described +above. While ultimately we will end up with a process that is fairly +simple computationally, understanding *why* various manipulations matter +is important to truly understanding the algorithm (as opposed to merely +computing with it). + +Adapting to Real World Data +--------------------------- + +The approach described above provides a nice theory for why a +neighborhood graph based approach should capture manifold structure when +doing dimension reduction. The problem tends to come when one tries to +put the theory into practice. The first obvious difficulty (and we can +see it even our example above) is that choosing the right radius for the +balls that make up the open cover is hard. If you choose something too +small the resulting simplicial complex splits into many connected +components. If you choose something too large the simplicial complex +turns into just a few very high dimensional simplices (and their faces +etc.) and fails to capture the manifold structure anymore. How should +one solve this? + +The dilemma is in part due to the theorem (called the `Nerve +theorem `__) that +provides our justification that this process captures the topology. +Specifically, the theorem says that the simplicial complex will be +(homtopically) equivalent to the union of the cover. In our case, +working with finite data, the cover, for certain radii, doesn't cover +the whole of the manifold that we imagine underlies the data -- it is +that lack of coverage that results in the disconnected components. +Similarly, where the points are too bunched up, our cover does cover +"too much" and we end up with higher dimensional simplices than we might +ideally like. If the data were *uniformly distributed* across the +manifold then selecting a suitable radius would be easy -- the average +distance between points would work well. Moreover with a uniform +distribution we would be guaranteed that our cover would actually cover +the whole manifold with no "gaps" and no unnecessarily disconnected +components. Similarly, we would not suffer from those unfortunate +bunching effects resulting in unnecessarily high dimensional simplices. + +If we consider data that is uniformly distributed along the same +manifold it is not hard to pick a good radius (a little above half the +average distance between points) and the resulting open cover looks +pretty good: + +.. figure:: images/how_umap_works_uniform_distribution_cover.png + :alt: Open balls over uniformly\_distributed\_data + + Open balls over uniformly\_distributed\_data + + + +Because the data is evenly spread we actually cover the underlying +manifold and don't end up with clumping. In other words, all this theory +works well assuming that the data is uniformly distributed over the +manifold. + +Unsurprisingly this uniform distribution assumption crops up elsewhere +in manifold learning. The proofs that Laplacian eigenmaps work well +require the assumption that the data is uniformly distributed on the +manifold. Clearly if we had a uniform distribution of points on the +manifold this would all work a lot better -- but we don't! Real world +data simply isn't that nicely behaved. How can we resolve this? By +turning the problem on its head: assume that the data is uniformly +distributed on the manifold, and ask what that tells us about the +manifold itself. If the data *looks* like it isn't uniformly distributed +that must simply be because the notion of distance is varying across the +manifold -- space itself is warping: stretching or shrinking according +to where the data appear sparser or denser. + +By assuming that the data is uniformly distributed we can actually +compute (an approximation of) a local notion of distance for each point +by making use of a little standard `Riemannian +geometry `__. In +practical terms, once you push the math through, this turns out to mean +that a unit ball about a point stretches to the *k*-th nearest neighbor +of the point, where *k* is the sample size we are using to approximate +the local sense of distance. Each point is given its own unique distance +function, and we can simply select balls of radius one with respect to +that local distance function! + +.. figure:: images/how_umap_works_local_metric_open_cover.png + :alt: Open balls of radius one with a locally varying metric + + Open balls of radius one with a locally varying metric + + + +This theoretically derived result matches well with many traditional +graph based algorithms: a standard approach for such algorithms is to +use a *k*-neighbor graph instead of using balls of some fixed radius to +define connectivity. What this means is that each point in the dataset +is given an edge to each of its *k* nearest neighbors -- the effective +result of our locally varying metric with balls of radius one. Now, +however, we can explain why this works in terms of simplicial complexes +and the Nerve theorem. + +Of course we have traded choosing the radius of the balls for choosing a +value for *k*. However it is often easier to pick a resolution scale in +terms of number of neighbors than it is to correctly choose a distance. +This is because choosing a distance is very dataset dependent: one needs +to look at the distribution of distances in the dataset to even begin to +select a good value. In contrast, while a *k* value is still dataset +dependent to some degree, there are reasonable default choices, such as +the 10 nearest neighbors, that should work acceptably for most datasets. + +At the same time the topological interpretation of all of this gives us +a more meaningful interpretation of *k*. The choice of *k* determines how +locally we wish to estimate the Riemannian metric. A small choice of *k* +means we want a very local interpretation which will more accurately +capture fine detail structure and variation of the Riemannian metric. +Choosing a large *k* means our estimates will be based on larger +regions, and thus, while missing some of the fine detail structure, they +will be more broadly accurate across the manifold as a whole, having +more data to make the estimate with. + +We also get a further benefit from this Riemannian metric based +approach: we actually have a local metric space associated to each +point, and can meaningfully measure distance, and thus we could weight +the edges of the graph we might generate by how far apart (in terms of +the local metric) the points on the edges are. In slightly more +mathematical terms we can think of this as working in a fuzzy topology +where being in an open set in a cover is no longer a binary yes or no, +but instead a fuzzy value between zero and one. Obviously the certainty +that points are in a ball of a given radius will decay as we move away +from the center of the ball. We could visualize such a fuzzy cover as +looking something like this + +.. figure:: images/how_umap_works_fuzzy_open_cover.png + :alt: Fuzzy open balls of radius one with a locally varying metric + + Fuzzy open balls of radius one with a locally varying metric + + + +None of that is very concrete or formal -- it is merely an intuitive +picture of what we would like to have happen. It turns out that we can +actually formalize all of this by stealing the `singular +set `__ +and `geometric +realization `__ +functors from algebraic topology and then adapting them to apply to +metric spaces and fuzzy simplicial sets. The mathematics involved in +this is outside the scope of this exposition, but for those interested +you can look at the `original work on this by David +Spivak `__ +and our `paper `__. It will have to +suffice to say that there is some mathematical machinery that lets us +realize this intuition in a well defined way. + +This resolves a number of issues, but a new problem presents itself when +we apply this sort of process to real data, especially in higher +dimensions: a lot of points become essentially totally isolated. One +would imagine that this shouldn't happen if the manifold the data was +sampled from isn't pathological. So what property are we expecting that +manifold to have that we are somehow missing with the current approach? +What we need to add is the idea of local connectivity. + +Note that this is not a requirement that the manifold as a whole be +connected -- it can be made up of many connected components. Instead it +is a requirement that at any point on the manifold there is some +sufficiently small neighborhood of the point that *is* connected (this +"in a sufficiently small neighborhood" is what the "local" part means). +For the practical problem we are working with, where we only have a +finite approximation of the manifold, this means that no point should be +*completely* isolated -- it should connect to at least one other point. +In terms of fuzzy open sets what this amounts to is that we should have +complete confidence that the open set extends as far as the closest +neighbor of each point. We can implement this by simply having the fuzzy +confidence decay in terms of distance *beyond* the first nearest +neighbor. We can visualize the result in terms of our example dataset +again. + +.. figure:: images/how_umap_works_umap_open_cover.png + :alt: Local connectivity and fuzzy open sets + + Local connectivity and fuzzy open sets + + +Again this can be formalized in terms of the aforementioned mathematical +machinery from algebraic topology. From a practical standpoint this +plays an important role for high dimensional data -- in high dimensions +distances tend to be larger, but also more similar to one another (see +`the curse of +dimensionality `__). +This means that the distance to the first nearest neighbor can be quite +large, but the distance to the tenth nearest neighbor can often be only +slightly larger (in relative terms). The local connectivity constraint +ensures that we focus on the difference in distances among nearest +neighbors rather than the absolute distance (which shows little +differentiation among neighbors). + +Just when we think we are almost there, having worked around some of the +issues of real world data, we run aground on a new obstruction: our +local metrics are not compatible! Each point has its own local metric +associated to it, and from point *a*'s perspective the distance from +point *a* to point *b* might be 1.5, but from the perspective of point +*b* the distance from point *b* to point *a* might only be 0.6. Which +point is right? How do we decide? Going back to our graph based +intuition we can think of this as having directed edges with varying +weights something like this. + +.. figure:: images/how_umap_works_raw_graph.png + :alt: Edges with incompatible weights + + Edges with incompatible weights + + +Between any two points we might have up to two edges and the weights on +those edges disagree with one another. There are a number of options for +what to do given two disagreeing weights -- we could take the maximum, +the minimum, the arithmetic mean, the geometric mean, or something else +entirely. What we would really like is some principled way to make the +decision. It is at this point that the mathematical machinery we built +comes into play. Mathematically we actually have a family of fuzzy +simplicial sets, and the obvious choice is to take their union -- a well +defined operation. There are a a few ways to define fuzzy unions, +depending on the nature of the logic involved, but here we have +relatively clear probabilistic semantics that make the choice +straightforward. In graph terms what we get is the following: if we want +to merge together two disagreeing edges with weight *a* and *b* then we +should have a single edge with combined weight :math:`a + b - a \cdot b`. +The way to think of this is that the weights are effectively the +probabilities that an edge (1-simplex) exists. The combined weight is +then the probability that at least one of the edges exists. + +If we apply this process to union together all the fuzzy simplicial sets +we end up with a single fuzzy simplicial complex, which we can again +think of as a weighted graph. In computational terms we are simply +applying the edge weight combination formula across the whole graph +(with non-edges having a weight of 0). In the end we have something that +looks like this. + +.. figure:: images/how_umap_works_umap_graph.png + :alt: Graph with combined edge weights + + Graph with combined edge weights + + + +So in some sense in the end we have simply constructed a weighted graph +(although we could make use of higher dimensional simplices if we +wished, just at significant extra computational cost). What the +mathematical theory lurking in the background did for us is determine +*why* we should construct *this* graph. It also helped make the +decisions about exactly *how* to compute things, and gives a concrete +interpretation of *what* this graph means. So while in the end we just +constructed a graph, the math answered the important questions to get us +here, and can help us determine what to do next. + +So given that we now have a fuzzy topological representation of the data +(which the math says will capture the topology of the manifold +underlying the data), how do we go about converting that into a low +dimensional representation? + +Finding a Low Dimensional Representation +---------------------------------------- + +Ideally we want the low dimensional representation to have as similar +a fuzzy topological structure as possible. The first question +is how do we determine the fuzzy topological structure of a low +dimensional representation, and the second question is how do we find a +good one. + +The first question is largely already answered -- we should presumably +follow the same procedure we just used to find the fuzzy topological +structure of our data. There is a quirk, however: this time around the +data won't be lying on some manifold, we'll have a low dimensional +representation that is lying on a very particular manifold. That +manifold is, of course, just the low dimensional euclidean space we are +trying to embed into. This means that all the effort we went to +previously to make vary the notion of distance across the manifold is +going to be misplaced when working with the low dimensional +representation. We explicitly *want* the distance on the manifold to be +standard euclidean distance with respect to the global coordinate +system, not a varying metric. That saves some trouble. The other quirk +is that we made use of the distance to the nearest neighbor, again +something we computed given the data. This is also a property we would +like to be globally true across the manifold as we optimize toward a +good low dimensional representation, so we will have to accept it as a +hyper-parameter ``min_dist`` to the algorithm. + +The second question, 'how do we find a good low dimensional +representation', hinges on our ability to measure how "close" a match we +have found in terms of fuzzy topological structures. Given such a +measure we can turn this into an optimization problem of finding the low +dimensional representation with the closest fuzzy topological structure. +Obviously if our measure of closeness turns out to have various +properties the nature of the optimization techniques we can apply will +differ. + +Going back to when we were merging together the conflicting weights +associated to simplices, we interpreted the weights as the probability +of the simplex existing. Thus, since both topological structures we are +comparing share the same 0-simplices, we can imagine that we are +comparing the two vectors of probabilities indexed by the 1-simplices. +Given that these are Bernoulli variables (ultimately the simplex either +exists or it doesn't, and the probability is the parameter of a +Bernoulli distribution), the right choice here is the cross entropy. + +Explicitly, if the set of all possible 1-simplices is :math:`E`, and we +have weight functions such that :math:`w_h(e)` is the weight of the +1-simplex :math:`e` in the high dimensional case and :math:`w_l(e)` is +the weight of :math:`e` in the low dimensional case, then the cross +entropy will be + +.. math:: + + + \sum_{e\in E} w_h(e) \log\left(\frac{w_h(e)}{w_l(e)}\right) + (1 - w_h(e)) \log\left(\frac{1 - w_h(e)}{1 - w_l(e)}\right) + +This might look complicated, but if we go back to thinking in terms of a +graph we can view minimizing the cross entropy as a kind of force +directed graph layout algorithm. + +The first term, :math:`w_h(e) \log\left(\frac{w_h(e)}{w_l(e)}\right)`, +provides an attractive force between the points :math:`e` spans whenever +there is a large weight associated to the high dimensional case. This is +because this term will be minimized when :math:`w_l(e)` is as large as +possible, which will occur when the distance between the points is as +small as possible. + +In contrast the second term, +:math:`(1 - w_h(e)) \log\left(\frac{1 - w_h(e)}{1 - w_l(e)}\right)`, +provides a repulsive force between the ends of :math:`e` whenever +:math:`w_h(e)` is small. This is because the term will be minimized by +making :math:`w_l(e)` as small as possible. + +On balance this process of pull and push, mediated by the weights on +edges of the topological representation of the high dimensional data, +will let the low dimensional representation settle into a state that +relatively accurately represents the overall topology of the source +data. + +The UMAP Algorithm +------------------ + +Putting all these pieces together we can construct the UMAP algorithm. +The first phase consists of constructing a fuzzy topological +representation, essentially as described above. The second phase is +simply optimizing the low dimensional representation to have as close +a fuzzy topological representation as possible as measured by cross +entropy. + +When constructing the initial fuzzy topological representation we can +take a few shortcuts. In practice, since fuzzy set membership strengths +decay away to be vanishingly small, we only need to compute them for the +nearest neighbors of each point. Ultimately that means we need a way to +quickly compute (approximate) nearest neighbors efficiently, even in +high dimensional spaces. We can do this by taking advantage of the +`Nearest-Neighbor-Descent algorithm of Dong et +al `__. The remaining +computations are now only dealing with local neighbors of each point and +are thus very efficient. + +In optimizing the low dimensional embedding we can again take some +shortcuts. We can use stochastic gradient descent for the optimization +process. To make the gradient descent problem easier it is beneficial if +the final objective function is differentiable. We can arrange for that +by using a smooth approximation of the actual membership strength +function for the low dimensional representation, selecting from a +suitably versatile family. In practice UMAP uses the family of curves of +the form :math:`\frac{1}{1 + a x^{2b}}`. Equally we don't want to have to +deal with all possible edges, so we can use the negative sampling trick +(as used by word2vec and LargeVis), to simply sample negative examples +as needed. Finally since the Laplacian of the topological representation +is an approximation of the Laplace-Beltrami operator of the manifold we +can use spectral embedding techniques to initialize the low dimensional +representation into a good state. + +Putting all these pieces together we arrive at an algorithm that is fast +and scalable, yet still built out of sound mathematical theory. +Hopefully this introduction has helped provide some intuition for that +underlying theory, and for how the UMAP algorithm works in practice. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/20newsgroups_hellinger_counts.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/20newsgroups_hellinger_counts.png new file mode 100644 index 00000000..3b822ddf Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/20newsgroups_hellinger_counts.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/20newsgroups_hellinger_tfidf.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/20newsgroups_hellinger_tfidf.png new file mode 100644 index 00000000..e37e5105 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/20newsgroups_hellinger_tfidf.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_13_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_13_1.png new file mode 100644 index 00000000..600189a8 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_13_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_18_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_18_0.png new file mode 100644 index 00000000..53fc3ee1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_18_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_20_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_20_1.png new file mode 100644 index 00000000..4853c4eb Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_20_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_26_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_26_1.png new file mode 100644 index 00000000..29cee7c3 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_26_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_6_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_6_1.png new file mode 100644 index 00000000..7dfabaf8 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/BasicUsage_6_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/Hyperbolic_tiling.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/Hyperbolic_tiling.png new file mode 100644 index 00000000..aa18c17d Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/Hyperbolic_tiling.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_10_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_10_1.png new file mode 100644 index 00000000..be818487 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_10_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_15_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_15_1.png new file mode 100644 index 00000000..53cce09b Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_15_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_22_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_22_1.png new file mode 100644 index 00000000..5e04356c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_22_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_31_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_31_0.png new file mode 100644 index 00000000..c10b5dca Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_31_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_33_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_33_0.png new file mode 100644 index 00000000..5f776e00 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/SupervisedUMAP_33_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAPTransform_15_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAPTransform_15_0.png new file mode 100644 index 00000000..9574c03d Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAPTransform_15_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAPTransform_21_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAPTransform_21_0.png new file mode 100644 index 00000000..3f9a3e04 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAPTransform_21_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAP_zoo.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAP_zoo.png new file mode 100644 index 00000000..c4306357 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/UMAP_zoo.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/activation_atlas.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/activation_atlas.png new file mode 100644 index 00000000..368a5e60 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/activation_atlas.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_15_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_15_0.png new file mode 100644 index 00000000..b8027aa1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_15_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_22_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_22_0.png new file mode 100644 index 00000000..0fa47492 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_22_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_29_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_29_0.png new file mode 100644 index 00000000..07b5b411 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_29_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_29_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_29_1.png new file mode 100644 index 00000000..571aa38c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_29_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_5_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_5_1.png new file mode 100644 index 00000000..edbd0262 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_basic_usage_5_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_3d.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_3d.png new file mode 100644 index 00000000..ce689667 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_3d.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_3d_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_3d_1.png new file mode 100644 index 00000000..8ca6b438 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_3d_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_anim.gif b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_anim.gif new file mode 100644 index 00000000..9cde241d Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_pendigits_anim.gif differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_politics_demo_31_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_politics_demo_31_0.png new file mode 100644 index 00000000..e444d0be Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_politics_demo_31_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_politics_demo_spaghetti.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_politics_demo_spaghetti.png new file mode 100644 index 00000000..8c453df3 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/aligned_umap_politics_demo_spaghetti.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/audio_explorer.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/audio_explorer.png new file mode 100644 index 00000000..c59e35ab Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/audio_explorer.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_17_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_17_1.png new file mode 100644 index 00000000..dcd794bb Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_17_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_22_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_22_0.png new file mode 100644 index 00000000..fe2e5917 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_22_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_24_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_24_2.png new file mode 100644 index 00000000..4c3de166 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_24_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_30_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_30_1.png new file mode 100644 index 00000000..6e8ab80a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_30_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_8_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_8_1.png new file mode 100644 index 00000000..49af4881 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/basic_usage_8_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/bert_embedding.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/bert_embedding.png new file mode 100644 index 00000000..59ab1239 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/bert_embedding.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/c_elegans_3d.jpg b/ext/TAL/applications/UMAP_fast/umap/doc/images/c_elegans_3d.jpg new file mode 100644 index 00000000..b7df4734 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/c_elegans_3d.jpg differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_10_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_10_1.png new file mode 100644 index 00000000..eca6e480 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_10_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_16_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_16_1.png new file mode 100644 index 00000000..c018ca0f Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_16_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_27_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_27_1.png new file mode 100644 index 00000000..bd059a31 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_27_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_31_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_31_1.png new file mode 100644 index 00000000..25f0abc0 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_31_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_6_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_6_1.png new file mode 100644 index 00000000..9579c2dd Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/clustering_6_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_11_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_11_1.png new file mode 100644 index 00000000..034f4a87 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_11_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_14_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_14_1.png new file mode 100644 index 00000000..ed6c1ca4 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_14_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_18_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_18_1.png new file mode 100644 index 00000000..0371ba78 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_18_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_20_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_20_1.png new file mode 100644 index 00000000..6f654c56 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_20_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_23_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_23_1.png new file mode 100644 index 00000000..e0174e01 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_23_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_36_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_36_1.png new file mode 100644 index 00000000..4c756302 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_36_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_38_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_38_1.png new file mode 100644 index 00000000..eaae7fe4 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_38_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_42_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_42_1.png new file mode 100644 index 00000000..d9000ba4 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_42_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_44_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_44_1.png new file mode 100644 index 00000000..06a403d5 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_44_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_47_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_47_1.png new file mode 100644 index 00000000..4b7a684e Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_47_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_50_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_50_1.png new file mode 100644 index 00000000..fcf288a1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_50_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_6_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_6_1.png new file mode 100644 index 00000000..d16a229f Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/composing_models_6_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_10_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_10_1.png new file mode 100644 index 00000000..84a954e7 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_10_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_13_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_13_1.png new file mode 100644 index 00000000..fa5452e4 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_13_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_16_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_16_1.png new file mode 100644 index 00000000..2feabc2e Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_16_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_19_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_19_1.png new file mode 100644 index 00000000..693540df Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_19_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_21_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_21_1.png new file mode 100644 index 00000000..ae8bf45b Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_21_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_24_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_24_1.png new file mode 100644 index 00000000..4eeeb141 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_24_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_6_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_6_1.png new file mode 100644 index 00000000..d16a229f Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/densmap_demo_6_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_projector.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_projector.png new file mode 100644 index 00000000..f3424d19 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_projector.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_11_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_11_1.png new file mode 100644 index 00000000..6adfe103 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_11_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_15_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_15_1.png new file mode 100644 index 00000000..b977b7ff Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_15_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_18_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_18_1.png new file mode 100644 index 00000000..21894f29 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_18_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_26_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_26_0.png new file mode 100644 index 00000000..0e9646ca Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_26_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_29_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_29_1.png new file mode 100644 index 00000000..890e5c11 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_29_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_33_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_33_1.png new file mode 100644 index 00000000..e0bf3dd4 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_33_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_37_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_37_1.png new file mode 100644 index 00000000..dc96bf19 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_37_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_39_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_39_1.png new file mode 100644 index 00000000..b74282df Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_39_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_47_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_47_1.png new file mode 100644 index 00000000..22114e55 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_47_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_52_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_52_1.png new file mode 100644 index 00000000..c62de66a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_52_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_55_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_55_0.png new file mode 100644 index 00000000..3fdd510e Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_55_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_59_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_59_0.png new file mode 100644 index 00000000..f7ecc3ba Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_59_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_7_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_7_1.png new file mode 100644 index 00000000..d5ad2864 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/embedding_space_7_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/exploring_fashion_mnist.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/exploring_fashion_mnist.png new file mode 100644 index 00000000..d5d933f8 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/exploring_fashion_mnist.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap.svg b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap.svg new file mode 100644 index 00000000..2fd158c9 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap.svg @@ -0,0 +1,1419 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap_supervised.svg b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap_supervised.svg new file mode 100644 index 00000000..49a97731 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap_supervised.svg @@ -0,0 +1,1417 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap_supervised_prediction.svg b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap_supervised_prediction.svg new file mode 100644 index 00000000..c337608c --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_densmap_supervised_prediction.svg @@ -0,0 +1,1417 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap.svg b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap.svg new file mode 100644 index 00000000..1ada0205 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap.svg @@ -0,0 +1,1354 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap_supervised.svg b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap_supervised.svg new file mode 100644 index 00000000..4a9b72df --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap_supervised.svg @@ -0,0 +1,1374 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap_supervised_prediction.svg b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap_supervised_prediction.svg new file mode 100644 index 00000000..2381a3c6 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_2D_umap_supervised_prediction.svg @@ -0,0 +1,1374 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_subset.svg b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_subset.svg new file mode 100644 index 00000000..60e3c0b2 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/images/galaxy10_subset.svg @@ -0,0 +1,912 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_basic_graph.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_basic_graph.png new file mode 100644 index 00000000..3537702c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_basic_graph.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_fuzzy_open_cover.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_fuzzy_open_cover.png new file mode 100644 index 00000000..2864a01c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_fuzzy_open_cover.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_local_metric_open_cover.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_local_metric_open_cover.png new file mode 100644 index 00000000..557f4c8b Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_local_metric_open_cover.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_open_cover.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_open_cover.png new file mode 100644 index 00000000..7d23019c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_open_cover.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_raw_data.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_raw_data.png new file mode 100644 index 00000000..a25e0e56 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_raw_data.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_raw_graph.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_raw_graph.png new file mode 100644 index 00000000..a3389878 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_raw_graph.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_graph.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_graph.png new file mode 100644 index 00000000..264a4aef Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_graph.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_graph_layout.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_graph_layout.png new file mode 100644 index 00000000..c1558668 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_graph_layout.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_layout.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_layout.png new file mode 100644 index 00000000..3f16abf3 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_layout.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_open_cover.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_open_cover.png new file mode 100644 index 00000000..9cfa12ba Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_umap_open_cover.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_uniform_distribution_cover.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_uniform_distribution_cover.png new file mode 100644 index 00000000..9820f66a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/how_umap_works_uniform_distribution_cover.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_13_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_13_0.png new file mode 100644 index 00000000..815cfbce Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_13_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_20_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_20_1.png new file mode 100644 index 00000000..55dc60dd Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_20_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_26_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_26_0.png new file mode 100644 index 00000000..59bae20b Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_26_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_7_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_7_1.png new file mode 100644 index 00000000..9282273a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/inverse_transform_7_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/organogenesis_paper.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/organogenesis_paper.png new file mode 100644 index 00000000..74b15d29 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/organogenesis_paper.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/orion_particles.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/orion_particles.png new file mode 100644 index 00000000..999947ff Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/orion_particles.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_10_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_10_0.png new file mode 100644 index 00000000..49a328e1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_10_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_12_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_12_2.png new file mode 100644 index 00000000..75489304 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_12_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_13_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_13_2.png new file mode 100644 index 00000000..8cf9f3ad Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_13_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_15_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_15_0.png new file mode 100644 index 00000000..7eb75d0e Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_15_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_19_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_19_0.png new file mode 100644 index 00000000..49a328e1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_19_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_22_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_22_2.png new file mode 100644 index 00000000..75489304 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_22_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_27_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_27_0.png new file mode 100644 index 00000000..7eb75d0e Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_27_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_5_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_5_0.png new file mode 100644 index 00000000..b6674f01 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_5_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_7_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_7_2.png new file mode 100644 index 00000000..8cf9f3ad Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_7_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_9_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_9_0.png new file mode 100644 index 00000000..b6674f01 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/outliers_9_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_1.png new file mode 100644 index 00000000..ef329fe3 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_2.png new file mode 100644 index 00000000..63ffdff6 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_3.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_3.png new file mode 100644 index 00000000..f677d817 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_3.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_4.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_4.png new file mode 100644 index 00000000..9aa1e5c1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_4.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_5.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_5.png new file mode 100644 index 00000000..fa1fa70a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_5.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_6.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_6.png new file mode 100644 index 00000000..189e9378 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_6.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_7.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_7.png new file mode 100644 index 00000000..76a0d50e Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_13_7.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_1.png new file mode 100644 index 00000000..cb0136dd Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_2.png new file mode 100644 index 00000000..fb111f67 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_3.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_3.png new file mode 100644 index 00000000..8da5df4c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_3.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_4.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_4.png new file mode 100644 index 00000000..5a21d401 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_4.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_5.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_5.png new file mode 100644 index 00000000..b4c61c36 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_5.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_6.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_6.png new file mode 100644 index 00000000..aa5cbb7c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_16_6.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_19_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_19_1.png new file mode 100644 index 00000000..7b80f314 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_19_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_21_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_21_1.png new file mode 100644 index 00000000..5408f0e4 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_21_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_1.png new file mode 100644 index 00000000..97105266 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_2.png new file mode 100644 index 00000000..de262db1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_3.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_3.png new file mode 100644 index 00000000..2a162d80 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_3.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_4.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_4.png new file mode 100644 index 00000000..e6874127 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_4.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_5.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_5.png new file mode 100644 index 00000000..977c29f0 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_32_5.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_8_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_8_1.png new file mode 100644 index 00000000..45ef6712 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/parameters_8_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_14_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_14_1.png new file mode 100644 index 00000000..7aac037a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_14_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_15_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_15_1.png new file mode 100644 index 00000000..2f01596c Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_15_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_17_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_17_1.png new file mode 100644 index 00000000..7d2a9cc0 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_17_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_18_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_18_1.png new file mode 100644 index 00000000..dd35a64f Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_18_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_20_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_20_1.png new file mode 100644 index 00000000..1791e6a1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_20_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_21_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_21_1.png new file mode 100644 index 00000000..edfadc85 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/performance_21_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/pixplot.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/pixplot.png new file mode 100644 index 00000000..2870c331 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/pixplot.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_10_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_10_1.png new file mode 100644 index 00000000..b841ec4f Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_10_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_12_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_12_1.png new file mode 100644 index 00000000..966479ac Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_12_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_14_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_14_1.png new file mode 100644 index 00000000..e1152770 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_14_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_19_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_19_2.png new file mode 100644 index 00000000..a06e28b6 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_19_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_21_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_21_2.png new file mode 100644 index 00000000..519239fe Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_21_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_32_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_32_2.png new file mode 100644 index 00000000..5338b137 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_32_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_34_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_34_2.png new file mode 100644 index 00000000..d8dd872b Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_34_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_38_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_38_1.png new file mode 100644 index 00000000..7c02bbde Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_38_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_40_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_40_1.png new file mode 100644 index 00000000..9ce3f558 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_40_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_42_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_42_0.png new file mode 100644 index 00000000..bc91f7e6 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_42_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_44_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_44_1.png new file mode 100644 index 00000000..305cdb77 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_44_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_8_2.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_8_2.png new file mode 100644 index 00000000..41c17886 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/plotting_8_2.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/population_umap.jpg b/ext/TAL/applications/UMAP_fast/umap/doc/images/population_umap.jpg new file mode 100644 index 00000000..94a701e9 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/population_umap.jpg differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/pumap-only.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/pumap-only.png new file mode 100644 index 00000000..3db864c5 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/pumap-only.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_10_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_10_1.png new file mode 100644 index 00000000..ba2340c9 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_10_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_14_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_14_1.png new file mode 100644 index 00000000..d14261b3 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_14_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_18_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_18_1.png new file mode 100644 index 00000000..d14261b3 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_18_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_6_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_6_1.png new file mode 100644 index 00000000..cc1f3b5b Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/reproducibility_6_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/simplices.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/simplices.png new file mode 100644 index 00000000..ff28e530 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/simplices.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/single_cell_umap.jpg b/ext/TAL/applications/UMAP_fast/umap/doc/images/single_cell_umap.jpg new file mode 100644 index 00000000..416ac956 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/single_cell_umap.jpg differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_11_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_11_1.png new file mode 100644 index 00000000..729304f7 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_11_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_18_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_18_0.png new file mode 100644 index 00000000..f80909ac Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_18_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_31_1.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_31_1.png new file mode 100644 index 00000000..d56cccf9 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_31_1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_35_0.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_35_0.png new file mode 100644 index 00000000..d4b6f4c5 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/sparse_35_0.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/structure_recent_phil.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/structure_recent_phil.png new file mode 100644 index 00000000..b3efb61a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/structure_recent_phil.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/syllabus_galaxy.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/syllabus_galaxy.png new file mode 100644 index 00000000..1e18a193 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/syllabus_galaxy.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/time_cluster.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/time_cluster.png new file mode 100644 index 00000000..2f7109ea Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/time_cluster.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/umap-loss.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap-loss.png new file mode 100644 index 00000000..e0412794 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap-loss.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/umap-only.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap-only.png new file mode 100644 index 00000000..5d72c61a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap-only.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_explorer.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_explorer.png new file mode 100644 index 00000000..b2ca6899 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_explorer.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_primes.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_primes.png new file mode 100644 index 00000000..6d388e24 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_primes.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_surrey.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_surrey.png new file mode 100644 index 00000000..389b65f9 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_surrey.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_vae_pca.png b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_vae_pca.png new file mode 100644 index 00000000..f200cde1 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/images/umap_vae_pca.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/index.rst b/ext/TAL/applications/UMAP_fast/umap/doc/index.rst new file mode 100644 index 00000000..e3a574d0 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/index.rst @@ -0,0 +1,104 @@ +.. umap documentation master file, created by + sphinx-quickstart on Fri Jun 8 10:09:40 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +.. image:: logo_large.png + :width: 600 + :align: center + +UMAP: Uniform Manifold Approximation and Projection for Dimension Reduction +=========================================================================== + +Uniform Manifold Approximation and Projection (UMAP) is a dimension reduction +technique that can be used for visualisation similarly to t-SNE, but also for +general non-linear dimension reduction. The algorithm is founded on three +assumptions about the data + +1. The data is uniformly distributed on Riemannian manifold; +2. The Riemannian metric is locally constant (or can be approximated as such); +3. The manifold is locally connected. + +From these assumptions it is possible to model the manifold with a fuzzy +topological structure. The embedding is found by searching for a low dimensional +projection of the data that has the closest possible equivalent fuzzy +topological structure. + +The details for the underlying mathematics can be found in +`our paper on ArXiv `_: + +McInnes, L, Healy, J, *UMAP: Uniform Manifold Approximation and Projection +for Dimension Reduction*, ArXiv e-prints 1802.03426, 2018 + +You can find the software `on github `_. + +**Installation** + +Conda install, via the excellent work of the conda-forge team: + +.. code:: bash + + conda install -c conda-forge umap-learn + +The conda-forge packages are available for linux, OS X, and Windows 64 bit. + +PyPI install, presuming you have numba and sklearn and all its requirements +(numpy and scipy) installed: + +.. code:: bash + + pip install umap-learn + + +.. toctree:: + :maxdepth: 2 + :caption: User Guide / Tutorial: + + basic_usage + parameters + plotting + reproducibility + transform + inverse_transform + parametric_umap + sparse + supervised + clustering + outliers + composing_models + densmap_demo + document_embedding + embedding_space + aligned_umap_basic_usage + aligned_umap_politics_demo + release_notes + faq + +.. toctree:: + :maxdepth: 2 + :caption: Background on UMAP: + + how_umap_works + performance + +.. toctree:: + :maxdepth: 2 + :caption: Examples of UMAP usage + + interactive_viz + exploratory_analysis + scientific_papers + +.. toctree:: + :caption: API Reference: + + api + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/interactive_viz.rst b/ext/TAL/applications/UMAP_fast/umap/doc/interactive_viz.rst new file mode 100644 index 00000000..65091103 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/interactive_viz.rst @@ -0,0 +1,117 @@ +Interactive Visualizations +========================== + +UMAP has found use in a number of interesting interactive visualization projects, analyzing everything from +images from photo archives, to word embedding, animal point clouds, and even sound. Sometimes it has also +been used in interesting interactive tools that simply help a user to get an intuition for what the algorithm +is doing (by applying it to intuitive 3D data). Below are some amazing projects that make use of UMAP. + +UMAP Zoo +-------- +An exploration of how UMAP behaves when dimension reducing point clouds of animals. It is +interactive, letting you switch between 2D and 3D representations and has a wide selection +of different animals. Attempting to guess the animal from the 2D UMAP representation is a +fun game. In practice this tool can go a long way to helping to build at least some intuitions +for what UMAP tends to do with data. + +.. image:: images/UMAP_zoo.png + :width: 400px + +`UMAP Zoo `__ + +Thanks to Douglas Duhaime. + +Tensorflow Embedding Projector +------------------------------ +If you just want to explore UMAP embeddings of datasets then the Embedding Projector +from Tensorflow is a great way to do that. As well as having a good interactive 3D view +it also has facilities for inspecting and searching labels and tags on the data. By default +it loads up word2vec vectors, but you can upload any data you wish. You can then select +the UMAP option among the tabs for embeddings choices (alongside PCA and t-SNE). + +.. image:: images/embedding_projector.png + :width: 400px + +`Embedding Projector `__ + +Thanks to Andy Coenen and the Embedding Projector team. + +PixPlot +------- +PixPlot provides an overview of large photo-collections. In the demonstration app +from Yale's Digital Humanities lab it provides a window on the Meserve-Kunhardt Collection +of historical photographs. The approach uses convolutional neural nets to reduce the images +to 2048 dimensions, and then uses UMAP to present them in a 2-dimensional map which the +user can interactive pan and zoom around in. This process results in similar photos +ending up in similar regions of the map allowing for easy perusal of large photo +collections. The PixPlot project is also available on github in case you wish to train +it on your own photo collection. + +.. image:: images/pixplot.png + :width: 400px + +`PixPlot `__ + +Thanks to Douglas Duhaime and the Digital Humanities lab at Yale. + +UMAP Explorer +------------- +A great demonstration of building a web based app for interactively exploring a UMAP embedding. +In this case it provides an exploration of UMAP run on the MNIST digits dataset. Each point in +the embedding is rendered as the digit image, and coloured according to the digit class. Mousing +over the images will make them larger and provide a view of the digit in the upper left. You can also pan +and zoom around the emebdding to get a better understanding of how UMAP has mapped the different styles of +handwritten digits down to 2 dimensions. + +.. image:: images/umap_explorer.png + :width: 400px + +`UMAP Explorer `__ + +Thanks for Grant Custer. + +Audio Explorer +-------------- +The Audio Explorer uses UMAP to embed sound samples into a 2 dimensional space for easy exploration. +The goal here is to take a large library of sounds samples and put similar sounds in similar regions +of the map, allowing a user to quickly mouse over and listen to various variations of a given sample +to quickly find exactly the right sound sample to use. Audio explorer uses MFCCs and/or WaveNet to +provide an initial useful vector representation of the sound samples, before applying UMAP to +generate the 2D embedding. + +.. image:: images/audio_explorer.png + :width: 400px + +`Audio Explorer `__ + +Thanks to Leon Fedden. + +Orion Search +------------ +Orion is an open source research measurement and knowledge discovery tool that enables you to monitor +progress in science, visually explore the scientific landscape and search for relevant publications. +Orion encodes bioRxiv paper abstracts to dense vectors with Sentence Transformers and projects them to +an interactive 3D visualisation with UMAP. You can filter the UMAP embeddings by topic and country. +You can also select a subset of the UMAP embeddings and retrieve those papers and their metadata. + +.. image:: images/orion_particles.png + :width: 400px + +`Orion Search `__ + +Thanks to Kostas Stathoulopoulos, Zac Ioannidis and Lilia Villafuerte. + +Exploring Fashion MNIST +----------------------- +A web based interactive exploration of a 3D UMAP embedding ran on the Fashion MNIST dataset. Users can +freely navigate the 3D space, jumping to a specific image by clicking an image or entering an image id. +Like Grant Custer's UMAP Explorer, each point is rendered as the actual image and colored according to +the label. It is also similar to the Tensorflow Embedding Projector, but designed more specifically for +Fashion MNIST, thus more efficient and capable of showing all the 70k images. + +.. image:: images/exploring_fashion_mnist.png + :width: 400px + +`Exploring Fashion MNIST `__ + +Thanks to stwind. \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/inverse_transform.rst b/ext/TAL/applications/UMAP_fast/umap/doc/inverse_transform.rst new file mode 100644 index 00000000..97ae928b --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/inverse_transform.rst @@ -0,0 +1,244 @@ + +Inverse transforms +================== + +UMAP has some support for inverse transforms -- generating a high +dimensional data sample given a location in the low dimensional +embedding space. To start let's load all the relevant libraries. + +.. code:: python3 + + import numpy as np + import matplotlib.pyplot as plt + from matplotlib.gridspec import GridSpec + import seaborn as sns + import sklearn.datasets + import umap + import umap.plot + +We will need some data to test with. To start we'll use the MNIST digits +dataset. This is a dataset of 70000 handwritten digits encoded as +grayscale 28x28 pixel images. Our goal is to use UMAP to reduce the +dimension of this dataset to something small, and then see if we can +generate new digits by sampling points from the embedding space. To load +the MNIST dataset we'll make use of sklearn's ``fetch_openml`` function. + +.. code:: python3 + + data, labels = sklearn.datasets.fetch_openml('mnist_784', version=1, return_X_y=True) + +Now we need to generate a reduced dimension representation of this data. +This is straightforward with UMAP, but in this case rather than using +``fit_transform`` we'll use the fit method so that we can retain the +trained model for later generating new digits based on samples from the +embedding space. + +.. code:: python3 + + mapper = umap.UMAP(random_state=42).fit(data) + +To ensure that things worked correctly we can plot the data (since we +reduced it to two dimensions). We'll use the ``umap.plot`` functionality +to do this. + +.. code:: python3 + + umap.plot.points(mapper, labels=labels) + + + +.. image:: images/inverse_transform_7_1.png + + +This looks much like we would expect. The different digit classes have +been decently separated. Now we need to create a set of samples in the +embedding space to apply the ``inverse_transform`` operation to. To do +this we'll generate a grid of samples linearly interpolating between +four corner points. To make our selection interesting we'll carefully +choose the corners to span over the dataset, and sample different digits +so that we can better see the transitions. + +.. code:: python3 + + corners = np.array([ + [-5, -10], # 1 + [-7, 6], # 7 + [2, -8], # 2 + [12, 4], # 0 + ]) + + test_pts = np.array([ + (corners[0]*(1-x) + corners[1]*x)*(1-y) + + (corners[2]*(1-x) + corners[3]*x)*y + for y in np.linspace(0, 1, 10) + for x in np.linspace(0, 1, 10) + ]) + +Now we can apply the ``inverse_transform`` method to this set of test +points. Each test point is a two dimensional point lying somewhere in +the embedding space. The ``inverse_transform`` method will convert this +into an approximation of the high dimensional representation that would +have been embedded into such a location. Following the sklearn API this +is as simple to use as calling the ``inverse_transform`` method of the +trained model and passing it the set of test points that we want to +convert into high dimensional representations. Be warned that this can +be quite expensive computationally. + +.. code:: python3 + + inv_transformed_points = mapper.inverse_transform(test_pts) + +Now the goal is to visualize how well we have done. Effectively what we +would like to do is show the test points in the embedding space, and +then show a grid of the corresponding images generated by the inverse +transform. To get all of this in a single matplotlib figure takes a +little setting up, but is quite manageable -- mostly it is just a matter +of managing ``GridSpec`` formatting. Once we have that setup we just +need a scatterplot of the embedding, a scatterplot of the test points, +and finally a grid of the images we generated (converting the inverse +transformed vectors into images is just a matter of reshaping them back +to 28 by 28 pixel grids and using ``imshow``). + +.. code:: python3 + + # Set up the grid + fig = plt.figure(figsize=(12,6)) + gs = GridSpec(10, 20, fig) + scatter_ax = fig.add_subplot(gs[:, :10]) + digit_axes = np.zeros((10, 10), dtype=object) + for i in range(10): + for j in range(10): + digit_axes[i, j] = fig.add_subplot(gs[i, 10 + j]) + + # Use umap.plot to plot to the major axis + # umap.plot.points(mapper, labels=labels, ax=scatter_ax) + scatter_ax.scatter(mapper.embedding_[:, 0], mapper.embedding_[:, 1], + c=labels.astype(np.int32), cmap='Spectral', s=0.1) + scatter_ax.set(xticks=[], yticks=[]) + + # Plot the locations of the text points + scatter_ax.scatter(test_pts[:, 0], test_pts[:, 1], marker='x', c='k', s=15) + + # Plot each of the generated digit images + for i in range(10): + for j in range(10): + digit_axes[i, j].imshow(inv_transformed_points[i*10 + j].reshape(28, 28)) + digit_axes[i, j].set(xticks=[], yticks=[]) + + + +.. image:: images/inverse_transform_13_0.png + + +The end result looks pretty good -- we did indeed generate plausible +looking digit images, and many of the transitions (from 1 to 7 across +the top row for example) seem pretty natural and make sense. This can +help you to understand the structure of the cluster of 1s (it +transitions on the angle, sloping toward what will eventually be 7s), +and why 7s and 9s are close together in the embedding. Of course there +are also some stranger transitions, especially where the test points +fell into large gaps between clusters in the embedding -- in some sense +it is hard to interpret what should go in some of those gaps as they +don't really represent anything resembling a smooth transition). + +A further note: None of the test points chosen fall outside the convex +hull of the embedding. This is deliberate -- the inverse transform +function operates poorly outside the bounds of that convex hull. Be +warned that if you select points to inverse transform that are outside +the bounds about the embedding you will likely get strange results +(often simply snapping to a particular source high dimensional vector). + +Let's continue the demonstration by looking at the Fashion MNIST +dataset. As before we can load this through sklearn. + +.. code:: python3 + + data, labels = sklearn.datasets.fetch_openml('Fashion-MNIST', version=1, return_X_y=True) + +Again we can fit this data with UMAP and get a mapper object. + +.. code:: python3 + + mapper = umap.UMAP(random_state=42).fit(data) + +Let's plot the embedding to see what we got as a result: + +.. code:: python3 + + umap.plot.points(mapper, labels=labels) + + + + +.. image:: images/inverse_transform_20_1.png + + +Again we'll generate a set of test points by making a grid interpolating +between four corners. As before we'll select the corners so that we can +stay within the convex hull of the embedding points and ensure nothing +too strange happens with the inverse transforms. + +.. code:: python3 + + corners = np.array([ + [-2, -6], # bags + [-9, 3], # boots? + [7, -5], # shirts/tops/dresses + [4, 10], # pants + ]) + + test_pts = np.array([ + (corners[0]*(1-x) + corners[1]*x)*(1-y) + + (corners[2]*(1-x) + corners[3]*x)*y + for y in np.linspace(0, 1, 10) + for x in np.linspace(0, 1, 10) + ]) + +Now we simply apply the inverse transform just as before. Again, be +warned, this is quite expensive computationally and may take some time +to complete. + +.. code:: python3 + + inv_transformed_points = mapper.inverse_transform(test_pts) + +And now we can use similar code as above to set up our plot of the +embedding with test points overlaid, and the generated images. + +.. code:: python3 + + # Set up the grid + fig = plt.figure(figsize=(12,6)) + gs = GridSpec(10, 20, fig) + scatter_ax = fig.add_subplot(gs[:, :10]) + digit_axes = np.zeros((10, 10), dtype=object) + for i in range(10): + for j in range(10): + digit_axes[i, j] = fig.add_subplot(gs[i, 10 + j]) + + # Use umap.plot to plot to the major axis + # umap.plot.points(mapper, labels=labels, ax=scatter_ax) + scatter_ax.scatter(mapper.embedding_[:, 0], mapper.embedding_[:, 1], + c=labels.astype(np.int32), cmap='Spectral', s=0.1) + scatter_ax.set(xticks=[], yticks=[]) + + # Plot the locations of the text points + scatter_ax.scatter(test_pts[:, 0], test_pts[:, 1], marker='x', c='k', s=15) + + # Plot each of the generated digit images + for i in range(10): + for j in range(10): + digit_axes[i, j].imshow(inv_transformed_points[i*10 + j].reshape(28, 28)) + digit_axes[i, j].set(xticks=[], yticks=[]) + + + +.. image:: images/inverse_transform_26_0.png + + +This time we see some of the interpolations between items looking rather +strange -- particularly the points that lie somewhere between shoes and +pants -- ultimately it is doing the best it can with a difficult +problem. At the same time many of the other transitions seem to work +pretty well, so it is, indeed, providing useful information about how +the embedding is structured. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/logo.png b/ext/TAL/applications/UMAP_fast/umap/doc/logo.png new file mode 100644 index 00000000..abece2e3 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/logo.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/logo_large.png b/ext/TAL/applications/UMAP_fast/umap/doc/logo_large.png new file mode 100644 index 00000000..7fdf198f Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/doc/logo_large.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/make.bat b/ext/TAL/applications/UMAP_fast/umap/doc/make.bat new file mode 100644 index 00000000..ef8c6601 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=umap + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/outliers.rst b/ext/TAL/applications/UMAP_fast/umap/doc/outliers.rst new file mode 100644 index 00000000..97b6bc8b --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/outliers.rst @@ -0,0 +1,278 @@ +Outlier detection using UMAP +============================ + +While an earlier tutorial looked at using `UMAP for +clustering `__, +it can also be used for outlier detection, providing that some care is +taken. This tutorial will look at how to use UMAP in this manner, and +what to look out for, by finding anomalous digits in the MNIST +handwritten digits dataset. To start with let's load the relevant +libraries: + +.. code:: python3 + + import numpy as np + import sklearn.datasets + import sklearn.neighbors + import umap + import umap.plot + import matplotlib.pyplot as plt + %matplotlib inline + +With this in hand, let's grab the MNIST digits dataset from the +internet, using the new ``fetch_ml`` loader in sklearn. + +.. code:: python3 + + data, labels = sklearn.datasets.fetch_openml('mnist_784', version=1, return_X_y=True) + +Before we get started we should try looking for outliers in terms of the +native 784 dimensional space that MNIST digits live in. To do this we +will make use of the `Local Outlier Factor +(LOF) `__ method for +determining outliers since sklearn has an easy to use implementation. +The essential intuition of LOF is to look for points that have a +(locally approximated) density that differs significantly from the +average density of their neighbors. In our case the actual details are +not so important -- it is enough to know that the algorithm is +reasonably robust and effective on vector space data. We can apply it +using the ``fit_predict`` method of the sklearn class. The LOF class +take a parameter ``contamination`` which specifies the percentage of +data that the user expects to be noise. For this use case we will set it +to 0.001428 since, given the 70,000 samples in MNIST, this will result +in 100 outliers, which we can then look at in more detail. + +.. code:: python3 + + %%time + outlier_scores = sklearn.neighbors.LocalOutlierFactor(contamination=0.001428).fit_predict(data) + + +.. parsed-literal:: + + CPU times: user 1h 29min 10s, sys: 12.4 s, total: 1h 29min 22s + Wall time: 1h 29min 53s + + +It is worth noting how long that took. Over an hour and a half! Why did +it take so long? Because LOF requires a notion of density, which in turn +relies on a nearest neighbor type computation -- which is expensive in +sklearn for high dimensional data. This alone is potentially a reason to +look at reducing the dimension of the data -- it makes it more amenable +to existing techniques like LOF. + +Now that we have a set of outlier scores we can find the actual outlying +digit images -- these are the ones with scores equal to -1. Let's +extract that data, and check that we got 100 different digit images. + +.. code:: python3 + + outlying_digits = data[outlier_scores == -1] + outlying_digits.shape + + + + +.. parsed-literal:: + + (100, 784) + + + +Now that we have the outlying digit images the first question we should +be asking is "what do they look like?". Fortunately for us we can +convert the 784 dimensional vectors back into image and plot them, +making it easier to look at. Since we extracted the 100 most outlying +digit images we can just display a 10x10 grid of them. + +.. code:: python3 + + fig, axes = plt.subplots(7, 10, figsize=(10,10)) + for i, ax in enumerate(axes.flatten()): + ax.imshow(outlying_digits[i].reshape((28,28))) + plt.setp(ax, xticks=[], yticks=[]) + plt.tight_layout() + + + +.. image:: images/outliers_9_0.png + + +These do certainly look like somewhat strange looking handwritten +digits, so our outlier detection seems to be working to some extent. + +Now let's try a naive approach using UMAP and see how far that gets us. +First let's just apply UMAP directly with default parameters to the +MNIST data. + +.. code:: python3 + + mapper = umap.UMAP().fit(data) + +Now we can see what we got using the new plotting tools in umap.plot. + +.. code:: python3 + + umap.plot.points(mapper, labels=labels) + + +.. parsed-literal:: + + + + + + +.. image:: images/outliers_13_2.png + + +That looks like what we have come to expect from a UMAP embedding of +MNIST. The question is have we managed to preserve outliers well enough +that LOF can still find the bizarre digit images, or has the embedding +lost that information and contracted the outliers into the individual +digit clusters? We can simply apply LOF to the embedding and see what +that returns. + +.. code:: python3 + + %%time + outlier_scores = sklearn.neighbors.LocalOutlierFactor(contamination=0.001428).fit_predict(mapper.embedding_) + +This was obviously much faster since we are operating in a much lower +dimensional space that is more amenable to the spatial indexing methods +that sklearn uses to find nearest neighbors. As before we extract the +outlying digit images, and verify that we got 100 of them, + +.. code:: python3 + + outlying_digits = data[outlier_scores == -1] + outlying_digits.shape + + + + +.. parsed-literal:: + + (100, 784) + + + +Now we need to plot the outlying digit images to see what kinds of digit +images this approach found to be particularly strange. + +.. code:: python3 + + fig, axes = plt.subplots(7, 10, figsize=(10,10)) + for i, ax in enumerate(axes.flatten()): + ax.imshow(outlying_digits[i].reshape((28,28))) + plt.setp(ax, xticks=[], yticks=[]) + plt.tight_layout() + + + +.. image:: images/outliers_19_0.png + + +In many ways this looks to be a *better* result than the original LOF in +the high dimensional space. While the digit images that the high +dimensional LOF found to be strange were indeed somewhat odd looking, +many of these digit images are considerably stranger -- significantly +odd line thickness, warped shapes, and images that are hard to even +recognise as digits. This helps to demonstrate a certain amount of +confirmation bias when examining outliers: since we expect things tagged +as outliers to be strange we tend to find aspects of them that justify +that classification, potentially unaware of how much stranger some of +the data may in fact be. This should make us wary of even this outlier +set: what else might lurk in the dataset? + +We can, in fact, potentially improve on this result by tuning the UMAP +embedding a little for the task of finding outliers. When UMAP combines +together the different local simplicial sets (see :doc:`how_umap_works` +for more details) the standard approach uses a union, but we could +instead take an intersection. An intersection ensures that outliers +remain disconnected, which is certainly beneficial when seeking to find +outliers. A downside of the intersection is that it tends to break up +the resulting simplicial set into many disconnected components and a lot +of the more non-local and global structure is lost, resulting in a lot +lower quality of the resulting embedding. We can, however, interpolate +between the union and intersection. In UMAP this is given by the +``set_op_mix_ratio``, where a value of 0.0 represents an intersection, +and a value of 1.0 represents a union (the default value is 1.0). By +setting this to a lower value, say 0.25, we can encourage the embedding +to do a better job of preserving outliers as outlying, while still +retaining the benefits of a union operation. + +.. code:: python3 + + mapper = umap.UMAP(set_op_mix_ratio=0.25).fit(data) + +.. code:: python3 + + umap.plot.points(mapper, labels=labels) + + +.. parsed-literal:: + + + + + + +.. image:: images/outliers_22_2.png + + +As you can see the embedding is not as well structured overall as when +we had a ``set_op_mix_ratio`` of 1.0, but we have potentially done a +better job of ensuring that outliers remain outlying. We can test that +hypothesis by running LOF on this embedding and looking at the resulting +digit images we get out. Ideally we should expect to find some +potentially even stranger results. + +.. code:: python3 + + %%time + outlier_scores = sklearn.neighbors.LocalOutlierFactor(contamination=0.001428).fit_predict(mapper.embedding_) + +.. code:: python3 + + outlying_digits = data[outlier_scores == -1] + outlying_digits.shape + + + + +.. parsed-literal:: + + (100, 784) + + + +We have the expected 100 most outlying digit images, so let's visualise +the results and see if they really are particularly strange. + +.. code:: python3 + + fig, axes = plt.subplots(10, 10, figsize=(10,10)) + for i, ax in enumerate(axes.flatten()): + ax.imshow(outlying_digits[i].reshape((28,28))) + plt.setp(ax, xticks=[], yticks=[]) + plt.tight_layout() + + + +.. image:: images/outliers_27_0.png + + +Here we see that the line thickness variation (particularly "fat" +digits, or particularly "fine" lines) that the original embedding helped +surface come through even more strongly here. We also see a number of +clearly corrupted images with extra lines, dots, or strange blurring +occurring. + +So, in summary, using UMAP to reduce dimension prior to running +classical outlier detection methods such as LOF can improve both the +speed with which the algorithm runs, and the quality of results the +outlier detection can find. Furthermore we have introduced the +``set_op_mix_ratio`` parameter, and explained how it can be used to +potentially improve the performance of outlier detection approaches +applied to UMAP embeddings. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/parameters.rst b/ext/TAL/applications/UMAP_fast/umap/doc/parameters.rst new file mode 100644 index 00000000..4fc60ca2 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/parameters.rst @@ -0,0 +1,472 @@ + +Basic UMAP Parameters +===================== + +UMAP is a fairly flexible non-linear dimension reduction algorithm. It +seeks to learn the manifold structure of your data and find a low +dimensional embedding that preserves the essential topological structure +of that manifold. In this notebook we will generate some visualisable +4-dimensional data, demonstrate how to use UMAP to provide a +2-dimensional representation of it, and then look at how various UMAP +parameters can impact the resulting embedding. This documentation is +based on the work of Philippe Rivière for visionscarto.net. + +To start we'll need some basic libraries. First ``numpy`` will be needed +for basic array manipulation. Since we will be visualising the results +we will need ``matplotlib`` and ``seaborn``. Finally we will need +``umap`` for doing the dimension reduction itself. + +.. code:: python3 + + import numpy as np + import matplotlib.pyplot as plt + from mpl_toolkits.mplot3d import Axes3D + import seaborn as sns + import umap + %matplotlib inline + +.. code:: python3 + + sns.set(style='white', context='poster', rc={'figure.figsize':(14,10)}) + +Next we will need some data to embed into a lower dimensional +representation. To make the 4-dimensional data "visualisable" we will +generate data uniformly at random from a 4-dimensional cube such that we +can interpret a sample as a tuple of (R,G,B,a) values specifying a color +(and translucency). Thus when we plot low dimensional representations +each point can be colored according to its 4-dimensional value. For this we +can use ``numpy``. We will fix a random seed for the sake of +consistency. + +.. code:: python3 + + np.random.seed(42) + data = np.random.rand(800, 4) + +Now we need to find a low dimensional representation of the data. As in +the Basic Usage documentation, we can do this by using the +:meth:`~umap.umap_.UMAP.fit_transform` method on a :class:`~umap.umap_.UMAP` object. + +.. code:: python3 + + fit = umap.UMAP() + %time u = fit.fit_transform(data) + + +.. parsed-literal:: + + CPU times: user 7.73 s, sys: 211 ms, total: 7.94 s + Wall time: 6.8 s + + +The resulting value ``u`` is a 2-dimensional representation of the data. +We can visualise the result by using ``matplotlib`` to draw a scatter +plot of ``u``. We can color each point of the scatter plot by the +associated 4-dimensional color from the source data. + +.. code:: python3 + + plt.scatter(u[:,0], u[:,1], c=data) + plt.title('UMAP embedding of random colours'); + +.. image:: images/parameters_8_1.png + + +As you can see the result is that the data is placed in 2-dimensional +space such that points that were nearby in 4-dimensional space (i.e. +are similar colors) are kept close together. Since we drew a random +selection of points in the color cube there is a certain amount of +induced structure from where the random points happened to clump up in +color space. + +UMAP has several hyperparameters that can have a significant impact on +the resulting embedding. In this notebook we will be covering the four +major ones: + +- ``n_neighbors`` +- ``min_dist`` +- ``n_components`` +- ``metric`` + +Each of these parameters has a distinct effect, and we will look at each +in turn. To make exploration simpler we will first write a short utility +function that can fit the data with UMAP given a set of parameter +choices, and plot the result. + +.. code:: python3 + + def draw_umap(n_neighbors=15, min_dist=0.1, n_components=2, metric='euclidean', title=''): + fit = umap.UMAP( + n_neighbors=n_neighbors, + min_dist=min_dist, + n_components=n_components, + metric=metric + ) + u = fit.fit_transform(data); + fig = plt.figure() + if n_components == 1: + ax = fig.add_subplot(111) + ax.scatter(u[:,0], range(len(u)), c=data) + if n_components == 2: + ax = fig.add_subplot(111) + ax.scatter(u[:,0], u[:,1], c=data) + if n_components == 3: + ax = fig.add_subplot(111, projection='3d') + ax.scatter(u[:,0], u[:,1], u[:,2], c=data, s=100) + plt.title(title, fontsize=18) + +``n_neighbors`` +~~~~~~~~~~~~~~~ + +This parameter controls how UMAP balances local versus global structure +in the data. It does this by constraining the size of the local +neighborhood UMAP will look at when attempting to learn the manifold +structure of the data. This means that low values of ``n_neighbors`` +will force UMAP to concentrate on very local structure (potentially to +the detriment of the big picture), while large values will push UMAP to +look at larger neighborhoods of each point when estimating the manifold +structure of the data, losing fine detail structure for the sake of +getting the broader of the data. + +We can see that in practice by fitting our dataset with UMAP using a +range of ``n_neighbors`` values. The default value of ``n_neighbors`` +for UMAP (as used above) is 15, but we will look at values ranging from +2 (a very local view of the manifold) up to 200 (a quarter of the data). + +.. code:: python3 + + for n in (2, 5, 10, 20, 50, 100, 200): + draw_umap(n_neighbors=n, title='n_neighbors = {}'.format(n)) + + + +.. image:: images/parameters_13_1.png + + + +.. image:: images/parameters_13_2.png + + + +.. image:: images/parameters_13_3.png + + + +.. image:: images/parameters_13_4.png + + + +.. image:: images/parameters_13_5.png + + + +.. image:: images/parameters_13_6.png + + + +.. image:: images/parameters_13_7.png + + +With a value of ``n_neighbors=2`` we see that UMAP merely glues together +small chains, but due to the narrow/local view, fails to see how those +connect together. It also leaves many different components (and even +singleton points). This represents the fact that from a fine detail +point of view the data is very disconnected and scattered throughout the +space. + +As ``n_neighbors`` is increased UMAP manages to see more of the overall +structure of the data, gluing more components together, and better +coverying the broader structure of the data. By the stage of +``n_neighbors=20`` we have a fairly good overall view of the data +showing how the various colors interelate to each other over the whole +dataset. + +As ``n_neighbors`` increases further more and more focus in placed on +the overall structure of the data. This results in, with +``n_neighbors=200`` a plot where the overall structure (blues, greens, +and reds; high luminance versus low) is well captured, but at the loss +of some of the finer local structure (individual colors are no longer +necessarily immediately near their closest color match). + +This effect well exemplifies the local/global tradeoff provided by +``n_neighbors``. + +``min_dist`` +~~~~~~~~~~~~ + +The ``min_dist`` parameter controls how tightly UMAP is allowed to pack +points together. It, quite literally, provides the minimum distance +apart that points are allowed to be in the low dimensional +representation. This means that low values of ``min_dist`` will result +in clumpier embeddings. This can be useful if you are interested in +clustering, or in finer topological structure. Larger values of +``min_dist`` will prevent UMAP from packing points together and will +focus on the preservation of the broad topological structure +instead. + +The default value for ``min_dist`` (as used above) is 0.1. We will look +at a range of values from 0.0 through to 0.99. + +.. code:: python3 + + for d in (0.0, 0.1, 0.25, 0.5, 0.8, 0.99): + draw_umap(min_dist=d, title='min_dist = {}'.format(d)) + + +.. image:: images/parameters_16_1.png + + + +.. image:: images/parameters_16_2.png + + + +.. image:: images/parameters_16_3.png + + + +.. image:: images/parameters_16_4.png + + + +.. image:: images/parameters_16_5.png + + + +.. image:: images/parameters_16_6.png + + +Here we see that with ``min_dist=0.0`` UMAP manages to find small +connected components, clumps and strings in the data, and emphasises +these features in the resulting embedding. As ``min_dist`` is increased +these structures are pushed apart into softer more general features, +providing a better overarching view of the data at the loss of the more +detailed topological structure. + +``n_components`` +~~~~~~~~~~~~~~~~ + +As is standard for many ``scikit-learn`` dimension reduction algorithms +UMAP provides a ``n_components`` parameter option that allows the user +to determine the dimensionality of the reduced dimension space we will +be embedding the data into. Unlike some other visualisation algorithms +such as t-SNE, UMAP scales well in the embedding dimension, so you can use it +for more than just visualisation in 2- or 3-dimensions. + +For the purposes of this demonstration (so that we can see the effects +of the parameter) we will only be looking at 1-dimensional and +3-dimensional embeddings, which we have some hope of visualizing. + +First of all we will set ``n_components`` to 1, forcing UMAP to embed +the data in a line. For visualisation purposes we will randomly +distribute the data on the y-axis to provide some separation between +points. + +.. code:: python3 + + draw_umap(n_components=1, title='n_components = 1') + + +.. image:: images/parameters_19_1.png + + +Now we will try ``n_components=3``. For visualisation we will make use +of ``matplotlib``'s basic 3-dimensional plotting. + +.. code:: python3 + + draw_umap(n_components=3, title='n_components = 3') + + +.. parsed-literal:: + + /opt/anaconda3/envs/umap_dev/lib/python3.6/site-packages/sklearn/metrics/pairwise.py:257: RuntimeWarning: invalid value encountered in sqrt + return distances if squared else np.sqrt(distances, out=distances) + + + +.. image:: images/parameters_21_1.png + + +Here we can see that with more dimensions in which to work UMAP has an +easier time separating out the colors in a way that respects the +topological structure of the data. + +As mentioned, there is really no requirement to stop at ``n_components=3``. If you are interested in (density based) clustering, or other +machine learning techniques, it can be beneficial to pick a larger +embedding dimension (say 10, or 50) closer to the the dimension of the +underlying manifold on which your data lies. + +``metric`` +~~~~~~~~~~ + +The final UMAP parameter we will be considering in this notebook is the +``metric`` parameter. This controls how distance is computed in the +ambient space of the input data. By default UMAP supports a wide variety +of metrics, including: + +**Minkowski style metrics** + +- euclidean +- manhattan +- chebyshev +- minkowski + +**Miscellaneous spatial metrics** + +- canberra +- braycurtis +- haversine + +**Normalized spatial metrics** + +- mahalanobis +- wminkowski +- seuclidean + +**Angular and correlation metrics** + +- cosine +- correlation + +**Metrics for binary data** + +- hamming +- jaccard +- dice +- russellrao +- kulsinski +- rogerstanimoto +- sokalmichener +- sokalsneath +- yule + +Any of which can be specified by setting ``metric=''``; for +example to use cosine distance as the metric you would use +``metric='cosine'``. + +UMAP offers more than this however -- it supports custom user defined +metrics as long as those metrics can be compiled in ``nopython`` mode by +numba. For this notebook we will be looking at such custom metrics. To +define such metrics we'll need numba ... + +.. code:: python3 + + import numba + +For our first custom metric we'll define the distance to be the absolute +value of difference in the red channel. + +.. code:: python3 + + @numba.njit() + def red_channel_dist(a,b): + return np.abs(a[0] - b[0]) + +To get more adventurous it will be useful to have some colorspace +conversion -- to keep things simple we'll just use HSL formulas to +extract the hue, saturation, and lightness from an (R,G,B) tuple. + +.. code:: python3 + + @numba.njit() + def hue(r, g, b): + cmax = max(r, g, b) + cmin = min(r, g, b) + delta = cmax - cmin + if cmax == r: + return ((g - b) / delta) % 6 + elif cmax == g: + return ((b - r) / delta) + 2 + else: + return ((r - g) / delta) + 4 + + @numba.njit() + def lightness(r, g, b): + cmax = max(r, g, b) + cmin = min(r, g, b) + return (cmax + cmin) / 2.0 + + @numba.njit() + def saturation(r, g, b): + cmax = max(r, g, b) + cmin = min(r, g, b) + chroma = cmax - cmin + light = lightness(r, g, b) + if light == 1: + return 0 + else: + return chroma / (1 - abs(2*light - 1)) + +With that in hand we can define three extra distances. The first simply +measures the difference in hue, the second measures the euclidean +distance in a combined saturation and lightness space, while the third +measures distance in the full HSL space. + +.. code:: python3 + + @numba.njit() + def hue_dist(a, b): + diff = (hue(a[0], a[1], a[2]) - hue(b[0], b[1], b[2])) % 6 + if diff < 0: + return diff + 6 + else: + return diff + + @numba.njit() + def sl_dist(a, b): + a_sat = saturation(a[0], a[1], a[2]) + b_sat = saturation(b[0], b[1], b[2]) + a_light = lightness(a[0], a[1], a[2]) + b_light = lightness(b[0], b[1], b[2]) + return (a_sat - b_sat)**2 + (a_light - b_light)**2 + + @numba.njit() + def hsl_dist(a, b): + a_sat = saturation(a[0], a[1], a[2]) + b_sat = saturation(b[0], b[1], b[2]) + a_light = lightness(a[0], a[1], a[2]) + b_light = lightness(b[0], b[1], b[2]) + a_hue = hue(a[0], a[1], a[2]) + b_hue = hue(b[0], b[1], b[2]) + return (a_sat - b_sat)**2 + (a_light - b_light)**2 + (((a_hue - b_hue) % 6) / 6.0) + +With such custom metrics in hand we can get UMAP to embed the data using +those metrics to measure the distance between our input data points. Note +that ``numba`` provides significant flexibility in what we can do in +defining distance functions. Despite this we retain the high performance +we expect from UMAP even using such custom functions. + +.. code:: python3 + + for m in ("euclidean", red_channel_dist, sl_dist, hue_dist, hsl_dist): + name = m if type(m) is str else m.__name__ + draw_umap(n_components=2, metric=m, title='metric = {}'.format(name)) + + +.. image:: images/parameters_32_1.png + + + +.. image:: images/parameters_32_2.png + + + +.. image:: images/parameters_32_3.png + + + +.. image:: images/parameters_32_4.png + + + +.. image:: images/parameters_32_5.png + + +And here we can see the effects of the metrics quite clearly. The pure +red channel correctly sees the data as living on a one dimensional +manifold, the hue metric interprets the data as living in a circle, and +the HSL metric fattens out the circle according to the saturation and +lightness. This provides a reasonable demonstration of the power and +flexibility of UMAP in understanding the underlying topology of data, +and finding a suitable low dimensional representation of that topology. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/parametric_umap.rst b/ext/TAL/applications/UMAP_fast/umap/doc/parametric_umap.rst new file mode 100644 index 00000000..fadc1be8 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/parametric_umap.rst @@ -0,0 +1,219 @@ +Parametric Embedding +============================= + +.. role:: python(code) + :language: python + +UMAP is comprised of two steps: First, compute a graph representing your data, second, learn an embedding for that graph: + +.. image:: images/umap-only.png + +Parametric UMAP replaces the second step, minimizing the same objective function as UMAP (we'll call it non-parametric UMAP here), but learning the relationship between the data and embedding using a neural network, rather than learning the embeddings directly: + +.. image:: images/pumap-only.png + +Parametric UMAP is simply a subclass of UMAP, so it can be used just like nonparametric UMAP, replacing :python:`umap.UMAP` with :python:`parametric_umap.ParametricUMAP`. The most basic usage of parametric UMAP would be to simply replace UMAP with ParametricUMAP in your code: + +.. code:: python3 + + from umap.parametric_umap import ParametricUMAP + embedder = ParametricUMAP() + embedding = embedder.fit_transform(my_data) + +In this implementation, we use Keras and Tensorflow as a backend to train that neural network. The added complexity of a learned embedding presents a number of configurable settings available in addition to those in non-parametric UMAP. A set of Jupyter notebooks walking you through these parameters are available on the `GitHub repository `_ + + +Defining your own network +--------------------------- + +By default, parametric UMAP uses 3-layer 100-neuron fully-connected neural network. To extend Parametric UMAP to use a more complex architecture, like a convolutional neural network, we simply need to define the network and pass it in as an argument to ParametricUMAP. This can be done easliy, using tf.keras.Sequential. Here's an example for MNIST: + +.. code:: python3 + + # define the network + import tensorflow as tf + dims = (28, 28, 1) + n_components = 2 + encoder = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=dims), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, strides=(2, 2), activation="relu", padding="same" + ), + tf.keras.layers.Conv2D( + filters=64, kernel_size=3, strides=(2, 2), activation="relu", padding="same" + ), + tf.keras.layers.Flatten(), + tf.keras.layers.Dense(units=256, activation="relu"), + tf.keras.layers.Dense(units=256, activation="relu"), + tf.keras.layers.Dense(units=n_components), + ]) + encoder.summary() + +To load pass the data into ParametricUMAP, we first need to flatten it from 28x28x1 images to a 784-dimensional vector. + +.. code:: python3 + + from tensorflow.keras.datasets import mnist + (train_images, Y_train), (test_images, Y_test) = mnist.load_data() + train_images = train_images.reshape((train_images.shape[0], -1))/255. + test_images = test_images.reshape((test_images.shape[0], -1))/255. + + +We can then the network into ParametricUMAP and train: + +.. code:: python3 + + # pass encoder network to ParametricUMAP + embedder = ParametricUMAP(encoder=encoder, dims=dims) + embedding = embedder.fit_transform(train_images) + +If you are unfamilar with Tensorflow/Keras and want to train your own model, we reccomend that you take a look at the `Tensorflow documentation `_. + + +Saving and loading your model +----------------------------- + +Unlike non-parametric UMAP Parametric UMAP cannot be saved simply by pickling the UMAP object because of the Keras networks it contains. To save Parametric UMAP, there is a build in function: + +.. code:: python3 + + embedder.save('/your/path/here') + +You can then load parametric UMAP elsewhere: + +.. code:: python3 + + from umap.parametric_umap import load_ParametricUMAP + embedder = load_ParametricUMAP('/your/path/here') + +This loads both the UMAP object and the parametric networks it contains. + + +Plotting loss +------------- +Parametric UMAP monitors loss during training using Keras. That loss will be printed after each epoch during training. This loss is saved in :python:`embedder.history`, and can be plotted: + +.. code:: python3 + + print(embedder._history) + fig, ax = plt.subplots() + ax.plot(embedder._history['loss']) + ax.set_ylabel('Cross Entropy') + ax.set_xlabel('Epoch') + +.. image:: images/umap-loss.png + +Parametric inverse_transform (reconstruction) +--------------------------------------------- +To use a second neural network to learn an inverse mapping between data and embeddings, we simply need to pass `parametric_reconstruction= True` to the ParametricUMAP. + + +Like the encoder, a custom decoder can also be passed to ParametricUMAP, e.g. + +.. code:: python3 + + decoder = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=(n_components)), + tf.keras.layers.Dense(units=256, activation="relu"), + tf.keras.layers.Dense(units=7 * 7 * 256, activation="relu"), + tf.keras.layers.Reshape(target_shape=(7, 7, 256)), + tf.keras.layers.UpSampling2D((2)), + tf.keras.layers.Conv2D( + filters=64, kernel_size=3, padding="same", activation="relu" + ), + tf.keras.layers.UpSampling2D((2)), + tf.keras.layers.Conv2D( + filters=32, kernel_size=3, padding="same", activation="relu" + ), + + ]) + +In addition, validation data can be used to test reconstruction loss on out-of-dataset samples: + +.. code:: python3 + + validation_images = test_images.reshape((test_images.shape[0], -1))/255. + +Finally, we can pass the validation data and the networks to ParametricUMAP and train: + + +.. code:: python3 + + embedder = ParametricUMAP( + encoder=encoder, + decoder=decoder, + dims=dims, + parametric_reconstruction= True, + reconstruction_validation=validation_images, + verbose=True, + ) + embedding = embedder.fit_transform(train_images) + + +Autoencoding UMAP +----------------- + + +In the example above, the encoder is trained to minimize UMAP loss, and the decoder is trained to minimize reconstruction loss. To train the encoder jointly on both UMAP loss and reconstruction loss, pass :python:`autoencoder_loss = True` into the ParametricUMAP. + + +.. code:: python3 + + embedder = ParametricUMAP( + encoder=encoder, + decoder=decoder, + dims=dims, + parametric_reconstruction= True, + reconstruction_validation=validation_images, + autoencoder_loss = True, + verbose=True, + ) + + +Early stopping and Keras callbacks +---------------------------------- + +It can sometimes be useful to train the embedder until some plateau in training loss is met. In deep learning, early stopping is one way to do this. Keras provides custom `callbacks `_ that allow you to implement checks during training, such as early stopping. We can use callbacks, such as early stopping, with ParametricUMAP to stop training early based on a predefined training threshold, using the :python:`keras_fit_kwargs` argument: + +.. code:: python3 + + keras_fit_kwargs = {"callbacks": [ + tf.keras.callbacks.EarlyStopping( + monitor='loss', + min_delta=10**-2, + patience=10, + verbose=1, + ) + ]} + + embedder = ParametricUMAP( + verbose=True, + keras_fit_kwargs = keras_fit_kwargs, + n_training_epochs=20 + ) + + +We also passed in :python:`n_training_epochs = 20`, allowing early stopping to end training before 20 epochs are reached. + + +Additional important parameters +------------------------------- + +* **batch_size:** ParametricUMAP in trained over batches of edges randomly sampled from the UMAP graph, and then trained via gradient descent. ParametricUMAP defaults to a batch size of 1000 edges, but can be adjusted to a value that fits better on your GPU or CPU. +* **loss_report_frequency:** If set to 1, an epoch in in the Keras embedding refers to a single iteration over the graph computed in UMAP. Setting :python:`loss_report_frequency` to 10, would split up that epoch into 10 seperate epochs, for more frequent reporting. +* **n_training_epochs:** The number of epochs over the UMAP graph to train for (irrespective of :python:`loss_report_frequency`). Training the network for multiple epochs will result in better embeddings, but take longer. This parameter is different than :python:`n_epochs` in the base UMAP class, which corresponds to the maximum number of times an edge is trained in a single ParametricUMAP epoch. +* **optimizer:** The optimizer used to train the neural network. by default Adam (:python:`tf.keras.optimizers.Adam(1e-3)`) is used. You might be able to speed up or improve training by using a different optimizer. +* **parametric_embedding:** If set to false, a non-parametric embedding is learned, using the same code as the parametric embedding, which can serve as a direct comparison between parametric and non-parametric embedding using the same optimizer. The parametric embeddings are performed over the entire dataset simultaneously. + + +Extending the model +------------------- +You may want to customize parametric UMAP beyond what we have implemented in this package. To make it as easy as possible to tinker around with Parametric UMAP, we made a few Jupyter notebooks that show you how to extend Parametric UMAP to your own use-cases. + +* [Link coming soon] [Colab link] + +Citing our work +--------------- +If you use Parametric UMAP in your work, please cite our paper: + +[link coming soon] \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/performance.rst b/ext/TAL/applications/UMAP_fast/umap/doc/performance.rst new file mode 100644 index 00000000..16dc09dc --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/performance.rst @@ -0,0 +1,292 @@ +Performance Comparison of Dimension Reduction Implementations +============================================================= + +Different dimension reduction techniques can have quite different +computational complexity. Beyond the algorithm itself there is also the +question of how exactly it is implemented. These two factors can have a +significant role in how long it actually takes to run a given dimension +reduction. Furthermore the nature of the data you are trying to reduce +can also matter – mostly the involves the dimensionality of the original +data. Here we will take a brief look at the performance characterstics +of a number of dimension reduction implementations. + +To start let’s get the basic tools we’ll need loaded up – numpy and +pandas obviously, but also tools to get and resample the data, and the +time module so we can perform some basic benchmarking. + +.. code:: python + + import numpy as np + import pandas as pd + from sklearn.datasets import fetch_openml + from sklearn.utils import resample + import time + +Next we’ll need the actual dimension reduction implementations. For the +purposes of this explanation we’ll mostly stick with +`scikit-learn `__, but for the sake of +comparison we’ll also include the +`MulticoreTSNE `__ +implementation of t-SNE, and +`openTSNE `__ both of which +have historically had significantly better performance than scikit-learn +t-SNE (more recent versions of scikit-learn have improved t-SNE +performance). + +.. code:: python + + from sklearn.manifold import TSNE, LocallyLinearEmbedding, Isomap, MDS, SpectralEmbedding + from sklearn.decomposition import PCA + from MulticoreTSNE import MulticoreTSNE + from openTSNE import TSNE as OpenTSNE + from umap import UMAP + +Next we’ll need out plotting tools, and, of course, some data to work +with. For this performance comparison we’ll default to the now standard +benchmark of manifold learning: the MNIST digits dataset. We can use +scikit-learn’s ``fetch_openml`` to grab it for us. + +.. code:: python + + import matplotlib.pyplot as plt + import seaborn as sns + %matplotlib inline + +.. code:: python + + sns.set(context='notebook', + rc={'figure.figsize':(12,10)}, + palette=sns.color_palette('tab10', 10)) + +.. code:: python + + mnist = fetch_openml('mnist_784', version=1, return_X_y=True) + +.. code:: python + + mnist_data = mnist[0] + mnist_labels = mnist[1].astype(int) + +Now it is time to start looking at performance. To start with let’s look +at how performance scales with increasing dataset size. + +Performance scaling by dataset size +----------------------------------- + +As the size of a dataset increases the runtime of a given dimension +reduction algorithm will increase at varying rates. If you ever want to +run your algorithm on larger datasets you will care not just about the +comparative runtime on a single small dataset, but how the performance +scales out as you move to larger datasets. We can similate this by +subsampling from MNIST digits (via scikit-learn’s convenient +``resample`` utility) and looking at the runtime for varying sized +subsamples. Since there is some randomness involved here (both in the +subsample selection, and in some of the algorithms which have stochastic +aspects) we will want to run a few examples for each dataset size. We +can easily package all of this up in a simple function that will return +a convenient pandas dataframe of dataset sizes and runtimes given an +algorithm. + +.. code:: python + + def data_size_scaling(algorithm, data, sizes=[100, 200, 400, 800, 1600], n_runs=5): + result = [] + for size in sizes: + for run in range(n_runs): + subsample = resample(data, n_samples=size) + start_time = time.time() + algorithm.fit(subsample) + elapsed_time = time.time() - start_time + del subsample + result.append((size, elapsed_time)) + return pd.DataFrame(result, columns=('dataset size', 'runtime (s)')) + +Now we just want to run this for each of the various dimension reduction +implementations so we can look at the results. Since we don’t know how +long these runs might take we’ll start off with a very small set of +samples, scaling up to only 1600 samples. + +.. code:: python + + all_algorithms = [ + PCA(), + UMAP(), + MulticoreTSNE(), + OpenTSNE(), + TSNE(), + LocallyLinearEmbedding(), + SpectralEmbedding(), + Isomap(), + MDS(), + ] + performance_data = {} + for algorithm in all_algorithms: + if 'openTSNE' in str(algorithm.__class__): + alg_name = "OpenTSNE" + elif 'MulticoreTSNE' in str(algorithm.__class__): + alg_name = "MulticoreTSNE" + else: + alg_name = str(algorithm).split('(')[0] + + performance_data[alg_name] = data_size_scaling(algorithm, mnist_data, n_runs=5) + + print(f"[{time.asctime(time.localtime())}] Completed {alg_name}") + + +.. parsed-literal:: + + [Sat Feb 22 09:50:24 2020] Completed PCA + [Sat Feb 22 09:51:23 2020] Completed UMAP + [Sat Feb 22 09:53:24 2020] Completed MulticoreTSNE + [Sat Feb 22 10:00:50 2020] Completed OpenTSNE + [Sat Feb 22 10:02:22 2020] Completed TSNE + [Sat Feb 22 10:02:44 2020] Completed LocallyLinearEmbedding + [Sat Feb 22 10:03:06 2020] Completed SpectralEmbedding + [Sat Feb 22 10:03:31 2020] Completed Isomap + [Sat Feb 22 10:11:45 2020] Completed MDS + + +Now let’s plot the results so we can see what is going on. We’ll use +seaborn’s regression plot to interpolate the effective scaling. For some +algorithms this can be a little noisy, especially in this relatively +small dataset regime, but it will give us a good idea of what is going +on. + +.. code:: python + + for alg_name, perf_data in performance_data.items(): + sns.regplot('dataset size', 'runtime (s)', perf_data, order=2, label=alg_name) + plt.legend() + + +.. image:: images/performance_15_1.png + + +We can see straight away that there are some outliers here. It is notable that +openTSNE does poorly on small datasets. It does not have the scaling properties of MDS however; for +larger dataset sizes MDS is going to quickly become completely +unmanageable which openTSNE has fairly flat scaling. At the same time +MulticoreTSNE demonstrates that t-SNE can run fairly efficiently. It is +hard to tell much about the other implementations other than the fact +that PCA is far and away the fastest option. To see more we’ll have to +look at runtimes on larger dataset sizes. Both MDS, Isomap and SpectralEmbedding +will actually take too long to run so let’s restrict ourselves to +the fastest performing implementations and see what happens as we extend +out to larger dataset sizes. + +.. code:: python + + fast_algorithms = [ + PCA(), + UMAP(), + MulticoreTSNE(), + OpenTSNE(), + TSNE(), + LocallyLinearEmbedding(), + ] + fast_performance_data = {} + for algorithm in fast_algorithms: + if 'openTSNE' in str(algorithm.__class__): + alg_name = "OpenTSNE" + elif 'MulticoreTSNE' in str(algorithm.__class__): + alg_name = "MulticoreTSNE" + else: + alg_name = str(algorithm).split('(')[0] + + fast_performance_data[alg_name] = data_size_scaling(algorithm, mnist_data, + sizes=[1600, 3200, 6400, 12800, 25600], n_runs=4) + + print(f"[{time.asctime(time.localtime())}] Completed {alg_name}") + + +.. parsed-literal:: + + [Sat Feb 22 10:12:15 2020] Completed PCA + [Sat Feb 22 10:14:51 2020] Completed UMAP + [Sat Feb 22 11:16:05 2020] Completed MulticoreTSNE + [Sat Feb 22 11:50:17 2020] Completed OpenTSNE + [Sat Feb 22 13:06:38 2020] Completed TSNE + [Sat Feb 22 14:14:36 2020] Completed LocallyLinearEmbedding + + +.. code:: python + + for alg_name, perf_data in fast_performance_data.items(): + sns.regplot('dataset size', 'runtime (s)', perf_data, order=2, label=alg_name) + plt.legend() + + + +.. image:: images/performance_18_1.png + + +At this point we begin to see some significant differentiation among the +different implementations. In the earlier plot OpenTSNE looked to be +performing relatively poorly, but now the scaling effects kick in, and +we see that is is faster than most. Similarly MulticoreTSNE looked to be +slower than some of the other algorithms in th earlier plot, but as we +scale out to larger datasets we see that its relative scaling +performance is superior to the scikit-learn implementations of +TSNE and locally linear embedding. + +It is probably worth extending out further – up to the full MNIST digits +dataset. To manage to do that in any reasonable amount of time we’ll +have to restrict out attention to an even smaller subset of +implementations. We will pare things down to just OpenTSNE, +MulticoreTSNE, PCA and UMAP. + +.. code:: python + + very_fast_algorithms = [ + PCA(), + UMAP(), + MulticoreTSNE(), + OpenTSNE(), + ] + vfast_performance_data = {} + for algorithm in very_fast_algorithms: + if 'openTSNE' in str(algorithm.__class__): + alg_name = "OpenTSNE" + elif 'MulticoreTSNE' in str(algorithm.__class__): + alg_name = "MulticoreTSNE" + else: + alg_name = str(algorithm).split('(')[0] + + vfast_performance_data[alg_name] = data_size_scaling(algorithm, mnist_data, + sizes=[3200, 6400, 12800, 25600, 51200, 70000], n_runs=2) + + print(f"[{time.asctime(time.localtime())}] Completed {alg_name}") + + +.. parsed-literal:: + + [Sat Feb 22 14:15:22 2020] Completed PCA + [Sat Feb 22 14:18:59 2020] Completed UMAP + [Sat Feb 22 17:04:58 2020] Completed MulticoreTSNE + [Sat Feb 22 17:54:14 2020] Completed OpenTSNE + + +.. code:: python + + for alg_name, perf_data in vfast_performance_data.items(): + sns.regplot('dataset size', 'runtime (s)', perf_data, order=2, label=alg_name) + plt.legend() + + +.. image:: images/performance_21_1.png + + +Here we see UMAP’s advantages over t-SNE really coming to the forefront. +While UMAP is clearly slower than PCA, its scaling performance is +dramatically better than MulticoreTSNE, and, despite the impressive +scaling performance of openTSNE, UMAP continues to outperform it. Based +on the slopes of the lines, for even larger datasets the difference +between UMAP and t-SNE is only going to grow. + +This concludes our look at scaling by dataset size. The short summary is +that PCA is far and away the fastest option, but you are potentially +giving up a lot for that speed. UMAP, while not competitive with PCA, is +clearly the next best option in terms of performance among the +implementations explored here. Given the quality of results that UMAP +can provide we feel it is clearly a good option for dimension reduction. + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/plotting.rst b/ext/TAL/applications/UMAP_fast/umap/doc/plotting.rst new file mode 100644 index 00000000..8bf1e2c8 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/plotting.rst @@ -0,0 +1,464 @@ +Plotting UMAP results +===================== + +UMAP is often used for visualization by reducing data to 2-dimensions. +Since this is such a common use case the umap package now includes +utility routines to make plotting UMAP results simple, and provide a +number of ways to view and diagnose the results. Rather than seeking to +provide a comprehensive solution that covers all possible plotting needs +this umap extension seeks to provide a simple to use interface to make +the majority of plotting needs easy, and help provide sensible plotting +choices wherever possible. To get started looking at the plotting +options let's load a variety of data to work with. + +.. code:: python3 + + import sklearn.datasets + import pandas as pd + import numpy as np + import umap + +.. code:: python3 + + pendigits = sklearn.datasets.load_digits() + mnist = sklearn.datasets.fetch_openml('mnist_784') + fmnist = sklearn.datasets.fetch_openml('Fashion-MNIST') + +To start we will fit a UMAP model to the pendigits data. This is as +simple as running the fit method and assigning the result to a variable. + +.. code:: python3 + + mapper = umap.UMAP().fit(pendigits.data) + +If we want to do plotting we will need the ``umap.plot`` package. While +the umap package has a fairly small set of requirements it is worth +noting that if you want to using ``umap.plot`` you will need a variety +of extra libraries that are not in the default requirements for umap. In +particular you will need: + +- `matplotlib `__ +- `pandas `__ +- `datashader `__ +- `bokeh `__ +- `holoviews `__ + +All should be either pip or conda installable. With those in hand you +can import the ``umap.plot`` package. + +.. code:: python3 + + import umap.plot + + +Now that we have the package loaded, how do we use it? The most +straightforward thing to do is plot the umap results as points. We can +achieve this via the function ``umap.plot.points``. In its most basic +form you can simply pass the trained UMAP model to ``umap.plot.points``: + +.. code:: python3 + + umap.plot.points(mapper) + + +.. image:: images/plotting_8_2.png + + +As you can see we immediately get a scatterplot of the UMAP embedding. +Note that the function automatically selects a point-size based on the +data density, and watermarks the image with the UMAP parameters that +were used (this will include the metric if it is non-standard). The +function also returns the matplotlib axes object associated to the plot, +so further matplotlib functions, such as adding titles, axis labels etc. +can be applied by the user if required. + +It is common for data passed to UMAP to have an associated set of +labels, which may have been derived from ground-truth, from clustering, +or via other means. In such cases it is desirable to be able to color +the scatterplot according to the labelling. We can do this by simply +passing the array of label information in with the ``labels`` keyword. +The ``umap.plot.points`` function will color the data with a +categorical colormap according to the labels provided. + +.. code:: python3 + + umap.plot.points(mapper, labels=pendigits.target) + + +.. image:: images/plotting_10_1.png + + +Alternatively you may have extra data that is continuous rather than +categorical. In this case you will want to use a continuous colormap to +shade the data. Again this is straightforward to do -- pass in the +continuous data with the ``values`` keyword and data will be colored +accordingly using a continuous colormap. + +Furthermore, if you don't like the default color choices the +``umap.plot.points`` function offers a number of 'themes' that provide +predefined color choices. Themes include: + +- fire +- viridis +- inferno +- blue +- red +- green +- darkblue +- darkred +- darkgreen + +Here we will make use of the 'fire' theme to demonstrate how simple it +is to change the aesthetics. + +.. code:: python3 + + umap.plot.points(mapper, values=pendigits.data.mean(axis=1), theme='fire') + +.. image:: images/plotting_12_1.png + + +If you want greater control you can specify exact colormaps and +background colors. For example here we want to color the data by label, +but use a black background and use the 'Paired' colormap for the +categorical coloring (passed as ``color_key_cmap``; the ``cmap`` keyword +defines the continuous colormap). + +.. code:: python3 + + umap.plot.points(mapper, labels=pendigits.target, color_key_cmap='Paired', background='black') + +.. image:: images/plotting_14_1.png + + +Many more options are available including a ``color_key`` to specify a +dictionary mapping of discrete labels to colors, ``cmap`` to specify the +continous colormap, or the width and height of the resulting plot. +Again, this does not provide comprehensive control of the plot +aesthetics, but the goal here is to provide a simple to use interface +rather than the ability for the user to fine tune all aspects -- users +seeking such control are far better served making use of the individual +underlying packages (matplotlib, datashader, and bokeh) by themselves. + +Plotting larger datasets +------------------------ + +Once you have a lot of data it becomes easier for a simple scatter plot +to lie to you. Most notably overplotting, where markers for points +overlap and pile up on top of each other, can deceive you into thinking +that extremely dense clumps may only contain a few points. While there +are things that can be done to help remedy this, such as reducing the +point size, or adding an alpha channel, few are sufficient to be sure +the plot isn't subtly lying to you in some way. `This essay +`_ in +the datashader documentation does an excellent job of describing the +issues with overplotting, why the obvious solutions are not quite +sufficient, and how to get around the problem. To make life easier for +users the ``umap.plot`` package will automatically switch to using +datashader for rendering once your dataset gets large enough. This helps +to ensure you don't get fooled by overplotting. We can see this in +action by working with one of the larger datasets such as Fashion-MNIST. + +.. code:: python3 + + mapper = umap.UMAP().fit(fmnist.data) + +Having fit the data with UMAP we can call ``umap.plot.points`` exactly +as before, but this time, since the data is large enough to have +potential overplotting, datashader will be used in the background for +rendering. + +.. code:: python3 + + umap.plot.points(mapper) + + +.. image:: images/plotting_19_2.png + + +All the same plot options as before hold, so we can color by labels, and +apply the same themes, and it will all seamlessly use datashader for the +actual rendering. Thus, regardless of how much data you have +``umap.plot.points`` will render it well with a transparent user +interface. You, as a user, don't need to worry about switching to +plotting with datashader, or how to convert your plotting to its +slightly different API -- you can just use the same API and trust the +results you get. + +.. code:: python3 + + umap.plot.points(mapper, labels=fmnist.target, theme='fire') + + +.. image:: images/plotting_21_2.png + + +Interactive plotting, and hover tools +------------------------------------- + +Rendering good looking static plots is important, but what if you want +to be able to interact with your data -- pan around, and zoom in on the +clusters to see the finer structure? What if you want to annotate your +data with more complex labels than merely colors? Wouldn't it be good to +be able to hover over data points and get more information about the +individual point? Since this is a very common use case ``umap.plot`` +tries to make it easy to quickly generate such plots, and provide basic +utilities to allow you to have annotated hover tools working quickly. +Again, the goal is not to provide a comprehensive solution that can do +everything, but rather a simple to use and consistent API to get users +up and running fast. + +To make a good example of this let's use a subset of the Fashion MNIST +dataset. We can quickly train a new mapper object on that. + +.. code:: python3 + + mapper = umap.UMAP().fit(fmnist.data[:30000]) + +The goal is to be able to hover over different points and see data +associated with the given point (or points) under the cursor. For this +simple demonstration we'll just use the target information of the point. +To create hover information you need to construct a dataframe of all the +data you would like to appear in the hover. Each row should correspond +to a source of data points (appearing in the same order), and the columns +can provide whatever extra data you would like to display in the hover +tooltip. In this case we'll need a dataframe that can include the index +of the point, its target number, and the actual name of the type of +fashion item that target corresponds to. This is easy to quickly put +together using pandas. + +.. code:: python3 + + hover_data = pd.DataFrame({'index':np.arange(30000), + 'label':fmnist.target[:30000]}) + hover_data['item'] = hover_data.label.map( + { + '0':'T-shirt/top', + '1':'Trouser', + '2':'Pullover', + '3':'Dress', + '4':'Coat', + '5':'Sandal', + '6':'Shirt', + '7':'Sneaker', + '8':'Bag', + '9':'Ankle Boot', + } + ) + +For interactive use the ``umap.plot`` package makes use of bokeh. Bokeh +has several output methods, but in the approach we'll be outputting +inline in a notebook. We have to enable this using the +``output_notebook`` function. Alteratively we could use ``output_file`` +or other similar options -- see the bokeh documentation for more +details. + +.. code:: python3 + + umap.plot.output_notebook() + + + +.. raw:: html + + +
+ + Loading BokehJS ... +
+ + + + +Now we can make an interactive plot using ``umap.plot.interactive``. +This has a very similar API to the ``umap.plot.points`` approach, but +also supports a ``hover_data`` keyword which, if passed a suitable +dataframe, will provide hover tooltips in the interactive plot. Since +bokeh allows different outputs, to display it in the notebook we will +have to take the extra stop of calling ``show`` on the result. + +.. code:: python3 + + p = umap.plot.interactive(mapper, labels=fmnist.target[:30000], hover_data=hover_data, point_size=2) + umap.plot.show(p) + + + +.. raw:: html + :file: plotting_interactive_example.html + + + + + +We get the sort of result one would like -- a fully interactive plot +that can be zoomed in on, and more, but we also now have an interactive +hover tool which presents the data from the dataframe we constructed. +This allows a quick and easy method to get up and running with a richer +interactive exploration of your UMAP plot. ``umap.plot.interactive`` +supports all the same aesthetic parameters as ``umap.plot.points`` so +you can theme your plot, color by label or value, and other similar +operations explained above for ``umap.plot.points``. + +Plotting connectivity +--------------------- + +UMAP works by constructing an intermediate topological representation of +the approximate manifold the data may have been sampled from. In +practice this structure can be simplified down to a weighted graph. +Sometimes it can be beneficial to see how that graph (representing +connectivity in the manifold) looks with respect to the resulting +embedding. It can be used to better understand the embedding, and for +diagnostic purposes. To see the connectivity you can use the +``umap.plot.connectivity`` function. It works very similarly to the +``umap.plot.points`` function, and has the option as to whether to +display the embedding point, or just the connectivity. To start let's do +a simple plot showing the points: + +.. code:: python3 + + umap.plot.connectivity(mapper, show_points=True) + + +.. image:: images/plotting_32_2.png + + +As with ``umap.plot.points`` there are options to control the basic +aesthetics, including theme options and an ``edge_cmap`` keyword +argument to specify the colormap used for displaying the edges. + +Since this approach already leverages datashader for edge plotting, we +can go a step further and make use of the edge-bundling options +available in datashader. This can provide a less busy view of +connectivity, but can be expensive to compute, particularly for larger +datasets. + +.. code:: python3 + + umap.plot.connectivity(mapper, edge_bundling='hammer') + + + +.. image:: images/plotting_34_2.png + + +Diagnostic plotting +------------------- + +Plotting the connectivity provides at least one basic diagnostic view +that helps a user understand what is going on with an embedding. More +views on data are better, of course, so ``umap.plot`` includes a +``umap.plot.diagnostic`` function that can provide various diagnostic +plots. We'll look at a few of them here. To do so we'll use the full +MNIST digits data set. + +.. code:: python3 + + mapper = umap.UMAP().fit(mnist.data) + +The first diagnostic type is a Principal Components Analysis based +diagnostic, which you can select with ``diagnostic_type='pca'``. The +essence of the approach is that we can use PCA, which preserves global +structure, to reduce the data to three dimensions. If we scale the +results to fit in a 3D cube we can convert the 3D PCA coordinates of +each point into an RGB description of a color. By then coloring the +points in the UMAP embedding with the colors induced by the PCA it is +possible to get a sense of how some of the more large scale global +structure has been represented in the embedding. + +.. code:: python3 + + umap.plot.diagnostic(mapper, diagnostic_type='pca') + + +.. image:: images/plotting_38_1.png + + +What we are looking for here is a generally smooth transition of colors, +and an overall layout that broadly respects the color transitions. In +this case the far left has a bottom cluster that transitions from dark +green at the bottom to blue at the top, and this matches well with the +cluster in the upper right which have a similar shade of blue at the +bottom before transitioning to more cyan and blue. In contast in the +right of the plot the lower cluster runs from purplish pink to green +from top to bottom, while the cluster above it has its bottom edge more +purple than green, suggesting that perhaps one or the other of these +clusters has been flipped vertically during the optimization process, +and this was never quite corrected. + +An alternative, but similar, approach is to use vector quantization as +the method to generate a 3D embedding to generate colors. Vector +quantization effectively finds 3 representative centers for the data, +and then describes each data point in terms of its distance to these +centers. Clearly this, again, captures a lot of the broad global +structure of the data. + +.. code:: python3 + + umap.plot.diagnostic(mapper, diagnostic_type='vq') + + +.. image:: images/plotting_40_1.png + + +Again we are looking for largely smooth transitions, and for related +colors to match up between clusters. This view supports the fact that +the left hand side of the embedding has worked well, but looking at the +right hand side it seems clear that it is the upper two of the clusters +that has been inadvertently flipped vertically. By contrasting views +like this one can get a better sense of how well the embedding is +working. + +For a different perspective we can look at approximations of the local +dimension around each data point. Ideally the local dimension should +match the embedding dimension (although this is often a lot to hope for. +In practice when the local dimension is high this represents points (or +areas of the space) that UMAP will have a harder time embedding as well. +Thus one can trust the embedding to be more accurate in regions where +the points have consistently lower local dimension. + +.. code:: python3 + + local_dims = umap.plot.diagnostic(mapper, diagnostic_type='local_dim') + + + +.. image:: images/plotting_42_0.png + + +As you can see, the local dimension of the data varies quite widely across +the data. In particular the lower left cluster has the lowest local +dimension -- this is actually unsurprising as this is the cluster +corresponding to the digits 1: there are relatively few degrees of +freedom over how a person draws a number one, and so the resulting local +dimension is lower. In contrast the clusters in the middle have a much +higher local dimension. We should expect the embedidng to be a little +less accurate in these regions: it is hard to represent seven +dimensional data well in only two dimensions, and compromises will need +to be made. + +The final diagnostic we'll look at is how well local neighborhoods are +preserved. We can measure this in terms of the Jaccard index of the +local neighborhood in the high dimensional space compared to the +equivalent neighborhood in the embedding. The Jaccard index is +essentially the ratio of the number of neighbors that the two +neighborhoods have in common over the total number of unique neighbors +across the two neighborhoods. Higher values mean that the local +neighborhood has been more accurately preserved. + +.. code:: python3 + + umap.plot.diagnostic(mapper, diagnostic_type='neighborhood') + + +.. image:: images/plotting_44_1.png + + +As one might expect the local neighborhood preservation tends to be a +lot better for those points that had a lower local dimension (as seen in +the last plot). There is also a tendency for the edges of clusters +(where there were clear boundaries to be followed) to have a better +preservation of neighborhoods than the centers of the clusters that had +higher local dimension. Again, this provides a view on which areas of +the embedding you can have greater trust in, and which regions had to +make compromises to embed into two dimensions. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/plotting_example_interactive.py b/ext/TAL/applications/UMAP_fast/umap/doc/plotting_example_interactive.py new file mode 100644 index 00000000..177f9af6 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/plotting_example_interactive.py @@ -0,0 +1,32 @@ +import sklearn.datasets +import pandas as pd +import numpy as np +import umap +import umap.plot + +fmnist = sklearn.datasets.fetch_openml("Fashion-MNIST") + +mapper = umap.UMAP().fit(fmnist.data[:30000]) + +hover_data = pd.DataFrame({"index": np.arange(30000), "label": fmnist.target[:30000]}) +hover_data["item"] = hover_data.label.map( + { + "0": "T-shirt/top", + "1": "Trouser", + "2": "Pullover", + "3": "Dress", + "4": "Coat", + "5": "Sandal", + "6": "Shirt", + "7": "Sneaker", + "8": "Bag", + "9": "Ankle Boot", + } +) + +umap.plot.output_file("plotting_interactive_example.html") + +p = umap.plot.interactive( + mapper, labels=fmnist.target[:30000], hover_data=hover_data, point_size=2 +) +umap.plot.show(p) diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/plotting_interactive_example.html b/ext/TAL/applications/UMAP_fast/umap/doc/plotting_interactive_example.html new file mode 100644 index 00000000..eb7244f1 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/plotting_interactive_example.html @@ -0,0 +1,99821 @@ + + + + + + + Bokeh Plot + + + + + + + + + + + + + + + +
+ + + + + + + + \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/release_notes.rst b/ext/TAL/applications/UMAP_fast/umap/doc/release_notes.rst new file mode 100644 index 00000000..fda2d6a6 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/release_notes.rst @@ -0,0 +1,45 @@ +Release Notes +============= + +Some notes on new features in various releases + +What's new in 0.5 +----------------- + +* ParametricUMAP learns embeddings with neural networks. +* AlignedUMAP can align multiple embeddings using relations between datasets. +* DensMAP can preserve local density information in embeddings. +* UMAP now depends on PyNNDescent, but has faster more parallel performance as a result. +* UMAP now supports an ``update`` method to add new data and retrain. +* Various performance improvements and bug fixes. +* Additional plotting support, including text searching in interactive plots. +* Support for "maximal distances" in neighbor graphs. + +What's new in 0.4 +----------------- + +* Inverse transform method. Generate points in the original space corresponding to points in embedded space. (Thanks to Joseph Courtney) +* Different embedding spaces. Support for embedding to a variety of different spaces other than Euclidean. (Thanks to Joseph Courtney) +* New metrics, including Hellinger distance for sparse count data. +* New discrete/label metrics, including hierarchical categories, counts, ordinal data, and string edit distance. +* Support for parallelism in neighbor search and layout optimization. (Thanks to Tom White) +* Support for alternative methods to handling duplicated data samples. (Thanks to John Healy) +* New plotting methods for fast and easy plots. +* Initial support for dataframe embedding -- still experimental, but worth trying. +* Support for transform methods with sparse data. +* Multithreading support when no random seed is set. + + +What's new in 0.3 +----------------- + +* Supervised and semi-supervised dimension reduction. Support for using labels or partial labels for dimension reduction. +* Transform method. Support for adding new unseen points to an existing embedding. +* Performance improvements. + + +What's new in 0.2 +----------------- + +* A new layout algorithm that handles large datasets (more) correctly. +* Performance improvements. \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/reproducibility.rst b/ext/TAL/applications/UMAP_fast/umap/doc/reproducibility.rst new file mode 100644 index 00000000..f70ff702 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/reproducibility.rst @@ -0,0 +1,197 @@ + +UMAP Reproducibility +==================== + +UMAP is a stochastic algorithm -- it makes use of randomness both to +speed up approximation steps, and to aid in solving hard optimization +problems. This means that different runs of UMAP can produce different +results. UMAP is relatively stable -- thus the variance between runs +should ideally be relatively small -- but different runs may have +variations none the less. To ensure that results can be reproduced +exactly UMAP allows the user to set a random seed state. + +Since version 0.4 UMAP also support multi-threading for faster +performance; when performing optimization this exploits the fact that +race conditions between the threads are acceptable within certain +optimization phases. Unfortunately this means that the randomness in +UMAP outputs for the multi-threaded case depends not only on the random +seed input, but also on race conditions between threads during +optimization, over which no control can be had. This means that +multi-threaded UMAP results cannot be explicitly reproduced. + +In this tutorial we'll look at how UMAP can be used in multi-threaded +mode for performance purposes, and alternatively how we can fix random +states to ensure exact reproducibility at the cost of some performance. +First let's load the relevant libraries and get some data; in this case +the MNIST digits dataset. + +.. code:: python3 + + import numpy as np + import sklearn.datasets + import umap + import umap.plot + +.. code:: python3 + + data, labels = sklearn.datasets.fetch_openml( + 'mnist_784', version=1, return_X_y=True + ) + +With data in hand let's run UMAP on it, and note how long it takes to +run: + +.. code:: python3 + + %%time + mapper1 = umap.UMAP().fit(data) + + +.. parsed-literal:: + + CPU times: user 3min 18s, sys: 3.84 s, total: 3min 22s + Wall time: 1min 29s + + +The thing to note here is that the "Wall time" is significantly smaller +than the CPU time -- this means that multiple CPU cores were used. For +this particular demonstration I am making use of the latest version of +PyNNDescent for nearest neighbor search (UMAP will use it if you have it +installed) which supports multi-threading as well. The result is a very +fast fitting to the data that does an effective job of using several +cores. If you are on a large server with many cores available and don't +wish to use them *all* (which is the default situation) you can +currently control the number of cores used by setting the numba +environment variable ``NUMBA_NUM_THREADS``; see the `numba +documentation `__ +for more details. + +Now let's plot our result to see what the embedding looks like: + +.. code:: python3 + + umap.plot.points(mapper1, labels=labels) + + +.. image:: images/reproducibility_6_1.png + + +Now, let's run UMAP again and compare the results to that of our first +run. + +.. code:: python3 + + %%time + mapper2 = umap.UMAP().fit(data) + + +.. parsed-literal:: + + CPU times: user 2min 53s, sys: 4.16 s, total: 2min 57s + Wall time: 1min 5s + + +You will note that this time we ran *even faster*. This is because +during the first run numba was still JIT compiling some of the code in +the background. In contrast, this time that work has already been done, +so it no longer takes up any of our run-time. We see that we are still +making use of mutliple cores well. + +Now let's plot the results of this second run and compare to the first: + +.. code:: python3 + + umap.plot.points(mapper2, labels=labels) + + +.. image:: images/reproducibility_10_1.png + + +Qualitatively this looks very similar, but a little closer inspection +will quickly show that the results are actually different between the +runs. Note that even in versions of UMAP prior to 0.4 this would have +been the case -- since we fixed no specific random seed, and were thus +using the current random state of the system which will naturally differ +between runs. This is the default behaviour, as is standard with sklearn +estimators that are stochastic. Rather than having a default random seed +the user is required to explicitly provide one should they want a +reproducible result. As noted by Vito Zanotelli + + ... setting a random seed is like signing a waiver "I am aware that + this is a stochastic algorithm and I have done sufficient tests to + confirm that my main conclusions are not affected by this + randomness". + +With that in mind, let's see what happens if we set an explicit +``random_state`` value: + +.. code:: python3 + + %%time + mapper3 = umap.UMAP(random_state=42).fit(data) + + +.. parsed-literal:: + + CPU times: user 2min 27s, sys: 4.16 s, total: 2min 31s + Wall time: 1min 56s + + +The first thing to note that that this run took significantly longer +(despite having all the functions JIT compiled by numba already). Then +note that the Wall time and CPU times are now much closer to each other +-- we are no longer exploiting multiple cores to anywhere near the same +degree. This is because by setting a ``random_state`` we are effectively +turning off any of the multi-threading that does not support explicit +reproducibility. Let's plot the results: + +.. code:: python3 + + umap.plot.points(mapper3, labels=labels) + + +.. image:: images/reproducibility_14_1.png + + +We arrive at much the same results as before from a qualitative point of +view, but again inspection will show that there are some differences. +More importantly this result should now be reproducible. Thus we can run +UMAP again, with the same ``random_state`` set ... + +.. code:: python3 + + %%time + mapper4 = umap.UMAP(random_state=42).fit(data) + + +.. parsed-literal:: + + CPU times: user 2min 26s, sys: 4.13 s, total: 2min 30s + Wall time: 1min 54s + + +Again, this takes longer than the earlier runs with no ``random_state`` +set. However when we plot the results of the second run we see that they +look not merely qualitatively similar, but instead appear to be almost +identical: + +.. code:: python3 + + umap.plot.points(mapper4, labels=labels) + +.. image:: images/reproducibility_18_1.png + + +We can, in fact, check that the results are identical by verifying that +each and every coordinate of the resulting embeddings match perfectly: + +.. code:: python3 + + np.all(mapper3.embedding_ == mapper4.embedding_) + + +.. parsed-literal:: + + True + +So we have, in fact, reproduced the embedding exactly. diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/scientific_papers.rst b/ext/TAL/applications/UMAP_fast/umap/doc/scientific_papers.rst new file mode 100644 index 00000000..a9028298 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/scientific_papers.rst @@ -0,0 +1,104 @@ +Scientific Papers +================= + +UMAP has been used in a wide variety of scientific publications from a diverse range of +fields. Here we will highlight a small selection of papers that demonstrate both +the depth of analysis, and breadth of subjects, UMAP can be used for. These range from biology, +to machine learning, and even social science. + + +The single-cell transcriptional landscape of mammalian organogenesis +-------------------------------------------------------------------- +A detailed look at the development of mouse embryos from a single-cell view. UMAP +is used as a core piece of The Monocle3 software suite for identifying cell types +and trajectories. This was a major paper in Nature, demonstrating the power +of UMAP for large scale scientific endeavours. + +.. image:: images/organogenesis_paper.png + :width: 400px + +`Link to the paper `__ + +A lineage-resolved molecular atlas of C. elegans embryogenesis at single-cell resolution +---------------------------------------------------------------------------------------- +Still in the realm of single cell biology this paper looks at the developmental +landscape of the round-word C. elegans. UMAP is used for detailed analysis of +the developmental trajectories of cells, looking at global scales, and then +digging down to look at individual organs. The result is an impressive +array of UMAP visualisations that tease out ever finer structures in +cellular development. + +.. image:: images/c_elegans_3d.jpg + :width: 400px + +`Link to the paper `__ + +Exploring Neural Networks with Activation Atlases +------------------------------------------------- +Understanding the image processing capabilities (and deficits!) of modern +convolutional neural networks is a challenge. This interactive paper from +Distill seeks to provide a way to "peek inside the black box" by looking +at the activations throughout the network. By mapping this high dimensional +data down to 2D with UMAP the authors can construct an "atlas" of how +different images are perceived by the network. + +.. image:: images/activation_atlas.png + :width: 400px + +`Link to the paper `__ + +TimeCluster: dimension reduction applied to temporal data for visual analytics +------------------------------------------------------------------------------ +An interesting approach to time-series analysis, targeted toward cases where the +time series has repeating patterns -- though no necessarily of a consistently +periodic nature. The approach involves dimension reduction and clustering +of sliding window blocks of the time-series. The result is a map where +repeating behaviour is exposed as loop structures. This can be useful +for both clustering similar blocks within a time-series, or finding +outliers. + +.. image:: images/time_cluster.png + :width: 400px + +`Link to the paper `__ + +Dimensionality reduction for visualizing single-cell data using UMAP +-------------------------------------------------------------------- +An early paper on applying UMAP to single-cell biology data. It looks at +both, gene-expression data and flow-cytometry data, and compares UMAP to +t-SNE both in terms of performance and quality of results. This is a good +introduction to using UMAP for single-cell biology data. + +.. image:: images/single_cell_umap.jpg + :width: 400px + +`Link to the paper `__ + + +Revealing multi-scale population structure in large cohorts +----------------------------------------------------------- +A paper looking at population genetics which uses UMAP as a means +to visualise population structures. This produced some intriguing +visualizations, and was one of the first of several papers taking +this visualization approach. It also includes some novel visualizations +using UMAP projections to 3D as RGB color specifications for +data points, allowing the UMAP structure to be visualized in +geographic maps based on where the samples were drawn from. + +.. image:: images/population_umap.jpg + :width: 400px + +`Link to the paper `__ + + +Understanding Vulnerability of Children in Surrey +-------------------------------------------------- +An example of the use of UMAP in sociological studies -- in this case +looking at children in Surrey, British Columbia. Here UMAP is used as +a tool to aid in general data analysis, and proves effective for the +tasks to which it was put. + +.. image:: images/umap_surrey.png + :width: 400px + +`Link to the paper `__ \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/sparse.rst b/ext/TAL/applications/UMAP_fast/umap/doc/sparse.rst new file mode 100644 index 00000000..44473fc4 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/sparse.rst @@ -0,0 +1,407 @@ + +UMAP on sparse data +=================== + +Sometimes datasets get very large, and potentially very very high +dimensional. In many such cases, however, the data itself is sparse -- +that is, while there are many many features, any given sample has only a +small number of non-zero features observed. In such cases the data can +be represented much more efficiently in terms of memory usage by a +sparse matrix data structure. It can be hard to find dimension reduction +techniques that work directly on such sparse data -- often one applies a +basic linear technique such as ``TruncatedSVD`` from sklearn (which does +accept sparse matrix input) to get the data in a format amenable to +other more advanced dimension reduction techniques. In the case of UMAP +this is not necessary -- UMAP can run directly on sparse matrix input. +This tutorial will walk through a couple of examples of doing this. +First we'll need some libraries loaded. We need ``numpy`` obviously, but +we'll also make use of ``scipy.sparse`` which provides sparse matrix +data structures. One of our examples will be purely mathematical, and +we'll make use of ``sympy`` for that; the other example is test based +and we'll use sklearn for that (specifically +``sklearn.feature_extraction.text``). Beyond that we'll need umap, and +plotting tools. + +.. code:: python3 + + import numpy as np + import scipy.sparse + import sympy + import sklearn.datasets + import sklearn.feature_extraction.text + import umap + import umap.plot + import matplotlib.pyplot as plt + %matplotlib inline + +A mathematical example +---------------------- + +Our first example constructs a sparse matrix of data out of pure math. +This example is inspired by the work of `John +Williamson `__, and +if you haven't looked at that work you are strongly encouraged to do so. +The dataset under consideration will be the integers. We will represent +each integer by a vector of its divisibility by distinct primes. Thus +our feature space is the space of prime numbers (less than or equal to +the largest integer we will be considering) -- potentially very high +dimensional. In practice a given integer is divisible by only a small +number of distinct primes, so each sample will be mostly made up of +zeros (all the primes that the number is not divisible by), and thus we +will have a very sparse dataset. + +To get started we'll need a list of all the primes. Fortunately we have +``sympy`` at our disposal and we can quickly get that information with a +single call to ``primerange``. We'll also need a dictionary mapping the +different primes to the column number they correspond to in our data +structure; effectively we'll just be enumerating the primes. + +.. code:: python3 + + primes = list(sympy.primerange(2, 110000)) + prime_to_column = {p:i for i, p in enumerate(primes)} + +Now we need to construct our data in a format we can put into a sparse +matrix easily. At this point a little background on sparse matrix data +structures is useful. For this purpose we'll be using the so called +`"LIL" +format `__. +LIL is short for "List of Lists", since that is how the data is +internally stored. There is a list of all the rows, and each row is +stored as a list giving the column indices of the non-zero entries. To +store the data values there is a parallel structure containing the value +of the entry corresponding to a given row and column. + +To put the data together in this sort of format we need to construct +such a list of lists. We can do that by iterating over all the integers +up to a fixed bound, and for each integer (i.e. each row in our dataset) +generating the list of column indices which will be non-zero. The column +indices will simply be the indices corresponding to the primes that +divide the number. Since ``sympy`` has a function ``primefactors`` which +returns a list of the unique prime factors of any integer we simply need +to map those through our dictionary to covert the primes into column +numbers. + +Parallel to that we'll construct the corresponding structure of values +to insert into a matrix. Since we are only concerned with divisibility +this will simply be a one in every non-zero entry, so we can just add a +list of ones of the appropriate length for each row. + +.. code:: python3 + + %%time + lil_matrix_rows = [] + lil_matrix_data = [] + for n in range(100000): + prime_factors = sympy.primefactors(n) + lil_matrix_rows.append([prime_to_column[p] for p in prime_factors]) + lil_matrix_data.append([1] * len(prime_factors)) + + +.. parsed-literal:: + + CPU times: user 2.07 s, sys: 26.4 ms, total: 2.1 s + Wall time: 2.1 s + + +Now we need to get that into a sparse matrix. Fortunately the +``scipy.sparse`` package makes this easy, and we've already built the +data in a fairly useful structure. First we create a sparse matrix of +the correct format (LIL) and the right shape (as many rows as we have +generated, and as many columns as there are primes). This is essentially +just an empty matrix however. We can fix that by setting the ``rows`` +attribute to be the rows we have generated, and the ``data`` attribute +to be the corresponding structure of values (all ones). The result is a +sparse matrix data structure which can then be easily manipulated and +converted into other sparse matrix formats easily. + +.. code:: python3 + + factor_matrix = scipy.sparse.lil_matrix((len(lil_matrix_rows), len(primes)), dtype=np.float32) + factor_matrix.rows = np.array(lil_matrix_rows) + factor_matrix.data = np.array(lil_matrix_data) + factor_matrix + + + + +.. parsed-literal:: + + <100000x10453 sparse matrix of type '' + with 266398 stored elements in LInked List format> + + + +As you can see we have a matrix with 100000 rows and over 10000 columns. +If we were storing that as a numpy array it would take a great deal of +memory. In practice, however, there are only 260000 or so entries that +are not zero, and that's all we really need to store, making it much +more compact. + +The question now is how can we feed that sparse matrix structure into +UMAP to have it learn an embedding. The answer is surprisingly +straightforward -- we just hand it directly to the fit method. Just like +other sklearn estimators that can handle sparse input UMAP will detect +the sparse matrix and just do the right thing. + +.. code:: python3 + + %%time + mapper = umap.UMAP(metric='cosine', random_state=42, low_memory=True).fit(factor_matrix) + + +.. parsed-literal:: + + CPU times: user 9min 36s, sys: 6.76 s, total: 9min 43s + Wall time: 9min 7s + + +That was easy! But is it really working? We can easily plot the results: + +.. code:: python3 + + umap.plot.points(mapper, values=np.arange(100000), theme='viridis') + + +.. image:: images/sparse_11_1.png + + +And this looks very much in line with the results `John Williamson +got `__ with the +proviso that we only used 100,000 integers instead of 1,000,000 to +ensure that most users should be able to run this example (the full +million may require a large memory compute node). So it seems like this +is working well. The next question is whether we can use the +``transform`` functionality to map new data into this space. To test +that we'll need some more data. Fortunately there are more integers. +We'll grab the next 10,000 and put them in a sparse matrix, much as we +did for the first 100,000. + +.. code:: python3 + + %%time + lil_matrix_rows = [] + lil_matrix_data = [] + for n in range(100000, 110000): + prime_factors = sympy.primefactors(n) + lil_matrix_rows.append([prime_to_column[p] for p in prime_factors]) + lil_matrix_data.append([1] * len(prime_factors)) + + +.. parsed-literal:: + + CPU times: user 214 ms, sys: 1.99 ms, total: 216 ms + Wall time: 222 ms + + +.. code:: python3 + + new_data = scipy.sparse.lil_matrix((len(lil_matrix_rows), len(primes)), dtype=np.float32) + new_data.rows = np.array(lil_matrix_rows) + new_data.data = np.array(lil_matrix_data) + new_data + + + + +.. parsed-literal:: + + <10000x10453 sparse matrix of type '' + with 27592 stored elements in LInked List format> + + + +To map the new data we generated we can simply hand it to the +``transform`` method of our trained model. This is a little slow, but it +does work. + +.. code:: python3 + + new_data_embedding = mapper.transform(new_data) + +And we can plot the results. Since we just got the locations of the +points this time (rather than a model) we'll have to resort to +matplotlib for plotting. + +.. code:: python3 + + fig = plt.figure(figsize=(12,12)) + ax = fig.add_subplot(111) + plt.scatter(new_data_embedding[:, 0], new_data_embedding[:, 1], s=0.1, c=np.arange(10000), cmap='viridis') + ax.set(xticks=[], yticks=[], facecolor='black'); + + + +.. image:: images/sparse_18_0.png + + +The color scale is different in this case, but you can see that the data +has been mapped into locations corresponding to the various structures +seen in the original embedding. Thus, even with large sparse data we can +embed the data, and even add new data to the embedding. + +A text analysis example +----------------------- + +Let's look at a more classical machine learning example of working with +sparse high dimensional data -- working with text documents. Machine +learning on text is hard, and there is a great deal of literature on the +subject, but for now we'll just consider a basic approach. Part of the +difficulty of machine learning with text is turning language into +numbers, since numbers are really all most machine learning algorithms +understand (at heart anyway). One of the most straightforward ways to do +this for documents is what is known as the `"bag-of-words" +model `__. In this +model we view a document as simply a multi-set of the words contained in +it -- we completely ignore word order. The result can be viewed as a +matrix of data by setting the feature space to be the set of all words +that appear in any document, and a document is represented by a vector +where the value of the *i*\ th entry is the number of times the *i*\ th +word occurs in that document. This is a very common approach, and is +what you will get if you apply sklearn's ``CountVectorizer`` to a text +dataset for example. The catch with this approach is that the feature +space is often *very* large, since we have a feature for each and every +word that ever occurs in the entire corpus of documents. The data is +sparse however, since most documents only use a small portion of the +total possible vocabulary. Thus the default output format of +``CountVectorizer`` (and other similar feature extraction tools in +sklearn) is a ``scipy.sparse`` format matrix. + +For this example we'll make use of the classic 20newsgroups dataset, a +sampling of newsgroup messages from the old NNTP newsgroup system +covering 20 different newsgroups. The ``sklearn.datasets`` module can +easily fetch the data, and, in fact, we can fetch a pre-vectorized +version to save us the trouble of running ``CountVectorizer`` ourselves. +We'll grab both the training set, and the test set for later use. + +.. code:: python3 + + news_train = sklearn.datasets.fetch_20newsgroups_vectorized(subset='train') + news_test = sklearn.datasets.fetch_20newsgroups_vectorized(subset='test') + +If we look at the actual data we have pulled back, we'll see that +sklearn has run a ``CountVectorizer`` and produced the data in the sparse +matrix format. + +.. code:: python3 + + news_train.data + + + + +.. parsed-literal:: + + <11314x130107 sparse matrix of type '' + with 1787565 stored elements in Compressed Sparse Row format> + + + +The value of the sparse matrix format is immediately obvious in this +case; while there are only 11,000 samples there are 130,000 features! If +the data were stored in a standard ``numpy`` array we would be using up +10GB of memory! And most of that memory would simply be storing the +number zero, over and over again. In the sparse matrix format it easily fits +in memory on most machines. This sort of dimensionality of data is very +common with text workloads. + +The raw counts are, however, not ideal since common words such as "the" and +"and" will dominate the counts for most documents, while contributing +very little information about the actual content of the document. We can +correct for this by using a ``TfidfTransformer`` from sklearn, which +will convert the data into `TF-IDF +format `__. There are lots +of ways to think about the transformation done by TF-IDF, but I like to +think of it intuitively as follows. The information content of a word +can be thought of as (roughly) proportional to the negative log of the +frequency of the word; the more often a word is used, the less +information it tends to carry, and infrequent words carry more +information. What TF-IDF is going to do can be thought of as akin to +re-weighting the columns according to the information content of the +word associated to that column. Thus the common words like "the" and +"and" will get down-weighted, as carrying less information about the +document, while infrequent words will be deemed more imporant and have +their associated columns up-weighted. We can apply this transformation +to both the train and test sets (using the same transformer trained on +the training set). + +.. code:: python3 + + tfidf = sklearn.feature_extraction.text.TfidfTransformer(norm='l1').fit(news_train.data) + train_data = tfidf.transform(news_train.data) + test_data = tfidf.transform(news_test.data) + +The result is still a sparse matrix, since TF-IDF doesn't change the +zero elements at all, nor the number of features. + +.. code:: python3 + + train_data + + + + +.. parsed-literal:: + + <11314x130107 sparse matrix of type '' + with 1787565 stored elements in Compressed Sparse Row format> + + + +Now we need to pass this very high dimensional data to UMAP. Unlike some +other non-linear dimension reduction techniques we don't need to apply +PCA first to get the data down to a reasonable dimensionality; nor do we +need to use other techniques to reduce the data to be able to be +represented as a dense ``numpy`` array; we can work directly on the +130,000 dimensional sparse matrix. + +.. code:: python3 + + %%time + mapper = umap.UMAP(metric='hellinger', random_state=42).fit(train_data) + + +.. parsed-literal:: + + CPU times: user 8min 40s, sys: 3.07 s, total: 8min 44s + Wall time: 8min 43s + + +Now we can plot the results, with labels according to the target +variable of the data -- which newsgroup the posting was drawn from. + +.. code:: python3 + + umap.plot.points(mapper, labels=news_train.target) + + + + +.. image:: images/sparse_31_1.png + + +We can see that even going directly from a 130,000 dimensional space +down to only 2 dimensions UMAP has done a decent job of separating out +many of the different newsgroups. + +We can now attempt to add the test data to the same space using the +``transform`` method. + +.. code:: python3 + + test_embedding = mapper.transform(test_data) + +While this is somewhat expensive computationally, it does work, and we +can plot the end result: + +.. code:: python3 + + fig = plt.figure(figsize=(12,12)) + ax = fig.add_subplot(111) + plt.scatter(test_embedding[:, 0], test_embedding[:, 1], s=1, c=news_test.target, cmap='Spectral') + ax.set(xticks=[], yticks=[]); + + + +.. image:: images/sparse_35_0.png + diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/supervised.rst b/ext/TAL/applications/UMAP_fast/umap/doc/supervised.rst new file mode 100644 index 00000000..7f334894 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/supervised.rst @@ -0,0 +1,497 @@ + +UMAP for Supervised Dimension Reduction and Metric Learning +=========================================================== + +While UMAP can be used for standard unsupervised dimension reduction the +algorithm offers significant flexibility allowing it to be extended to +perform other tasks, including making use of categorical label +information to do supervised dimension reduction, and even metric +learning. We'll look at some examples of how to do that below. + +First we will need to load some base libraries -- ``numpy``, obviously, +but also ``mnist`` to read in the Fashion-MNIST data, and matplotlib and +seaborn for plotting. + +.. code:: python3 + + import numpy as np + from mnist.loader import MNIST + import matplotlib.pyplot as plt + %matplotlib inline + import seaborn as sns + sns.set(style='white', context='poster') + +Our example dataset for this exploration will be the `Fashion-MNIST +dataset from Zalando +Research `__. It is +designed to be a drop-in replacement for the classic MNIST digits +dataset, but uses images of fashion items (dresses, coats, shoes, bags, +etc.) instead of handwritten digits. Since the images are more complex +it provides a greater challenge than MNIST digits. We can load it in +(after downloading the dataset) using the `mnist +library `__. We can then package +up the train and test sets into one large dataset, normalise the values +(to be in the range [0,1]), and set up labels for the 10 classes. + +.. code:: python3 + + mndata = MNIST('fashion-mnist/data/fashion') + train, train_labels = mndata.load_training() + test, test_labels = mndata.load_testing() + data = np.array(np.vstack([train, test]), dtype=np.float64) / 255.0 + target = np.hstack([train_labels, test_labels]) + classes = [ + 'T-shirt/top', + 'Trouser', + 'Pullover', + 'Dress', + 'Coat', + 'Sandal', + 'Shirt', + 'Sneaker', + 'Bag', + 'Ankle boot'] + +Next we'll load the ``umap`` library so we can perform dimension reduction on +this dataset. + +.. code:: python3 + + import umap + +UMAP on Fashion MNIST +--------------------- + +First we'll just do standard unsupervised dimension reduction using UMAP +so we have a baseline of what the results look like for later +comparison. This is simply a matter of instantiating a :class:`~umap.umap_.UMAP` object (in +this case setting the :attr:`~umap.umap_.UMAP.n_neighbors` parameter to be 5 -- we are +interested mostly in very local information), then calling the +:meth:`~umap.umap_.UMAP.fit_transform` method with the data we wish to reduce. By default +UMAP reduces to two dimensions, so we'll be able to view the results as +a scatterplot. + +.. code:: python3 + + %%time + embedding = umap.UMAP(n_neighbors=5).fit_transform(data) + + +.. parsed-literal:: + + CPU times: user 1min 45s, sys: 7.22 s, total: 1min 52s + Wall time: 1min 26s + + +That took a little time, but not all that long considering it is 70,000 +data points in 784 dimensional space. We can simply plot the results as +a scatterplot, colored by the class of the fashion item. We can use +matplotlib's colorbar with suitable tick-labels to give us the color key. + +.. code:: python3 + + fig, ax = plt.subplots(1, figsize=(14, 10)) + plt.scatter(*embedding.T, s=0.3, c=target, cmap='Spectral', alpha=1.0) + plt.setp(ax, xticks=[], yticks=[]) + cbar = plt.colorbar(boundaries=np.arange(11)-0.5) + cbar.set_ticks(np.arange(10)) + cbar.set_ticklabels(classes) + plt.title('Fashion MNIST Embedded via UMAP'); + +.. image:: images/SupervisedUMAP_10_1.png + + +The result is fairly good. We successfully separated a number of the +classes, and the global structure (separating pants and footwear from +shirts, coats and dresses) is well preserved as well. Unlike results for +MNIST digits, however, there were a number of classes that did not +separate quite so cleanly. In particular T-shirts, shirts, dresses, +pullovers, and coats are all a little mixed. At the very least the +dresses are largely separated, and the T-shirts are mostly in one large +clump, but they are not well distinguished from the others. Worse still +are the coats, shirts, and pullovers (somewhat unsurprisingly as these +can certainly look very similar) which all have significant overlap with +one another. Ideally we would like much better class separation. Since +we have the label information we can actually give that to UMAP to use! + +Using Labels to Separate Classes (Supervised UMAP) +-------------------------------------------------- + +How do we go about coercing UMAP to make use of target labels? If you +are familiar with the sklearn API you'll know that the :meth:`~umap.umap_.UMAP.fit` method +takes a target parameter ``y`` that specifies supervised target +information (for example when training a supervised classification +model). We can simply pass the :class:`~umap.umap_.UMAP` model that target data when +fitting and it will make use of it to perform supervised dimension +reduction! + +.. code:: python3 + + %%time + embedding = umap.UMAP().fit_transform(data, y=target) + + +.. parsed-literal:: + + CPU times: user 3min 28s, sys: 9.17 s, total: 3min 37s + Wall time: 2min 45s + + +This took a little longer -- both because we are using a larger +:py:obj:`~umap.umap_.UMAP.n_neighbors` value (which is suggested when doing supervised +dimension reduction; here we are using the default value of 15), and +because we need to condition on the label data. As before we have +reduced the data down to two dimensions so we can again visualize the +data with a scatterplot, coloring by class. + +.. code:: python3 + + fig, ax = plt.subplots(1, figsize=(14, 10)) + plt.scatter(*embedding.T, s=0.1, c=target, cmap='Spectral', alpha=1.0) + plt.setp(ax, xticks=[], yticks=[]) + cbar = plt.colorbar(boundaries=np.arange(11)-0.5) + cbar.set_ticks(np.arange(10)) + cbar.set_ticklabels(classes) + plt.title('Fashion MNIST Embedded via UMAP using Labels'); + + +.. image:: images/SupervisedUMAP_15_1.png + + +The result is a cleanly separated set of classes (and a little bit of +stray noise -- points that were sufficiently different from their class +as to not be grouped with the rest). Aside from the clear class +separation however (which is expected -- we gave the algorithm all the +class information), there are a couple of important points to note. The +first point to note is that we have retained the internal structure of +the individual classes. Both the shirts and pullovers still have the +distinct banding pattern that was visible in the original unsupervised +case; the pants, t-shirts and bags both retained their shape and +internal structure; etc. The second point to note is that we have also +retained the global structure. While the individual classes have been +cleanly separated from one another, the inter-relationships among the +classes have been preserved: footwear classes are all near one another; +trousers and bags are at opposite sides of the plot; and the arc of +pullover, shirts, t-shirts and dresses is still in place. + +The key point is this: the important structural properties of the data +have been retained while the known classes have been cleanly pulled +apart and isolated. If you have data with known classes and want to +separate them while still having a meaningful embedding of individual +points then supervised UMAP can provide exactly what you need. + +Using Partial Labelling (Semi-Supervised UMAP) +---------------------------------------------- + +What if we only have some of our data labelled, however, and a number of +items are without labels. Can we still make use of the label information +we do have? This is now a semi-supervised learning problem, and yes, we +can work with those cases too. To set up the example we'll mask some of +the target information -- we'll do this by using the sklearn standard of +giving unlabelled points a label of -1 (such as, for example, +the noise points from a DBSCAN clustering). + +.. code:: python3 + + masked_target = target.copy().astype(np.int8) + masked_target[np.random.choice(70000, size=10000, replace=False)] = -1 + +Now that we have randomly masked some of the labels we can try to +perform supervised learning again. Everything works as before, but UMAP +will interpret the -1 label as being an unlabelled point and learn +accordingly. + +.. code:: python3 + + %%time + fitter = umap.UMAP().fit(data, y=masked_target) + embedding = fitter.embedding_ + + +.. parsed-literal:: + + CPU times: user 3min 8s, sys: 7.85 s, total: 3min 16s + Wall time: 2min 40s + + +Again we can look at a scatterplot of the data colored by class. + +.. code:: python3 + + fig, ax = plt.subplots(1, figsize=(14, 10)) + plt.scatter(*embedding.T, s=0.1, c=target, cmap='Spectral', alpha=1.0) + plt.setp(ax, xticks=[], yticks=[]) + cbar = plt.colorbar(boundaries=np.arange(11)-0.5) + cbar.set_ticks(np.arange(10)) + cbar.set_ticklabels(classes) + plt.title('Fashion MNIST Embedded via UMAP using Partial Labels'); + + +.. image:: images/SupervisedUMAP_22_1.png + + +The result is much as we would expect -- while we haven't cleanly +separated the data as we did in the totally supervised case, the classes +have been made cleaner and more distinct. This semi-supervised approach +provides a powerful tool when labelling is potentially expensive, or +when you have more data than labels, but want to make use of that extra +data. + +Training with Labels and Embedding Unlabelled Test Data (Metric Learning with UMAP) +----------------------------------------------------------------------------------- + +If we have learned a supervised embedding, can we use that to embed new +previously unseen (and now unlabelled) points into the space? This would +provide an algorithm for `metric +learning `__, +where we can use a labelled set of points to learn a metric on data, and +then use that learned metric as a measure of distance between new +unlabelled points. This can be particularly useful as part of a machine +learning pipline where we learn a supervised embedding as a form of +supervised feature engineering, and then build a classifier on that new +space -- this is viable as long as we can pass new data to the embedding +model to be transformed to the new space. + +To try this out with UMAP let's use the train/test split provided by +Fashion MNIST: + +.. code:: python3 + + train_data = np.array(train) + test_data = np.array(test) + +Now we can fit a model to the training data, making use of the training +labels to learn a supervised embedding. + +.. code:: python3 + + %%time + mapper = umap.UMAP(n_neighbors=10).fit(train_data, np.array(train_labels)) + + +.. parsed-literal:: + + CPU times: user 2min 18s, sys: 7.53 s, total: 2min 26s + Wall time: 1min 52s + + +Next we can use the :meth:`~umap.umap_.UMAP.transform` method on that model to transform the +test set into the learned space. This time we won't pass the label +information and let the model attempt to place the data correctly. + +.. code:: python3 + + %%time + test_embedding = mapper.transform(test_data) + + +.. parsed-literal:: + + CPU times: user 17.3 s, sys: 986 ms, total: 18.3 s + Wall time: 15.4 s + + +UMAP transforms are not as fast as some approaches, but as you can see +this was still fairly efficient. The important question is how well we +managed to embed the test data into the existing learned space. To start +let's visualise the embedding of the training data so we can get a sense +of where things *should* go. + +.. code:: python3 + + fig, ax = plt.subplots(1, figsize=(14, 10)) + plt.scatter(*mapper.embedding_.T, s=0.3, c=np.array(train_labels), cmap='Spectral', alpha=1.0) + plt.setp(ax, xticks=[], yticks=[]) + cbar = plt.colorbar(boundaries=np.arange(11)-0.5) + cbar.set_ticks(np.arange(10)) + cbar.set_ticklabels(classes) + plt.title('Fashion MNIST Train Digits Embedded via UMAP Transform'); + + + +.. image:: images/SupervisedUMAP_31_0.png + + +As you can see this has done a similar job as before, successfully +embedding the separate classes while retaining both the internal +structure and the overall global structure. We can now look at how the +test set, for which we provided no label information, was embedded via +the :meth:`~umap.umap_.UMAP.transform` method. + +.. code:: python3 + + fig, ax = plt.subplots(1, figsize=(14, 10)) + plt.scatter(*test_embedding.T, s=2, c=np.array(test_labels), cmap='Spectral', alpha=1.0) + plt.setp(ax, xticks=[], yticks=[]) + cbar = plt.colorbar(boundaries=np.arange(11)-0.5) + cbar.set_ticks(np.arange(10)) + cbar.set_ticklabels(classes) + plt.title('Fashion MNIST Train Digits Embedded via UMAP'); + + + +.. image:: images/SupervisedUMAP_33_0.png + + +As you can see we have replicated the layout of the training data, +including much of the internal structure of the classes. For the most +part assignment of new points follows the classes well. The greatest +source of confusion are some t-shirts that ended up mixed with the +shirts, and some pullovers which are confused with the coats. Given the +difficulty of the problem this is a good result, particularly when +compared with current state-of-the-art approaches such as `siamese and +triplet +networks `__. + + +Supervised UMAP on the Galaxy10SDSS dataset +------------------------------------------- + +The `Galaxy10SDSS dataset `__ +is a crowd sourced human labelled dataset of galaxy images, +which have been separated in to ten classes. Umap can +learn an embedding that partially separates the data. To +keep runtime small, UMAP is applied to a subset of the +data. + +.. code:: python3 + + import numpy as np + import h5py + import matplotlib.pyplot as plt + import umap + import os + import math + import requests + + if not os.path.isfile("Galaxy10.h5"): + url = "http://astro.utoronto.ca/~bovy/Galaxy10/Galaxy10.h5" + r = requests.get(url, allow_redirects=True) + open("Galaxy10.h5", "wb").write(r.content) + + # To get the images and labels from file + with h5py.File("Galaxy10.h5", "r") as F: + images = np.array(F["images"]) + labels = np.array(F["ans"]) + + X_train = np.empty([math.floor(len(labels) / 100), 14283], dtype=np.float64) + y_train = np.empty([math.floor(len(labels) / 100)], dtype=np.float64) + X_test = X_train + y_test = y_train + # Get a subset of the data + for i in range(math.floor(len(labels) / 100)): + X_train[i, :] = np.array(np.ndarray.flatten(images[i, :, :, :]), dtype=np.float64) + y_train[i] = labels[i] + X_test[i, :] = np.array( + np.ndarray.flatten(images[i + math.floor(len(labels) / 100), :, :, :]), + dtype=np.float64, + ) + y_test[i] = labels[i + math.floor(len(labels) / 100)] + + # Plot distribution + classes, frequency = np.unique(y_train, return_counts=True) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.bar(classes, frequency) + plt.xlabel("Class") + plt.ylabel("Frequency") + plt.title("Data Subset") + plt.savefig("galaxy10_subset.svg") + + + +.. image:: images/galaxy10_subset.svg + + +The figure shows that the selected subset of the data set is +unbalanced, but the entire dataset is also unbalanced, so +this experiment will still use this subset. The next step is +to examine the output of the standard UMAP algorithm. + +.. code:: python3 + + reducer = umap.UMAP( + n_components=2, n_neighbors=5, random_state=42, transform_seed=42, verbose=False + ) + reducer.fit(X_train) + + galaxy10_umap = reducer.transform(X_train) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.scatter( + galaxy10_umap[:, 0], + galaxy10_umap[:, 1], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, + ) + plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) + plt.savefig("galaxy10_2D_umap.svg") + + + +.. image:: images/galaxy10_2D_umap.svg + + +The standard UMAP algorithm does not separate the galaxies +according to their type. Supervised UMAP can do better. + +.. code:: python3 + + reducer = umap.UMAP( + n_components=2, n_neighbors=15, random_state=42, transform_seed=42, verbose=False + ) + reducer.fit(X_train, y_train) + + galaxy10_umap_supervised = reducer.transform(X_train) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.scatter( + galaxy10_umap_supervised[:, 0], + galaxy10_umap_supervised[:, 1], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, + ) + plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) + plt.savefig("galaxy10_2D_umap_supervised.svg") + + + +.. image:: images/galaxy10_2D_umap_supervised.svg + + +Supervised UMAP does indeed do better. There is a litle overlap +between some of the classes, but the original dataset +also has some ambiguities in the classification. The best +check of this method is to project the testing data onto the +learned embedding. + +.. code:: python3 + + galaxy10_umap_supervised_prediction = reducer.transform(X_test) + fig = plt.figure(1, figsize=(4, 4)) + plt.clf() + plt.scatter( + galaxy10_umap_supervised_prediction[:, 0], + galaxy10_umap_supervised_prediction[:, 1], + c=y_test, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_test, + ) + plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) + plt.savefig("galaxy10_2D_umap_supervised_prediction.svg") + + + +.. image:: images/galaxy10_2D_umap_supervised_prediction.svg + + +This shows that the learned embedding can be used on new data +sets, and so this method may be helpful for examining images +of galaxies. Try out this method on the full 200 Mb dataset +as well as the newer 2.54 Gb +`Galaxy 10 DECals dataset `__ diff --git a/ext/TAL/applications/UMAP_fast/umap/doc/transform.rst b/ext/TAL/applications/UMAP_fast/umap/doc/transform.rst new file mode 100644 index 00000000..742af001 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/doc/transform.rst @@ -0,0 +1,244 @@ + +Transforming New Data with UMAP +=============================== + +UMAP is useful for generating visualisations, but if you want to make +use of UMAP more generally for machine learning tasks it is important to +be be able to train a model and then later pass new data to the model +and have it transform that data into the learned space. For example if +we use UMAP to learn a latent space and then train a classifier on data +transformed into the latent space then the classifier is only useful for +prediction if we can transform data for which we want a prediction into +the latent space the classifier uses. Fortunately UMAP makes this +possible, albeit more slowly than some other transformers that allow +this. + +To demonstrate this functionality we'll make use of +`scikit-learn `__ and the +digits dataset contained therein (see :doc:`basic_usage` for an example +of the digits dataset). First let's load all the modules we'll need to +get this done. + +.. code:: python3 + + import numpy as np + from sklearn.datasets import load_digits + from sklearn.model_selection import train_test_split, cross_val_score + from sklearn.neighbors import KNeighborsClassifier + from sklearn.svm import SVC + + import matplotlib.pyplot as plt + import seaborn as sns + %matplotlib inline + +.. code:: python3 + + sns.set(context='notebook', style='white', rc={'figure.figsize':(14,10)}) + +.. code:: python3 + + digits = load_digits() + +To keep everything honest let's use sklearn ``train_test_split`` to +separate out a training and test set (stratified over the different +digit types). By default ``train_test_split`` will carve off 25% of the +data for testing, which seems suitable in this case. + +.. code:: python3 + + X_train, X_test, y_train, y_test = train_test_split(digits.data, + digits.target, + stratify=digits.target, + random_state=42) + +Now to get a benchmark idea of what we are looking at let's train a +couple of different classifiers and then see how well they score on the +test set. For this example let's try a support vector classifier and a +KNN classifier. Ideally we should be tuning hyper-parameters (perhaps a +grid search using k-fold cross validation), but for the purposes of this +simple demo we will simply use default parameters for both classifiers. + +.. code:: python3 + + svc = SVC().fit(X_train, y_train) + knn = KNeighborsClassifier().fit(X_train, y_train) + +The next question is how well these classifiers perform on the test set. +Conveniently sklearn provides a ``score`` method that can output the +accuracy on the test set. + +.. code:: python3 + + svc.score(X_test, y_test), knn.score(X_test, y_test) + + + + +.. parsed-literal:: + + (0.62, 0.9844444444444445) + + + +The result is that the support vector classifier apparently had poor +hyper-parameters for this case (I expect with some tuning we could build +a much more accurate model) and the KNN classifier is doing very well. + +The goal now is to make use of UMAP as a preprocessing step that one +could potentially fit into a pipeline. We will therefore obviously need +the ``umap`` module loaded. + +.. code:: python3 + + import umap + +To make use of UMAP as a data transformer we first need to fit the model +with the training data. This works exactly as in the :doc:`basic_usage` +example using the fit method. In this case we simply hand it the +training data and it will learn an appropriate (two dimensional by +default) embedding. + +.. code:: python3 + + trans = umap.UMAP(n_neighbors=5, random_state=42).fit(X_train) + + +Since we embedded to two dimensions we can visualise the results to +ensure that we are getting a potential benefit out of this approach. +This is simply a matter of generating a scatterplot with data points +colored by the class they come from. Note that the embedded training +data can be accessed as the ``.embedding_`` attribute of the UMAP model +once we have fit the model to some data. + +.. code:: python3 + + plt.scatter(trans.embedding_[:, 0], trans.embedding_[:, 1], s= 5, c=y_train, cmap='Spectral') + plt.title('Embedding of the training set by UMAP', fontsize=24); + + + +.. image:: images/UMAPTransform_15_0.png + + +This looks very promising! Most of the classes got very cleanly +separated, and that gives us some hope that it could help with +classifier performance. It is worth noting that this was a completely +unsupervised data transform; we could have used the training label +information, but that is the subject of :doc:`a later tutorial `. + +We can now train some new models (again an SVC and a KNN classifier) on +the embedded training data. This looks exactly as before but now we pass +it the embedded data. Note that calling ``transform`` on input identical +to what the model was trained on will simply return the ``embedding_`` +attribute, so sklearn pipelines will work as expected. + +.. code:: python3 + + svc = SVC().fit(trans.embedding_, y_train) + knn = KNeighborsClassifier().fit(trans.embedding_, y_train) + +Now we want to work with the test data which none of the models (UMAP or +the classifiers) have seen. To do this we use the standard sklearn API +and make use of the ``transform`` method, this time handing it the new +unseen test data. We will assign this to ``test_embedding`` so that we +can take a closer look at the result of applying an existing UMAP model +to new data. + +.. code:: python3 + + %time test_embedding = trans.transform(X_test) + + +.. parsed-literal:: + + CPU times: user 867 ms, sys: 70.7 ms, total: 938 ms + Wall time: 335 ms + + +Note that the transform operations works very efficiently -- taking less +than half a second. Compared to some other transformers this is a little +on the slow side, but it is fast enough for many uses. Note that as the +size of the training and/or test sets increase the performance will slow +proportionally. It's also worth noting that the first call to transform +may be slow due to Numba JIT overhead -- further runs will be very fast. + +The next important question is what the transform did to our test data. +In principle we have a new two dimensional representation of the +test-set, and ideally this should be based on the existing embedding of +the training set. We can check this by visualising the data (since we +are in two dimensions) to see if this is true. A simple scatterplot as +before will suffice. + +.. code:: python3 + + plt.scatter(test_embedding[:, 0], test_embedding[:, 1], s= 5, c=y_test, cmap='Spectral') + plt.title('Embedding of the test set by UMAP', fontsize=24); + + + +.. image:: images/UMAPTransform_21_0.png + + +The results look like what we should expect; the test data has been +embedded into two dimensions in exactly the locations we should expect +(by class) given the embedding of the training data visualised above. +This means we can now try out models that were trained on the +embedded training data by handing them the newly transformed test set. + +.. code:: python3 + + svc.score(trans.transform(X_test), y_test), knn.score(trans.transform(X_test), y_test) + + + + +.. parsed-literal:: + + (0.9844444444444445, 0.9844444444444445) + + + +The results are pretty good. While the accuracy of the KNN classifier +did not improve there was not a lot of scope for improvement given the +data. On the other hand the SVC has improved to have equal accuracy to +the KNN classifier. Of course we could probably have achieved this level +of accuracy by better setting SVC hyper-parameters, but the point here +is that we can use UMAP as if it were a standard sklearn transformer as +part of an sklearn machine learning pipeline. + +Just for fun we can run the same experiments, but this time reduce to +ten dimensions (where we can no longer visualise). In practice this will +have little gain in this case -- for the digits dataset two dimensions +is plenty for UMAP and more dimensions won't help. On the other hand for +more complex datasets where more dimensions may allow for a much more +faithful embedding it is worth noting that we are not restricted to only +two dimension. + +.. code:: python3 + + trans = umap.UMAP(n_neighbors=5, n_components=10, random_state=42).fit(X_train) + + +.. code:: python3 + + svc = SVC().fit(trans.embedding_, y_train) + knn = KNeighborsClassifier().fit(trans.embedding_, y_train) + +.. code:: python3 + + svc.score(trans.transform(X_test), y_test), knn.score(trans.transform(X_test), y_test) + + + + +.. parsed-literal:: + + (0.9822222222222222, 0.9822222222222222) + + + +And we see that in this case we actually marginally lowered our accuracy +scores (within the potential noise in such scoring mind you). However +for more interesting datasets the larger dimensional embedding might have +been a significant gain -- it is certainly worth exploring as one of the +parameters in a grid search across a pipeline that includes UMAP. diff --git a/ext/TAL/applications/UMAP_fast/umap/docs_requirements.txt b/ext/TAL/applications/UMAP_fast/umap/docs_requirements.txt new file mode 100644 index 00000000..970621d7 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/docs_requirements.txt @@ -0,0 +1,5 @@ +sphinx +sphinx_gallery +matplotlib +pillow +sphinx_rtd_theme \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/README.txt b/ext/TAL/applications/UMAP_fast/umap/examples/README.txt new file mode 100644 index 00000000..eebf8c5c --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/README.txt @@ -0,0 +1,6 @@ +Gallery of Examples of UMAP usage +--------------------------------- + +A small gallery collection examples of UMAP usage. Do you +have an interesting UMAP plot that uses publicly available +data? Submit a pull request to have it added as an example! \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/digits/digits.html b/ext/TAL/applications/UMAP_fast/umap/examples/digits/digits.html new file mode 100644 index 00000000..0ba4bcc7 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/digits/digits.html @@ -0,0 +1,69 @@ + + + + + + Bokeh Plot + + + + + + + + + +
+
+
+ + + + \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/digits/digits.py b/ext/TAL/applications/UMAP_fast/umap/examples/digits/digits.py new file mode 100644 index 00000000..95e71b15 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/digits/digits.py @@ -0,0 +1,34 @@ +from bokeh.plotting import figure, output_file, show +from bokeh.models import CategoricalColorMapper, ColumnDataSource +from bokeh.palettes import Category10 + +import umap +from sklearn.datasets import load_digits + +digits = load_digits() +embedding = umap.UMAP().fit_transform(digits.data) + +output_file("digits.html") + +targets = [str(d) for d in digits.target_names] + +source = ColumnDataSource( + dict( + x=[e[0] for e in embedding], + y=[e[1] for e in embedding], + label=[targets[d] for d in digits.target], + ) +) + +cmap = CategoricalColorMapper(factors=targets, palette=Category10[10]) + +p = figure(title="test umap") +p.circle( + x="x", + y="y", + source=source, + color={"field": "label", "transform": cmap}, + legend="label", +) + +show(p) diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/galaxy10sdss.py b/ext/TAL/applications/UMAP_fast/umap/examples/galaxy10sdss.py new file mode 100644 index 00000000..9a2152a6 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/galaxy10sdss.py @@ -0,0 +1,275 @@ +""" +UMAP on the Galaxy10SDSS dataset +--------------------------------------------------------- + +This is an example of using UMAP on the Galaxy10SDSS +dataset. The goal of this example is largely to +demonstrate the use of supervised learning as an +effective tool for visualizing and reducing complex data. +In addition, hdbscan is used to classify the processed +data. +""" + +import numpy as np +import h5py +import matplotlib.pyplot as plt +import umap +import os + +# from sklearn.model_selection import train_test_split +import math +import requests + +# libraries for clustering +import hdbscan +import sklearn.cluster as cluster +from sklearn.metrics import adjusted_rand_score, adjusted_mutual_info_score + +if not os.path.isfile("Galaxy10.h5"): + url = "http://astro.utoronto.ca/~bovy/Galaxy10/Galaxy10.h5" + r = requests.get(url, allow_redirects=True) + open("Galaxy10.h5", "wb").write(r.content) + +# To get the images and labels from file +with h5py.File("Galaxy10.h5", "r") as F: + images = np.array(F["images"]) + labels = np.array(F["ans"]) + +X_train = np.empty([math.floor(len(labels) / 100), 14283], dtype=np.float64) +y_train = np.empty([math.floor(len(labels) / 100)], dtype=np.float64) +X_test = X_train +y_test = y_train +# Get a subset of the data +for i in range(math.floor(len(labels) / 100)): + X_train[i, :] = np.array(np.ndarray.flatten(images[i, :, :, :]), dtype=np.float64) + y_train[i] = labels[i] + X_test[i, :] = np.array( + np.ndarray.flatten(images[i + math.floor(len(labels) / 100), :, :, :]), + dtype=np.float64, + ) + y_test[i] = labels[i + math.floor(len(labels) / 100)] + +# Plot distribution +classes, frequency = np.unique(y_train, return_counts=True) +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +plt.bar(classes, frequency) +plt.xlabel("Class") +plt.ylabel("Frequency") +plt.title("Data Subset") +plt.savefig("galaxy10_subset.svg") +# 2D Embedding +## UMAP +reducer = umap.UMAP( + n_components=2, n_neighbors=20, random_state=42, transform_seed=42, verbose=False +) +reducer.fit(X_train) + +galaxy10_umap = reducer.transform(X_train) +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +plt.scatter( + galaxy10_umap[:, 0], + galaxy10_umap[:, 1], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, +) +plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_2D_umap.svg") +### UMAP - Supervised +reducer = umap.UMAP( + n_components=2, n_neighbors=15, random_state=42, transform_seed=42, verbose=False +) +reducer.fit(X_train, y_train) + +galaxy10_umap_supervised = reducer.transform(X_train) +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +plt.scatter( + galaxy10_umap_supervised[:, 0], + galaxy10_umap_supervised[:, 1], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, +) +plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_2D_umap_supervised.svg") +### UMAP - Supervised prediction +galaxy10_umap_supervised_prediction = reducer.transform(X_test) +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +plt.scatter( + galaxy10_umap_supervised_prediction[:, 0], + galaxy10_umap_supervised_prediction[:, 1], + c=y_test, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_test, +) +plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_2D_umap_supervised_prediction.svg") + +# cluster the data +labels = hdbscan.HDBSCAN( + min_samples=10, + min_cluster_size=10, +).fit_predict(galaxy10_umap_supervised_prediction) +clustered = labels >= 0 +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +plt.scatter( + galaxy10_umap_supervised_prediction[~clustered, 0], + galaxy10_umap_supervised_prediction[~clustered, 1], + color=(0.5, 0.5, 0.5), + alpha=0.5, +) +plt.scatter( + galaxy10_umap_supervised_prediction[clustered, 0], + galaxy10_umap_supervised_prediction[clustered, 1], + c=y_test[clustered], + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_test[clustered], +) +plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_2D_umap_supervised_prediction_clustered.svg") + +# Print out information on quality of clustering +print("2D Supervised Embedding with Clustering") +print(adjusted_rand_score(y_test, labels), adjusted_mutual_info_score(y_test, labels)) + +print( + adjusted_rand_score(y_test[clustered], labels[clustered]), + adjusted_mutual_info_score(y_test[clustered], labels[clustered]), +) + +print(np.sum(clustered) / y_test.shape[0]) + +# 3D Embedding +## UMAP +reducer = umap.UMAP( + n_components=3, n_neighbors=20, random_state=42, transform_seed=42, verbose=False +) +reducer.fit(X_train) +galaxy10_umap = reducer.transform(X_train) +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +ax = fig.add_subplot(projection="3d") +p = ax.scatter( + galaxy10_umap[:, 0], + galaxy10_umap[:, 1], + galaxy10_umap[:, 2], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, +) +fig.colorbar(p, ax=ax, boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_3D_umap.svg") +## UMAP - Supervised +reducer = umap.UMAP( + n_components=3, n_neighbors=20, random_state=42, transform_seed=42, verbose=False +) +reducer.fit(X_train, y_train) +galaxy10_umap_supervised = reducer.transform(X_train) +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +ax = fig.add_subplot(projection="3d") +p = ax.scatter( + galaxy10_umap_supervised[:, 0], + galaxy10_umap_supervised[:, 1], + galaxy10_umap_supervised[:, 2], + c=y_train, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_train, +) +fig.colorbar(p, ax=ax, boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_3D_umap_supervised.svg") +## UMAP - Supervised prediction +galaxy10_umap_supervised_prediction = reducer.transform(X_test) +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +ax = fig.add_subplot(projection="3d") +p = ax.scatter( + galaxy10_umap_supervised_prediction[:, 0], + galaxy10_umap_supervised_prediction[:, 1], + galaxy10_umap_supervised_prediction[:, 2], + c=y_test, + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_test, +) +fig.colorbar(p, ax=ax, boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_3D_umap_supervised_prediction.svg") + +# cluster the data +labels = hdbscan.HDBSCAN( + min_samples=10, + min_cluster_size=10, +).fit_predict(galaxy10_umap_supervised_prediction) +clustered = labels >= 0 +fig = plt.figure(1, figsize=(4, 4)) +plt.clf() +ax = fig.add_subplot(projection="3d") +q = ax.scatter( + galaxy10_umap_supervised_prediction[~clustered, 0], + galaxy10_umap_supervised_prediction[~clustered, 1], + galaxy10_umap_supervised_prediction[~clustered, 2], + color=(0.5, 0.5, 0.5), + alpha=0.5, +) +p = ax.scatter( + galaxy10_umap_supervised_prediction[clustered, 0], + galaxy10_umap_supervised_prediction[clustered, 1], + galaxy10_umap_supervised_prediction[clustered, 2], + c=y_test[clustered], + cmap=plt.cm.nipy_spectral, + edgecolor="k", + label=y_test[clustered], +) +fig.colorbar(p, ax=ax, boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.savefig("galaxy10_3D_umap_supervised_prediction_clustered.svg") + +# Print out information on quality of clustering +print("3D Supervised Embedding with Clustering") +print(adjusted_rand_score(y_test, labels), adjusted_mutual_info_score(y_test, labels)) + +print( + adjusted_rand_score(y_test[clustered], labels[clustered]), + adjusted_mutual_info_score(y_test[clustered], labels[clustered]), +) + +print(np.sum(clustered) / y_test.shape[0]) +# Dimensions 4 to 25 +for dimensions in range(4, 26): + reducer = umap.UMAP( + n_components=dimensions, + n_neighbors=20, + random_state=42, + transform_seed=42, + verbose=False, + ) + reducer.fit(X_train, y_train) + galaxy10_umap_supervised = reducer.transform(X_train) + # UMAP - Supervised prediction + galaxy10_umap_supervised_prediction = reducer.transform(X_test) + # cluster the data + labels = hdbscan.HDBSCAN( + min_samples=10, + min_cluster_size=10, + ).fit_predict(galaxy10_umap_supervised_prediction) + clustered = labels >= 0 + # Print out information on quality of clustering + print(str(dimensions) + "D Supervised Embedding with Clustering") + print( + adjusted_rand_score(y_test, labels), adjusted_mutual_info_score(y_test, labels) + ) + print( + adjusted_rand_score(y_test[clustered], labels[clustered]), + adjusted_mutual_info_score(y_test[clustered], labels[clustered]), + ) + print(np.sum(clustered) / y_test.shape[0]) diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/inverse_transform_example.py b/ext/TAL/applications/UMAP_fast/umap/examples/inverse_transform_example.py new file mode 100755 index 00000000..2574cb7b --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/inverse_transform_example.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +import matplotlib.pyplot as plt +import numpy as np +from sklearn.datasets import fetch_openml + +import umap + +mnist = fetch_openml('Fashion-MNIST', version=1) + + +trans = umap.UMAP( + n_neighbors=10, + random_state=42, + metric="euclidean", + output_metric="euclidean", + init="spectral", + verbose=True, +).fit(mnist.data) + +corners = np.array( + [[-5.1, 2.9], [-1.9, 6.4], [-5.4, -6.3], [8.3, 4.0],] # 7 # 4 # 1 # 0 +) + +test_pts = np.array( + [ + (corners[0] * (1 - x) + corners[1] * x) * (1 - y) + + (corners[2] * (1 - x) + corners[3] * x) * y + for y in np.linspace(0, 1, 10) + for x in np.linspace(0, 1, 10) + ] +) + +inv_transformed_points = trans.inverse_transform(test_pts) + +plt.scatter( + trans.embedding_[:, 0], + trans.embedding_[:, 1], + c=mnist.target, + cmap="Spectral", + s=0.25, +) +plt.colorbar(boundaries=np.arange(11) - 0.5).set_ticks(np.arange(10)) +plt.scatter(test_pts[:, 0], test_pts[:, 1], marker="x", c="k") + +fig, ax = plt.subplots(10, 10) +for i in range(10): + for j in range(10): + ax[i, j].imshow( + inv_transformed_points[i * 10 + j].reshape(28, 28), origin="upper" + ) + ax[i, j].get_xaxis().set_visible(False) + ax[i, j].get_yaxis().set_visible(False) + +plt.show() diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/iris/iris.html b/ext/TAL/applications/UMAP_fast/umap/examples/iris/iris.html new file mode 100644 index 00000000..6e9eea40 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/iris/iris.html @@ -0,0 +1,69 @@ + + + + + + Bokeh Plot + + + + + + + + + +
+
+
+ + + + \ No newline at end of file diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/iris/iris.py b/ext/TAL/applications/UMAP_fast/umap/examples/iris/iris.py new file mode 100644 index 00000000..495d8a12 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/iris/iris.py @@ -0,0 +1,37 @@ +from bokeh.plotting import figure, output_file, show +from bokeh.models import CategoricalColorMapper, ColumnDataSource +from bokeh.palettes import Category10 + +import umap +from sklearn.datasets import load_iris + +iris = load_iris() +embedding = umap.UMAP( + n_neighbors=50, learning_rate=0.5, init="random", min_dist=0.001 +).fit_transform(iris.data) + +output_file("iris.html") + + +targets = [str(d) for d in iris.target_names] + +source = ColumnDataSource( + dict( + x=[e[0] for e in embedding], + y=[e[1] for e in embedding], + label=[targets[d] for d in iris.target], + ) +) + +cmap = CategoricalColorMapper(factors=targets, palette=Category10[10]) + +p = figure(title="Test UMAP on Iris dataset") +p.circle( + x="x", + y="y", + source=source, + color={"field": "label", "transform": cmap}, + legend="label", +) + +show(p) diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/mnist_torus_sphere_example.py b/ext/TAL/applications/UMAP_fast/umap/examples/mnist_torus_sphere_example.py new file mode 100755 index 00000000..26d6d594 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/mnist_torus_sphere_example.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +import matplotlib.pyplot as plt +import numba +import numpy as np +from mayavi import mlab +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split + +import umap + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split( + digits.data, digits.target, stratify=digits.target, random_state=42 +) + +target_spaces = ["plane", "torus", "sphere"] + +if "plane" in target_spaces: + # embed onto a plane + + trans = umap.UMAP( + n_neighbors=10, + random_state=42, + metric="euclidean", + output_metric="euclidean", + init="spectral", + verbose=True, + ).fit(X_train) + + plt.scatter( + trans.embedding_[:, 0], trans.embedding_[:, 1], c=y_train, cmap="Spectral" + ) + plt.show() + +if "torus" in target_spaces: + # embed onto a torus + # note: this is a topological torus, not a geometric torus. Think + # Pacman, not donut. + + @numba.njit(fastmath=True) + def torus_euclidean_grad(x, y, torus_dimensions=(2 * np.pi, 2 * np.pi)): + """Standard euclidean distance. + + ..math:: + D(x, y) = \sqrt{\sum_i (x_i - y_i)^2} + """ + distance_sqr = 0.0 + g = np.zeros_like(x) + for i in range(x.shape[0]): + a = abs(x[i] - y[i]) + if 2 * a < torus_dimensions[i]: + distance_sqr += a ** 2 + g[i] = x[i] - y[i] + else: + distance_sqr += (torus_dimensions[i] - a) ** 2 + g[i] = (x[i] - y[i]) * (a - torus_dimensions[i]) / a + distance = np.sqrt(distance_sqr) + return distance, g / (1e-6 + distance) + + trans = umap.UMAP( + n_neighbors=10, + random_state=42, + metric="euclidean", + output_metric=torus_euclidean_grad, + init="spectral", + min_dist=0.15, # requires adjustment since the torus has limited space + verbose=True, + ).fit(X_train) + + mlab.clf() + x, y, z = np.mgrid[-3:3:50j, -3:3:50j, -3:3:50j] + + # Plot a torus + R = 2 + r = 1 + values = (R - np.sqrt(x ** 2 + y ** 2)) ** 2 + z ** 2 - r ** 2 + mlab.contour3d(x, y, z, values, color=(1.0, 1.0, 1.0), contours=[0]) + + # torus angles -> 3D + x = (R + r * np.cos(trans.embedding_[:, 0])) * np.cos(trans.embedding_[:, 1]) + y = (R + r * np.cos(trans.embedding_[:, 0])) * np.sin(trans.embedding_[:, 1]) + z = r * np.sin(trans.embedding_[:, 0]) + + pts = mlab.points3d( + x, y, z, y_train, colormap="spectral", scale_mode="none", scale_factor=0.1 + ) + + mlab.show() + +if "sphere" in target_spaces: + # embed onto a sphere + trans = umap.UMAP( + n_neighbors=10, + random_state=42, + metric="euclidean", + output_metric="haversine", + init="spectral", + min_dist=0.15, # requires adjustment since the sphere has limited space + verbose=True, + ).fit(X_train) + + mlab.clf() + x, y, z = np.mgrid[-3:3:50j, -3:3:50j, -3:3:50j] + + # Plot a sphere + r = 3 + values = x ** 2 + y ** 2 + z ** 2 - r ** 2 + mlab.contour3d(x, y, z, values, color=(1.0, 1.0, 1.0), contours=[0]) + + # latitude, longitude -> 3D + x = r * np.sin(trans.embedding_[:, 0]) * np.cos(trans.embedding_[:, 1]) + y = r * np.sin(trans.embedding_[:, 0]) * np.sin(trans.embedding_[:, 1]) + z = r * np.cos(trans.embedding_[:, 0]) + + pts = mlab.points3d( + x, y, z, y_train, colormap="spectral", scale_mode="none", scale_factor=0.2 + ) + + mlab.show() diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/mnist_transform_new_data.py b/ext/TAL/applications/UMAP_fast/umap/examples/mnist_transform_new_data.py new file mode 100755 index 00000000..8c45236a --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/mnist_transform_new_data.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +""" +UMAP on the MNIST Digits dataset +-------------------------------- + +A simple example demonstrating how to use UMAP on a larger +dataset such as MNIST. We first pull the MNIST dataset and +then use UMAP to reduce it to only 2-dimensions for +easy visualisation. + +Note that UMAP manages to both group the individual digit +classes, but also to retain the overall global structure +among the different digit classes -- keeping 1 far from +0, and grouping triplets of 3,5,8 and 4,7,9 which can +blend into one another in some cases. +""" +import matplotlib.pyplot as plt +import seaborn as sns +from sklearn.datasets import fetch_openml +from sklearn.model_selection import train_test_split + +import umap + +sns.set(context="paper", style="white") + +mnist = fetch_openml("mnist_784", version=1) +X_train, X_test, y_train, y_test = train_test_split( + mnist.data, mnist.target, stratify=mnist.target, random_state=42 +) + +reducer = umap.UMAP(random_state=42) +embedding_train = reducer.fit_transform(X_train) +embedding_test = reducer.transform(X_test) + +fig, ax = plt.subplots(1, 2, sharex=True, sharey=True, figsize=(12, 10)) +ax[0].scatter( + embedding_train[:, 0], embedding_train[:, 1], c=y_train, cmap="Spectral" # , s=0.1 +) +ax[1].scatter( + embedding_test[:, 0], embedding_test[:, 1], c=y_test, cmap="Spectral" # , s=0.1 +) +plt.setp(ax[0], xticks=[], yticks=[]) +plt.setp(ax[1], xticks=[], yticks=[]) +plt.suptitle("MNIST data embedded into two dimensions by UMAP", fontsize=18) +ax[0].set_title("Training Set", fontsize=12) +ax[1].set_title("Test Set", fontsize=12) +plt.show() diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/plot_algorithm_comparison.py b/ext/TAL/applications/UMAP_fast/umap/examples/plot_algorithm_comparison.py new file mode 100644 index 00000000..107b845c --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/plot_algorithm_comparison.py @@ -0,0 +1,139 @@ +""" +Comparison of Dimension Reduction Techniques +-------------------------------------------- + +A comparison of several different dimension reduction +techniques on a variety of toy datasets. The datasets +are all toy datasets, but should provide a representative +range of the strengths and weaknesses of the different +algorithms. + +The time to perform the dimension reduction with each +algorithm and each dataset is recorded in the lower +right of each plot. + +Things to note about the datasets: + +- Blobs: A set of five gaussian blobs in 10 dimensional + space. This should be a prototypical example + of something that should clearly separate + even in a reduced dimension space. +- Iris: a classic small dataset with one distinct class + and two classes that are not clearly separated. +- Digits: handwritten digits -- ideally different digit + classes should form distinct groups. Due to + the nature of handwriting digits may have several + forms (crossed or uncrossed sevens, capped or + straight line oes, etc.) +- Wine: wine characteristics ideally used for a toy + regression. Ultimately the data is essentially + one dimensional in nature. +- Swiss Roll: data is essentially a rectangle, but + has been "rolled up" like a swiss roll + in three dimensional space. Ideally a + dimension reduction technique should + be able to "unroll" it. The data + has been coloured according to one dimension + of the rectangle, so should form + a rectangle of smooth color variation. +- Sphere: the two dimensional surface of a three + dimensional sphere. This cannot be represented + accurately in two dimensions without tearing. + The sphere has been coloured with hue around + the equator and black to white from the south + to north pole. +""" +import numpy as np +import matplotlib.pyplot as plt +import seaborn as sns +import time + +from sklearn import datasets, decomposition, manifold, preprocessing +from colorsys import hsv_to_rgb + +import umap + +sns.set(context="paper", style="white") + +blobs, blob_labels = datasets.make_blobs( + n_samples=500, n_features=10, centers=5, random_state=42 +) +iris = datasets.load_iris() +digits = datasets.load_digits(n_class=10) +wine = datasets.load_wine() +swissroll, swissroll_labels = datasets.make_swiss_roll( + n_samples=1000, noise=0.1, random_state=42 +) +sphere = np.random.normal(size=(600, 3)) +sphere = preprocessing.normalize(sphere) +sphere_hsv = np.array( + [ + ( + (np.arctan2(c[1], c[0]) + np.pi) / (2 * np.pi), + np.abs(c[2]), + min((c[2] + 1.1), 1.0), + ) + for c in sphere + ] +) +sphere_colors = np.array([hsv_to_rgb(*c) for c in sphere_hsv]) + +reducers = [ + (manifold.TSNE, {"perplexity": 50}), + # (manifold.LocallyLinearEmbedding, {'n_neighbors':10, 'method':'hessian'}), + (manifold.Isomap, {"n_neighbors": 30}), + (manifold.MDS, {}), + (decomposition.PCA, {}), + (umap.UMAP, {"n_neighbors": 30, "min_dist": 0.3}), +] + +test_data = [ + (blobs, blob_labels), + (iris.data, iris.target), + (digits.data, digits.target), + (wine.data, wine.target), + (swissroll, swissroll_labels), + (sphere, sphere_colors), +] +dataset_names = ["Blobs", "Iris", "Digits", "Wine", "Swiss Roll", "Sphere"] + +n_rows = len(test_data) +n_cols = len(reducers) +ax_index = 1 +ax_list = [] + +# plt.figure(figsize=(9 * 2 + 3, 12.5)) +plt.figure(figsize=(10, 8)) +plt.subplots_adjust( + left=0.02, right=0.98, bottom=0.001, top=0.96, wspace=0.05, hspace=0.01 +) +for data, labels in test_data: + for reducer, args in reducers: + start_time = time.time() + embedding = reducer(n_components=2, **args).fit_transform(data) + elapsed_time = time.time() - start_time + ax = plt.subplot(n_rows, n_cols, ax_index) + if isinstance(labels[0], tuple): + ax.scatter(*embedding.T, s=10, c=labels, alpha=0.5) + else: + ax.scatter(*embedding.T, s=10, c=labels, cmap="Spectral", alpha=0.5) + ax.text( + 0.99, + 0.01, + "{:.2f} s".format(elapsed_time), + transform=ax.transAxes, + size=14, + horizontalalignment="right", + ) + ax_list.append(ax) + ax_index += 1 +plt.setp(ax_list, xticks=[], yticks=[]) + +for i in np.arange(n_rows) * n_cols: + ax_list[i].set_ylabel(dataset_names[i // n_cols], size=16) +for i in range(n_cols): + ax_list[i].set_xlabel(repr(reducers[i][0]()).split("(")[0], size=16) + ax_list[i].xaxis.set_label_position("top") + +plt.tight_layout() +plt.show() diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/plot_fashion-mnist_example.py b/ext/TAL/applications/UMAP_fast/umap/examples/plot_fashion-mnist_example.py new file mode 100644 index 00000000..863d2b8e --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/plot_fashion-mnist_example.py @@ -0,0 +1,73 @@ +""" +UMAP on the Fashion MNIST Digits dataset using Datashader +--------------------------------------------------------- + +This is a simple example of using UMAP on the Fashion-MNIST +dataset. The goal of this example is largely to demonstrate +the use of datashader as an effective tool for visualising +UMAP results. In particular datashader allows visualisation +of very large datasets where overplotting can be a serious +problem. It supports coloring by categorical variables +(as shown in this example), or by continuous variables, +or by density (as is common in datashader examples). +""" +import umap +import numpy as np +import pandas as pd +import requests +import os +import datashader as ds +import datashader.utils as utils +import datashader.transfer_functions as tf +import matplotlib.pyplot as plt +import seaborn as sns + +sns.set(context="paper", style="white") + +if not os.path.isfile("fashion-mnist.csv"): + csv_data = requests.get("https://www.openml.org/data/get_csv/18238735/phpnBqZGZ") + with open("fashion-mnist.csv", "w") as f: + f.write(csv_data.text) +source_df = pd.read_csv("fashion-mnist.csv") + +data = source_df.iloc[:, :784].values.astype(np.float32) +target = source_df["class"].values + +pal = [ + "#9e0142", + "#d8434e", + "#f67a49", + "#fdbf6f", + "#feeda1", + "#f1f9a9", + "#bfe5a0", + "#74c7a5", + "#378ebb", + "#5e4fa2", +] +color_key = {str(d): c for d, c in enumerate(pal)} + +reducer = umap.UMAP(random_state=42) +embedding = reducer.fit_transform(data) + +df = pd.DataFrame(embedding, columns=("x", "y")) +df["class"] = pd.Series([str(x) for x in target], dtype="category") + +cvs = ds.Canvas(plot_width=400, plot_height=400) +agg = cvs.points(df, "x", "y", ds.count_cat("class")) +img = tf.shade(agg, color_key=color_key, how="eq_hist") + +utils.export_image(img, filename="fashion-mnist", background="black") + +image = plt.imread("fashion-mnist.png") +fig, ax = plt.subplots(figsize=(6, 6)) +plt.imshow(image) +plt.setp(ax, xticks=[], yticks=[]) +plt.title( + "Fashion MNIST data embedded\n" + "into two dimensions by UMAP\n" + "visualised with Datashader", + fontsize=12, +) + +plt.show() diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/plot_feature_extraction_classification.py b/ext/TAL/applications/UMAP_fast/umap/examples/plot_feature_extraction_classification.py new file mode 100644 index 00000000..89aa9a6a --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/plot_feature_extraction_classification.py @@ -0,0 +1,71 @@ +""" +UMAP as a Feature Extraction Technique for Classification +--------------------------------------------------------- + +The following script shows how UMAP can be used as a feature extraction +technique to improve the accuracy on a classification task. It also shows +how UMAP can be integrated in standard scikit-learn pipelines. + +The first step is to create a dataset for a classification task, which is +performed with the function ``sklearn.datasets.make_classification``. The +dataset is then split into a training set and a test set using the +``sklearn.model_selection.train_test_split`` function. + +Second, a linear SVM is fitted on the training set. To choose the best +hyperparameters automatically, a gridsearch is performed on the training set. +The performance of the model is then evaluated on the test set with the +accuracy metric. + + Third, the previous step is repeated with a slight modification: UMAP is + used as a feature extraction technique. This small change results in a + substantial improvement compared to the model where raw data is used. +""" +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split, GridSearchCV +from sklearn.pipeline import Pipeline +from sklearn.svm import LinearSVC + +from umap import UMAP + +# Make a toy dataset +X, y = make_classification( + n_samples=1000, + n_features=300, + n_informative=250, + n_redundant=0, + n_repeated=0, + n_classes=2, + random_state=1212, +) + +# Split the dataset into a training set and a test set +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=42 +) + +# Classification with a linear SVM +svc = LinearSVC(dual=False, random_state=123) +params_grid = {"C": [10 ** k for k in range(-3, 4)]} +clf = GridSearchCV(svc, params_grid) +clf.fit(X_train, y_train) +print( + "Accuracy on the test set with raw data: {:.3f}".format(clf.score(X_test, y_test)) +) + +# Transformation with UMAP followed by classification with a linear SVM +umap = UMAP(random_state=456) +pipeline = Pipeline([("umap", umap), ("svc", svc)]) +params_grid_pipeline = { + "umap__n_neighbors": [5, 20], + "umap__n_components": [15, 25, 50], + "svc__C": [10 ** k for k in range(-3, 4)], +} + + +clf_pipeline = GridSearchCV(pipeline, params_grid_pipeline) +clf_pipeline.fit(X_train, y_train) +print( + "Accuracy on the test set with UMAP transformation: {:.3f}".format( + clf_pipeline.score(X_test, y_test) + ) +) diff --git a/ext/TAL/applications/UMAP_fast/umap/examples/plot_mnist_example.py b/ext/TAL/applications/UMAP_fast/umap/examples/plot_mnist_example.py new file mode 100644 index 00000000..44b412b4 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/examples/plot_mnist_example.py @@ -0,0 +1,34 @@ +""" +UMAP on the MNIST Digits dataset +-------------------------------- + +A simple example demonstrating how to use UMAP on a larger +dataset such as MNIST. We first pull the MNIST dataset and +then use UMAP to reduce it to only 2-dimensions for +easy visualisation. + +Note that UMAP manages to both group the individual digit +classes, but also to retain the overall global structure +among the different digit classes -- keeping 1 far from +0, and grouping triplets of 3,5,8 and 4,7,9 which can +blend into one another in some cases. +""" +import umap +from sklearn.datasets import fetch_openml +import matplotlib.pyplot as plt +import seaborn as sns + +sns.set(context="paper", style="white") + +mnist = fetch_openml("mnist_784", version=1) + +reducer = umap.UMAP(random_state=42) +embedding = reducer.fit_transform(mnist.data) + +fig, ax = plt.subplots(figsize=(12, 10)) +color = mnist.target.astype(int) +plt.scatter(embedding[:, 0], embedding[:, 1], c=color, cmap="Spectral", s=0.1) +plt.setp(ax, xticks=[], yticks=[]) +plt.title("MNIST data embedded into two dimensions by UMAP", fontsize=18) + +plt.show() diff --git a/ext/TAL/applications/UMAP_fast/umap/images/densmap_example_mnist.png b/ext/TAL/applications/UMAP_fast/umap/images/densmap_example_mnist.png new file mode 100644 index 00000000..4c75b312 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/images/densmap_example_mnist.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/images/iris.png b/ext/TAL/applications/UMAP_fast/umap/images/iris.png new file mode 100644 index 00000000..148c5f7b Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/images/iris.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/images/mnist_digits.png b/ext/TAL/applications/UMAP_fast/umap/images/mnist_digits.png new file mode 100644 index 00000000..824969d6 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/images/mnist_digits.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/images/sklearn_digits.png b/ext/TAL/applications/UMAP_fast/umap/images/sklearn_digits.png new file mode 100644 index 00000000..6ac13ddd Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/images/sklearn_digits.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/images/umap_example_fashion_mnist1.png b/ext/TAL/applications/UMAP_fast/umap/images/umap_example_fashion_mnist1.png new file mode 100644 index 00000000..338c181a Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/images/umap_example_fashion_mnist1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/images/umap_example_mnist1.png b/ext/TAL/applications/UMAP_fast/umap/images/umap_example_mnist1.png new file mode 100644 index 00000000..9699b414 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/images/umap_example_mnist1.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/images/umap_example_shuttle.png b/ext/TAL/applications/UMAP_fast/umap/images/umap_example_shuttle.png new file mode 100644 index 00000000..ecc60fd8 Binary files /dev/null and b/ext/TAL/applications/UMAP_fast/umap/images/umap_example_shuttle.png differ diff --git a/ext/TAL/applications/UMAP_fast/umap/kwcheck.out b/ext/TAL/applications/UMAP_fast/umap/kwcheck.out new file mode 100644 index 00000000..e69de29b diff --git a/ext/TAL/applications/UMAP_fast/umap/notebooks/AnimatingUMAP.ipynb b/ext/TAL/applications/UMAP_fast/umap/notebooks/AnimatingUMAP.ipynb new file mode 100644 index 00000000..49fbe1ab --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/notebooks/AnimatingUMAP.ipynb @@ -0,0 +1,5347 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Making Animations of UMAP Hyper-parameters\n", + "\n", + "Sometimes one of the best ways to see the effects of hyperparameters is simply to visualise what happens as they change. We can do that in practice with UMAP by simply creating an animation that transitions between embeddings generated with variations of hyperparameters. To do this we'll make use of matplotlib and its animation capabilities. Jake Vanderplas has [a great tutorial](https://jakevdp.github.io/blog/2012/08/18/matplotlib-animation-tutorial/) if you want to know more about creating animations with matplotlib." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note:**\n", + "This is a self contained example of how to use UMAP and the impact of individual hyper-parameters. To make sure everything works correctly please use `conda`.\n", + "For install and usage details see [here](https://docs.conda.io/en/latest/miniconda.html)\n", + "\n", + "To create animations we need `ffmpeg`. It can be installed with `conda`.\n", + "\n", + "If you already have `ffmpeg` installed on your machine and you know what you are doing you do not need conda. It is only used to install `ffmpeg`.\n", + "\n", + "=> Remove the next two cells if you are not using `conda`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "conda 4.7.12\n" + ] + } + ], + "source": [ + "!conda --version" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting package metadata (current_repodata.json): ...working... done\n", + "Solving environment: ...working... done\n", + "\n", + "## Package Plan ##\n", + "\n", + " environment location: C:\\Progams\\Miniconda\\envs\\tf\n", + "\n", + " added / updated specs:\n", + " - ffmpeg\n", + "\n", + "\n", + "The following packages will be downloaded:\n", + "\n", + " package | build\n", + " ---------------------------|-----------------\n", + " ca-certificates-2019.9.11 | hecc5488_0 181 KB conda-forge\n", + " certifi-2019.9.11 | py37_0 155 KB\n", + " ffmpeg-4.2 | h6538335_0 23.4 MB conda-forge\n", + " openssl-1.1.1c | hfa6e2cd_0 4.7 MB conda-forge\n", + " ------------------------------------------------------------\n", + " Total: 28.5 MB\n", + "\n", + "The following NEW packages will be INSTALLED:\n", + "\n", + " ffmpeg conda-forge/win-64::ffmpeg-4.2-h6538335_0\n", + "\n", + "The following packages will be UPDATED:\n", + "\n", + " ca-certificates pkgs/main::ca-certificates-2019.5.15-1 --> conda-forge::ca-certificates-2019.9.11-hecc5488_0\n", + " certifi 2019.6.16-py37_1 --> 2019.9.11-py37_0\n", + "\n", + "The following packages will be SUPERSEDED by a higher-priority channel:\n", + "\n", + " openssl pkgs/main::openssl-1.1.1c-he774522_1 --> conda-forge::openssl-1.1.1c-hfa6e2cd_0\n", + "\n", + "\n", + "\n", + "Downloading and Extracting Packages\n", + "\n", + "openssl-1.1.1c | 4.7 MB | | 0% \n", + "openssl-1.1.1c | 4.7 MB | | 0% \n", + "openssl-1.1.1c | 4.7 MB | #5 | 15% \n", + "openssl-1.1.1c | 4.7 MB | ### | 30% \n", + "openssl-1.1.1c | 4.7 MB | ####5 | 46% \n", + "openssl-1.1.1c | 4.7 MB | ###### | 61% \n", + "openssl-1.1.1c | 4.7 MB | #######5 | 75% \n", + "openssl-1.1.1c | 4.7 MB | ######### | 90% \n", + "openssl-1.1.1c | 4.7 MB | ########## | 100% \n", + "\n", + "ca-certificates-2019 | 181 KB | | 0% \n", + "ca-certificates-2019 | 181 KB | ########## | 100% \n", + "\n", + "certifi-2019.9.11 | 155 KB | | 0% \n", + "certifi-2019.9.11 | 155 KB | ########## | 100% \n", + "\n", + "ffmpeg-4.2 | 23.4 MB | | 0% \n", + "ffmpeg-4.2 | 23.4 MB | 2 | 2% \n", + "ffmpeg-4.2 | 23.4 MB | 5 | 5% \n", + "ffmpeg-4.2 | 23.4 MB | 8 | 8% \n", + "ffmpeg-4.2 | 23.4 MB | #1 | 11% \n", + "ffmpeg-4.2 | 23.4 MB | #4 | 14% \n", + "ffmpeg-4.2 | 23.4 MB | #7 | 17% \n", + "ffmpeg-4.2 | 23.4 MB | ## | 20% \n", + "ffmpeg-4.2 | 23.4 MB | ##3 | 23% \n", + "ffmpeg-4.2 | 23.4 MB | ##6 | 26% \n", + "ffmpeg-4.2 | 23.4 MB | ##9 | 29% \n", + "ffmpeg-4.2 | 23.4 MB | ###2 | 32% \n", + "ffmpeg-4.2 | 23.4 MB | ###5 | 35% \n", + "ffmpeg-4.2 | 23.4 MB | ###8 | 38% \n", + "ffmpeg-4.2 | 23.4 MB | ####1 | 41% \n", + "ffmpeg-4.2 | 23.4 MB | ####4 | 44% \n", + "ffmpeg-4.2 | 23.4 MB | ####7 | 47% \n", + "ffmpeg-4.2 | 23.4 MB | ##### | 50% \n", + "ffmpeg-4.2 | 23.4 MB | #####3 | 53% \n", + "ffmpeg-4.2 | 23.4 MB | #####6 | 56% \n", + "ffmpeg-4.2 | 23.4 MB | #####9 | 59% \n", + "ffmpeg-4.2 | 23.4 MB | ######2 | 62% \n", + "ffmpeg-4.2 | 23.4 MB | ######5 | 65% \n", + "ffmpeg-4.2 | 23.4 MB | ######8 | 68% \n", + "ffmpeg-4.2 | 23.4 MB | #######1 | 71% \n", + "ffmpeg-4.2 | 23.4 MB | #######4 | 74% \n", + "ffmpeg-4.2 | 23.4 MB | #######7 | 77% \n", + "ffmpeg-4.2 | 23.4 MB | ######## | 81% \n", + "ffmpeg-4.2 | 23.4 MB | ########3 | 84% \n", + "ffmpeg-4.2 | 23.4 MB | ########6 | 87% \n", + "ffmpeg-4.2 | 23.4 MB | ########9 | 90% \n", + "ffmpeg-4.2 | 23.4 MB | #########2 | 93% \n", + "ffmpeg-4.2 | 23.4 MB | #########5 | 96% \n", + "ffmpeg-4.2 | 23.4 MB | #########8 | 99% \n", + "ffmpeg-4.2 | 23.4 MB | ########## | 100% \n", + "Preparing transaction: ...working... done\n", + "Verifying transaction: ...working... done\n", + "Executing transaction: ...working... done\n" + ] + } + ], + "source": [ + "!conda install -c conda-forge ffmpeg -y" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.7.3\n" + ] + } + ], + "source": [ + "!python --version" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To start we'll need some basic libraries. First ``numpy`` will be needed for basic array manipulation. Since we will be visualising the results we will need ``matplotlib`` and ``seaborn``. Finally we will need ``umap`` for doing the dimension reduction itself." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: numpy in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (1.17.1)\n", + "Requirement already satisfied: matplotlib in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (3.1.1)\n", + "Requirement already satisfied: seaborn in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (0.9.0)\n", + "Requirement already satisfied: umap-learn in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (0.3.10)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from matplotlib) (1.1.0)\n", + "Requirement already satisfied: cycler>=0.10 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from matplotlib) (0.10.0)\n", + "Requirement already satisfied: python-dateutil>=2.1 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from matplotlib) (2.8.0)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from matplotlib) (2.4.2)\n", + "Requirement already satisfied: pandas>=0.15.2 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from seaborn) (0.25.1)\n", + "Requirement already satisfied: scipy>=0.14.0 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from seaborn) (1.3.1)\n", + "Requirement already satisfied: scikit-learn>=0.16 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from umap-learn) (0.21.3)\n", + "Requirement already satisfied: numba>=0.37 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from umap-learn) (0.45.0)\n", + "Requirement already satisfied: setuptools in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from kiwisolver>=1.0.1->matplotlib) (41.0.1)\n", + "Requirement already satisfied: six in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from cycler>=0.10->matplotlib) (1.12.0)\n", + "Requirement already satisfied: pytz>=2017.2 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from pandas>=0.15.2->seaborn) (2019.2)\n", + "Requirement already satisfied: joblib>=0.11 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from scikit-learn>=0.16->umap-learn) (0.13.2)\n", + "Requirement already satisfied: llvmlite>=0.29.0dev0 in c:\\progams\\miniconda\\envs\\tf\\lib\\site-packages (from numba>=0.37->umap-learn) (0.29.0)\n" + ] + } + ], + "source": [ + "!pip install numpy matplotlib seaborn umap-learn" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To start let's load everything we'll need" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.axes_grid1 import make_axes_locatable\n", + "from matplotlib import animation\n", + "from IPython.display import HTML\n", + "import seaborn as sns\n", + "import itertools\n", + "sns.set(style='white', rc={'figure.figsize':(14, 12), 'animation.html': 'html5'})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Ignore UserWarnings\n", + "import warnings\n", + "warnings.simplefilter('ignore', UserWarning)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import load_digits" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from umap import UMAP" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To try this out we'll needs a reasonably small dataset (so embedding runs don't take *too* long since we'll be doing a lot of them). For ease of reproducibility for everyone else I'll use the digits dataset from sklearn. If you want to try other datasets just drop them in here -- COIL20 might be interesting, or you might have your own data." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[ 0., 0., 5., ..., 0., 0., 0.],\n", + " [ 0., 0., 0., ..., 10., 0., 0.],\n", + " [ 0., 0., 0., ..., 16., 9., 0.],\n", + " ...,\n", + " [ 0., 0., 1., ..., 6., 0., 0.],\n", + " [ 0., 0., 2., ..., 12., 0., 0.],\n", + " [ 0., 0., 10., ..., 12., 1., 0.]])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "digits = load_digits()\n", + "data = digits.data\n", + "data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to move the points in between the embeddings given by different parameter values. There are potentially fancy ways to do this (Something using rotation and reflection to get an initial alignment might be interesting), but we'll use straighforward linear interpolation between the two embeddings. To do this we'll need a simple function that can turn out intermediate embeddings for the in-between frames of the animation." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def tween(e1, e2, n_frames=20):\n", + " for i in range(5):\n", + " yield e1\n", + " for i in range(n_frames):\n", + " alpha = i / float(n_frames - 1)\n", + " yield (1 - alpha) * e1 + alpha * e2\n", + " for i in range(5):\n", + " yield(e2)\n", + " return" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we can fill in intermediate frame we just need to generate all the embeddings. We'll create a function that can take an argument and set of parameter values and then generate all the embeddings including the in-between frames." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_frame_data(data, arg_name='n_neighbors', arg_list=[]):\n", + " result = []\n", + " es = []\n", + " for arg in arg_list:\n", + " kwargs = {arg_name:arg}\n", + " if len(es) > 0:\n", + " es.append(UMAP(init=es[-1], negative_sample_rate=3, **kwargs).fit_transform(data))\n", + " else:\n", + " es.append(UMAP(negative_sample_rate=3, **kwargs).fit_transform(data))\n", + " \n", + " for e1, e2 in zip(es[:-1], es[1:]):\n", + " result.extend(list(tween(e1, e2)))\n", + " \n", + " return result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we just need to create a function to actually generate the animation given a list of embeddings (one for each frame). This is really just a matter of workign through the details of how matplotlib generates animations -- I would refer you again to Jake's tutorial if you are interested in the detailed mechanics of this." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def create_animation(frame_data, arg_name='n_neighbors', arg_list=[]):\n", + " fig, ax = plt.subplots()\n", + " all_data = np.vstack(frame_data)\n", + " frame_bounds = (all_data[:, 0].min() * 1.1, \n", + " all_data[:, 0].max() * 1.1,\n", + " all_data[:, 1].min() * 1.1, \n", + " all_data[:, 1].max() * 1.1)\n", + " ax.set_xlim(frame_bounds[0], frame_bounds[1])\n", + " ax.set_ylim(frame_bounds[2], frame_bounds[3])\n", + " points = ax.scatter(frame_data[0][:, 0], frame_data[0][:, 1], \n", + " s=5, c=digits.target, cmap='Spectral', animated=True)\n", + " title = ax.set_title('', fontsize=24)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + "\n", + " cbar = fig.colorbar(\n", + " points,\n", + " cax=make_axes_locatable(ax).append_axes(\"right\", size=\"5%\", pad=0.05),\n", + " orientation=\"vertical\",\n", + " values=np.arange(10),\n", + " boundaries=np.arange(11)-0.5,\n", + " ticks=np.arange(10),\n", + " drawedges=True,\n", + " )\n", + " cbar.ax.yaxis.set_ticklabels(np.arange(10), fontsize=18)\n", + "\n", + " def init():\n", + " points.set_offsets(frame_data[0])\n", + " arg = arg_list[0]\n", + " arg_str = f'{arg:.3f}' if isinstance(arg, float) else f'{arg}'\n", + " title.set_text(f'UMAP with {arg_name}={arg_str}')\n", + " return (points,)\n", + "\n", + " def animate(i):\n", + " points.set_offsets(frame_data[i])\n", + " if (i + 15) % 30 == 0:\n", + " arg = arg_list[(i + 15) // 30]\n", + " arg_str = f'{arg:.3f}' if isinstance(arg, float) else f'{arg}'\n", + " title.set_text(f'UMAP with {arg_name}={arg_str}')\n", + " return (points,)\n", + "\n", + " anim = animation.FuncAnimation(fig, animate, init_func=init, frames=len(frame_data), interval=20, blit=True)\n", + " plt.close()\n", + " return anim" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally a little bit of glue to make it all go together." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def animate_param(data, arg_name='n_neighbors', arg_list=[]):\n", + " frame_data = generate_frame_data(data, arg_name, arg_list)\n", + " return create_animation(frame_data, arg_name, arg_list)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can create an animation. It will be embedded as an HTML5 video into this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "animate_param(data, 'n_neighbors', [3, 4, 5, 7, 10, 15, 25, 50, 100, 200])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "animate_param(data, 'min_dist', [0.0, 0.01, 0.1, 0.2, 0.4, 0.6, 0.9])" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "animate_param(data, 'local_connectivity', [0.1, 0.2, 0.5, 1, 2, 5, 10])" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "animate_param(data, 'set_op_mix_ratio', np.linspace(0.0, 1.0, 10))" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/ext/TAL/applications/UMAP_fast/umap/notebooks/Document embedding using UMAP.ipynb b/ext/TAL/applications/UMAP_fast/umap/notebooks/Document embedding using UMAP.ipynb new file mode 100644 index 00000000..cf3739b6 --- /dev/null +++ b/ext/TAL/applications/UMAP_fast/umap/notebooks/Document embedding using UMAP.ipynb @@ -0,0 +1,1916 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Overview\n", + "\n", + "This is an example notebook of using UMAP to embed text (but this can be extended to any collection of tokens). We are going to use the [20 newsgroups dataset](http://qwone.com/~jason/20Newsgroups/) which is a collection of forum posts labelled by topic. We are going to embed these documents and see that similar documents (i.e. posts in the same subforum) will end up close together. You can use this embedding for other downstream tasks such as visualizing your corpus or run a clustering algorithm (e.g. HDBSCAN). We will use a bag of words model and use UMAP on the count vectors as well as the TF-IDF vectors." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Environment setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.8.1\r\n" + ] + } + ], + "source": [ + "!python --version" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First install the dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install numpy scipy pandas scikit-learn datashader holoviews numba" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You will need umap-learn >- 0.4.0 which is currently (at the time of writing) not available via pip/conda. If it is, great! Just run the command below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install umap-learn" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You may need to install it from the master branch of the [github repo](https://github.com/lmcinnes/umap) by following the instructions in the README" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Importing packages" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + " var JS_MIME_TYPE = 'application/javascript';\n", + " var HTML_MIME_TYPE = 'text/html';\n", + " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", + " var CLASS_NAME = 'output_bokeh rendered_html';\n", + "\n", + " /**\n", + " * Render data to the DOM node\n", + " */\n", + " function render(props, node) {\n", + " var script = document.createElement(\"script\");\n", + " node.appendChild(script);\n", + " }\n", + "\n", + " /**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + " function handleClearOutput(event, handle) {\n", + " var cell = handle.cell;\n", + "\n", + " var id = cell.output_area._bokeh_element_id;\n", + " var server_id = cell.output_area._bokeh_server_id;\n", + " // Clean up Bokeh references\n", + " if (id != null && id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + "\n", + " if (server_id !== undefined) {\n", + " // Clean up Bokeh references\n", + " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", + " cell.notebook.kernel.execute(cmd, {\n", + " iopub: {\n", + " output: function(msg) {\n", + " var id = msg.content.text.trim();\n", + " if (id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + " }\n", + " }\n", + " });\n", + " // Destroy server and session\n", + " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", + " cell.notebook.kernel.execute(cmd);\n", + " }\n", + " }\n", + "\n", + " /**\n", + " * Handle when a new output is added\n", + " */\n", + " function handleAddOutput(event, handle) {\n", + " var output_area = handle.output_area;\n", + " var output = handle.output;\n", + "\n", + " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", + " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + "\n", + " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + "\n", + " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", + " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", + " // store reference to embed id on output_area\n", + " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " }\n", + " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " var bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var script_attrs = bk_div.children[0].attributes;\n", + " for (var i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + " }\n", + "\n", + " function register_renderer(events, OutputArea) {\n", + "\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " var toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[toinsert.length - 1]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " /* Handle when an output is cleared or removed */\n", + " events.on('clear_output.CodeCell', handleClearOutput);\n", + " events.on('delete.Cell', handleClearOutput);\n", + "\n", + " /* Handle when a new output is added */\n", + " events.on('output_added.OutputArea', handleAddOutput);\n", + "\n", + " /**\n", + " * Register the mime type and append_mime function with output_area\n", + " */\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " /* Is output safe? */\n", + " safe: true,\n", + " /* Index of renderer in `output_area.display_order` */\n", + " index: 0\n", + " });\n", + " }\n", + "\n", + " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", + " if (root.Jupyter !== undefined) {\n", + " var events = require('base/js/events');\n", + " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", + "\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " }\n", + "\n", + " \n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " var NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"
    \\n\"+\n", + " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", + " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", + " \"
\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " var el = document.getElementById(\"1001\");\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS is loading...\";\n", + " }\n", + " if (root.Bokeh !== undefined) {\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error() {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (var i = 0; i < css_urls.length; i++) {\n", + " var url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " };var element = document.getElementById(\"1001\");\n", + " if (element == null) {\n", + " console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n", + " return false;\n", + " }\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " \n", + " var js_urls = [];\n", + " var css_urls = [];\n", + " \n", + "\n", + " var inline_js = [\n", + " function(Bokeh) {\n", + " /* BEGIN bokeh.min.js */\n", + " /*!\n", + " * Copyright (c) 2012 - 2019, Anaconda, Inc., and Bokeh Contributors\n", + " * All rights reserved.\n", + " * \n", + " * Redistribution and use in source and binary forms, with or without modification,\n", + " * are permitted provided that the following conditions are met:\n", + " * \n", + " * Redistributions of source code must retain the above copyright notice,\n", + " * this list of conditions and the following disclaimer.\n", + " * \n", + " * Redistributions in binary form must reproduce the above copyright notice,\n", + " * this list of conditions and the following disclaimer in the documentation\n", + " * and/or other materials provided with the distribution.\n", + " * \n", + " * Neither the name of Anaconda nor the names of any contributors\n", + " * may be used to endorse or promote products derived from this software\n", + " * without specific prior written permission.\n", + " * \n", + " * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n", + " * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n", + " * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n", + " * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE\n", + " * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\n", + " * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n", + " * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS\n", + " * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN\n", + " * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\n", + " * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF\n", + " * THE POSSIBILITY OF SUCH DAMAGE.\n", + " */\n", + " (function(root, factory) {\n", + " root[\"Bokeh\"] = factory();\n", + " })(this, function() {\n", + " var define;\n", + " var parent_require = typeof require === \"function\" && require\n", + " return (function(modules, entry, aliases, externals) {\n", + " if (aliases === undefined) aliases = {};\n", + " if (externals === undefined) externals = {};\n", + "\n", + " var cache = {};\n", + "\n", + " var normalize = function(name) {\n", + " if (typeof name === \"number\")\n", + " return name;\n", + "\n", + " if (name === \"bokehjs\")\n", + " return entry;\n", + "\n", + " var prefix = \"@bokehjs/\"\n", + " if (name.slice(0, prefix.length) === prefix)\n", + " name = name.slice(prefix.length)\n", + "\n", + " var alias = aliases[name]\n", + " if (alias != null)\n", + " return alias;\n", + "\n", + " var trailing = name.length > 0 && name[name.lenght-1] === \"/\";\n", + " var index = aliases[name + (trailing ? \"\" : \"/\") + \"index\"];\n", + " if (index != null)\n", + " return index;\n", + "\n", + " return name;\n", + " }\n", + "\n", + " var require = function(name) {\n", + " var mod = cache[name];\n", + " if (!mod) {\n", + " var id = normalize(name);\n", + "\n", + " mod = cache[id];\n", + " if (!mod) {\n", + " if (!modules[id]) {\n", + " if (parent_require && externals[id]) {\n", + " try {\n", + " mod = {exports: parent_require(id)};\n", + " cache[id] = cache[name] = mod;\n", + " return mod.exports;\n", + " } catch (e) {}\n", + " }\n", + "\n", + " var err = new Error(\"Cannot find module '\" + name + \"'\");\n", + " err.code = 'MODULE_NOT_FOUND';\n", + " throw err;\n", + " }\n", + "\n", + " mod = {exports: {}};\n", + " cache[id] = cache[name] = mod;\n", + " modules[id].call(mod.exports, require, mod, mod.exports);\n", + " } else\n", + " cache[name] = mod;\n", + " }\n", + "\n", + " return mod.exports;\n", + " }\n", + "\n", + " var main = require(entry);\n", + " main.require = require;\n", + "\n", + " main.register_plugin = function(plugin_modules, plugin_entry, plugin_aliases, plugin_externals) {\n", + " if (plugin_aliases === undefined) plugin_aliases = {};\n", + " if (plugin_externals === undefined) plugin_externals = {};\n", + "\n", + " for (var name in plugin_modules) {\n", + " modules[name] = plugin_modules[name];\n", + " }\n", + "\n", + " for (var name in plugin_aliases) {\n", + " aliases[name] = plugin_aliases[name];\n", + " }\n", + "\n", + " for (var name in plugin_externals) {\n", + " externals[name] = plugin_externals[name];\n", + " }\n", + "\n", + " var plugin = require(plugin_entry);\n", + "\n", + " for (var name in plugin) {\n", + " main[name] = plugin[name];\n", + " }\n", + "\n", + " return plugin;\n", + " }\n", + "\n", + " return main;\n", + " })\n", + " ([\n", + " function _(n,o,r){n(1),function(n){for(var o in n)r.hasOwnProperty(o)||(r[o]=n[o])}(n(102))},\n", + " function _(n,c,f){n(2),n(11),n(14),n(21),n(49),n(52),n(87),n(94),n(100)},\n", + " function _(e,n,a){e(3)()||Object.defineProperty(Object,\"assign\",{value:e(4),configurable:!0,enumerable:!1,writable:!0})},\n", + " function _(r,t,o){t.exports=function(){var r,t=Object.assign;return\"function\"==typeof t&&(t(r={foo:\"raz\"},{bar:\"dwa\"},{trzy:\"trzy\"}),r.foo+r.bar+r.trzy===\"razdwatrzy\")}},\n", + " function _(t,r,n){var o=t(5),c=t(10),a=Math.max;r.exports=function(t,r){var n,f,h,i=a(arguments.length,2);for(t=Object(c(t)),h=function(o){try{t[o]=r[o]}catch(t){n||(n=t)}},f=1;f= 0\");if(!isFinite(r))throw new RangeError(\"Count must be < ∞\");for(n=\"\";r;)r%2&&(n+=t),r>1&&(t+=t),r>>=1;return n}},\n", + " function _(t,i,n){var r=t(18),a=Math.abs,o=Math.floor;i.exports=function(t){return isNaN(t)?0:0!==(t=Number(t))&&isFinite(t)?r(t)*o(a(t)):t}},\n", + " function _(n,t,i){t.exports=n(19)()?Math.sign:n(20)},\n", + " function _(n,t,o){t.exports=function(){var n=Math.sign;return\"function\"==typeof n&&(1===n(10)&&-1===n(-20))}},\n", + " function _(n,r,t){r.exports=function(n){return n=Number(n),isNaN(n)||0===n?n:n>0?1:-1}},\n", + " function _(e,r,a){e(22)()||Object.defineProperty(Array,\"from\",{value:e(23),configurable:!0,enumerable:!1,writable:!0})},\n", + " function _(n,o,r){o.exports=function(){var n,o,r=Array.from;return\"function\"==typeof r&&(o=r(n=[\"raz\",\"dwa\"]),Boolean(o&&o!==n&&\"dwa\"===o[1]))}},\n", + " function _(e,l,r){var n=e(24).iterator,t=e(44),a=e(45),i=e(46),u=e(47),o=e(10),f=e(8),c=e(48),v=Array.isArray,h=Function.prototype.call,y={configurable:!0,enumerable:!0,writable:!0,value:null},s=Object.defineProperty;l.exports=function(e){var l,r,A,g,p,w,b,d,x,j,O=arguments[1],m=arguments[2];if(e=Object(o(e)),f(O)&&u(O),this&&this!==Array&&a(this))l=this;else{if(!O){if(t(e))return 1!==(p=e.length)?Array.apply(null,e):((g=new Array(1))[0]=e[0],g);if(v(e)){for(g=new Array(p=e.length),r=0;r=55296&&w<=56319&&(j+=e[++r]),j=O?h.call(O,m,j,A):j,l?(y.value=j,s(g,A,y)):g[A]=j,++A;p=A}if(void 0===p)for(p=i(e.length),l&&(g=new l(p)),r=0;r-1}},\n", + " function _(r,n,o){var t=r(40);n.exports=function(r){if(!t(r))throw new TypeError(r+\" is not a symbol\");return r}},\n", + " function _(o,t,n){t.exports=function(o){return!!o&&(\"symbol\"==typeof o||!!o.constructor&&(\"Symbol\"===o.constructor.name&&\"Symbol\"===o[o.constructor.toStringTag]))}},\n", + " function _(t,e,n){var r=t(28),o=Object.create,c=Object.defineProperty,u=Object.prototype,f=o(null);e.exports=function(t){for(var e,n,o=0;f[t+(o||\"\")];)++o;return f[t+=o||\"\"]=!0,c(u,e=\"@@\"+t,r.gs(null,function(t){n||(n=!0,c(this,e,r(t)),n=!1)})),e}},\n", + " function _(e,t,a){var s=e(28),i=e(26).Symbol;t.exports=function(e){return Object.defineProperties(e,{hasInstance:s(\"\",i&&i.hasInstance||e(\"hasInstance\")),isConcatSpreadable:s(\"\",i&&i.isConcatSpreadable||e(\"isConcatSpreadable\")),iterator:s(\"\",i&&i.iterator||e(\"iterator\")),match:s(\"\",i&&i.match||e(\"match\")),replace:s(\"\",i&&i.replace||e(\"replace\")),search:s(\"\",i&&i.search||e(\"search\")),species:s(\"\",i&&i.species||e(\"species\")),split:s(\"\",i&&i.split||e(\"split\")),toPrimitive:s(\"\",i&&i.toPrimitive||e(\"toPrimitive\")),toStringTag:s(\"\",i&&i.toStringTag||e(\"toStringTag\")),unscopables:s(\"\",i&&i.unscopables||e(\"unscopables\"))})}},\n", + " function _(r,n,e){var t=r(28),i=r(39),o=Object.create(null);n.exports=function(r){return Object.defineProperties(r,{for:t(function(n){return o[n]?o[n]:o[n]=r(String(n))}),keyFor:t(function(r){var n;for(n in i(r),o)if(o[n]===r)return n})})}},\n", + " function _(t,n,r){var o=Object.prototype.toString,c=o.call(function(){return arguments}());n.exports=function(t){return o.call(t)===c}},\n", + " function _(t,o,n){var e=Object.prototype.toString,c=RegExp.prototype.test.bind(/^[object [A-Za-z0-9]*Function]$/);o.exports=function(t){return\"function\"==typeof t&&c(e.call(t))}},\n", + " function _(n,t,r){var a=n(17),o=Math.max;t.exports=function(n){return o(0,a(n))}},\n", + " function _(n,o,t){o.exports=function(n){if(\"function\"!=typeof n)throw new TypeError(n+\" is not a function\");return n}},\n", + " function _(t,n,o){var e=Object.prototype.toString,r=e.call(\"\");n.exports=function(t){return\"string\"==typeof t||t&&\"object\"==typeof t&&(t instanceof String||e.call(t)===r)||!1}},\n", + " function _(e,a,l){e(50)()||Object.defineProperty(Math,\"log10\",{value:e(51),configurable:!0,enumerable:!1,writable:!0})},\n", + " function _(n,t,o){t.exports=function(){var n=Math.log10;return\"function\"==typeof n&&.3010299956639812===n(2)}},\n", + " function _(N,a,t){var n=Math.log,r=Math.LOG10E;a.exports=function(N){return isNaN(N)?NaN:(N=Number(N))<0?NaN:0===N?-1/0:1===N?0:N===1/0?1/0:n(N)*r}},\n", + " function _(e,n,r){e(53)()||Object.defineProperty(e(26),\"Set\",{value:e(54),configurable:!0,enumerable:!1,writable:!0})},\n", + " function _(t,e,n){e.exports=function(){var t,e;return\"function\"==typeof Set&&(t=new Set([\"raz\",\"dwa\",\"trzy\"]),\"[object Set]\"===String(t)&&(3===t.size&&(\"function\"==typeof t.add&&(\"function\"==typeof t.clear&&(\"function\"==typeof t.delete&&(\"function\"==typeof t.entries&&(\"function\"==typeof t.forEach&&(\"function\"==typeof t.has&&(\"function\"==typeof t.keys&&(\"function\"==typeof t.values&&(!1===(e=t.values().next()).done&&\"raz\"===e.value)))))))))))}},\n", + " function _(t,e,n){var r,i,s,o=t(55),a=t(56),_=t(60),c=t(47),u=t(28),h=t(65),l=t(24),f=t(66),p=t(68),y=t(85),v=t(86),d=Function.prototype.call,D=Object.defineProperty,g=Object.getPrototypeOf;v&&(s=Set),e.exports=r=function(){var t,e=arguments[0];if(!(this instanceof r))throw new TypeError(\"Constructor requires 'new'\");return t=v&&_?_(new s,g(this)):this,null!=e&&f(e),D(t,\"__setData__\",u(\"c\",[])),e?(p(e,function(t){-1===a.call(this,t)&&this.push(t)},t.__setData__),t):t},v&&(_&&_(r,s),r.prototype=Object.create(s.prototype,{constructor:u(r)})),h(Object.defineProperties(r.prototype,{add:u(function(t){return this.has(t)?this:(this.emit(\"_add\",this.__setData__.push(t)-1,t),this)}),clear:u(function(){this.__setData__.length&&(o.call(this.__setData__),this.emit(\"_clear\"))}),delete:u(function(t){var e=a.call(this.__setData__,t);return-1!==e&&(this.__setData__.splice(e,1),this.emit(\"_delete\",e,t),!0)}),entries:u(function(){return new y(this,\"key+value\")}),forEach:u(function(t){var e,n,r,i=arguments[1];for(c(t),n=(e=this.values())._next();void 0!==n;)r=e._resolve(n),d.call(t,i,r,r,this),n=e._next()}),has:u(function(t){return-1!==a.call(this.__setData__,t)}),keys:u(i=function(){return this.values()}),size:u.gs(function(){return this.__setData__.length}),values:u(function(){return new y(this)}),toString:u(function(){return\"[object Set]\"})})),D(r.prototype,l.iterator,u(i)),D(r.prototype,l.toStringTag,u(\"c\",\"Set\"))},\n", + " function _(t,n,i){var r=t(10);n.exports=function(){return r(this).length=0,this}},\n", + " function _(t,r,e){var i=t(57),n=t(46),o=t(10),a=Array.prototype.indexOf,h=Object.prototype.hasOwnProperty,s=Math.abs,p=Math.floor;r.exports=function(t){var r,e,f,l;if(!i(t))return a.apply(this,arguments);for(e=n(o(this).length),f=arguments[1],r=f=isNaN(f)?0:f>=0?p(f):n(this.length)-p(s(f));r=55296&&v<=56319&&(g+=r[++p]),i.call(n,x,g,s),!y);++p);else f.call(r,function(r){return i.call(n,x,r,s),y})}},\n", + " function _(n,t,e){var o=n(44),r=n(48),f=n(70),i=n(84),u=n(66),c=n(24).iterator;t.exports=function(n){return\"function\"==typeof u(n)[c]?n[c]():o(n)?new f(n):r(n)?new i(n):new f(n)}},\n", + " function _(t,e,r){var o,_=t(60),i=t(36),n=t(28),l=t(24),a=t(71),s=Object.defineProperty;o=e.exports=function(t,e){if(!(this instanceof o))throw new TypeError(\"Constructor requires 'new'\");a.call(this,t),e=e?i.call(e,\"key+value\")?\"key+value\":i.call(e,\"key\")?\"key\":\"value\":\"value\",s(this,\"__kind__\",n(\"\",e))},_&&_(o,a),delete o.prototype.constructor,o.prototype=Object.create(a.prototype,{_resolve:n(function(t){return\"value\"===this.__kind__?this.__list__[t]:\"key+value\"===this.__kind__?[t,this.__list__[t]]:t})}),s(o.prototype,l.toStringTag,n(\"c\",\"Array Iterator\"))},\n", + " function _(_,t,e){var n,i=_(55),o=_(34),s=_(47),r=_(10),h=_(28),d=_(72),c=_(24),u=Object.defineProperty,l=Object.defineProperties;t.exports=n=function(_,t){if(!(this instanceof n))throw new TypeError(\"Constructor requires 'new'\");l(this,{__list__:h(\"w\",r(_)),__context__:h(\"w\",t),__nextIndex__:h(\"w\",0)}),t&&(s(t.on),t.on(\"_add\",this._onAdd),t.on(\"_delete\",this._onDelete),t.on(\"_clear\",this._onClear))},delete n.prototype.constructor,l(n.prototype,o({_next:h(function(){var _;if(this.__list__)return this.__redo__&&void 0!==(_=this.__redo__.shift())?_:this.__nextIndex__=this.__nextIndex__||(++this.__nextIndex__,this.__redo__?(this.__redo__.forEach(function(t,e){t>=_&&(this.__redo__[e]=++t)},this),this.__redo__.push(_)):u(this,\"__redo__\",h(\"c\",[_])))}),_onDelete:h(function(_){var t;_>=this.__nextIndex__||(--this.__nextIndex__,this.__redo__&&(-1!==(t=this.__redo__.indexOf(_))&&this.__redo__.splice(t,1),this.__redo__.forEach(function(t,e){t>_&&(this.__redo__[e]=--t)},this)))}),_onClear:h(function(){this.__redo__&&i.call(this.__redo__),this.__nextIndex__=0})}))),u(n.prototype,c.iterator,h(function(){return this}))},\n", + " function _(e,t,n){var r,o=e(29),i=e(73),l=e(78),u=e(79),s=e(35),v=e(81),a=Function.prototype.bind,c=Object.defineProperty,f=Object.prototype.hasOwnProperty;r=function(e,t,n){var r,o=i(t)&&l(t.value);return delete(r=u(t)).writable,delete r.value,r.get=function(){return!n.overwriteDefinition&&f.call(this,e)?o:(t.value=a.call(o,n.resolveContext?n.resolveContext(this):this),c(this,e,t),this[e])},r},t.exports=function(e){var t=s(arguments[1]);return o(t.resolveContext)&&l(t.resolveContext),v(e,function(e,n){return r(n,e,t)})}},\n", + " function _(n,t,o){var r=n(74),u=n(29);t.exports=function(n){return u(n)?n:r(n,\"Cannot use %v\",arguments[1])}},\n", + " function _(r,e,n){var t=r(29),i=r(33),o=r(75),f=r(76),u=function(r,e){return r.replace(\"%v\",f(e))};e.exports=function(r,e,n){if(!i(n))throw new TypeError(u(e,r));if(!t(r)){if(\"default\"in n)return n.default;if(n.isOptional)return null}var f=o(n.errorMessage);throw t(f)||(f=e),new TypeError(u(f,r))}},\n", + " function _(t,n,r){var u=t(29),e=t(33),i=Object.prototype.toString;n.exports=function(t){if(!u(t))return null;if(e(t)){var n=t.toString;if(\"function\"!=typeof n)return null;if(n===i)return null}try{return\"\"+t}catch(t){return null}}},\n", + " function _(r,e,n){var t=r(77),u=/[\\n\\r\\u2028\\u2029]/g;e.exports=function(r){var e=t(r);return null===e?\"\":(e.length>100&&(e=e.slice(0,99)+\"…\"),e=e.replace(u,function(r){switch(r){case\"\\n\":return\"\\\\n\";case\"\\r\":return\"\\\\r\";case\"\\u2028\":return\"\\\\u2028\";case\"\\u2029\":return\"\\\\u2029\";default:throw new Error(\"Unexpected character\")}}))}},\n", + " function _(t,r,n){r.exports=function(t){try{return t.toString()}catch(r){try{return String(t)}catch(t){return null}}}},\n", + " function _(n,t,i){var o=n(74),r=n(30);t.exports=function(n){return r(n)?n:o(n,\"%v is not a plain function\",arguments[1])}},\n", + " function _(n,r,t){var e=n(80),u=n(34),c=n(10);r.exports=function(n){var r=Object(c(n)),t=arguments[1],i=Object(arguments[2]);if(r!==n&&!t)return r;var f={};return t?e(t,function(r){(i.ensure||r in n)&&(f[r]=n[r])}):u(f,n),f}},\n", + " function _(r,o,f){o.exports=r(22)()?Array.from:r(23)},\n", + " function _(n,t,o){var c=n(47),r=n(82),u=Function.prototype.call;t.exports=function(n,t){var o={},a=arguments[2];return c(t),r(n,function(n,c,r,i){o[c]=u.call(t,a,n,c,r,i)}),o}},\n", + " function _(o,c,f){c.exports=o(83)(\"forEach\")},\n", + " function _(t,n,o){var c=t(47),e=t(10),r=Function.prototype.bind,u=Function.prototype.call,l=Object.keys,p=Object.prototype.propertyIsEnumerable;n.exports=function(t,n){return function(o,i){var a,f=arguments[2],y=arguments[3];return o=Object(e(o)),c(i),a=l(o),y&&a.sort(\"function\"==typeof y?r.call(y,o):void 0),\"function\"!=typeof t&&(t=a[t]),u.call(t,a,function(t,c){return p.call(o,t)?u.call(i,f,o[t],t,o,c):n})}}},\n", + " function _(t,_,e){var n,r=t(60),i=t(28),o=t(24),s=t(71),h=Object.defineProperty;n=_.exports=function(t){if(!(this instanceof n))throw new TypeError(\"Constructor requires 'new'\");t=String(t),s.call(this,t),h(this,\"__length__\",i(\"\",t.length))},r&&r(n,s),delete n.prototype.constructor,n.prototype=Object.create(s.prototype,{_next:i(function(){if(this.__list__)return this.__nextIndex__=55296&&_<=56319?e+this.__list__[this.__nextIndex__++]:e})}),h(n.prototype,o.toStringTag,i(\"c\",\"String Iterator\"))},\n", + " function _(t,e,_){var r,i=t(60),o=t(36),n=t(28),s=t(71),a=t(24).toStringTag,c=Object.defineProperty;r=e.exports=function(t,e){if(!(this instanceof r))return new r(t,e);s.call(this,t.__setData__,t),e=e&&o.call(e,\"key+value\")?\"key+value\":\"value\",c(this,\"__kind__\",n(\"\",e))},i&&i(r,s),r.prototype=Object.create(s.prototype,{constructor:n(r),_resolve:n(function(t){return\"value\"===this.__kind__?this.__list__[t]:[this.__list__[t],this.__list__[t]]}),toString:n(function(){return\"[object Set Iterator]\"})}),c(r.prototype,a,n(\"c\",\"Set Iterator\"))},\n", + " function _(t,e,o){e.exports=\"undefined\"!=typeof Set&&\"[object Set]\"===Object.prototype.toString.call(Set.prototype)},\n", + " function _(e,a,n){e(88)()||Object.defineProperty(e(26),\"Map\",{value:e(89),configurable:!0,enumerable:!1,writable:!0})},\n", + " function _(t,e,n){e.exports=function(){var t,e;if(\"function\"!=typeof Map)return!1;try{t=new Map([[\"raz\",\"one\"],[\"dwa\",\"two\"],[\"trzy\",\"three\"]])}catch(t){return!1}return\"[object Map]\"===String(t)&&(3===t.size&&(\"function\"==typeof t.clear&&(\"function\"==typeof t.delete&&(\"function\"==typeof t.entries&&(\"function\"==typeof t.forEach&&(\"function\"==typeof t.get&&(\"function\"==typeof t.has&&(\"function\"==typeof t.keys&&(\"function\"==typeof t.set&&(\"function\"==typeof t.values&&(!1===(e=t.entries().next()).done&&(!!e.value&&(\"raz\"===e.value[0]&&\"one\"===e.value[1])))))))))))))}},\n", + " function _(t,e,a){var _,n=t(55),i=t(56),r=t(60),s=t(47),o=t(10),p=t(28),c=t(65),u=t(24),l=t(66),h=t(68),f=t(90),y=t(93),m=Function.prototype.call,D=Object.defineProperties,v=Object.getPrototypeOf;e.exports=_=function(){var t,e,a,n=arguments[0];if(!(this instanceof _))throw new TypeError(\"Constructor requires 'new'\");return a=y&&r&&Map!==_?r(new Map,v(this)):this,null!=n&&l(n),D(a,{__mapKeysData__:p(\"c\",t=[]),__mapValuesData__:p(\"c\",e=[])}),n?(h(n,function(a){var _=o(a)[0];a=a[1],-1===i.call(t,_)&&(t.push(_),e.push(a))},a),a):a},y&&(r&&r(_,Map),_.prototype=Object.create(Map.prototype,{constructor:p(_)})),c(D(_.prototype,{clear:p(function(){this.__mapKeysData__.length&&(n.call(this.__mapKeysData__),n.call(this.__mapValuesData__),this.emit(\"_clear\"))}),delete:p(function(t){var e=i.call(this.__mapKeysData__,t);return-1!==e&&(this.__mapKeysData__.splice(e,1),this.__mapValuesData__.splice(e,1),this.emit(\"_delete\",e,t),!0)}),entries:p(function(){return new f(this,\"key+value\")}),forEach:p(function(t){var e,a,_=arguments[1];for(s(t),a=(e=this.entries())._next();void 0!==a;)m.call(t,_,this.__mapValuesData__[a],this.__mapKeysData__[a],this),a=e._next()}),get:p(function(t){var e=i.call(this.__mapKeysData__,t);if(-1!==e)return this.__mapValuesData__[e]}),has:p(function(t){return-1!==i.call(this.__mapKeysData__,t)}),keys:p(function(){return new f(this,\"key\")}),set:p(function(t,e){var a,_=i.call(this.__mapKeysData__,t);return-1===_&&(_=this.__mapKeysData__.push(t)-1,a=!0),this.__mapValuesData__[_]=e,a&&this.emit(\"_add\",_,t),this}),size:p.gs(function(){return this.__mapKeysData__.length}),values:p(function(){return new f(this,\"value\")}),toString:p(function(){return\"[object Map]\"})})),Object.defineProperty(_.prototype,u.iterator,p(function(){return this.entries()})),Object.defineProperty(_.prototype,u.toStringTag,p(\"c\",\"Map\"))},\n", + " function _(t,_,e){var i,n=t(60),r=t(28),o=t(71),s=t(24).toStringTag,a=t(91),u=Object.defineProperties,c=o.prototype._unBind;i=_.exports=function(t,_){if(!(this instanceof i))return new i(t,_);o.call(this,t.__mapKeysData__,t),_&&a[_]||(_=\"key+value\"),u(this,{__kind__:r(\"\",_),__values__:r(\"w\",t.__mapValuesData__)})},n&&n(i,o),i.prototype=Object.create(o.prototype,{constructor:r(i),_resolve:r(function(t){return\"value\"===this.__kind__?this.__values__[t]:\"key\"===this.__kind__?this.__list__[t]:[this.__list__[t],this.__values__[t]]}),_unBind:r(function(){this.__values__=null,c.call(this)}),toString:r(function(){return\"[object Map Iterator]\"})}),Object.defineProperty(i.prototype,s,r(\"c\",\"Map Iterator\"))},\n", + " function _(e,u,a){u.exports=e(92)(\"key\",\"value\",\"key+value\")},\n", + " function _(r,t,n){var c=Array.prototype.forEach,o=Object.create;t.exports=function(r){var t=o(null);return c.call(arguments,function(r){t[r]=!0}),t}},\n", + " function _(t,e,o){e.exports=\"undefined\"!=typeof Map&&\"[object Map]\"===Object.prototype.toString.call(new Map)},\n", + " function _(e,a,n){e(95)()||Object.defineProperty(e(26),\"WeakMap\",{value:e(96),configurable:!0,enumerable:!1,writable:!0})},\n", + " function _(t,e,n){e.exports=function(){var t,e;if(\"function\"!=typeof WeakMap)return!1;try{t=new WeakMap([[e={},\"one\"],[{},\"two\"],[{},\"three\"]])}catch(t){return!1}return\"[object WeakMap]\"===String(t)&&(\"function\"==typeof t.set&&(t.set({},1)===t&&(\"function\"==typeof t.delete&&(\"function\"==typeof t.has&&\"one\"===t.get(e)))))}},\n", + " function _(t,e,a){var r,n=t(8),o=t(60),p=t(97),_=t(10),i=t(98),c=t(28),s=t(69),u=t(68),f=t(24).toStringTag,k=t(99),M=Array.isArray,h=Object.defineProperty,w=Object.prototype.hasOwnProperty,y=Object.getPrototypeOf;e.exports=r=function(){var t,e=arguments[0];if(!(this instanceof r))throw new TypeError(\"Constructor requires 'new'\");return t=k&&o&&WeakMap!==r?o(new WeakMap,y(this)):this,n(e)&&(M(e)||(e=s(e))),h(t,\"__weakMapData__\",c(\"c\",\"$weakMap$\"+i())),e?(u(e,function(e){_(e),t.set(e[0],e[1])}),t):t},k&&(o&&o(r,WeakMap),r.prototype=Object.create(WeakMap.prototype,{constructor:c(r)})),Object.defineProperties(r.prototype,{delete:c(function(t){return!!w.call(p(t),this.__weakMapData__)&&(delete t[this.__weakMapData__],!0)}),get:c(function(t){if(w.call(p(t),this.__weakMapData__))return t[this.__weakMapData__]}),has:c(function(t){return w.call(p(t),this.__weakMapData__)}),set:c(function(t,e){return h(p(t),this.__weakMapData__,c(\"c\",e)),this}),toString:c(function(){return\"[object WeakMap]\"})}),h(r.prototype,f,c(\"c\",\"WeakMap\"))},\n", + " function _(n,r,t){var o=n(63);r.exports=function(n){if(!o(n))throw new TypeError(n+\" is not an Object\");return n}},\n", + " function _(t,n,r){var e=Object.create(null),o=Math.random;n.exports=function(){var t;do{t=o().toString(36).slice(2)}while(e[t]);return t}},\n", + " function _(t,e,o){e.exports=\"function\"==typeof WeakMap&&\"[object WeakMap]\"===Object.prototype.toString.call(new WeakMap)},\n", + " function _(l,o,f){o.exports=l(101).polyfill()},\n", + " function _(t,e,r){\n", + " /*!\n", + " * @overview es6-promise - a tiny implementation of Promises/A+.\n", + " * @copyright Copyright (c) 2014 Yehuda Katz, Tom Dale, Stefan Penner and contributors (Conversion to ES6 API by Jake Archibald)\n", + " * @license Licensed under MIT license\n", + " * See https://raw.githubusercontent.com/stefanpenner/es6-promise/master/LICENSE\n", + " * @version v4.2.6+9869a4bc\n", + " */\n", + " !function(t,n){\"object\"==typeof r&&void 0!==e?e.exports=n():\"function\"==typeof define&&define.amd?define(n):t.ES6Promise=n()}(this,function(){\"use strict\";function e(t){return\"function\"==typeof t}var r=Array.isArray?Array.isArray:function(t){return\"[object Array]\"===Object.prototype.toString.call(t)},n=0,o=void 0,i=void 0,s=function(t,e){v[n]=t,v[n+1]=e,2===(n+=2)&&(i?i(p):b())};var u=\"undefined\"!=typeof window?window:void 0,c=u||{},a=c.MutationObserver||c.WebKitMutationObserver,f=\"undefined\"==typeof self&&\"undefined\"!=typeof process&&\"[object process]\"==={}.toString.call(process),l=\"undefined\"!=typeof Uint8ClampedArray&&\"undefined\"!=typeof importScripts&&\"undefined\"!=typeof MessageChannel;function h(){var t=setTimeout;return function(){return t(p,1)}}var v=new Array(1e3);function p(){for(var t=0;t0;)this.remove_root(this._roots[0])}finally{this._pop_all_models_freeze()}},e.prototype.interactive_start=function(e){null==this._interactive_plot&&(this._interactive_plot=e,this._interactive_plot.trigger_event(new s.LODStart)),this._interactive_timestamp=Date.now()},e.prototype.interactive_stop=function(e){null!=this._interactive_plot&&this._interactive_plot.id===e.id&&this._interactive_plot.trigger_event(new s.LODEnd),this._interactive_plot=null,this._interactive_timestamp=null},e.prototype.interactive_duration=function(){return null==this._interactive_timestamp?-1:Date.now()-this._interactive_timestamp},e.prototype.destructively_move=function(e){if(e===this)throw new Error(\"Attempted to overwrite a document with itself\");e.clear();var t=d.copy(this._roots);this.clear();for(var n=0,o=t;n=0&&this._callbacks.splice(t,1)},e.prototype._trigger_on_change=function(e){for(var t=0,n=this._callbacks;t0||d.difference(f,a).length>0)throw new Error(\"Not implemented: computing add/remove of document roots\");var g={},y=[];for(var w in n._all_models)if(w in i){var b=e._events_to_sync_objects(i[w],c[w],n,g);y=y.concat(b)}return{references:e._references_json(h.values(g),!1),events:y}},e.prototype.to_json_string=function(e){return void 0===e&&(e=!0),JSON.stringify(this.to_json(e))},e.prototype.to_json=function(t){void 0===t&&(t=!0);var n=this._roots.map(function(e){return e.id}),o=h.values(this._all_models);return{version:r.version,title:this._title,roots:{root_ids:n,references:e._references_json(o,t)}}},e.from_json_string=function(t){var n=JSON.parse(t);return e.from_json(n)},e.from_json=function(t){i.logger.debug(\"Creating Document from JSON\");var n=t.version,o=-1!==n.indexOf(\"+\")||-1!==n.indexOf(\"-\"),s=\"Library versions: JS (\"+r.version+\") / Python (\"+n+\")\";o||r.version===n?i.logger.debug(s):(i.logger.warn(\"JS/Python version mismatch\"),i.logger.warn(s));var a=t.roots,_=a.root_ids,l=a.references,c=e._instantiate_references_json(l,{});e._initialize_references_json(l,{},c);for(var u=new e,d=0,h=_;d0,\"'step' must be a positive number\"),null==r&&(r=n,n=0);for(var t=n<=r?e:-e,i=(0,Math.max)((0,Math.ceil)((0,Math.abs)(r-n)/e),0),a=Array(i),o=0;o=0?r:n.length+r]},e.zip=function(){for(var n=[],r=0;rt||void 0===e)return 1;if(e2*Math.PI;)n-=2*Math.PI;return n}function o(n,r){return a(n-r)}function u(){return Math.random()}t.angle_norm=a,t.angle_dist=o,t.angle_between=function(n,r,t,u){var e=o(r,t);if(0==e)return!1;if(e==2*Math.PI)return!0;var f=a(n),i=o(r,f)<=e&&o(f,t)<=e;return 0==u?i:!i},t.random=u,t.randomIn=function(n,r){return null==r&&(r=n,n=0),n+Math.floor(Math.random()*(r-n+1))},t.atan2=function(n,r){return Math.atan2(r[1]-n[1],r[0]-n[0])},t.rnorm=function(n,r){for(var t,a;t=u(),a=(2*(a=u())-1)*Math.sqrt(1/Math.E*2),!(-4*t*t*Math.log(t)>=a*a););var o=a/t;return o=n+r*o},t.clamp=function(n,r,t){return n>t?t:n=0;u--)(o=t[u])&&(c=(a<3?o(c):a>3?o(e,n,c):o(e,n))||c);return a>3&&c&&Object.defineProperty(e,n,c),c},u=function(t,e){return function(n,r){e(n,r,t)}},i=function(t,e){if(\"object\"==typeof Reflect&&\"function\"==typeof Reflect.metadata)return Reflect.metadata(t,e)},f=function(t,e,n,r){return new(n||(n=Promise))(function(o,a){function c(t){try{i(r.next(t))}catch(t){a(t)}}function u(t){try{i(r.throw(t))}catch(t){a(t)}}function i(t){t.done?o(t.value):new n(function(e){e(t.value)}).then(c,u)}i((r=r.apply(t,e||[])).next())})},l=function(t,e){var n,r,o,a,c={label:0,sent:function(){if(1&o[0])throw o[1];return o[1]},trys:[],ops:[]};return a={next:u(0),throw:u(1),return:u(2)},\"function\"==typeof Symbol&&(a[Symbol.iterator]=function(){return this}),a;function u(a){return function(u){return function(a){if(n)throw new TypeError(\"Generator is already executing.\");for(;c;)try{if(n=1,r&&(o=2&a[0]?r.return:a[0]?r.throw||((o=r.return)&&o.call(r),0):r.next)&&!(o=o.call(r,a[1])).done)return o;switch(r=0,o&&(a=[2&a[0],o.value]),a[0]){case 0:case 1:o=a;break;case 4:return c.label++,{value:a[1],done:!1};case 5:c.label++,r=a[1],a=[0];continue;case 7:a=c.ops.pop(),c.trys.pop();continue;default:if(!(o=(o=c.trys).length>0&&o[o.length-1])&&(6===a[0]||2===a[0])){c=0;continue}if(3===a[0]&&(!o||a[1]>o[0]&&a[1]=t.length&&(t=void 0),{value:t&&t[n++],done:!t}}}},p=function(t,e){var n=\"function\"==typeof Symbol&&t[Symbol.iterator];if(!n)return t;var r,o,a=n.call(t),c=[];try{for(;(void 0===e||e-- >0)&&!(r=a.next()).done;)c.push(r.value)}catch(t){o={error:t}}finally{try{r&&!r.done&&(n=a.return)&&n.call(a)}finally{if(o)throw o.error}}return c},_=function(){for(var t=[],e=0;e1||u(t,e)})})}function u(t,e){try{(n=o[t](e)).value instanceof h?Promise.resolve(n.value.v).then(i,f):l(a[0][2],n)}catch(t){l(a[0][3],t)}var n}function i(t){u(\"next\",t)}function f(t){u(\"throw\",t)}function l(t,e){t(e),a.shift(),a.length&&u(a[0][0],a[0][1])}},d=function(t){var e,n;return e={},r(\"next\"),r(\"throw\",function(t){throw t}),r(\"return\"),e[Symbol.iterator]=function(){return this},e;function r(r,o){e[r]=t[r]?function(e){return(n=!n)?{value:h(t[r](e)),done:\"return\"===r}:o?o(e):e}:o}},w=function(t){if(!Symbol.asyncIterator)throw new TypeError(\"Symbol.asyncIterator is not defined.\");var e,n=t[Symbol.asyncIterator];return n?n.call(t):(t=y(t),e={},r(\"next\"),r(\"throw\"),r(\"return\"),e[Symbol.asyncIterator]=function(){return this},e);function r(n){e[n]=t[n]&&function(e){return new Promise(function(r,o){(function(t,e,n,r){Promise.resolve(r).then(function(e){t({value:e,done:n})},e)})(r,o,(e=t[n](e)).done,e.value)})}}},m=function(t,e){return Object.defineProperty?Object.defineProperty(t,\"raw\",{value:e}):t.raw=e,t},O=function(t){if(t&&t.__esModule)return t;var e={};if(null!=t)for(var n in t)Object.hasOwnProperty.call(t,n)&&(e[n]=t[n]);return e.default=t,e},j=function(t){return t&&t.__esModule?t:{default:t}},t(\"__extends\",r),t(\"__assign\",o),t(\"__rest\",a),t(\"__decorate\",c),t(\"__param\",u),t(\"__metadata\",i),t(\"__awaiter\",f),t(\"__generator\",l),t(\"__exportStar\",s),t(\"__values\",y),t(\"__read\",p),t(\"__spread\",_),t(\"__spreadArrays\",b),t(\"__await\",h),t(\"__asyncGenerator\",v),t(\"__asyncDelegator\",d),t(\"__asyncValues\",w),t(\"__makeTemplateObject\",m),t(\"__importStar\",O),t(\"__importDefault\",j)})},\n", + " function _(n,r,t){function e(n,r,t){for(var e=[],o=3;ou&&(r=u),null==t||t>u-r?t=u-r:t<0&&(t=0);for(var i=u-t+e.length,f=new n.constructor(i),a=0;a0?0:e-1;o>=0&&ot&&(t=r);return t},t.max_by=function(n,r){if(0==n.length)throw new Error(\"max_by() called with an empty array\");for(var t=n[0],e=r(t),o=1,u=n.length;oe&&(t=i,e=f)}return t},t.sum=function(n){for(var r=0,t=0,e=n.length;t0&&(this._pending=!0);for(var p=0;p0?this._dict[t]=s:delete this._dict[t]}else i.isEqual(e,n)&&delete this._dict[t]},t.prototype.get_one=function(t,n){var e=this._existing(t);if(o.isArray(e)){if(1===e.length)return e[0];throw new Error(n)}return e},t}();e.MultiDict=s,s.__name__=\"MultiDict\";var a=function(){function t(n){if(null==n)this._values=[];else if(n instanceof t)this._values=r.copy(n._values);else{this._values=[];for(var e=0,i=n;et?(a&&(clearTimeout(a),a=null),o=c,i=n.apply(r,u),a||(r=u=null)):a||!1===e.trailing||(a=setTimeout(l,f)),i}},e.once=function(n){var t,e=!1;return function(){return e||(e=!0,t=n()),t}}},\n", + " function _(e,t,n){var r=e(121),a=e(125);function l(e,t){var n={};for(var r in e){var a=e[r];n[t+r]=a}return n}var i={line_color:[r.ColorSpec,\"black\"],line_width:[r.NumberSpec,1],line_alpha:[r.NumberSpec,1],line_join:[r.LineJoin,\"bevel\"],line_cap:[r.LineCap,\"butt\"],line_dash:[r.Array,[]],line_dash_offset:[r.Number,0]};n.line=function(e){return void 0===e&&(e=\"\"),l(i,e)};var o={fill_color:[r.ColorSpec,\"gray\"],fill_alpha:[r.NumberSpec,1]};n.fill=function(e){return void 0===e&&(e=\"\"),l(o,e)};var c={hatch_color:[r.ColorSpec,\"black\"],hatch_alpha:[r.NumberSpec,1],hatch_scale:[r.NumberSpec,12],hatch_pattern:[r.StringSpec,null],hatch_weight:[r.NumberSpec,1],hatch_extra:[r.Any,{}]};n.hatch=function(e){return void 0===e&&(e=\"\"),l(c,e)};var h={text_font:[r.Font,\"helvetica\"],text_font_size:[r.FontSizeSpec,\"12pt\"],text_font_style:[r.FontStyle,\"normal\"],text_color:[r.ColorSpec,\"#444444\"],text_alpha:[r.NumberSpec,1],text_align:[r.TextAlign,\"left\"],text_baseline:[r.TextBaseline,\"bottom\"],text_line_height:[r.Number,1.2]};n.text=function(e){return void 0===e&&(e=\"\"),l(h,e)},n.create=function(e){for(var t={},r=0,l=e;r\",\"*\"],n.HTTPMethod=[\"POST\",\"GET\"],n.HexTileOrientation=[\"pointytop\",\"flattop\"],n.HoverMode=[\"mouse\",\"hline\",\"vline\"],n.LatLon=[\"lat\",\"lon\"],n.LegendClickPolicy=[\"none\",\"hide\",\"mute\"],n.LegendLocation=n.Anchor,n.LineCap=[\"butt\",\"round\",\"square\"],n.LineJoin=[\"miter\",\"round\",\"bevel\"],n.LinePolicy=[\"prev\",\"next\",\"nearest\",\"interp\",\"none\"],n.Location=[\"above\",\"below\",\"left\",\"right\"],n.Logo=[\"normal\",\"grey\"],n.MarkerType=[\"asterisk\",\"circle\",\"circle_cross\",\"circle_x\",\"cross\",\"dash\",\"diamond\",\"diamond_cross\",\"hex\",\"inverted_triangle\",\"square\",\"square_cross\",\"square_x\",\"triangle\",\"x\"],n.Orientation=[\"vertical\",\"horizontal\"],n.OutputBackend=[\"canvas\",\"svg\",\"webgl\"],n.PaddingUnits=[\"percent\",\"absolute\"],n.Place=[\"above\",\"below\",\"left\",\"right\",\"center\"],n.PointPolicy=[\"snap_to_data\",\"follow_mouse\",\"none\"],n.RadiusDimension=[\"x\",\"y\",\"max\",\"min\"],n.RenderLevel=[\"image\",\"underlay\",\"glyph\",\"annotation\",\"overlay\"],n.RenderMode=[\"canvas\",\"css\"],n.ResetPolicy=[\"standard\",\"event_only\"],n.RoundingFunction=[\"round\",\"nearest\",\"floor\",\"rounddown\",\"ceil\",\"roundup\"],n.Side=[\"above\",\"below\",\"left\",\"right\"],n.SizingMode=[\"stretch_width\",\"stretch_height\",\"stretch_both\",\"scale_width\",\"scale_height\",\"scale_both\",\"fixed\"],n.SliderCallbackPolicy=[\"continuous\",\"throttle\",\"mouseup\"],n.Sort=[\"ascending\",\"descending\"],n.SpatialUnits=[\"screen\",\"data\"],n.StartEnd=[\"start\",\"end\"],n.StepMode=[\"after\",\"before\",\"center\"],n.TapBehavior=[\"select\",\"inspect\"],n.TextAlign=[\"left\",\"right\",\"center\"],n.TextBaseline=[\"top\",\"middle\",\"bottom\",\"alphabetic\",\"hanging\",\"ideographic\"],n.TextureRepetition=[\"repeat\",\"repeat_x\",\"repeat_y\",\"no_repeat\"],n.TickLabelOrientation=[\"vertical\",\"horizontal\",\"parallel\",\"normal\"],n.TooltipAttachment=[\"horizontal\",\"vertical\",\"left\",\"right\",\"above\",\"below\"],n.UpdateMode=[\"replace\",\"append\"],n.VerticalAlign=[\"top\",\"middle\",\"bottom\"]},\n", + " function _(r,e,t){var n=r(124),a=r(110);function o(r){var e=Number(r).toString(16);return 1==e.length?\"0\"+e:e}function l(r){if(0==(r+=\"\").indexOf(\"#\"))return r;if(n.is_svg_color(r))return n.svg_colors[r];if(0==r.indexOf(\"rgb\")){var e=r.replace(/^rgba?\\(|\\s+|\\)$/g,\"\").split(\",\"),t=e.slice(0,3).map(o).join(\"\");return 4==e.length&&(t+=o(Math.floor(255*parseFloat(e[3])))),\"#\"+t.slice(0,8)}return r}function i(r){var e;switch(r.substring(0,4)){case\"rgba\":e={start:\"rgba(\",len:4,alpha:!0};break;case\"rgb(\":e={start:\"rgb(\",len:3,alpha:!1};break;default:return!1}if(new RegExp(\".*?(\\\\.).*(,)\").test(r))throw new Error(\"color expects integers for rgb in rgb/rgba tuple, received \"+r);var t=r.replace(e.start,\"\").replace(\")\",\"\").split(\",\").map(parseFloat);if(t.length!=e.len)throw new Error(\"color expects rgba \"+e.len+\"-tuple, received \"+r);if(e.alpha&&!(0<=t[3]&&t[3]<=1))throw new Error(\"color expects rgba 4-tuple to have alpha value between 0 and 1\");if(a.includes(t.slice(0,3).map(function(r){return 0<=r&&r<=255}),!1))throw new Error(\"color expects rgb to have value between 0 and 255\");return!0}t.is_color=function(r){return n.is_svg_color(r.toLowerCase())||\"#\"==r.substring(0,1)||i(r)},t.rgb2hex=function(r,e,t){return\"#\"+o(255&r)+o(255&e)+o(255&t)},t.color2hex=l,t.color2rgba=function(r,e){if(void 0===e&&(e=1),!r)return[0,0,0,0];var t=l(r);(t=t.replace(/ |#/g,\"\")).length<=4&&(t=t.replace(/(.)/g,\"$1$1\"));for(var n=t.match(/../g).map(function(r){return parseInt(r,16)/255});n.length<3;)n.push(0);return n.length<4&&n.push(e),n.slice(0,4)},t.valid_rgb=i},\n", + " function _(F,e,r){r.svg_colors={indianred:\"#CD5C5C\",lightcoral:\"#F08080\",salmon:\"#FA8072\",darksalmon:\"#E9967A\",lightsalmon:\"#FFA07A\",crimson:\"#DC143C\",red:\"#FF0000\",firebrick:\"#B22222\",darkred:\"#8B0000\",pink:\"#FFC0CB\",lightpink:\"#FFB6C1\",hotpink:\"#FF69B4\",deeppink:\"#FF1493\",mediumvioletred:\"#C71585\",palevioletred:\"#DB7093\",coral:\"#FF7F50\",tomato:\"#FF6347\",orangered:\"#FF4500\",darkorange:\"#FF8C00\",orange:\"#FFA500\",gold:\"#FFD700\",yellow:\"#FFFF00\",lightyellow:\"#FFFFE0\",lemonchiffon:\"#FFFACD\",lightgoldenrodyellow:\"#FAFAD2\",papayawhip:\"#FFEFD5\",moccasin:\"#FFE4B5\",peachpuff:\"#FFDAB9\",palegoldenrod:\"#EEE8AA\",khaki:\"#F0E68C\",darkkhaki:\"#BDB76B\",lavender:\"#E6E6FA\",thistle:\"#D8BFD8\",plum:\"#DDA0DD\",violet:\"#EE82EE\",orchid:\"#DA70D6\",fuchsia:\"#FF00FF\",magenta:\"#FF00FF\",mediumorchid:\"#BA55D3\",mediumpurple:\"#9370DB\",blueviolet:\"#8A2BE2\",darkviolet:\"#9400D3\",darkorchid:\"#9932CC\",darkmagenta:\"#8B008B\",purple:\"#800080\",indigo:\"#4B0082\",slateblue:\"#6A5ACD\",darkslateblue:\"#483D8B\",mediumslateblue:\"#7B68EE\",greenyellow:\"#ADFF2F\",chartreuse:\"#7FFF00\",lawngreen:\"#7CFC00\",lime:\"#00FF00\",limegreen:\"#32CD32\",palegreen:\"#98FB98\",lightgreen:\"#90EE90\",mediumspringgreen:\"#00FA9A\",springgreen:\"#00FF7F\",mediumseagreen:\"#3CB371\",seagreen:\"#2E8B57\",forestgreen:\"#228B22\",green:\"#008000\",darkgreen:\"#006400\",yellowgreen:\"#9ACD32\",olivedrab:\"#6B8E23\",olive:\"#808000\",darkolivegreen:\"#556B2F\",mediumaquamarine:\"#66CDAA\",darkseagreen:\"#8FBC8F\",lightseagreen:\"#20B2AA\",darkcyan:\"#008B8B\",teal:\"#008080\",aqua:\"#00FFFF\",cyan:\"#00FFFF\",lightcyan:\"#E0FFFF\",paleturquoise:\"#AFEEEE\",aquamarine:\"#7FFFD4\",turquoise:\"#40E0D0\",mediumturquoise:\"#48D1CC\",darkturquoise:\"#00CED1\",cadetblue:\"#5F9EA0\",steelblue:\"#4682B4\",lightsteelblue:\"#B0C4DE\",powderblue:\"#B0E0E6\",lightblue:\"#ADD8E6\",skyblue:\"#87CEEB\",lightskyblue:\"#87CEFA\",deepskyblue:\"#00BFFF\",dodgerblue:\"#1E90FF\",cornflowerblue:\"#6495ED\",royalblue:\"#4169E1\",blue:\"#0000FF\",mediumblue:\"#0000CD\",darkblue:\"#00008B\",navy:\"#000080\",midnightblue:\"#191970\",cornsilk:\"#FFF8DC\",blanchedalmond:\"#FFEBCD\",bisque:\"#FFE4C4\",navajowhite:\"#FFDEAD\",wheat:\"#F5DEB3\",burlywood:\"#DEB887\",tan:\"#D2B48C\",rosybrown:\"#BC8F8F\",sandybrown:\"#F4A460\",goldenrod:\"#DAA520\",darkgoldenrod:\"#B8860B\",peru:\"#CD853F\",chocolate:\"#D2691E\",saddlebrown:\"#8B4513\",sienna:\"#A0522D\",brown:\"#A52A2A\",maroon:\"#800000\",white:\"#FFFFFF\",snow:\"#FFFAFA\",honeydew:\"#F0FFF0\",mintcream:\"#F5FFFA\",azure:\"#F0FFFF\",aliceblue:\"#F0F8FF\",ghostwhite:\"#F8F8FF\",whitesmoke:\"#F5F5F5\",seashell:\"#FFF5EE\",beige:\"#F5F5DC\",oldlace:\"#FDF5E6\",floralwhite:\"#FFFAF0\",ivory:\"#FFFFF0\",antiquewhite:\"#FAEBD7\",linen:\"#FAF0E6\",lavenderblush:\"#FFF0F5\",mistyrose:\"#FFE4E1\",gainsboro:\"#DCDCDC\",lightgray:\"#D3D3D3\",lightgrey:\"#D3D3D3\",silver:\"#C0C0C0\",darkgray:\"#A9A9A9\",darkgrey:\"#A9A9A9\",gray:\"#808080\",grey:\"#808080\",dimgray:\"#696969\",dimgrey:\"#696969\",lightslategray:\"#778899\",lightslategrey:\"#778899\",slategray:\"#708090\",slategrey:\"#708090\",darkslategray:\"#2F4F4F\",darkslategrey:\"#2F4F4F\",black:\"#000000\"},r.is_svg_color=function(F){return F in r.svg_colors}},\n", + " function _(e,n,t){var r=e(113),c=e(110);function o(e,n){return r.__assign(e,n)}function u(e){return Object.keys(e).length}t.keys=Object.keys,t.values=function(e){for(var n=Object.keys(e),t=n.length,r=new Array(t),c=0;c\"'`])/g,function(r){switch(r){case\"&\":return\"&\";case\"<\":return\"<\";case\">\":return\">\";case'\"':return\""\";case\"'\":return\"'\";case\"`\":return\"`\";default:return r}})},e.unescape=function(r){return r.replace(/&(amp|lt|gt|quot|#x27|#x60);/g,function(r,t){switch(t){case\"amp\":return\"&\";case\"lt\":return\"<\";case\"gt\":return\">\";case\"quot\":return'\"';case\"#x27\":return\"'\";case\"#x60\":return\"`\";default:return t}})},e.use_strict=function(r){return\"'use strict';\\n\"+r}},\n", + " function _(e,t,n){var i=function(){function e(){this._dev=!1}return Object.defineProperty(e.prototype,\"dev\",{get:function(){return this._dev},set:function(e){this._dev=e},enumerable:!0,configurable:!0}),e}();n.Settings=i,i.__name__=\"Settings\",n.settings=new i},\n", + " function _(n,o,r){function f(n){for(var o in n)r.hasOwnProperty(o)||(r[o]=n[o])}f(n(130)),f(n(242)),f(n(269)),f(n(273)),f(n(288)),f(n(292)),f(n(298)),f(n(302)),f(n(332)),f(n(335)),f(n(337)),f(n(350)),f(n(217)),f(n(356)),f(n(360)),f(n(383)),f(n(384)),f(n(385)),f(n(386)),f(n(387)),f(n(393)),f(n(395)),f(n(405)),f(n(409))},\n", + " function _(a,e,o){var r=a(131);o.Annotation=r.Annotation;var n=a(168);o.Arrow=n.Arrow;var t=a(169);o.ArrowHead=t.ArrowHead;var v=a(169);o.OpenHead=v.OpenHead;var l=a(169);o.NormalHead=l.NormalHead;var d=a(169);o.TeeHead=d.TeeHead;var i=a(169);o.VeeHead=i.VeeHead;var A=a(200);o.Band=A.Band;var H=a(201);o.BoxAnnotation=H.BoxAnnotation;var T=a(203);o.ColorBar=T.ColorBar;var p=a(227);o.Label=p.Label;var L=a(229);o.LabelSet=L.LabelSet;var b=a(230);o.Legend=b.Legend;var B=a(231);o.LegendItem=B.LegendItem;var S=a(233);o.PolyAnnotation=S.PolyAnnotation;var g=a(234);o.Slope=g.Slope;var m=a(235);o.Span=m.Span;var w=a(228);o.TextAnnotation=w.TextAnnotation;var x=a(236);o.Title=x.Title;var P=a(237);o.ToolbarPanel=P.ToolbarPanel;var h=a(238);o.Tooltip=h.Tooltip;var k=a(241);o.Whisker=k.Whisker},\n", + " function _(t,e,n){var i=t(113),o=t(132),r=t(125),s=t(160),a=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return i.__extends(e,t),Object.defineProperty(e.prototype,\"panel\",{get:function(){return this.layout},enumerable:!0,configurable:!0}),e.prototype.get_size=function(){if(this.model.visible){var t=this._get_size(),e=t.width,n=t.height;return{width:Math.round(e),height:Math.round(n)}}return{width:0,height:0}},e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this);var n=this.model.properties;this.on_change(n.visible,function(){return e.plot_view.request_layout()})},e.prototype._get_size=function(){throw new Error(\"not implemented\")},Object.defineProperty(e.prototype,\"ctx\",{get:function(){return this.plot_view.canvas_view.ctx},enumerable:!0,configurable:!0}),e.prototype.set_data=function(t){var e,n,i=this.model.materialize_dataspecs(t);if(r.extend(this,i),this.plot_model.use_map){null!=this._x&&(e=o.project_xy(this._x,this._y),this._x=e[0],this._y=e[1]),null!=this._xs&&(n=o.project_xsys(this._xs,this._ys),this._xs=n[0],this._ys=n[1])}},Object.defineProperty(e.prototype,\"needs_clip\",{get:function(){return null==this.layout},enumerable:!0,configurable:!0}),e.prototype.serializable_state=function(){var e=t.prototype.serializable_state.call(this);return null==this.layout?e:Object.assign(Object.assign({},e),{bbox:this.layout.bbox.box})},e}(s.RendererView);n.AnnotationView=a,a.__name__=\"AnnotationView\";var l=function(t){function e(e){return t.call(this,e)||this}return i.__extends(e,t),e.init_Annotation=function(){this.override({level:\"annotation\"})},e}(s.Renderer);n.Annotation=l,l.__name__=\"Annotation\",l.init_Annotation()},\n", + " function _(r,n,t){var a=r(133),e=r(134),o=new e(\"GOOGLE\"),c=new e(\"WGS84\");t.wgs84_mercator=a(c,o);var i={lon:[-20026376.39,20026376.39],lat:[-20048966.1,20048966.1]},u={lon:[-180,180],lat:[-85.06,85.06]};function l(r,n){for(var a=Math.min(r.length,n.length),e=new Array(a),o=new Array(a),c=0;cu[n][0]&&r-1})}(n)?i(n):function(n){return\"+\"===n[0]}(n)?o(n):void 0:n}},\n", + " function _(r,n,i){var t=r(137),e=r(138),a=r(141);function f(r){var n=this;if(2===arguments.length){var i=arguments[1];\"string\"==typeof i?\"+\"===i.charAt(0)?f[r]=e(arguments[1]):f[r]=a(arguments[1]):f[r]=i}else if(1===arguments.length){if(Array.isArray(r))return r.map(function(r){Array.isArray(r)?f.apply(n,r):f(r)});if(\"string\"==typeof r){if(r in f)return f[r]}else\"EPSG\"in r?f[\"EPSG:\"+r.EPSG]=r:\"ESRI\"in r?f[\"ESRI:\"+r.ESRI]=r:\"IAU2000\"in r?f[\"IAU2000:\"+r.IAU2000]=r:console.log(r);return}}t(f),n.exports=f},\n", + " function _(t,l,G){l.exports=function(t){t(\"EPSG:4326\",\"+title=WGS 84 (long/lat) +proj=longlat +ellps=WGS84 +datum=WGS84 +units=degrees\"),t(\"EPSG:4269\",\"+title=NAD83 (long/lat) +proj=longlat +a=6378137.0 +b=6356752.31414036 +ellps=GRS80 +datum=NAD83 +units=degrees\"),t(\"EPSG:3857\",\"+title=WGS 84 / Pseudo-Mercator +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs\"),t.WGS84=t[\"EPSG:4326\"],t[\"EPSG:3785\"]=t[\"EPSG:3857\"],t.GOOGLE=t[\"EPSG:3857\"],t[\"EPSG:900913\"]=t[\"EPSG:3857\"],t[\"EPSG:102113\"]=t[\"EPSG:3857\"]}},\n", + " function _(n,t,o){var a=.017453292519943295,u=n(139),e=n(140);t.exports=function(n){var t,o,r,i={},f=n.split(\"+\").map(function(n){return n.trim()}).filter(function(n){return n}).reduce(function(n,t){var o=t.split(\"=\");return o.push(!0),n[o[0].toLowerCase()]=o[1],n},{}),s={proj:\"projName\",datum:\"datumCode\",rf:function(n){i.rf=parseFloat(n)},lat_0:function(n){i.lat0=n*a},lat_1:function(n){i.lat1=n*a},lat_2:function(n){i.lat2=n*a},lat_ts:function(n){i.lat_ts=n*a},lon_0:function(n){i.long0=n*a},lon_1:function(n){i.long1=n*a},lon_2:function(n){i.long2=n*a},alpha:function(n){i.alpha=parseFloat(n)*a},lonc:function(n){i.longc=n*a},x_0:function(n){i.x0=parseFloat(n)},y_0:function(n){i.y0=parseFloat(n)},k_0:function(n){i.k0=parseFloat(n)},k:function(n){i.k0=parseFloat(n)},a:function(n){i.a=parseFloat(n)},b:function(n){i.b=parseFloat(n)},r_a:function(){i.R_A=!0},zone:function(n){i.zone=parseInt(n,10)},south:function(){i.utmSouth=!0},towgs84:function(n){i.datum_params=n.split(\",\").map(function(n){return parseFloat(n)})},to_meter:function(n){i.to_meter=parseFloat(n)},units:function(n){i.units=n,e[n]&&(i.to_meter=e[n].to_meter)},from_greenwich:function(n){i.from_greenwich=n*a},pm:function(n){i.from_greenwich=(u[n]?u[n]:parseFloat(n))*a},nadgrids:function(n){\"@null\"===n?i.datumCode=\"none\":i.nadgrids=n},axis:function(n){3===n.length&&-1!==\"ewnsud\".indexOf(n.substr(0,1))&&-1!==\"ewnsud\".indexOf(n.substr(1,1))&&-1!==\"ewnsud\".indexOf(n.substr(2,1))&&(i.axis=n)}};for(t in f)o=f[t],t in s?\"function\"==typeof(r=s[t])?r(o):i[r]=o:i[t]=o;return\"string\"==typeof i.datumCode&&\"WGS84\"!==i.datumCode&&(i.datumCode=i.datumCode.toLowerCase()),i}},\n", + " function _(o,r,s){s.greenwich=0,s.lisbon=-9.131906111111,s.paris=2.337229166667,s.bogota=-74.080916666667,s.madrid=-3.687938888889,s.rome=12.452333333333,s.bern=7.439583333333,s.jakarta=106.807719444444,s.ferro=-17.666666666667,s.brussels=4.367975,s.stockholm=18.058277777778,s.athens=23.7163375,s.oslo=10.722916666667},\n", + " function _(t,e,f){f.ft={to_meter:.3048},f[\"us-ft\"]={to_meter:1200/3937}},\n", + " function _(e,a,t){var r=.017453292519943295,n=e(142);function o(e,a,t){e[a]=t.map(function(e){var a={};return l(e,a),a}).reduce(function(e,a){return n(e,a)},{})}function l(e,a){var t;Array.isArray(e)?(\"PARAMETER\"===(t=e.shift())&&(t=e.shift()),1===e.length?Array.isArray(e[0])?(a[t]={},l(e[0],a[t])):a[t]=e[0]:e.length?\"TOWGS84\"===t?a[t]=e:(a[t]={},[\"UNIT\",\"PRIMEM\",\"VERT_DATUM\"].indexOf(t)>-1?(a[t]={name:e[0].toLowerCase(),convert:e[1]},3===e.length&&(a[t].auth=e[2])):\"SPHEROID\"===t?(a[t]={name:e[0],a:e[1],rf:e[2]},4===e.length&&(a[t].auth=e[3])):[\"GEOGCS\",\"GEOCCS\",\"DATUM\",\"VERT_CS\",\"COMPD_CS\",\"LOCAL_CS\",\"FITTED_CS\",\"LOCAL_DATUM\"].indexOf(t)>-1?(e[0]=[\"name\",e[0]],o(a,t,e)):e.every(function(e){return Array.isArray(e)})?o(a,t,e):l(e,a[t])):a[t]=!0):a[e]=!0}function i(e){return e*r}a.exports=function(e,a){var t=JSON.parse((\",\"+e).replace(/\\s*\\,\\s*([A-Z_0-9]+?)(\\[)/g,',[\"$1\",').slice(1).replace(/\\s*\\,\\s*([A-Z_0-9]+?)\\]/g,',\"$1\"]').replace(/,\\[\"VERTCS\".+/,\"\")),r=t.shift(),o=t.shift();t.unshift([\"name\",o]),t.unshift([\"type\",r]),t.unshift(\"output\");var _={};return l(t,_),function(e){function a(a){var t=e.to_meter||1;return parseFloat(a,10)*t}\"GEOGCS\"===e.type?e.projName=\"longlat\":\"LOCAL_CS\"===e.type?(e.projName=\"identity\",e.local=!0):\"object\"==typeof e.PROJECTION?e.projName=Object.keys(e.PROJECTION)[0]:e.projName=e.PROJECTION,e.UNIT&&(e.units=e.UNIT.name.toLowerCase(),\"metre\"===e.units&&(e.units=\"meter\"),e.UNIT.convert&&(\"GEOGCS\"===e.type?e.DATUM&&e.DATUM.SPHEROID&&(e.to_meter=parseFloat(e.UNIT.convert,10)*e.DATUM.SPHEROID.a):e.to_meter=parseFloat(e.UNIT.convert,10))),e.GEOGCS&&(e.GEOGCS.DATUM?e.datumCode=e.GEOGCS.DATUM.name.toLowerCase():e.datumCode=e.GEOGCS.name.toLowerCase(),\"d_\"===e.datumCode.slice(0,2)&&(e.datumCode=e.datumCode.slice(2)),\"new_zealand_geodetic_datum_1949\"!==e.datumCode&&\"new_zealand_1949\"!==e.datumCode||(e.datumCode=\"nzgd49\"),\"wgs_1984\"===e.datumCode&&(\"Mercator_Auxiliary_Sphere\"===e.PROJECTION&&(e.sphere=!0),e.datumCode=\"wgs84\"),\"_ferro\"===e.datumCode.slice(-6)&&(e.datumCode=e.datumCode.slice(0,-6)),\"_jakarta\"===e.datumCode.slice(-8)&&(e.datumCode=e.datumCode.slice(0,-8)),~e.datumCode.indexOf(\"belge\")&&(e.datumCode=\"rnb72\"),e.GEOGCS.DATUM&&e.GEOGCS.DATUM.SPHEROID&&(e.ellps=e.GEOGCS.DATUM.SPHEROID.name.replace(\"_19\",\"\").replace(/[Cc]larke\\_18/,\"clrk\"),\"international\"===e.ellps.toLowerCase().slice(0,13)&&(e.ellps=\"intl\"),e.a=e.GEOGCS.DATUM.SPHEROID.a,e.rf=parseFloat(e.GEOGCS.DATUM.SPHEROID.rf,10)),~e.datumCode.indexOf(\"osgb_1936\")&&(e.datumCode=\"osgb36\")),e.b&&!isFinite(e.b)&&(e.b=e.a),[[\"standard_parallel_1\",\"Standard_Parallel_1\"],[\"standard_parallel_2\",\"Standard_Parallel_2\"],[\"false_easting\",\"False_Easting\"],[\"false_northing\",\"False_Northing\"],[\"central_meridian\",\"Central_Meridian\"],[\"latitude_of_origin\",\"Latitude_Of_Origin\"],[\"latitude_of_origin\",\"Central_Parallel\"],[\"scale_factor\",\"Scale_Factor\"],[\"k0\",\"scale_factor\"],[\"latitude_of_center\",\"Latitude_of_center\"],[\"lat0\",\"latitude_of_center\",i],[\"longitude_of_center\",\"Longitude_Of_Center\"],[\"longc\",\"longitude_of_center\",i],[\"x0\",\"false_easting\",a],[\"y0\",\"false_northing\",a],[\"long0\",\"central_meridian\",i],[\"lat0\",\"latitude_of_origin\",i],[\"lat0\",\"standard_parallel_1\",i],[\"lat1\",\"standard_parallel_1\",i],[\"lat2\",\"standard_parallel_2\",i],[\"alpha\",\"azimuth\",i],[\"srsCode\",\"name\"]].forEach(function(a){return t=e,n=(r=a)[0],o=r[1],void(!(n in t)&&o in t&&(t[n]=t[o],3===r.length&&(t[n]=r[2](t[n]))));var t,r,n,o}),e.long0||!e.longc||\"Albers_Conic_Equal_Area\"!==e.projName&&\"Lambert_Azimuthal_Equal_Area\"!==e.projName||(e.long0=e.longc),e.lat_ts||!e.lat1||\"Stereographic_South_Pole\"!==e.projName&&\"Polar Stereographic (variant B)\"!==e.projName||(e.lat0=i(e.lat1>0?90:-90),e.lat_ts=e.lat1)}(_.output),n(a,_.output)}},\n", + " function _(n,r,i){r.exports=function(n,r){var i,o;if(n=n||{},!r)return n;for(o in r)void 0!==(i=r[o])&&(n[o]=i);return n}},\n", + " function _(n,o,t){var r=[n(144),n(150)],e={},a=[];function i(n,o){var t=a.length;return n.names?(a[t]=n,n.names.forEach(function(n){e[n.toLowerCase()]=t}),this):(console.log(o),!0)}t.add=i,t.get=function(n){if(!n)return!1;var o=n.toLowerCase();return void 0!==e[o]&&a[e[o]]?a[e[o]]:void 0},t.start=function(){r.forEach(i)}},\n", + " function _(t,s,i){var h=t(145),a=Math.PI/2,e=57.29577951308232,r=t(146),n=Math.PI/4,l=t(148),o=t(149);i.init=function(){var t=this.b/this.a;this.es=1-t*t,\"x0\"in this||(this.x0=0),\"y0\"in this||(this.y0=0),this.e=Math.sqrt(this.es),this.lat_ts?this.sphere?this.k0=Math.cos(this.lat_ts):this.k0=h(this.e,Math.sin(this.lat_ts),Math.cos(this.lat_ts)):this.k0||(this.k?this.k0=this.k:this.k0=1)},i.forward=function(t){var s,i,h=t.x,o=t.y;if(o*e>90&&o*e<-90&&h*e>180&&h*e<-180)return null;if(Math.abs(Math.abs(o)-a)<=1e-10)return null;if(this.sphere)s=this.x0+this.a*this.k0*r(h-this.long0),i=this.y0+this.a*this.k0*Math.log(Math.tan(n+.5*o));else{var M=Math.sin(o),u=l(this.e,o,M);s=this.x0+this.a*this.k0*r(h-this.long0),i=this.y0-this.a*this.k0*Math.log(u)}return t.x=s,t.y=i,t},i.inverse=function(t){var s,i,h=t.x-this.x0,e=t.y-this.y0;if(this.sphere)i=a-2*Math.atan(Math.exp(-e/(this.a*this.k0)));else{var n=Math.exp(-e/(this.a*this.k0));if(-9999===(i=o(this.e,n)))return null}return s=r(this.long0+h/(this.a*this.k0)),t.x=s,t.y=i,t},i.names=[\"Mercator\",\"Popular Visualisation Pseudo Mercator\",\"Mercator_1SP\",\"Mercator_Auxiliary_Sphere\",\"merc\"]},\n", + " function _(t,n,r){n.exports=function(t,n,r){var o=t*n;return r/Math.sqrt(1-o*o)}},\n", + " function _(t,n,a){var r=2*Math.PI,o=t(147);n.exports=function(t){return Math.abs(t)<=3.14159265359?t:t-o(t)*r}},\n", + " function _(n,t,o){t.exports=function(n){return n<0?-1:1}},\n", + " function _(t,a,n){var r=Math.PI/2;a.exports=function(t,a,n){var o=t*n,h=.5*t;return o=Math.pow((1-o)/(1+o),h),Math.tan(.5*(r-a))/o}},\n", + " function _(a,t,n){var r=Math.PI/2;t.exports=function(a,t){for(var n,h,M=.5*a,o=r-2*Math.atan(t),e=0;e<=15;e++)if(n=a*Math.sin(o),o+=h=r-2*Math.atan(t*Math.pow((1-n)/(1+n),M))-o,Math.abs(h)<=1e-10)return o;return-9999}},\n", + " function _(n,i,t){function e(n){return n}t.init=function(){},t.forward=e,t.inverse=e,t.names=[\"longlat\",\"identity\"]},\n", + " function _(r,e,t){var n=r(152);t.eccentricity=function(r,e,t,n){var a=r*r,c=e*e,f=(a-c)/a,i=0;return n?(a=(r*=1-f*(.16666666666666666+f*(.04722222222222222+.022156084656084655*f)))*r,f=0):i=Math.sqrt(f),{es:f,e:i,ep2:(a-c)/c}},t.sphere=function(r,e,t,a,c){if(!r){var f=n[a];f||(f=n.WGS84),r=f.a,e=f.b,t=f.rf}return t&&!e&&(e=(1-1/t)*r),(0===t||Math.abs(r-e)<1e-10)&&(c=!0,e=r),{a:r,b:e,rf:t,sphere:c}}},\n", + " function _(e,a,l){l.MERIT={a:6378137,rf:298.257,ellipseName:\"MERIT 1983\"},l.SGS85={a:6378136,rf:298.257,ellipseName:\"Soviet Geodetic System 85\"},l.GRS80={a:6378137,rf:298.257222101,ellipseName:\"GRS 1980(IUGG, 1980)\"},l.IAU76={a:6378140,rf:298.257,ellipseName:\"IAU 1976\"},l.airy={a:6377563.396,b:6356256.91,ellipseName:\"Airy 1830\"},l.APL4={a:6378137,rf:298.25,ellipseName:\"Appl. Physics. 1965\"},l.NWL9D={a:6378145,rf:298.25,ellipseName:\"Naval Weapons Lab., 1965\"},l.mod_airy={a:6377340.189,b:6356034.446,ellipseName:\"Modified Airy\"},l.andrae={a:6377104.43,rf:300,ellipseName:\"Andrae 1876 (Den., Iclnd.)\"},l.aust_SA={a:6378160,rf:298.25,ellipseName:\"Australian Natl & S. Amer. 1969\"},l.GRS67={a:6378160,rf:298.247167427,ellipseName:\"GRS 67(IUGG 1967)\"},l.bessel={a:6377397.155,rf:299.1528128,ellipseName:\"Bessel 1841\"},l.bess_nam={a:6377483.865,rf:299.1528128,ellipseName:\"Bessel 1841 (Namibia)\"},l.clrk66={a:6378206.4,b:6356583.8,ellipseName:\"Clarke 1866\"},l.clrk80={a:6378249.145,rf:293.4663,ellipseName:\"Clarke 1880 mod.\"},l.clrk58={a:6378293.645208759,rf:294.2606763692654,ellipseName:\"Clarke 1858\"},l.CPM={a:6375738.7,rf:334.29,ellipseName:\"Comm. des Poids et Mesures 1799\"},l.delmbr={a:6376428,rf:311.5,ellipseName:\"Delambre 1810 (Belgium)\"},l.engelis={a:6378136.05,rf:298.2566,ellipseName:\"Engelis 1985\"},l.evrst30={a:6377276.345,rf:300.8017,ellipseName:\"Everest 1830\"},l.evrst48={a:6377304.063,rf:300.8017,ellipseName:\"Everest 1948\"},l.evrst56={a:6377301.243,rf:300.8017,ellipseName:\"Everest 1956\"},l.evrst69={a:6377295.664,rf:300.8017,ellipseName:\"Everest 1969\"},l.evrstSS={a:6377298.556,rf:300.8017,ellipseName:\"Everest (Sabah & Sarawak)\"},l.fschr60={a:6378166,rf:298.3,ellipseName:\"Fischer (Mercury Datum) 1960\"},l.fschr60m={a:6378155,rf:298.3,ellipseName:\"Fischer 1960\"},l.fschr68={a:6378150,rf:298.3,ellipseName:\"Fischer 1968\"},l.helmert={a:6378200,rf:298.3,ellipseName:\"Helmert 1906\"},l.hough={a:6378270,rf:297,ellipseName:\"Hough\"},l.intl={a:6378388,rf:297,ellipseName:\"International 1909 (Hayford)\"},l.kaula={a:6378163,rf:298.24,ellipseName:\"Kaula 1961\"},l.lerch={a:6378139,rf:298.257,ellipseName:\"Lerch 1979\"},l.mprts={a:6397300,rf:191,ellipseName:\"Maupertius 1738\"},l.new_intl={a:6378157.5,b:6356772.2,ellipseName:\"New International 1967\"},l.plessis={a:6376523,rf:6355863,ellipseName:\"Plessis 1817 (France)\"},l.krass={a:6378245,rf:298.3,ellipseName:\"Krassovsky, 1942\"},l.SEasia={a:6378155,b:6356773.3205,ellipseName:\"Southeast Asia\"},l.walbeck={a:6376896,b:6355834.8467,ellipseName:\"Walbeck\"},l.WGS60={a:6378165,rf:298.3,ellipseName:\"WGS 60\"},l.WGS66={a:6378145,rf:298.25,ellipseName:\"WGS 66\"},l.WGS7={a:6378135,rf:298.26,ellipseName:\"WGS 72\"},l.WGS84={a:6378137,rf:298.257223563,ellipseName:\"WGS 84\"},l.sphere={a:6370997,b:6370997,ellipseName:\"Normal Sphere (r=6370997)\"}},\n", + " function _(e,a,s){s.wgs84={towgs84:\"0,0,0\",ellipse:\"WGS84\",datumName:\"WGS84\"},s.ch1903={towgs84:\"674.374,15.056,405.346\",ellipse:\"bessel\",datumName:\"swiss\"},s.ggrs87={towgs84:\"-199.87,74.79,246.62\",ellipse:\"GRS80\",datumName:\"Greek_Geodetic_Reference_System_1987\"},s.nad83={towgs84:\"0,0,0\",ellipse:\"GRS80\",datumName:\"North_American_Datum_1983\"},s.nad27={nadgrids:\"@conus,@alaska,@ntv2_0.gsb,@ntv1_can.dat\",ellipse:\"clrk66\",datumName:\"North_American_Datum_1927\"},s.potsdam={towgs84:\"606.0,23.0,413.0\",ellipse:\"bessel\",datumName:\"Potsdam Rauenberg 1950 DHDN\"},s.carthage={towgs84:\"-263.0,6.0,431.0\",ellipse:\"clark80\",datumName:\"Carthage 1934 Tunisia\"},s.hermannskogel={towgs84:\"653.0,-212.0,449.0\",ellipse:\"bessel\",datumName:\"Hermannskogel\"},s.ire65={towgs84:\"482.530,-130.596,564.557,-1.042,-0.214,-0.631,8.15\",ellipse:\"mod_airy\",datumName:\"Ireland 1965\"},s.rassadiran={towgs84:\"-133.63,-157.5,-158.62\",ellipse:\"intl\",datumName:\"Rassadiran\"},s.nzgd49={towgs84:\"59.47,-5.04,187.44,0.47,-0.1,1.024,-4.5993\",ellipse:\"intl\",datumName:\"New Zealand Geodetic Datum 1949\"},s.osgb36={towgs84:\"446.448,-125.157,542.060,0.1502,0.2470,0.8421,-20.4894\",ellipse:\"airy\",datumName:\"Airy 1830\"},s.s_jtsk={towgs84:\"589,76,480\",ellipse:\"bessel\",datumName:\"S-JTSK (Ferro)\"},s.beduaram={towgs84:\"-106,-87,188\",ellipse:\"clrk80\",datumName:\"Beduaram\"},s.gunung_segara={towgs84:\"-403,684,41\",ellipse:\"bessel\",datumName:\"Gunung Segara Jakarta\"},s.rnb72={towgs84:\"106.869,-52.2978,103.724,-0.33657,0.456955,-1.84218,1\",ellipse:\"intl\",datumName:\"Reseau National Belge 1972\"}},\n", + " function _(a,m,t){var p=1,u=2,r=4,_=5,d=484813681109536e-20;m.exports=function(a,m,t,s,e,n){var o={};return o.datum_type=r,a&&\"none\"===a&&(o.datum_type=_),m&&(o.datum_params=m.map(parseFloat),0===o.datum_params[0]&&0===o.datum_params[1]&&0===o.datum_params[2]||(o.datum_type=p),o.datum_params.length>3&&(0===o.datum_params[3]&&0===o.datum_params[4]&&0===o.datum_params[5]&&0===o.datum_params[6]||(o.datum_type=u,o.datum_params[3]*=d,o.datum_params[4]*=d,o.datum_params[5]*=d,o.datum_params[6]=o.datum_params[6]/1e6+1))),o.a=t,o.b=s,o.es=e,o.ep2=n,o}},\n", + " function _(t,e,r){var m=.017453292519943295,a=57.29577951308232,o=1,u=2,n=t(156),d=t(158),y=t(134),_=t(159);e.exports=function t(e,r,x){var i;return Array.isArray(x)&&(x=_(x)),e.datum&&r.datum&&function(t,e){return(t.datum.datum_type===o||t.datum.datum_type===u)&&\"WGS84\"!==e.datumCode||(e.datum.datum_type===o||e.datum.datum_type===u)&&\"WGS84\"!==t.datumCode}(e,r)&&(x=t(e,i=new y(\"WGS84\"),x),e=i),\"enu\"!==e.axis&&(x=d(e,!1,x)),\"longlat\"===e.projName?x={x:x.x*m,y:x.y*m}:(e.to_meter&&(x={x:x.x*e.to_meter,y:x.y*e.to_meter}),x=e.inverse(x)),e.from_greenwich&&(x.x+=e.from_greenwich),x=n(e.datum,r.datum,x),r.from_greenwich&&(x={x:x.x-r.grom_greenwich,y:x.y}),\"longlat\"===r.projName?x={x:x.x*a,y:x.y*a}:(x=r.forward(x),r.to_meter&&(x={x:x.x/r.to_meter,y:x.y/r.to_meter})),\"enu\"!==r.axis?d(r,!0,x):x}},\n", + " function _(t,e,a){var u=1,m=2,o=t(157);function c(t){return t===u||t===m}e.exports=function(t,e,a){return o.compareDatums(t,e)?a:5===t.datum_type||5===e.datum_type?a:t.es!==e.es||t.a!==e.a||c(t.datum_type)||c(e.datum_type)?(a=o.geodeticToGeocentric(a,t.es,t.a),c(t.datum_type)&&(a=o.geocentricToWgs84(a,t.datum_type,t.datum_params)),c(e.datum_type)&&(a=o.geocentricFromWgs84(a,e.datum_type,e.datum_params)),o.geocentricToGeodetic(a,e.es,e.a,e.b)):a}},\n", + " function _(a,t,r){var m=Math.PI/2;r.compareDatums=function(a,t){return a.datum_type===t.datum_type&&(!(a.a!==t.a||Math.abs(this.es-t.es)>5e-11)&&(1===a.datum_type?this.datum_params[0]===t.datum_params[0]&&a.datum_params[1]===t.datum_params[1]&&a.datum_params[2]===t.datum_params[2]:2!==a.datum_type||a.datum_params[0]===t.datum_params[0]&&a.datum_params[1]===t.datum_params[1]&&a.datum_params[2]===t.datum_params[2]&&a.datum_params[3]===t.datum_params[3]&&a.datum_params[4]===t.datum_params[4]&&a.datum_params[5]===t.datum_params[5]&&a.datum_params[6]===t.datum_params[6]))},r.geodeticToGeocentric=function(a,t,r){var s,u,e,n,d=a.x,i=a.y,p=a.z?a.z:0;if(i<-m&&i>-1.001*m)i=-m;else if(i>m&&i<1.001*m)i=m;else if(i<-m||i>m)return null;return d>Math.PI&&(d-=2*Math.PI),u=Math.sin(i),n=Math.cos(i),e=u*u,{x:((s=r/Math.sqrt(1-t*e))+p)*n*Math.cos(d),y:(s+p)*n*Math.sin(d),z:(s*(1-t)+p)*u}},r.geocentricToGeodetic=function(a,t,r,s){var u,e,n,d,i,p,_,h,o,y,c,z,M,x,f,g=a.x,l=a.y,q=a.z?a.z:0;if(u=Math.sqrt(g*g+l*l),e=Math.sqrt(g*g+l*l+q*q),u/r<1e-12){if(x=0,e/r<1e-12)return m,f=-s,{x:a.x,y:a.y,z:a.z}}else x=Math.atan2(l,g);n=q/e,h=(d=u/e)*(1-t)*(i=1/Math.sqrt(1-t*(2-t)*d*d)),o=n*i,M=0;do{M++,p=t*(_=r/Math.sqrt(1-t*o*o))/(_+(f=u*h+q*o-_*(1-t*o*o))),z=(c=n*(i=1/Math.sqrt(1-p*(2-p)*d*d)))*h-(y=d*(1-p)*i)*o,h=y,o=c}while(z*z>1e-24&&M<30);return{x:x,y:Math.atan(c/Math.abs(y)),z:f}},r.geocentricToWgs84=function(a,t,r){if(1===t)return{x:a.x+r[0],y:a.y+r[1],z:a.z+r[2]};if(2===t){var m=r[0],s=r[1],u=r[2],e=r[3],n=r[4],d=r[5],i=r[6];return{x:i*(a.x-d*a.y+n*a.z)+m,y:i*(d*a.x+a.y-e*a.z)+s,z:i*(-n*a.x+e*a.y+a.z)+u}}},r.geocentricFromWgs84=function(a,t,r){if(1===t)return{x:a.x-r[0],y:a.y-r[1],z:a.z-r[2]};if(2===t){var m=r[0],s=r[1],u=r[2],e=r[3],n=r[4],d=r[5],i=r[6],p=(a.x-m)/i,_=(a.y-s)/i,h=(a.z-u)/i;return{x:p+d*_-n*h,y:-d*p+_+e*h,z:n*p-e*_+h}}}},\n", + " function _(e,a,r){a.exports=function(e,a,r){var s,c,i,n=r.x,o=r.y,t=r.z||0,u={};for(i=0;i<3;i++)if(!a||2!==i||void 0!==r.z)switch(0===i?(s=n,c=\"x\"):1===i?(s=o,c=\"y\"):(s=t,c=\"z\"),e.axis[i]){case\"e\":u[c]=s;break;case\"w\":u[c]=-s;break;case\"n\":u[c]=s;break;case\"s\":u[c]=-s;break;case\"u\":void 0!==r[c]&&(u.z=s);break;case\"d\":void 0!==r[c]&&(u.z=-s);break;default:return null}return u}},\n", + " function _(n,t,e){t.exports=function(n){var t={x:n[0],y:n[1]};return n.length>2&&(t.z=n[2]),n.length>3&&(t.m=n[3]),t}},\n", + " function _(e,t,n){var i=e(113),r=e(161),o=e(165),l=e(121),u=e(166),_=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(t,e),t.prototype.initialize=function(){e.prototype.initialize.call(this),this.visuals=new o.Visuals(this.model),this._has_finished=!0},Object.defineProperty(t.prototype,\"plot_view\",{get:function(){return this.parent},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"plot_model\",{get:function(){return this.parent.model},enumerable:!0,configurable:!0}),t.prototype.request_render=function(){this.plot_view.request_render()},t.prototype.map_to_screen=function(e,t){return this.plot_view.map_to_screen(e,t,this.model.x_range_name,this.model.y_range_name)},Object.defineProperty(t.prototype,\"needs_clip\",{get:function(){return!1},enumerable:!0,configurable:!0}),t.prototype.notify_finished=function(){this.plot_view.notify_finished()},Object.defineProperty(t.prototype,\"has_webgl\",{get:function(){return!1},enumerable:!0,configurable:!0}),t}(r.DOMView);n.RendererView=_,_.__name__=\"RendererView\";var p=function(e){function t(t){return e.call(this,t)||this}return i.__extends(t,e),t.init_Renderer=function(){this.define({level:[l.RenderLevel],visible:[l.Boolean,!0]})},t}(u.Model);n.Renderer=p,p.__name__=\"Renderer\",p.init_Renderer()},\n", + " function _(e,t,n){var i=e(113),r=e(162),o=e(163),s=e(164),p=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(t,e),t.prototype.initialize=function(){e.prototype.initialize.call(this),this._has_finished=!1,this.el=this._createElement()},t.prototype.remove=function(){o.removeElement(this.el),e.prototype.remove.call(this)},t.prototype.css_classes=function(){return[]},t.prototype.cursor=function(e,t){return null},t.prototype.render=function(){},t.prototype.renderTo=function(e){e.appendChild(this.el),this.render()},t.prototype.has_finished=function(){return this._has_finished},Object.defineProperty(t.prototype,\"_root_element\",{get:function(){return o.parent(this.el,\".\"+s.bk_root)||document.body},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"is_idle\",{get:function(){return this.has_finished()},enumerable:!0,configurable:!0}),t.prototype._createElement=function(){return o.createElement(this.tagName,{class:this.css_classes()})},t}(r.View);n.DOMView=p,p.__name__=\"DOMView\",p.prototype.tagName=\"div\"},\n", + " function _(t,e,n){var o=t(113),i=t(116),r=t(109),a=t(127),s=function(t){function e(e){var n=t.call(this)||this;if(n.removed=new i.Signal0(n,\"removed\"),null==e.model)throw new Error(\"model of a view wasn't configured\");return n.model=e.model,n._parent=e.parent,n.id=e.id||a.uniqueId(),n.initialize(),!1!==e.connect_signals&&n.connect_signals(),n}return o.__extends(e,t),e.prototype.initialize=function(){},e.prototype.remove=function(){this._parent=void 0,this.disconnect_signals(),this.removed.emit()},e.prototype.toString=function(){return this.model.type+\"View(\"+this.id+\")\"},e.prototype.serializable_state=function(){return{type:this.model.type}},Object.defineProperty(e.prototype,\"parent\",{get:function(){if(void 0!==this._parent)return this._parent;throw new Error(\"parent of a view wasn't configured\")},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"is_root\",{get:function(){return null===this.parent},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"root\",{get:function(){return this.is_root?this:this.parent.root},enumerable:!0,configurable:!0}),e.prototype.assert_root=function(){if(!this.is_root)throw new Error(this.toString()+\" is not a root layout\")},e.prototype.connect_signals=function(){},e.prototype.disconnect_signals=function(){i.Signal.disconnectReceiver(this)},e.prototype.on_change=function(t,e){for(var n=0,o=r.isArray(t)?t:[t];n\":case\"vertical_wave\":_.moveTo(n,0),_.lineTo(3*n,c),_.lineTo(n,l),_.stroke();break;case\"*\":case\"criss_cross\":h(_,l),o(_,l,c),s(_,l,c)}return r}var r=function(){function e(e,t){void 0===t&&(t=\"\"),this.obj=e,this.prefix=t,this.cache={};for(var a=0,i=this.attrs;a0){var n=t[l];return null==n&&(t[l]=n=new e(l,o)),n}throw new TypeError(\"Logger.get() expects a non-empty string name and an optional log-level\")},Object.defineProperty(e.prototype,\"level\",{get:function(){return this.get_level()},enumerable:!0,configurable:!0}),e.prototype.get_level=function(){return this._log_level},e.prototype.set_level=function(l){if(l instanceof r)this._log_level=l;else{if(!n.isString(l)||null==e.log_levels[l])throw new Error(\"Logger.set_level() expects a log-level object or a string name of a log-level\");this._log_level=e.log_levels[l]}var o=\"[\"+this._name+\"]\";for(var t in e.log_levels){e.log_levels[t].levele?a.slice(-e):a}if(l.isTypedArray(t)){var i=t.length+n.length;if(null!=e&&i>e){var r=i-e,o=t.length;a=void 0;t.length0?this.selected_glyphs[0]:null},enumerable:!0,configurable:!0}),e.prototype.add_to_selected_glyphs=function(i){this.selected_glyphs.push(i)},e.prototype.update=function(i,e,t){this.final=e,t?this.update_through_union(i):(this.indices=i.indices,this.line_indices=i.line_indices,this.selected_glyphs=i.selected_glyphs,this.get_view=i.get_view,this.multiline_indices=i.multiline_indices,this.image_indices=i.image_indices)},e.prototype.clear=function(){this.final=!0,this.indices=[],this.line_indices=[],this.multiline_indices={},this.get_view=function(){return null},this.selected_glyphs=[]},e.prototype.is_empty=function(){return 0==this.indices.length&&0==this.line_indices.length&&0==this.image_indices.length},e.prototype.update_through_union=function(i){this.indices=l.union(i.indices,this.indices),this.selected_glyphs=l.union(i.selected_glyphs,this.selected_glyphs),this.line_indices=l.union(i.line_indices,this.line_indices),this.get_view()||(this.get_view=i.get_view),this.multiline_indices=h.merge(i.multiline_indices,this.multiline_indices)},e.prototype.update_through_intersection=function(i){this.indices=l.intersection(i.indices,this.indices),this.selected_glyphs=l.union(i.selected_glyphs,this.selected_glyphs),this.line_indices=l.union(i.line_indices,this.line_indices),this.get_view()||(this.get_view=i.get_view),this.multiline_indices=h.merge(i.multiline_indices,this.multiline_indices)},e}(s.Model);t.Selection=d,d.__name__=\"Selection\",d.init_Selection()},\n", + " function _(e,t,i){var n=e(113),o=e(115),r=e(173),s=e(175),c=e(192),l=e(121),p=function(e){function t(t){var i=e.call(this,t)||this;return i.inspectors={},i}return n.__extends(t,e),t.init_SelectionManager=function(){this.internal({source:[l.Any]})},t.prototype.select=function(e,t,i,n){void 0===n&&(n=!1);for(var o=[],r=[],l=0,p=e;l0){d=this.source.selection_policy.hit_test(t,o);a=a||this.source.selection_policy.do_selection(d,this.source,i,n)}return a},t.prototype.inspect=function(e,t){var i=!1;if(e instanceof s.GlyphRendererView){if(null!=(o=e.hit_test(t))){i=!o.is_empty();var n=this.get_or_create_inspector(e.model);n.update(o,!0,!1),this.source.setv({inspected:n},{silent:!0}),this.source.inspect.emit([e,{geometry:t}])}}else if(e instanceof c.GraphRendererView){var o=e.model.inspection_policy.hit_test(t,e);i=i||e.model.inspection_policy.do_inspection(o,t,e,!1,!1)}return i},t.prototype.clear=function(e){this.source.selected.clear(),null!=e&&this.get_or_create_inspector(e.model).clear()},t.prototype.get_or_create_inspector=function(e){return null==this.inspectors[e.id]&&(this.inspectors[e.id]=new r.Selection),this.inspectors[e.id]},t}(o.HasProps);i.SelectionManager=p,p.__name__=\"SelectionManager\",p.init_SelectionManager()},\n", + " function _(e,t,i){var n=e(113),l=e(176),s=e(177),h=e(187),r=e(188),o=e(190),a=e(191),d=e(167),c=e(121),_=e(114),p=e(110),u=e(125),g=e(184),y={fill:{},line:{}},m={fill:{fill_alpha:.3,fill_color:\"grey\"},line:{line_alpha:.3,line_color:\"grey\"}},v={fill:{fill_alpha:.2},line:{}},f=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return n.__extends(t,e),t.prototype.initialize=function(){e.prototype.initialize.call(this);var t=this.model.glyph,i=p.includes(t.mixins,\"fill\"),n=p.includes(t.mixins,\"line\"),l=u.clone(t.attributes);function s(e){var s=u.clone(l);return i&&u.extend(s,e.fill),n&&u.extend(s,e.line),new t.constructor(s)}delete l.id,this.glyph=this.build_glyph_view(t);var h=this.model.selection_glyph;null==h?h=s({fill:{},line:{}}):\"auto\"===h&&(h=s(y)),this.selection_glyph=this.build_glyph_view(h);var r=this.model.nonselection_glyph;null==r?r=s({fill:{},line:{}}):\"auto\"===r&&(r=s(v)),this.nonselection_glyph=this.build_glyph_view(r);var o=this.model.hover_glyph;null!=o&&(this.hover_glyph=this.build_glyph_view(o));var a=this.model.muted_glyph;null!=a&&(this.muted_glyph=this.build_glyph_view(a));var d=s(m);this.decimated_glyph=this.build_glyph_view(d),this.xscale=this.plot_view.frame.xscales[this.model.x_range_name],this.yscale=this.plot_view.frame.yscales[this.model.y_range_name],this.set_data(!1)},t.prototype.build_glyph_view=function(e){return new e.default_view({model:e,parent:this})},t.prototype.connect_signals=function(){var t=this;e.prototype.connect_signals.call(this),this.connect(this.model.change,function(){return t.request_render()}),this.connect(this.model.glyph.change,function(){return t.set_data()}),this.connect(this.model.data_source.change,function(){return t.set_data()}),this.connect(this.model.data_source.streaming,function(){return t.set_data()}),this.connect(this.model.data_source.patching,function(e){return t.set_data(!0,e)}),this.connect(this.model.data_source.selected.change,function(){return t.request_render()}),this.connect(this.model.data_source._select,function(){return t.request_render()}),null!=this.hover_glyph&&this.connect(this.model.data_source.inspect,function(){return t.request_render()}),this.connect(this.model.properties.view.change,function(){return t.set_data()}),this.connect(this.model.view.change,function(){return t.set_data()}),this.connect(this.model.properties.visible.change,function(){return t.plot_view.update_dataranges()});var i=this.plot_view.frame,n=i.x_ranges,l=i.y_ranges;for(var s in n){(h=n[s])instanceof g.FactorRange&&this.connect(h.change,function(){return t.set_data()})}for(var s in l){var h;(h=l[s])instanceof g.FactorRange&&this.connect(h.change,function(){return t.set_data()})}this.connect(this.model.glyph.transformchange,function(){return t.set_data()})},t.prototype.have_selection_glyphs=function(){return null!=this.selection_glyph&&null!=this.nonselection_glyph},t.prototype.set_data=function(e,t){void 0===e&&(e=!0),void 0===t&&(t=null);var i=Date.now(),n=this.model.data_source;this.all_indices=this.model.view.indices,this.glyph.model.setv({x_range_name:this.model.x_range_name,y_range_name:this.model.y_range_name},{silent:!0}),this.glyph.set_data(n,this.all_indices,t),this.glyph.set_visuals(n),this.decimated_glyph.set_visuals(n),this.have_selection_glyphs()&&(this.selection_glyph.set_visuals(n),this.nonselection_glyph.set_visuals(n)),null!=this.hover_glyph&&this.hover_glyph.set_visuals(n),null!=this.muted_glyph&&this.muted_glyph.set_visuals(n);var l=this.plot_model.lod_factor;this.decimated=[];for(var s=0,h=Math.floor(this.all_indices.length/l);s0?w[\"1d\"].indices:_.map(Object.keys(w[\"2d\"].indices),function(e){return parseInt(e)})),x=_.filter(a,function(t){return b.has(e.all_indices[t])}),D=this.plot_model.lod_threshold;null!=this.model.document&&this.model.document.interactive_duration()>0&&!i&&null!=D&&this.all_indices.length>D?(a=this.decimated,m=this.decimated_glyph,v=this.decimated_glyph,f=this.selection_glyph):(m=this.model.muted&&null!=this.muted_glyph?this.muted_glyph:this.glyph,v=this.nonselection_glyph,f=this.selection_glyph),null!=this.hover_glyph&&x.length&&(a=p.difference(a,x));var R,V=null;if(g.length&&this.have_selection_glyphs()){for(var G=Date.now(),A={},I=0,q=g;I1&&(t.stroke(),r=!1)}r?t.lineTo(n[l],s[l]):(t.beginPath(),t.moveTo(n[l],s[l]),r=!0),_=l}r&&t.stroke()},e.prototype._hit_point=function(t){for(var e=this,i=_.create_empty_hit_test_result(),n={x:t.sx,y:t.sy},s=9999,r=Math.max(2,this.visuals.line.line_width.value()/2),o=0,h=this.sx.length-1;o0){this.index=new e(n.length);for(var t=0,i=n;to&&(e=(t=[o,e])[0],o=t[1]),r>a&&(r=(i=[a,r])[0],a=i[1]),{x0:e,y0:r,x1:o,y1:a}},Object.defineProperty(n.prototype,\"bbox\",{get:function(){if(null==this.index)return r.empty();var n=this.index;return{x0:n.minX,y0:n.minY,x1:n.maxX,y1:n.maxY}},enumerable:!0,configurable:!0}),n.prototype.search=function(n){var t=this;if(null==this.index)return[];var i=this._normalize(n),e=i.x0,r=i.y0,o=i.x1,a=i.y1;return this.index.search(e,r,o,a).map(function(n){return t.points[n]})},n.prototype.indices=function(n){return this.search(n).map(function(n){return n.i})},n}();i.SpatialIndex=o,o.__name__=\"SpatialIndex\"},\n", + " function _(t,s,i){var e,h;e=this,h=function(){\"use strict\";var t=function(){this.ids=[],this.values=[],this.length=0};t.prototype.clear=function(){this.length=this.ids.length=this.values.length=0},t.prototype.push=function(t,s){this.ids.push(t),this.values.push(s);for(var i=this.length++;i>0;){var e=i-1>>1,h=this.values[e];if(s>=h)break;this.ids[i]=this.ids[e],this.values[i]=h,i=e}this.ids[i]=t,this.values[i]=s},t.prototype.pop=function(){if(0!==this.length){var t=this.ids[0];if(this.length--,this.length>0){for(var s=this.ids[0]=this.ids[this.length],i=this.values[0]=this.values[this.length],e=this.length>>1,h=0;h=i)break;this.ids[h]=o,this.values[h]=a,h=r}this.ids[h]=s,this.values[h]=i}return this.ids.pop(),this.values.pop(),t}},t.prototype.peek=function(){return this.ids[0]},t.prototype.peekValue=function(){return this.values[0]};var s=[Int8Array,Uint8Array,Uint8ClampedArray,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array],i=function(i,e,h,r){if(void 0===e&&(e=16),void 0===h&&(h=Float64Array),void 0===i)throw new Error(\"Missing required argument: numItems.\");if(isNaN(i)||i<=0)throw new Error(\"Unpexpected numItems value: \"+i+\".\");this.numItems=+i,this.nodeSize=Math.min(Math.max(+e,2),65535);var n=i,o=n;this._levelBounds=[4*n];do{o+=n=Math.ceil(n/this.nodeSize),this._levelBounds.push(4*o)}while(1!==n);this.ArrayType=h||Float64Array,this.IndexArrayType=o<16384?Uint16Array:Uint32Array;var a=s.indexOf(this.ArrayType),u=4*o*this.ArrayType.BYTES_PER_ELEMENT;if(a<0)throw new Error(\"Unexpected typed array class: \"+h+\".\");r&&r instanceof ArrayBuffer?(this.data=r,this._boxes=new this.ArrayType(this.data,8,4*o),this._indices=new this.IndexArrayType(this.data,8+u,o),this._pos=4*o,this.minX=this._boxes[this._pos-4],this.minY=this._boxes[this._pos-3],this.maxX=this._boxes[this._pos-2],this.maxY=this._boxes[this._pos-1]):(this.data=new ArrayBuffer(8+u+o*this.IndexArrayType.BYTES_PER_ELEMENT),this._boxes=new this.ArrayType(this.data,8,4*o),this._indices=new this.IndexArrayType(this.data,8+u,o),this._pos=0,this.minX=1/0,this.minY=1/0,this.maxX=-1/0,this.maxY=-1/0,new Uint8Array(this.data,0,2).set([251,48+a]),new Uint16Array(this.data,2,1)[0]=e,new Uint32Array(this.data,4,1)[0]=i),this._queue=new t};function e(t,s,i){return t>1;s[h]>t?e=h:i=h+1}return s[i]}function r(t,s,i,e,h){var r=t[e];t[e]=t[h],t[h]=r;var n=4*e,o=4*h,a=s[n],u=s[n+1],p=s[n+2],d=s[n+3];s[n]=s[o],s[n+1]=s[o+1],s[n+2]=s[o+2],s[n+3]=s[o+3],s[o]=a,s[o+1]=u,s[o+2]=p,s[o+3]=d;var _=i[e];i[e]=i[h],i[h]=_}function n(t,s){var i=t^s,e=65535^i,h=65535^(t|s),r=t&(65535^s),n=i|e>>1,o=i>>1^i,a=h>>1^e&r>>1^h,u=i&h>>1^r>>1^r;o=(i=n)&(e=o)>>2^e&(i^e)>>2,a^=i&(h=a)>>2^e&(r=u)>>2,u^=e&h>>2^(i^e)&r>>2,o=(i=n=i&i>>2^e&e>>2)&(e=o)>>4^e&(i^e)>>4,a^=i&(h=a)>>4^e&(r=u)>>4,u^=e&h>>4^(i^e)&r>>4,a^=(i=n=i&i>>4^e&e>>4)&(h=a)>>8^(e=o)&(r=u)>>8;var p=t^s,d=(e=(u^=e&h>>8^(i^e)&r>>8)^u>>1)|65535^(p|(i=a^a>>1));return((d=1431655765&((d=858993459&((d=252645135&((d=16711935&(d|d<<8))|d<<4))|d<<2))|d<<1))<<1|(p=1431655765&((p=858993459&((p=252645135&((p=16711935&(p|p<<8))|p<<4))|p<<2))|p<<1)))>>>0}return i.from=function(t){if(!(t instanceof ArrayBuffer))throw new Error(\"Data must be an instance of ArrayBuffer.\");var e=new Uint8Array(t,0,2),h=e[0],r=e[1];if(251!==h)throw new Error(\"Data does not appear to be in a Flatbush format.\");if(r>>4!=3)throw new Error(\"Got v\"+(r>>4)+\" data when expected v3.\");var n=new Uint16Array(t,2,1)[0],o=new Uint32Array(t,4,1)[0];return new i(o,n,s[15&r],t)},i.prototype.add=function(t,s,i,e){var h=this._pos>>2;this._indices[h]=h,this._boxes[this._pos++]=t,this._boxes[this._pos++]=s,this._boxes[this._pos++]=i,this._boxes[this._pos++]=e,tthis.maxX&&(this.maxX=i),e>this.maxY&&(this.maxY=e)},i.prototype.finish=function(){if(this._pos>>2!==this.numItems)throw new Error(\"Added \"+(this._pos>>2)+\" items when expected \"+this.numItems+\".\");for(var t=this.maxX-this.minX,s=this.maxY-this.minY,i=new Uint32Array(this.numItems),e=0;e=n)return;var o=s[h+n>>1];var a=h-1;var u=n+1;for(;;){do{a++}while(s[a]o);if(a>=u)break;r(s,i,e,a,u)}t(s,i,e,h,u);t(s,i,e,u+1,n)}(i,this._boxes,this._indices,0,this.numItems-1);for(var f=0,l=0;fm&&(m=E),I>c&&(c=I)}this._indices[this._pos>>2]=b,this._boxes[this._pos++]=x,this._boxes[this._pos++]=y,this._boxes[this._pos++]=m,this._boxes[this._pos++]=c}},i.prototype.search=function(t,s,i,e,h){if(this._pos!==this._boxes.length)throw new Error(\"Data not yet indexed - call index.finish().\");for(var r=this._boxes.length-4,n=this._levelBounds.length-1,o=[],a=[];void 0!==r;){for(var u=Math.min(r+4*this.nodeSize,this._levelBounds[n]),p=r;p>2];ithis._boxes[p+2]||s>this._boxes[p+3]||(r<4*this.numItems?(void 0===h||h(d))&&a.push(d):(o.push(d),o.push(n-1))))}n=o.pop(),r=o.pop()}return a},i.prototype.neighbors=function(t,s,i,r,n){if(void 0===i&&(i=1/0),void 0===r&&(r=1/0),this._pos!==this._boxes.length)throw new Error(\"Data not yet indexed - call index.finish().\");for(var o=this._boxes.length-4,a=this._queue,u=[],p=r*r;void 0!==o;){for(var d=Math.min(o+4*this.nodeSize,h(o,this._levelBounds)),_=o;_>2],l=e(t,this._boxes[_],this._boxes[_+2]),v=e(s,this._boxes[_+1],this._boxes[_+3]),x=l*l+v*v;o<4*this.numItems?(void 0===n||n(f))&&a.push(-f-1,x):a.push(f,x)}for(;a.length&&a.peek()<0;){if(a.peekValue()>p)return a.clear(),u;if(u.push(-a.pop()-1),u.length===i)return a.clear(),u}o=a.pop()}return a.clear(),u},i},\"object\"==typeof i&&void 0!==s?s.exports=h():\"function\"==typeof define&&define.amd?define(h):(e=e||self).Flatbush=h()},\n", + " function _(t,e,r){var i=Math.min,n=Math.max;r.empty=function(){return{x0:1/0,y0:1/0,x1:-1/0,y1:-1/0}},r.positive_x=function(){return{x0:Number.MIN_VALUE,y0:-1/0,x1:1/0,y1:1/0}},r.positive_y=function(){return{x0:-1/0,y0:Number.MIN_VALUE,x1:1/0,y1:1/0}},r.union=function(t,e){return{x0:i(t.x0,e.x0),x1:n(t.x1,e.x1),y0:i(t.y0,e.y0),y1:n(t.y1,e.y1)}};var o=function(){function t(t){if(null==t)this.x0=0,this.y0=0,this.x1=0,this.y1=0;else if(\"x0\"in t){var e=t.x0,r=t.y0,i=t.x1,n=t.y1;if(!(e<=i&&r<=n))throw new Error(\"invalid bbox {x0: \"+e+\", y0: \"+r+\", x1: \"+i+\", y1: \"+n+\"}\");this.x0=e,this.y0=r,this.x1=i,this.y1=n}else if(\"x\"in t){var o=t.x,h=t.y,u=t.width,y=t.height;if(!(u>=0&&y>=0))throw new Error(\"invalid bbox {x: \"+o+\", y: \"+h+\", width: \"+u+\", height: \"+y+\"}\");this.x0=o,this.y0=h,this.x1=o+u,this.y1=h+y}else{var f=void 0,s=void 0,c=void 0,p=void 0;if(\"width\"in t)if(\"left\"in t)s=(f=t.left)+t.width;else if(\"right\"in t)f=(s=t.right)-t.width;else{var b=t.width/2;f=t.hcenter-b,s=t.hcenter+b}else f=t.left,s=t.right;if(\"height\"in t)if(\"top\"in t)p=(c=t.top)+t.height;else if(\"bottom\"in t)c=(p=t.bottom)-t.height;else{var a=t.height/2;c=t.vcenter-a,p=t.vcenter+a}else c=t.top,p=t.bottom;if(!(f<=s&&c<=p))throw new Error(\"invalid bbox {left: \"+f+\", top: \"+c+\", right: \"+s+\", bottom: \"+p+\"}\");this.x0=f,this.y0=c,this.x1=s,this.y1=p}}return t.prototype.toString=function(){return\"BBox({left: \"+this.left+\", top: \"+this.top+\", width: \"+this.width+\", height: \"+this.height+\"})\"},Object.defineProperty(t.prototype,\"left\",{get:function(){return this.x0},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"top\",{get:function(){return this.y0},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"right\",{get:function(){return this.x1},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"bottom\",{get:function(){return this.y1},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"p0\",{get:function(){return[this.x0,this.y0]},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"p1\",{get:function(){return[this.x1,this.y1]},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"x\",{get:function(){return this.x0},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"y\",{get:function(){return this.y0},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"width\",{get:function(){return this.x1-this.x0},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"height\",{get:function(){return this.y1-this.y0},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"rect\",{get:function(){return{x0:this.x0,y0:this.y0,x1:this.x1,y1:this.y1}},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"box\",{get:function(){return{x:this.x,y:this.y,width:this.width,height:this.height}},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"h_range\",{get:function(){return{start:this.x0,end:this.x1}},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"v_range\",{get:function(){return{start:this.y0,end:this.y1}},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"ranges\",{get:function(){return[this.h_range,this.v_range]},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"aspect\",{get:function(){return this.width/this.height},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"hcenter\",{get:function(){return(this.left+this.right)/2},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"vcenter\",{get:function(){return(this.top+this.bottom)/2},enumerable:!0,configurable:!0}),t.prototype.contains=function(t,e){return t>=this.x0&&t<=this.x1&&e>=this.y0&&e<=this.y1},t.prototype.clip=function(t,e){return tthis.x1&&(t=this.x1),ethis.y1&&(e=this.y1),[t,e]},t.prototype.union=function(e){return new t({x0:i(this.x0,e.x0),y0:i(this.y0,e.y0),x1:n(this.x1,e.x1),y1:n(this.y1,e.y1)})},t.prototype.equals=function(t){return this.x0==t.x0&&this.y0==t.y0&&this.x1==t.x1&&this.y1==t.y1},Object.defineProperty(t.prototype,\"xview\",{get:function(){var t=this;return{compute:function(e){return t.left+e},v_compute:function(e){for(var r=new Float64Array(e.length),i=t.left,n=0;nt.x1&&(t.x1=n.x1)}for(var r=0,s=this.index.search(o.positive_y());rt.y1&&(t.y1=a.y1)}return this._bounds(t)},i.prototype.get_anchor_point=function(t,e,i){var n=i[0],r=i[1];switch(t){case\"center\":return{x:this.scenterx(e,n,r),y:this.scentery(e,n,r)};default:return null}},i.prototype.sdist=function(t,e,i,n,r){var s,o;void 0===n&&(n=\"edge\"),void 0===r&&(r=!1);var a=e.length;if(\"center\"==n){var h=c.map(i,function(t){return t/2});s=new Float64Array(a);for(var _=0;_1?r:{x:n.x+i*(r.x-n.x),y:n.y+i*(r.y-n.y)})}r.point_in_poly=function(t,n,r,e){for(var i=!1,o=r[r.length-1],u=e[e.length-1],a=0;a0&&_<1&&h>0&&h<1,x:t+_*(r-t),y:n+_*(e-n)}}},\n", + " function _(t,n,r){var e=t(113),i=t(185),a=t(121),s=t(114),o=t(110),p=t(109);function u(t,n,r){void 0===r&&(r=0);for(var e={},i=0;ithis.end},enumerable:!0,configurable:!0}),n}(a.Model);e.Range=r,r.__name__=\"Range\",r.init_Range()},\n", + " function _(e,t,i){var n=e(183);i.generic_line_legend=function(e,t,i,n){var r=i.x0,a=i.x1,l=i.y0,c=i.y1;t.save(),t.beginPath(),t.moveTo(r,(l+c)/2),t.lineTo(a,(l+c)/2),e.line.doit&&(e.line.set_vectorize(t,n),t.stroke()),t.restore()},i.generic_area_legend=function(e,t,i,n){var r=i.x0,a=i.x1,l=i.y0,c=i.y1,o=.1*Math.abs(a-r),s=.1*Math.abs(c-l),_=r+o,v=a-o,h=l+s,x=c-s;e.fill.doit&&(e.fill.set_vectorize(t,n),t.fillRect(_,h,v-_,x-h)),null!=e.hatch&&e.hatch.doit&&(e.hatch.set_vectorize(t,n),t.fillRect(_,h,v-_,x-h)),e.line&&e.line.doit&&(t.beginPath(),t.rect(_,h,v-_,x-h),e.line.set_vectorize(t,n),t.stroke())},i.line_interpolation=function(e,t,i,r,a,l){var c,o,s,_,v,h,x,y,f,d,g=t.sx,m=t.sy;\"point\"==t.type?(f=(c=e.yscale.r_invert(m-1,m+1))[0],d=c[1],x=(o=e.xscale.r_invert(g-1,g+1))[0],y=o[1]):\"v\"==t.direction?(f=(s=e.yscale.r_invert(m,m))[0],d=s[1],x=(_=[Math.min(i-1,a-1),Math.max(i+1,a+1)])[0],y=_[1]):(x=(v=e.xscale.r_invert(g,g))[0],y=v[1],f=(h=[Math.min(r-1,l-1),Math.max(r+1,l+1)])[0],d=h[1]);var u=n.check_2_segments_intersect(x,f,y,d,i,r,a,l);return[u.x,u.y]}},\n", + " function _(t,i,e){var n=t(113),s=t(178),l=t(186),o=t(183),r=function(t){function i(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(i,t),i.prototype._inner_loop=function(t,i,e,n,s){for(var l=0,o=i;l=0;s--)t.lineTo(i[s],n[s]);t.closePath(),r.call(t)},e.prototype._render=function(t,e,i){var n=this,r=i.sx1,s=i.sx2,o=i.sy;this.visuals.fill.doit&&(this.visuals.fill.set_value(t),this._inner(t,r,s,o,t.fill)),this.visuals.hatch.doit2(t,0,function(){return n._inner(t,r,s,o,t.fill)},function(){return n.renderer.request_render()})},e.prototype._hit_point=function(t){for(var e=this,i=o.create_empty_hit_test_result(),n=this.sy.length,r=new Float64Array(2*n),s=new Float64Array(2*n),a=0,h=n;a=0;s--)t.lineTo(e[s],n[s]);t.closePath(),r.call(t)},e.prototype._render=function(t,e,i){var n=this,r=i.sx,s=i.sy1,o=i.sy2;this.visuals.fill.doit&&(this.visuals.fill.set_value(t),this._inner(t,r,s,o,t.fill)),this.visuals.hatch.doit2(t,0,function(){return n._inner(t,r,s,o,t.fill)},function(){return n.renderer.request_render()})},e.prototype.scenterx=function(t){return this.sx[t]},e.prototype.scentery=function(t){return(this.sy1[t]+this.sy2[t])/2},e.prototype._hit_point=function(t){for(var e=this,i=o.create_empty_hit_test_result(),n=this.sx.length,r=new Float64Array(2*n),s=new Float64Array(2*n),a=0,h=n;a0?this.indices=r.intersection.apply(this,n):this.source instanceof u.ColumnarDataSource&&(this.indices=this.source.get_indices()),this.indices_map_to_subset()},n.prototype.indices_map_to_subset=function(){this.indices_map={};for(var i=0;i0){for(var l=n[0],o=0,_=n;o<_.length;o++){var s=_[o];l.update_through_intersection(s)}return l}return null},e}(u);n.IntersectRenderers=i,i.__name__=\"IntersectRenderers\";var l=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(e,t),e.prototype.hit_test=function(t,e){for(var n=[],r=0,u=e;r0){for(var l=n[0],o=0,_=n;o<_.length;o++){var s=_[o];l.update_through_union(s)}return l}return null},e}(u);n.UnionRenderers=l,l.__name__=\"UnionRenderers\"},\n", + " function _(r,n,t){var a=r(109),e=r(197);function i(r){for(var n=new Uint8Array(r.buffer,r.byteOffset,2*r.length),t=0,a=n.length;t=0||r.indexOf(\"Trident\")>0||r.indexOf(\"Edge\")>0,e.is_mobile=\"undefined\"!=typeof window&&(\"ontouchstart\"in window||navigator.maxTouchPoints>0),e.is_little_endian=function(){var n=new ArrayBuffer(4),i=new Uint8Array(n);new Uint32Array(n)[1]=168496141;var e=!0;return 10==i[4]&&11==i[5]&&12==i[6]&&13==i[7]&&(e=!1),e}()},\n", + " function _(n,t,r){r.concat=function(n){for(var t=[],r=1;r=0;t--)e.lineTo(this._upper_sx[t],this._upper_sy[t]);e.closePath(),this.visuals.fill.doit&&(this.visuals.fill.set_value(e),e.fill()),e.beginPath(),e.moveTo(this._lower_sx[0],this._lower_sy[0]);for(t=0,i=this._lower_sx.length;tthis.sleft&&tthis.stop&&is||(d[r].push(h[p]),d[a].push(0));for(p=0,f=m.length;ps||(c[r].push(m[p]),c[a].push(0));var g={major:this._format_major_labels(d[r],h)},v={major:[[],[]],minor:[[],[]]};return v.major[r]=i.v_compute(d[r]),v.minor[r]=i.v_compute(c[r]),v.major[a]=d[a],v.minor[a]=c[a],\"vertical\"==this.model.orientation&&(v.major[r]=u.map(v.major[r],function(e){return t-e}),v.minor[r]=u.map(v.minor[r],function(e){return t-e})),{coords:v,labels:g}},e}(r.AnnotationView);i.ColorBarView=g,g.__name__=\"ColorBarView\";var v=function(t){function e(e){return t.call(this,e)||this}return o.__extends(e,t),e.init_ColorBar=function(){this.prototype.default_view=g,this.mixins([\"text:major_label_\",\"text:title_\",\"line:major_tick_\",\"line:minor_tick_\",\"line:border_\",\"line:bar_\",\"fill:background_\"]),this.define({location:[m.Any,\"top_right\"],orientation:[m.Orientation,\"vertical\"],title:[m.String],title_standoff:[m.Number,2],width:[m.Any,\"auto\"],height:[m.Any,\"auto\"],scale_alpha:[m.Number,1],ticker:[m.Instance,function(){return new a.BasicTicker}],formatter:[m.Instance,function(){return new n.BasicTickFormatter}],major_label_overrides:[m.Any,{}],color_mapper:[m.Instance],label_standoff:[m.Number,5],margin:[m.Number,30],padding:[m.Number,10],major_tick_in:[m.Number,5],major_tick_out:[m.Number,0],minor_tick_in:[m.Number,0],minor_tick_out:[m.Number,0]}),this.override({background_fill_color:\"#ffffff\",background_fill_alpha:.95,bar_line_color:null,border_line_color:null,major_label_text_align:\"center\",major_label_text_baseline:\"middle\",major_label_text_font_size:\"8pt\",major_tick_line_color:\"#ffffff\",minor_tick_line_color:null,title_text_font_size:\"10pt\",title_text_font_style:\"italic\"})},e}(r.Annotation);i.ColorBar=v,v.__name__=\"ColorBar\",v.init_ColorBar()},\n", + " function _(i,n,c){var e=i(113),t=function(i){function n(n){return i.call(this,n)||this}return e.__extends(n,i),n}(i(205).AdaptiveTicker);c.BasicTicker=t,t.__name__=\"BasicTicker\"},\n", + " function _(t,i,a){var e=t(113),n=t(206),s=t(110),r=t(121);var h=function(t){function i(i){return t.call(this,i)||this}return e.__extends(i,t),i.init_AdaptiveTicker=function(){this.define({base:[r.Number,10],mantissas:[r.Array,[1,2,5]],min_interval:[r.Number,0],max_interval:[r.Number]})},i.prototype.initialize=function(){t.prototype.initialize.call(this);var i=s.nth(this.mantissas,-1)/this.base,a=s.nth(this.mantissas,0)*this.base;this.extended_mantissas=e.__spreadArrays([i],this.mantissas,[a]),this.base_factor=0===this.get_min_interval()?1:this.get_min_interval()},i.prototype.get_interval=function(t,i,a){var e,n,r=i-t,h=this.get_ideal_interval(t,i,a),_=Math.floor((e=h/this.base_factor,void 0===(n=this.base)&&(n=Math.E),Math.log(e)/Math.log(n))),o=Math.pow(this.base,_)*this.base_factor,m=this.extended_mantissas,c=m.map(function(t){return Math.abs(a-r/(t*o))});return function(t,i,a){return Math.max(i,Math.min(a,t))}(m[s.argmin(c)]*o,this.get_min_interval(),this.get_max_interval())},i}(n.ContinuousTicker);a.AdaptiveTicker=h,h.__name__=\"AdaptiveTicker\",h.init_AdaptiveTicker()},\n", + " function _(t,n,i){var r=t(113),e=t(207),o=t(121),u=t(110),_=t(109),s=function(t){function n(n){return t.call(this,n)||this}return r.__extends(n,t),n.init_ContinuousTicker=function(){this.define({num_minor_ticks:[o.Number,5],desired_num_ticks:[o.Number,6]})},n.prototype.get_ticks=function(t,n,i,r,e){return this.get_ticks_no_defaults(t,n,r,this.desired_num_ticks)},n.prototype.get_ticks_no_defaults=function(t,n,i,r){var e=this.get_interval(t,n,r),o=Math.floor(t/e),s=Math.ceil(n/e),a=(_.isStrictNaN(o)||_.isStrictNaN(s)?[]:u.range(o,s+1)).map(function(t){return t*e}).filter(function(i){return t<=i&&i<=n}),c=this.num_minor_ticks,l=[];if(c>0&&a.length>0){for(var f=e/c,h=u.range(0,c).map(function(t){return t*f}),m=0,p=h.slice(1);m=2&&(t=Math.abs(i[1]-i[0])/1e4);var r=!1;if(this.use_scientific)for(var n=0,o=i;nt&&(l>=this.scientific_limit_high||l<=this.scientific_limit_low)){r=!0;break}}var s=new Array(i.length),f=this.precision;if(null==f||a.isNumber(f))if(r)for(var h=0,_=i.length;h<_;h++)s[h]=i[h].toExponential(f||void 0);else for(h=0,_=i.length;h<_;h++)s[h]=i[h].toFixed(f||void 0).replace(/(\\.[0-9]*?)0+$/,\"$1\").replace(/\\.$/,\"\");else for(var p=this.last_precision,u=this.last_precision<=15;u?p<=15:p>=15;u?p++:p--){var m=!0;if(r){for(h=0,_=i.length;h<_;h++)if(s[h]=i[h].toExponential(p),h>0&&s[h]===s[h-1]){m=!1;break}if(m)break}else{for(h=0,_=i.length;h<_;h++)if(s[h]=i[h].toFixed(p).replace(/(\\.[0-9]*?)0+$/,\"$1\").replace(/\\.$/,\"\"),h>0&&s[h]==s[h-1]){m=!1;break}if(m)break}if(m){this.last_precision=p;break}}return s},e}(n.TickFormatter);t.BasicTickFormatter=c,c.__name__=\"BasicTickFormatter\",c.init_BasicTickFormatter()},\n", + " function _(t,n,r){var e=t(113),i=function(t){function n(n){return t.call(this,n)||this}return e.__extends(n,t),n}(t(166).Model);r.TickFormatter=i,i.__name__=\"TickFormatter\"},\n", + " function _(o,n,l){var r=o(113),t=o(211),i=o(114),e=function(o){function n(n){return o.call(this,n)||this}return r.__extends(n,o),n.prototype._v_compute=function(o,n,l,r){for(var t=r.nan_color,e=r.low_color,h=r.high_color,a=null!=this.low?this.low:i.min(o),u=null!=this.high?this.high:i.max(o),_=l.length-1,s=1/(u-a),c=1/l.length,p=0,f=o.length;p_?null!=h?h:l[_]:l[m]}else n[p]=l[_]}},n}(t.ContinuousColorMapper);l.LinearColorMapper=e,e.__name__=\"LinearColorMapper\"},\n", + " function _(o,r,i){var l=o(113),n=o(212),t=o(121),u=function(o){function r(r){return o.call(this,r)||this}return l.__extends(r,o),r.init_ContinuousColorMapper=function(){this.define({high:[t.Number],low:[t.Number],high_color:[t.Color],low_color:[t.Color]})},r.prototype._colors=function(r){return Object.assign(Object.assign({},o.prototype._colors.call(this,r)),{low_color:null!=this.low_color?r(this.low_color):void 0,high_color:null!=this.high_color?r(this.high_color):void 0})},r}(n.ColorMapper);i.ContinuousColorMapper=u,u.__name__=\"ContinuousColorMapper\",u.init_ContinuousColorMapper()},\n", + " function _(t,r,n){var e=t(113),o=t(213),i=t(121),a=t(109),u=t(123),_=t(197);function c(t){return a.isNumber(t)?t:(\"#\"!=t[0]&&(t=u.color2hex(t)),9!=t.length&&(t+=\"ff\"),parseInt(t.slice(1),16))}function l(t){for(var r=new Uint32Array(t.length),n=0,e=t.length;nr.x?-1:t.x==r.x?0:1}):o.sort(function(t,r){return t.xthis._x_sorted[this._x_sorted.length-1])return NaN}else{if(tthis._x_sorted[this._x_sorted.length-1])return this._y_sorted[this._y_sorted.length-1]}if(t==this._x_sorted[0])return this._y_sorted[0];var r=s.find_last_index(this._x_sorted,function(r){return rthis._x_sorted[this._x_sorted.length-1])return NaN}else{if(tthis._x_sorted[this._x_sorted.length-1])return this._y_sorted[this._y_sorted.length-1]}var e;switch(this.mode){case\"after\":e=s.find_last_index(this._x_sorted,function(e){return t>=e});break;case\"before\":e=s.find_index(this._x_sorted,function(e){return t<=e});break;case\"center\":var r=this._x_sorted.map(function(e){return Math.abs(e-t)}),n=s.min(r);e=s.find_index(r,function(t){return n===t});break;default:throw new Error(\"unknown mode: \"+this.mode)}return-1!=e?this._y_sorted[e]:NaN},e}(i.Interpolator);r.StepInterpolator=_,_.__name__=\"StepInterpolator\",_.init_StepInterpolator()},\n", + " function _(t,e,a){var r=t(113),o=function(t){function e(e){return t.call(this,e)||this}return r.__extends(e,t),e.prototype.compute=function(t){var e,a=this._compute_state(),r=a[0],o=a[1],n=a[2],i=a[3];if(0==n)e=0;else{var h=(Math.log(t)-i)/n;e=isFinite(h)?h*r+o:NaN}return e},e.prototype.v_compute=function(t){var e=this._compute_state(),a=e[0],r=e[1],o=e[2],n=e[3],i=new Float64Array(t.length);if(0==o)for(var h=0;h0?(this.el.style.top=y+\"px\",this.el.style.left=b+\"px\"):l.undisplay(this.el)}},e}(o.AnnotationView);i.TooltipView=c,c.__name__=\"TooltipView\";var d=function(t){function e(e){return t.call(this,e)||this}return s.__extends(e,t),e.init_Tooltip=function(){this.prototype.default_view=c,this.define({attachment:[a.TooltipAttachment,\"horizontal\"],inner_only:[a.Boolean,!0],show_arrow:[a.Boolean,!0]}),this.override({level:\"overlay\"}),this.internal({data:[a.Any,[]],custom:[a.Any]})},e.prototype.clear=function(){this.data=[]},e.prototype.add=function(t,e,i){this.data=this.data.concat([[t,e,i]])},e}(o.Annotation);i.Tooltip=d,d.__name__=\"Tooltip\",d.init_Tooltip()},\n", + " function _(o,t,n){o(164),o(163).styles.append('.bk-root {\\n /* Same border color used everywhere */\\n /* Gray of icons */\\n}\\n.bk-root .bk-tooltip {\\n font-weight: 300;\\n font-size: 12px;\\n position: absolute;\\n padding: 5px;\\n border: 1px solid #e5e5e5;\\n color: #2f2f2f;\\n background-color: white;\\n pointer-events: none;\\n opacity: 0.95;\\n z-index: 100;\\n}\\n.bk-root .bk-tooltip > div:not(:first-child) {\\n /* gives space when multiple elements are being hovered over */\\n margin-top: 5px;\\n border-top: #e5e5e5 1px dashed;\\n}\\n.bk-root .bk-tooltip.bk-left.bk-tooltip-arrow::before {\\n position: absolute;\\n margin: -7px 0 0 0;\\n top: 50%;\\n width: 0;\\n height: 0;\\n border-style: solid;\\n border-width: 7px 0 7px 0;\\n border-color: transparent;\\n content: \" \";\\n display: block;\\n left: -10px;\\n border-right-width: 10px;\\n border-right-color: #909599;\\n}\\n.bk-root .bk-tooltip.bk-left::before {\\n left: -10px;\\n border-right-width: 10px;\\n border-right-color: #909599;\\n}\\n.bk-root .bk-tooltip.bk-right.bk-tooltip-arrow::after {\\n position: absolute;\\n margin: -7px 0 0 0;\\n top: 50%;\\n width: 0;\\n height: 0;\\n border-style: solid;\\n border-width: 7px 0 7px 0;\\n border-color: transparent;\\n content: \" \";\\n display: block;\\n right: -10px;\\n border-left-width: 10px;\\n border-left-color: #909599;\\n}\\n.bk-root .bk-tooltip.bk-right::after {\\n right: -10px;\\n border-left-width: 10px;\\n border-left-color: #909599;\\n}\\n.bk-root .bk-tooltip.bk-above::before {\\n position: absolute;\\n margin: 0 0 0 -7px;\\n left: 50%;\\n width: 0;\\n height: 0;\\n border-style: solid;\\n border-width: 0 7px 0 7px;\\n border-color: transparent;\\n content: \" \";\\n display: block;\\n top: -10px;\\n border-bottom-width: 10px;\\n border-bottom-color: #909599;\\n}\\n.bk-root .bk-tooltip.bk-below::after {\\n position: absolute;\\n margin: 0 0 0 -7px;\\n left: 50%;\\n width: 0;\\n height: 0;\\n border-style: solid;\\n border-width: 0 7px 0 7px;\\n border-color: transparent;\\n content: \" \";\\n display: block;\\n bottom: -10px;\\n border-top-width: 10px;\\n border-top-color: #909599;\\n}\\n.bk-root .bk-tooltip-row-label {\\n text-align: right;\\n color: #26aae1;\\n /* blue from toolbar highlighting */\\n}\\n.bk-root .bk-tooltip-row-value {\\n color: default;\\n /* seems to be necessary for notebook */\\n}\\n.bk-root .bk-tooltip-color-block {\\n width: 12px;\\n height: 12px;\\n margin-left: 5px;\\n margin-right: 5px;\\n outline: #dddddd solid 1px;\\n display: inline-block;\\n}\\n'),n.bk_tooltip=\"bk-tooltip\",n.bk_tooltip_arrow=\"bk-tooltip-arrow\",n.bk_tooltip_custom=\"bk-tooltip-custom\",n.bk_tooltip_row_label=\"bk-tooltip-row-label\",n.bk_tooltip_row_value=\"bk-tooltip-row-value\",n.bk_tooltip_color_block=\"bk-tooltip-color-block\"},\n", + " function _(b,e,k){b(163).styles.append(\"\"),k.bk_active=\"bk-active\",k.bk_inline=\"bk-inline\",k.bk_left=\"bk-left\",k.bk_right=\"bk-right\",k.bk_above=\"bk-above\",k.bk_below=\"bk-below\",k.bk_up=\"bk-up\",k.bk_down=\"bk-down\",k.bk_side=function(b){switch(b){case\"above\":return k.bk_above;case\"below\":return k.bk_below;case\"left\":return k.bk_left;case\"right\":return k.bk_right}}},\n", + " function _(e,t,i){var s=e(113),n=e(131),r=e(170),o=e(169),a=e(121),h=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return s.__extends(t,e),t.prototype.initialize=function(){e.prototype.initialize.call(this),this.set_data(this.model.source)},t.prototype.connect_signals=function(){var t=this;e.prototype.connect_signals.call(this),this.connect(this.model.source.streaming,function(){return t.set_data(t.model.source)}),this.connect(this.model.source.patching,function(){return t.set_data(t.model.source)}),this.connect(this.model.source.change,function(){return t.set_data(t.model.source)})},t.prototype.set_data=function(t){e.prototype.set_data.call(this,t),this.visuals.warm_cache(t),this.plot_view.request_render()},t.prototype._map_data=function(){var e,t,i,s=this.plot_view.frame,n=this.model.dimension,r=s.xscales[this.model.x_range_name],o=s.yscales[this.model.y_range_name],a=\"height\"==n?o:r,h=\"height\"==n?r:o,_=\"height\"==n?s.yview:s.xview,l=\"height\"==n?s.xview:s.yview;e=\"data\"==this.model.properties.lower.units?a.v_compute(this._lower):_.v_compute(this._lower),t=\"data\"==this.model.properties.upper.units?a.v_compute(this._upper):_.v_compute(this._upper),i=\"data\"==this.model.properties.base.units?h.v_compute(this._base):l.v_compute(this._base);var u=\"height\"==n?[1,0]:[0,1],p=u[0],c=u[1],d=[e,i],m=[t,i];this._lower_sx=d[p],this._lower_sy=d[c],this._upper_sx=m[p],this._upper_sy=m[c]},t.prototype.render=function(){if(this.model.visible){this._map_data();var e=this.plot_view.canvas_view.ctx;if(this.visuals.line.doit)for(var t=0,i=this._lower_sx.length;tu&&(u=b)}return u>0&&(u+=a),u},Object.defineProperty(t.prototype,\"normals\",{get:function(){return this.panel.normals},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"dimension\",{get:function(){return this.panel.dimension},enumerable:!0,configurable:!0}),t.prototype.compute_labels=function(e){for(var t=this.model.formatter.doFormat(e,this),i=0;i_(l-c)?(a=u(h(n,o),l),r=h(u(n,o),c)):(a=h(n,o),r=u(n,o)),[a,r]}throw new Error(\"user bounds '\"+t+\"' not understood\")},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"rule_coords\",{get:function(){var e=this.dimension,t=(e+1)%2,i=this.ranges[0],a=this.computed_bounds,r=a[0],n=a[1],o=[new Array(2),new Array(2)];return o[e][0]=Math.max(r,i.min),o[e][1]=Math.min(n,i.max),o[e][0]>o[e][1]&&(o[e][0]=o[e][1]=NaN),o[t][0]=this.loc,o[t][1]=this.loc,o},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"tick_coords\",{get:function(){for(var e=this.dimension,t=(e+1)%2,i=this.ranges[0],a=this.computed_bounds,r=a[0],n=a[1],o=this.model.ticker.get_ticks(r,n,i,this.loc,{}),s=o.major,l=o.minor,_=[[],[]],h=[[],[]],u=[i.min,i.max],c=u[0],d=u[1],m=0;md||(_[e].push(s[m]),_[t].push(this.loc));for(m=0;md||(h[e].push(l[m]),h[t].push(this.loc));return{major:_,minor:h}},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"loc\",{get:function(){var e=this.model.fixed_location;if(null!=e){if(s.isNumber(e))return e;var t=this.ranges[1];if(t instanceof l.FactorRange)return t.synthetic(e);throw new Error(\"unexpected\")}var i=this.ranges[1];switch(this.panel.side){case\"left\":case\"below\":return i.start;case\"right\":case\"above\":return i.end}},enumerable:!0,configurable:!0}),t.prototype.serializable_state=function(){return Object.assign(Object.assign({},e.prototype.serializable_state.call(this)),{bbox:this.layout.bbox.box})},t}(r.GuideRendererView);i.AxisView=c,c.__name__=\"AxisView\";var d=function(e){function t(t){return e.call(this,t)||this}return a.__extends(t,e),t.init_Axis=function(){this.prototype.default_view=c,this.mixins([\"line:axis_\",\"line:major_tick_\",\"line:minor_tick_\",\"text:major_label_\",\"text:axis_label_\"]),this.define({bounds:[n.Any,\"auto\"],ticker:[n.Instance],formatter:[n.Instance],x_range_name:[n.String,\"default\"],y_range_name:[n.String,\"default\"],axis_label:[n.String,\"\"],axis_label_standoff:[n.Int,5],major_label_standoff:[n.Int,5],major_label_orientation:[n.Any,\"horizontal\"],major_label_overrides:[n.Any,{}],major_tick_in:[n.Number,2],major_tick_out:[n.Number,6],minor_tick_in:[n.Number,0],minor_tick_out:[n.Number,4],fixed_location:[n.Any,null]}),this.override({axis_line_color:\"black\",major_tick_line_color:\"black\",minor_tick_line_color:\"black\",major_label_text_font_size:\"8pt\",major_label_text_align:\"center\",major_label_text_baseline:\"alphabetic\",axis_label_text_font_size:\"10pt\",axis_label_text_font_style:\"italic\"})},t}(r.GuideRenderer);i.Axis=d,d.__name__=\"Axis\",d.init_Axis()},\n", + " function _(e,n,r){var i=e(113),t=e(160),d=function(e){function n(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(n,e),n}(t.RendererView);r.GuideRendererView=d,d.__name__=\"GuideRendererView\";var u=function(e){function n(n){return e.call(this,n)||this}return i.__extends(n,e),n.init_GuideRenderer=function(){this.override({level:\"overlay\"})},n}(t.Renderer);r.GuideRenderer=u,u.__name__=\"GuideRenderer\",u.init_GuideRenderer()},\n", + " function _(t,o,e){var i=t(113),r=t(243),s=t(246),a=t(247),n=t(121),l=function(t){function o(){return null!==t&&t.apply(this,arguments)||this}return i.__extends(o,t),o.prototype._render=function(t,o,e){this._draw_group_separators(t,o,e)},o.prototype._draw_group_separators=function(t,o,e){var i,r=this.ranges[0],s=this.computed_bounds,a=s[0],n=s[1];if(r.tops&&!(r.tops.length<2)&&this.visuals.separator_line.doit){for(var l=this.dimension,_=(l+1)%2,u=[[],[]],p=0,h=0;ha&&f1&&(l.tops[o]=n.tops,l.tops[e]=n.tops.map(function(o){return t.loc})),l},enumerable:!0,configurable:!0}),o}(r.AxisView);e.CategoricalAxisView=l,l.__name__=\"CategoricalAxisView\";var _=function(t){function o(o){return t.call(this,o)||this}return i.__extends(o,t),o.init_CategoricalAxis=function(){this.prototype.default_view=l,this.mixins([\"line:separator_\",\"text:group_\",\"text:subgroup_\"]),this.define({group_label_orientation:[n.Any,\"parallel\"],subgroup_label_orientation:[n.Any,\"parallel\"]}),this.override({ticker:function(){return new s.CategoricalTicker},formatter:function(){return new a.CategoricalTickFormatter},separator_line_color:\"lightgrey\",separator_line_width:2,group_text_font_style:\"bold\",group_text_font_size:\"8pt\",group_text_color:\"grey\",subgroup_text_font_style:\"bold\",subgroup_text_font_size:\"8pt\"})},o}(r.Axis);e.CategoricalAxis=_,_.__name__=\"CategoricalAxis\",_.init_CategoricalAxis()},\n", + " function _(t,c,r){var e=t(113),o=function(t){function c(c){return t.call(this,c)||this}return e.__extends(c,t),c.prototype.get_ticks=function(t,c,r,e,o){return{major:this._collect(r.factors,r,t,c),minor:[],tops:this._collect(r.tops||[],r,t,c),mids:this._collect(r.mids||[],r,t,c)}},c.prototype._collect=function(t,c,r,e){for(var o=[],i=0,n=t;ir&&l=60?\"minsec\":\"seconds\";case!(e<3600):return r>=3600?\"hourmin\":\"minutes\";case!(e<86400):return\"hours\";case!(e<2678400):return\"days\";case!(e<31536e3):return\"months\";default:return\"years\"}},r.prototype.doFormat=function(t,r){if(0==t.length)return[];for(var e=Math.abs(t[t.length-1]-t[0])/1e3,s=e/(t.length-1),i=this._get_resolution_str(s,e),n=this._width_formats[i][1][0],a=[],u=f.indexOf(i),c={},m=0,l=f;m=T-g;--c)for(o=0,a=s.length;o=h[o][n]&&h[o][h[o].clock]>u[h[o].clock]&&(i=h[o])}return i&&((l=/^(.*)\\/(.*)$/.exec(u.format))?i.abbrev=l[i.save?2:1]:i.abbrev=u.format.replace(/%s/,i.rule.letter)),i||u}function n(e,n){return\"UTC\"==e.zone?n:(e.entry=t(e,\"posix\",n),n+e.entry.offset+e.entry.save)}function r(e,n){return\"UTC\"==e.zone?n:(e.entry=r=t(e,\"wallclock\",n),0<(o=n-r.wallclock)&&o9)t+=s*l[c-10];else{if(a=new Date(n(e,t)),c<7)for(;s;)a.setUTCDate(a.getUTCDate()+i),a.getUTCDay()==c&&(s-=i);else 7==c?a.setUTCFullYear(a.getUTCFullYear()+s):8==c?a.setUTCMonth(a.getUTCMonth()+s):a.setUTCDate(a.getUTCDate()+s);null==(t=r(e,a.getTime()))&&(t=r(e,a.getTime()+864e5*i)-864e5*i)}return t}var a={clock:function(){return+new Date},zone:\"UTC\",entry:{abbrev:\"UTC\",offset:0,save:0},UTC:1,z:function(e,t,n,r){var o,a,u=this.entry.offset+this.entry.save,i=Math.abs(u/1e3),l=[],s=3600;for(o=0;o<3;o++)l.push((\"0\"+Math.floor(i/s)).slice(-2)),i%=s,s/=60;return\"^\"!=n||u?(\"^\"==n&&(r=3),3==r?(a=(a=l.join(\":\")).replace(/:00$/,\"\"),\"^\"!=n&&(a=a.replace(/:00$/,\"\"))):r?(a=l.slice(0,r+1).join(\":\"),\"^\"==n&&(a=a.replace(/:00$/,\"\"))):a=l.slice(0,2).join(\"\"),a=(a=(u<0?\"-\":\"+\")+a).replace(/([-+])(0)/,{_:\" $1\",\"-\":\"$1\"}[n]||\"$1$2\")):\"Z\"},\"%\":function(e){return\"%\"},n:function(e){return\"\\n\"},t:function(e){return\"\\t\"},U:function(e){return s(e,0)},W:function(e){return s(e,1)},V:function(e){return c(e)[0]},G:function(e){return c(e)[1]},g:function(e){return c(e)[1]%100},j:function(e){return Math.floor((e.getTime()-Date.UTC(e.getUTCFullYear(),0))/864e5)+1},s:function(e){return Math.floor(e.getTime()/1e3)},C:function(e){return Math.floor(e.getUTCFullYear()/100)},N:function(e){return e.getTime()%1e3*1e6},m:function(e){return e.getUTCMonth()+1},Y:function(e){return e.getUTCFullYear()},y:function(e){return e.getUTCFullYear()%100},H:function(e){return e.getUTCHours()},M:function(e){return e.getUTCMinutes()},S:function(e){return e.getUTCSeconds()},e:function(e){return e.getUTCDate()},d:function(e){return e.getUTCDate()},u:function(e){return e.getUTCDay()||7},w:function(e){return e.getUTCDay()},l:function(e){return e.getUTCHours()%12||12},I:function(e){return e.getUTCHours()%12||12},k:function(e){return e.getUTCHours()},Z:function(e){return this.entry.abbrev},a:function(e){return this[this.locale].day.abbrev[e.getUTCDay()]},A:function(e){return this[this.locale].day.full[e.getUTCDay()]},h:function(e){return this[this.locale].month.abbrev[e.getUTCMonth()]},b:function(e){return this[this.locale].month.abbrev[e.getUTCMonth()]},B:function(e){return this[this.locale].month.full[e.getUTCMonth()]},P:function(e){return this[this.locale].meridiem[Math.floor(e.getUTCHours()/12)].toLowerCase()},p:function(e){return this[this.locale].meridiem[Math.floor(e.getUTCHours()/12)]},R:function(e,t){return this.convert([t,\"%H:%M\"])},T:function(e,t){return this.convert([t,\"%H:%M:%S\"])},D:function(e,t){return this.convert([t,\"%m/%d/%y\"])},F:function(e,t){return this.convert([t,\"%Y-%m-%d\"])},x:function(e,t){return this.convert([t,this[this.locale].date])},r:function(e,t){return this.convert([t,this[this.locale].time12||\"%I:%M:%S\"])},X:function(e,t){return this.convert([t,this[this.locale].time24])},c:function(e,t){return this.convert([t,this[this.locale].dateTime])},convert:function(e){if(!e.length)return\"1.0.22\";var t,a,u,l,s,c=Object.create(this),f=[];for(t=0;t=o?Math.floor((n-o)/7)+1:0}function c(e){var t,n,r;return n=e.getUTCFullYear(),t=new Date(Date.UTC(n,0)).getUTCDay(),(r=s(e,1)+(t>1&&t<=4?1:0))?53!=r||4==t||3==t&&29==new Date(n,1,29).getDate()?[r,e.getUTCFullYear()]:[1,e.getUTCFullYear()+1]:(n=e.getUTCFullYear()-1,[r=4==(t=new Date(Date.UTC(n,0)).getUTCDay())||3==t&&29==new Date(n,1,29).getDate()?53:52,e.getUTCFullYear()-1])}return u=u.toLowerCase().split(\"|\"),\"delmHMSUWVgCIky\".replace(/./g,function(e){a[e].pad=2}),a.N.pad=9,a.j.pad=3,a.k.style=\"_\",a.l.style=\"_\",a.e.style=\"_\",function(){return a.convert(arguments)}})},\n", + " function _(r,n,e){var t=r(113),i=r(254),u=r(255),a=r(252),f=r(127),o=r(109);function l(r){for(var n=[],e=1;e.1&&Math.abs(r)<1e3):return\"%0.3f\";default:return\"%0.3e\"}}(),r):\"\"+r}function s(r,n,t,i){if(null==t)return c;if(null!=i&&(r in i||n in i)){var u=i[n in i?n:r];if(o.isString(u)){if(u in e.DEFAULT_FORMATTERS)return e.DEFAULT_FORMATTERS[u];throw new Error(\"Unknown tooltip field formatter type '\"+u+\"'\")}return function(r,n,e){return u.format(r,n,e)}}return e.DEFAULT_FORMATTERS.numeral}function p(r,n,e,t){if(\"$\"==r[0]){if(r.substring(1)in t)return t[r.substring(1)];throw new Error(\"Unknown special variable '\"+r+\"'\")}var i=n.get_column(r);if(null==i)return null;if(o.isNumber(e))return i[e];var u=i[e.index];return o.isTypedArray(u)||o.isArray(u)?o.isArray(u[0])?u[e.dim2][e.dim1]:u[e.flat_index]:u}e.sprintf=l,e.DEFAULT_FORMATTERS={numeral:function(r,n,e){return u.format(r,n)},datetime:function(r,n,e){return a(r,n)},printf:function(r,n,e){return l(n,r)}},e.basic_formatter=c,e.get_formatter=s,e.get_value=p,e.replace_placeholders=function(r,n,e,t,i){void 0===i&&(i={});var u=r.replace(/(?:^|[^@])([@|\\$](?:\\w+|{[^{}]+}))(?:{[^{}]+})?/g,function(r,n,e){return\"\"+n});return r=(r=(r=r.replace(/@\\$name/g,function(r){return\"@{\"+i.name+\"}\"})).replace(/(^|[^\\$])\\$(\\w+)/g,function(r,n,e){return n+\"@$\"+e})).replace(/(^|[^@])@(?:(\\$?\\w+)|{([^{}]+)})(?:{([^{}]+)})?/g,function(r,a,o,l,c){var m=p(o=null!=l?l:o,n,e,i);if(null==m)return\"\"+a+f.escape(\"???\");if(\"safe\"==c)return\"\"+a+m;var T=s(o,u,c,t);return\"\"+a+f.escape(T(m,c,i))})}},\n", + " function _(e,n,t){!function(){\"use strict\";var e={not_string:/[^s]/,not_bool:/[^t]/,not_type:/[^T]/,not_primitive:/[^v]/,number:/[diefg]/,numeric_arg:/[bcdiefguxX]/,json:/[j]/,not_json:/[^j]/,text:/^[^\\x25]+/,modulo:/^\\x25{2}/,placeholder:/^\\x25(?:([1-9]\\d*)\\$|\\(([^)]+)\\))?(\\+)?(0|'[^$])?(-)?(\\d+)?(?:\\.(\\d+))?([b-gijostTuvxX])/,key:/^([a-z_][a-z_\\d]*)/i,key_access:/^\\.([a-z_][a-z_\\d]*)/i,index_access:/^\\[(\\d+)\\]/,sign:/^[+-]/};function n(t){return function(t,r){var i,s,a,o,p,c,l,u,f,d=1,g=t.length,y=\"\";for(s=0;s=0),o.type){case\"b\":i=parseInt(i,10).toString(2);break;case\"c\":i=String.fromCharCode(parseInt(i,10));break;case\"d\":case\"i\":i=parseInt(i,10);break;case\"j\":i=JSON.stringify(i,null,o.width?parseInt(o.width):0);break;case\"e\":i=o.precision?parseFloat(i).toExponential(o.precision):parseFloat(i).toExponential();break;case\"f\":i=o.precision?parseFloat(i).toFixed(o.precision):parseFloat(i);break;case\"g\":i=o.precision?String(Number(i.toPrecision(o.precision))):parseFloat(i);break;case\"o\":i=(parseInt(i,10)>>>0).toString(8);break;case\"s\":i=String(i),i=o.precision?i.substring(0,o.precision):i;break;case\"t\":i=String(!!i),i=o.precision?i.substring(0,o.precision):i;break;case\"T\":i=Object.prototype.toString.call(i).slice(8,-1).toLowerCase(),i=o.precision?i.substring(0,o.precision):i;break;case\"u\":i=parseInt(i,10)>>>0;break;case\"v\":i=i.valueOf(),i=o.precision?i.substring(0,o.precision):i;break;case\"x\":i=(parseInt(i,10)>>>0).toString(16);break;case\"X\":i=(parseInt(i,10)>>>0).toString(16).toUpperCase()}e.json.test(o.type)?y+=i:(!e.number.test(o.type)||u&&!o.sign?f=\"\":(f=u?\"+\":\"-\",i=i.toString().replace(e.sign,\"\")),c=o.pad_char?\"0\"===o.pad_char?\"0\":o.pad_char.charAt(1):\" \",l=o.width-(f+i).length,p=o.width&&l>0?c.repeat(l):\"\",y+=o.align?f+i+p:\"0\"===c?f+p+i:p+f+i)}return y}(function(n){if(i[n])return i[n];var t,r=n,s=[],a=0;for(;r;){if(null!==(t=e.text.exec(r)))s.push(t[0]);else if(null!==(t=e.modulo.exec(r)))s.push(\"%\");else{if(null===(t=e.placeholder.exec(r)))throw new SyntaxError(\"[sprintf] unexpected placeholder\");if(t[2]){a|=1;var o=[],p=t[2],c=[];if(null===(c=e.key.exec(p)))throw new SyntaxError(\"[sprintf] failed to parse named argument key\");for(o.push(c[1]);\"\"!==(p=p.substring(c[0].length));)if(null!==(c=e.key_access.exec(p)))o.push(c[1]);else{if(null===(c=e.index_access.exec(p)))throw new SyntaxError(\"[sprintf] failed to parse named argument key\");o.push(c[1])}t[2]=o}else a|=2;if(3===a)throw new Error(\"[sprintf] mixing positional and named placeholders is not (yet) supported\");s.push({placeholder:t[0],param_no:t[1],keys:t[2],sign:t[3],pad_char:t[4],align:t[5],width:t[6],precision:t[7],type:t[8]})}r=r.substring(t[0].length)}return i[n]=s}(t),arguments)}function r(e,t){return n.apply(null,[e].concat(t||[]))}var i=Object.create(null);void 0!==t&&(t.sprintf=n,t.vsprintf=r),\"undefined\"!=typeof window&&(window.sprintf=n,window.vsprintf=r,\"function\"==typeof define&&define.amd&&define(function(){return{sprintf:n,vsprintf:r}}))}()},\n", + " function _(e,n,t){\n", + " /*!\n", + " * numbro.js\n", + " * version : 1.6.2\n", + " * author : Företagsplatsen AB\n", + " * license : MIT\n", + " * http://www.foretagsplatsen.se\n", + " */\n", + " var r,i={},a=i,o=\"en-US\",l=null,u=\"0,0\";void 0!==n&&n.exports;function c(e){this._value=e}function s(e){var n,t=\"\";for(n=0;n-1?function(e,n){var t,r,i,a;return t=(a=e.toString()).split(\"e\")[0],i=a.split(\"e\")[1],a=t.split(\".\")[0]+(r=t.split(\".\")[1]||\"\")+s(i-r.length),n>0&&(a+=\".\"+s(n)),a}(e,n):(t(e*o)/o).toFixed(n),r&&(i=new RegExp(\"0{1,\"+r+\"}$\"),a=a.replace(i,\"\")),a}function d(e,n,t){return n.indexOf(\"$\")>-1?function(e,n,t){var r,a,l=n,u=l.indexOf(\"$\"),c=l.indexOf(\"(\"),s=l.indexOf(\"+\"),f=l.indexOf(\"-\"),d=\"\",p=\"\";-1===l.indexOf(\"$\")?\"infix\"===i[o].currency.position?(p=i[o].currency.symbol,i[o].currency.spaceSeparated&&(p=\" \"+p+\" \")):i[o].currency.spaceSeparated&&(d=\" \"):l.indexOf(\" $\")>-1?(d=\" \",l=l.replace(\" $\",\"\")):l.indexOf(\"$ \")>-1?(d=\" \",l=l.replace(\"$ \",\"\")):l=l.replace(\"$\",\"\");if(a=h(e,l,t,p),-1===n.indexOf(\"$\"))switch(i[o].currency.position){case\"postfix\":a.indexOf(\")\")>-1?((a=a.split(\"\")).splice(-1,0,d+i[o].currency.symbol),a=a.join(\"\")):a=a+d+i[o].currency.symbol;break;case\"infix\":break;case\"prefix\":a.indexOf(\"(\")>-1||a.indexOf(\"-\")>-1?(a=a.split(\"\"),r=Math.max(c,f)+1,a.splice(r,0,i[o].currency.symbol+d),a=a.join(\"\")):a=i[o].currency.symbol+d+a;break;default:throw Error('Currency position should be among [\"prefix\", \"infix\", \"postfix\"]')}else u<=1?a.indexOf(\"(\")>-1||a.indexOf(\"+\")>-1||a.indexOf(\"-\")>-1?(a=a.split(\"\"),r=1,(u-1?((a=a.split(\"\")).splice(-1,0,d+i[o].currency.symbol),a=a.join(\"\")):a=a+d+i[o].currency.symbol;return a}(e,n,t):n.indexOf(\"%\")>-1?function(e,n,t){var r,i=\"\";e*=100,n.indexOf(\" %\")>-1?(i=\" \",n=n.replace(\" %\",\"\")):n=n.replace(\"%\",\"\");(r=h(e,n,t)).indexOf(\")\")>-1?((r=r.split(\"\")).splice(-1,0,i+\"%\"),r=r.join(\"\")):r=r+i+\"%\";return r}(e,n,t):n.indexOf(\":\")>-1?function(e){var n=Math.floor(e/60/60),t=Math.floor((e-60*n*60)/60),r=Math.round(e-60*n*60-60*t);return n+\":\"+(t<10?\"0\"+t:t)+\":\"+(r<10?\"0\"+r:r)}(e):h(e,n,t)}function h(e,n,t,r){var a,u,c,s,d,h,p,m,x,g,O,b,w,y,M,v,$,B=!1,E=!1,F=!1,k=\"\",U=!1,N=!1,S=!1,j=!1,D=!1,C=\"\",L=\"\",T=Math.abs(e),K=[\"B\",\"KiB\",\"MiB\",\"GiB\",\"TiB\",\"PiB\",\"EiB\",\"ZiB\",\"YiB\"],G=[\"B\",\"KB\",\"MB\",\"GB\",\"TB\",\"PB\",\"EB\",\"ZB\",\"YB\"],I=\"\",P=!1,R=!1;if(0===e&&null!==l)return l;if(!isFinite(e))return\"\"+e;if(0===n.indexOf(\"{\")){var W=n.indexOf(\"}\");if(-1===W)throw Error('Format should also contain a \"}\"');b=n.slice(1,W),n=n.slice(W+1)}else b=\"\";if(n.indexOf(\"}\")===n.length-1){var Y=n.indexOf(\"{\");if(-1===Y)throw Error('Format should also contain a \"{\"');w=n.slice(Y+1,-1),n=n.slice(0,Y+1)}else w=\"\";if(v=null===($=-1===n.indexOf(\".\")?n.match(/([0-9]+).*/):n.match(/([0-9]+)\\..*/))?-1:$[1].length,-1!==n.indexOf(\"-\")&&(P=!0),n.indexOf(\"(\")>-1?(B=!0,n=n.slice(1,-1)):n.indexOf(\"+\")>-1&&(E=!0,n=n.replace(/\\+/g,\"\")),n.indexOf(\"a\")>-1){if(g=n.split(\".\")[0].match(/[0-9]+/g)||[\"0\"],g=parseInt(g[0],10),U=n.indexOf(\"aK\")>=0,N=n.indexOf(\"aM\")>=0,S=n.indexOf(\"aB\")>=0,j=n.indexOf(\"aT\")>=0,D=U||N||S||j,n.indexOf(\" a\")>-1?(k=\" \",n=n.replace(\" a\",\"\")):n=n.replace(\"a\",\"\"),p=0===(p=(d=Math.floor(Math.log(T)/Math.LN10)+1)%3)?3:p,g&&0!==T&&(h=Math.floor(Math.log(T)/Math.LN10)+1-g,m=3*~~((Math.min(g,d)-p)/3),T/=Math.pow(10,m),-1===n.indexOf(\".\")&&g>3))for(n+=\"[.]\",M=(M=0===h?0:3*~~(h/3)-h)<0?M+3:M,a=0;a=Math.pow(10,12)&&!D||j?(k+=i[o].abbreviations.trillion,e/=Math.pow(10,12)):T=Math.pow(10,9)&&!D||S?(k+=i[o].abbreviations.billion,e/=Math.pow(10,9)):T=Math.pow(10,6)&&!D||N?(k+=i[o].abbreviations.million,e/=Math.pow(10,6)):(T=Math.pow(10,3)&&!D||U)&&(k+=i[o].abbreviations.thousand,e/=Math.pow(10,3)))}if(n.indexOf(\"b\")>-1)for(n.indexOf(\" b\")>-1?(C=\" \",n=n.replace(\" b\",\"\")):n=n.replace(\"b\",\"\"),s=0;s<=K.length;s++)if(u=Math.pow(1024,s),c=Math.pow(1024,s+1),e>=u&&e0&&(e/=u);break}if(n.indexOf(\"d\")>-1)for(n.indexOf(\" d\")>-1?(C=\" \",n=n.replace(\" d\",\"\")):n=n.replace(\"d\",\"\"),s=0;s<=G.length;s++)if(u=Math.pow(1e3,s),c=Math.pow(1e3,s+1),e>=u&&e0&&(e/=u);break}if(n.indexOf(\"o\")>-1&&(n.indexOf(\" o\")>-1?(L=\" \",n=n.replace(\" o\",\"\")):n=n.replace(\"o\",\"\"),i[o].ordinal&&(L+=i[o].ordinal(e))),n.indexOf(\"[.]\")>-1&&(F=!0,n=n.replace(\"[.]\",\".\")),x=e.toString().split(\".\")[0],O=n.split(\".\")[1],y=n.indexOf(\",\"),O){if(x=(I=-1!==O.indexOf(\"*\")?f(e,e.toString().split(\".\")[1].length,t):O.indexOf(\"[\")>-1?f(e,(O=(O=O.replace(\"]\",\"\")).split(\"[\"))[0].length+O[1].length,t,O[1].length):f(e,O.length,t)).split(\".\")[0],I.split(\".\")[1].length)I=(r?k+r:i[o].delimiters.decimal)+I.split(\".\")[1];else I=\"\";F&&0===Number(I.slice(1))&&(I=\"\")}else x=f(e,null,t);return x.indexOf(\"-\")>-1&&(x=x.slice(1),R=!0),x.length-1&&(x=x.toString().replace(/(\\d)(?=(\\d{3})+(?!\\d))/g,\"$1\"+i[o].delimiters.thousands)),0===n.indexOf(\".\")&&(x=\"\"),b+(n.indexOf(\"(\")2)&&(o.length<2?!!o[0].match(/^\\d+.*\\d$/)&&!o[0].match(u):1===o[0].length?!!o[0].match(/^\\d+$/)&&!o[0].match(u)&&!!o[1].match(/^\\d+$/):!!o[0].match(/^\\d+.*\\d$/)&&!o[0].match(u)&&!!o[1].match(/^\\d+$/)))))},n.exports={format:function(e,n,t,i){return null!=t&&t!==r.culture()&&r.setCulture(t),d(Number(e),null!=n?n:u,null==i?Math.round:i)}}},\n", + " function _(e,n,i){var t=e(113),r=e(110),a=e(205),s=e(257),c=e(258),_=e(261),m=e(262),k=e(260),o=function(e){function n(n){return e.call(this,n)||this}return t.__extends(n,e),n.init_DatetimeTicker=function(){this.override({num_minor_ticks:0,tickers:function(){return[new a.AdaptiveTicker({mantissas:[1,2,5],base:10,min_interval:0,max_interval:500*k.ONE_MILLI,num_minor_ticks:0}),new a.AdaptiveTicker({mantissas:[1,2,5,10,15,20,30],base:60,min_interval:k.ONE_SECOND,max_interval:30*k.ONE_MINUTE,num_minor_ticks:0}),new a.AdaptiveTicker({mantissas:[1,2,4,6,8,12],base:24,min_interval:k.ONE_HOUR,max_interval:12*k.ONE_HOUR,num_minor_ticks:0}),new c.DaysTicker({days:r.range(1,32)}),new c.DaysTicker({days:r.range(1,31,3)}),new c.DaysTicker({days:[1,8,15,22]}),new c.DaysTicker({days:[1,15]}),new _.MonthsTicker({months:r.range(0,12,1)}),new _.MonthsTicker({months:r.range(0,12,2)}),new _.MonthsTicker({months:r.range(0,12,4)}),new _.MonthsTicker({months:r.range(0,12,6)}),new m.YearsTicker({})]}})},n}(s.CompositeTicker);i.DatetimeTicker=o,o.__name__=\"DatetimeTicker\",o.init_DatetimeTicker()},\n", + " function _(t,e,i){var n=t(113),r=t(206),o=t(121),s=t(110),a=t(125),_=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.init_CompositeTicker=function(){this.define({tickers:[o.Array,[]]})},Object.defineProperty(e.prototype,\"min_intervals\",{get:function(){return this.tickers.map(function(t){return t.get_min_interval()})},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"max_intervals\",{get:function(){return this.tickers.map(function(t){return t.get_max_interval()})},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"min_interval\",{get:function(){return this.min_intervals[0]},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"max_interval\",{get:function(){return this.max_intervals[0]},enumerable:!0,configurable:!0}),e.prototype.get_best_ticker=function(t,e,i){var n,r=e-t,o=this.get_ideal_interval(t,e,i),_=[s.sorted_index(this.min_intervals,o)-1,s.sorted_index(this.max_intervals,o)],u=[this.min_intervals[_[0]],this.max_intervals[_[1]]].map(function(t){return Math.abs(i-r/t)});if(a.isEmpty(u.filter(function(t){return!isNaN(t)})))n=this.tickers[0];else{var c=_[s.argmin(u)];n=this.tickers[c]}return n},e.prototype.get_interval=function(t,e,i){return this.get_best_ticker(t,e,i).get_interval(t,e,i)},e.prototype.get_ticks_no_defaults=function(t,e,i,n){return this.get_best_ticker(t,e,n).get_ticks_no_defaults(t,e,i,n)},e}(r.ContinuousTicker);i.CompositeTicker=_,_.__name__=\"CompositeTicker\",_.init_CompositeTicker()},\n", + " function _(t,n,e){var i=t(113),r=t(259),a=t(260),o=t(121),s=t(110);var _=function(t){function n(n){return t.call(this,n)||this}return i.__extends(n,t),n.init_DaysTicker=function(){this.define({days:[o.Array,[]]}),this.override({num_minor_ticks:0})},n.prototype.initialize=function(){t.prototype.initialize.call(this);var n=this.days;n.length>1?this.interval=(n[1]-n[0])*a.ONE_DAY:this.interval=31*a.ONE_DAY},n.prototype.get_ticks_no_defaults=function(t,n,e,i){var r=function(t,n){var e=a.last_month_no_later_than(new Date(t)),i=a.last_month_no_later_than(new Date(n));i.setUTCMonth(i.getUTCMonth()+1);for(var r=[],o=e;r.push(a.copy_date(o)),o.setUTCMonth(o.getUTCMonth()+1),!(o>i););return r}(t,n),o=this.days,_=this.interval;return{major:s.concat(r.map(function(t){return function(t,n){for(var e=t.getUTCMonth(),i=[],r=0,s=o;r1?this.interval=(n[1]-n[0])*a.ONE_MONTH:this.interval=12*a.ONE_MONTH},n.prototype.get_ticks_no_defaults=function(t,n,e,r){var i=function(t,n){var e=a.last_year_no_later_than(new Date(t)),r=a.last_year_no_later_than(new Date(n));r.setUTCFullYear(r.getUTCFullYear()+1);for(var i=[],o=e;i.push(a.copy_date(o)),o.setUTCFullYear(o.getUTCFullYear()+1),!(o>r););return i}(t,n),o=this.months;return{major:l.concat(i.map(function(t){return o.map(function(n){var e=a.copy_date(t);return e.setUTCMonth(n),e})})).map(function(t){return t.getTime()}).filter(function(e){return t<=e&&e<=n}),minor:[]}},n}(i.SingleIntervalTicker);e.MonthsTicker=u,u.__name__=\"MonthsTicker\",u.init_MonthsTicker()},\n", + " function _(t,e,i){var n=t(113),r=t(204),a=t(259),_=t(260),c=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.prototype.initialize=function(){t.prototype.initialize.call(this),this.interval=_.ONE_YEAR,this.basic_ticker=new r.BasicTicker({num_minor_ticks:0})},e.prototype.get_ticks_no_defaults=function(t,e,i,n){var r=_.last_year_no_later_than(new Date(t)).getUTCFullYear(),a=_.last_year_no_later_than(new Date(e)).getUTCFullYear();return{major:this.basic_ticker.get_ticks_no_defaults(r,a,i,n).major.map(function(t){return Date.UTC(t,0,1)}).filter(function(i){return t<=i&&i<=e}),minor:[]}},e}(a.SingleIntervalTicker);i.YearsTicker=c,c.__name__=\"YearsTicker\"},\n", + " function _(i,n,t){var e=i(113),o=i(243),r=i(248),u=i(264),s=i(265),_=function(i){function n(){return null!==i&&i.apply(this,arguments)||this}return e.__extends(n,i),n}(o.AxisView);t.LogAxisView=_,_.__name__=\"LogAxisView\";var c=function(i){function n(n){return i.call(this,n)||this}return e.__extends(n,i),n.init_LogAxis=function(){this.prototype.default_view=_,this.override({ticker:function(){return new s.LogTicker},formatter:function(){return new u.LogTickFormatter}})},n}(r.ContinuousAxis);t.LogAxis=c,c.__name__=\"LogAxis\",c.init_LogAxis()},\n", + " function _(t,i,r){var e=t(113),n=t(209),o=t(208),a=t(167),c=t(121),l=function(t){function i(i){return t.call(this,i)||this}return e.__extends(i,t),i.init_LogTickFormatter=function(){this.define({ticker:[c.Instance,null]})},i.prototype.initialize=function(){t.prototype.initialize.call(this),this.basic_formatter=new o.BasicTickFormatter,null==this.ticker&&a.logger.warn(\"LogTickFormatter not configured with a ticker, using default base of 10 (labels will be incorrect if ticker base is not 10)\")},i.prototype.doFormat=function(t,i){if(0==t.length)return[];for(var r=null!=this.ticker?this.ticker.base:10,e=!1,n=new Array(t.length),o=0,a=t.length;o0&&n[o]==n[o-1]){e=!0;break}return e?this.basic_formatter.doFormat(t,i):n},i}(n.TickFormatter);r.LogTickFormatter=l,l.__name__=\"LogTickFormatter\",l.init_LogTickFormatter()},\n", + " function _(t,r,n){var e=t(113),i=t(205),o=t(110),a=function(t){function r(r){return t.call(this,r)||this}return e.__extends(r,t),r.init_LogTicker=function(){this.override({mantissas:[1,5]})},r.prototype.get_ticks_no_defaults=function(t,r,n,e){var i,a=this.num_minor_ticks,u=[],f=this.base,h=Math.log(t)/Math.log(f),l=Math.log(r)/Math.log(f),c=l-h;if(isFinite(c))if(c<2){var s=this.get_interval(t,r,e),g=Math.floor(t/s),_=Math.ceil(r/s);if(i=o.range(g,_+1).filter(function(t){return 0!=t}).map(function(t){return t*s}).filter(function(n){return t<=n&&n<=r}),a>0&&i.length>0){for(var p=s/a,v=0,M=(y=o.range(0,a).map(function(t){return t*p})).slice(1);v0&&i.length>0){for(var y,A=Math.pow(f,x)/a,F=0,q=y=o.range(1,a+1).map(function(t){return t*A});F1?((e=i).width=arguments[0],e.height=arguments[1]):e=t||i,!(this instanceof r))return new r(e);this.width=e.width||i.width,this.height=e.height||i.height,this.enableMirroring=void 0!==e.enableMirroring?e.enableMirroring:i.enableMirroring,this.canvas=this,this.__document=e.document||document,e.ctx?this.__ctx=e.ctx:(this.__canvas=this.__document.createElement(\"canvas\"),this.__ctx=this.__canvas.getContext(\"2d\")),this.__setDefaultStyles(),this.__stack=[this.__getStyleState()],this.__groupStack=[],this.__root=this.__document.createElementNS(\"http://www.w3.org/2000/svg\",\"svg\"),this.__root.setAttribute(\"version\",1.1),this.__root.setAttribute(\"xmlns\",\"http://www.w3.org/2000/svg\"),this.__root.setAttributeNS(\"http://www.w3.org/2000/xmlns/\",\"xmlns:xlink\",\"http://www.w3.org/1999/xlink\"),this.__root.setAttribute(\"width\",this.width),this.__root.setAttribute(\"height\",this.height),this.__ids={},this.__defs=this.__document.createElementNS(\"http://www.w3.org/2000/svg\",\"defs\"),this.__root.appendChild(this.__defs),this.__currentElement=this.__document.createElementNS(\"http://www.w3.org/2000/svg\",\"g\"),this.__root.appendChild(this.__currentElement)}).prototype.__createElement=function(t,e,r){void 0===e&&(e={});var i,n,s=this.__document.createElementNS(\"http://www.w3.org/2000/svg\",t),a=Object.keys(e);for(r&&(s.setAttribute(\"fill\",\"none\"),s.setAttribute(\"stroke\",\"none\")),i=0;i0){\"path\"===this.__currentElement.nodeName&&(this.__currentElementsToStyle||(this.__currentElementsToStyle={element:e,children:[]}),this.__currentElementsToStyle.children.push(this.__currentElement),this.__applyCurrentDefaultPath());var r=this.__createElement(\"g\");e.appendChild(r),this.__currentElement=r}var i=this.__currentElement.getAttribute(\"transform\");i?i+=\" \":i=\"\",i+=t,this.__currentElement.setAttribute(\"transform\",i)},r.prototype.scale=function(t,e){void 0===e&&(e=t),this.__addTransform(a(\"scale({x},{y})\",{x:t,y:e}))},r.prototype.rotate=function(t){var e=180*t/Math.PI;this.__addTransform(a(\"rotate({angle},{cx},{cy})\",{angle:e,cx:0,cy:0}))},r.prototype.translate=function(t,e){this.__addTransform(a(\"translate({x},{y})\",{x:t,y:e}))},r.prototype.transform=function(t,e,r,i,n,s){this.__addTransform(a(\"matrix({a},{b},{c},{d},{e},{f})\",{a:t,b:e,c:r,d:i,e:n,f:s}))},r.prototype.beginPath=function(){var t;this.__currentDefaultPath=\"\",this.__currentPosition={},t=this.__createElement(\"path\",{},!0),this.__closestGroupOrSvg().appendChild(t),this.__currentElement=t},r.prototype.__applyCurrentDefaultPath=function(){var t=this.__currentElement;\"path\"===t.nodeName?t.setAttribute(\"d\",this.__currentDefaultPath):console.error(\"Attempted to apply path command to node\",t.nodeName)},r.prototype.__addPathCommand=function(t){this.__currentDefaultPath+=\" \",this.__currentDefaultPath+=t},r.prototype.moveTo=function(t,e){\"path\"!==this.__currentElement.nodeName&&this.beginPath(),this.__currentPosition={x:t,y:e},this.__addPathCommand(a(\"M {x} {y}\",{x:t,y:e}))},r.prototype.closePath=function(){this.__currentDefaultPath&&this.__addPathCommand(\"Z\")},r.prototype.lineTo=function(t,e){this.__currentPosition={x:t,y:e},this.__currentDefaultPath.indexOf(\"M\")>-1?this.__addPathCommand(a(\"L {x} {y}\",{x:t,y:e})):this.__addPathCommand(a(\"M {x} {y}\",{x:t,y:e}))},r.prototype.bezierCurveTo=function(t,e,r,i,n,s){this.__currentPosition={x:n,y:s},this.__addPathCommand(a(\"C {cp1x} {cp1y} {cp2x} {cp2y} {x} {y}\",{cp1x:t,cp1y:e,cp2x:r,cp2y:i,x:n,y:s}))},r.prototype.quadraticCurveTo=function(t,e,r,i){this.__currentPosition={x:r,y:i},this.__addPathCommand(a(\"Q {cpx} {cpy} {x} {y}\",{cpx:t,cpy:e,x:r,y:i}))};var l=function(t){var e=Math.sqrt(t[0]*t[0]+t[1]*t[1]);return[t[0]/e,t[1]/e]};r.prototype.arcTo=function(t,e,r,i,n){var s=this.__currentPosition&&this.__currentPosition.x,a=this.__currentPosition&&this.__currentPosition.y;if(void 0!==s&&void 0!==a){if(n<0)throw new Error(\"IndexSizeError: The radius provided (\"+n+\") is negative.\");if(s===t&&a===e||t===r&&e===i||0===n)this.lineTo(t,e);else{var o=l([s-t,a-e]),h=l([r-t,i-e]);if(o[0]*h[1]!=o[1]*h[0]){var c=o[0]*h[0]+o[1]*h[1],p=Math.acos(Math.abs(c)),_=l([o[0]+h[0],o[1]+h[1]]),u=n/Math.sin(p/2),d=t+u*_[0],g=e+u*_[1],m=[-o[1],o[0]],f=[h[1],-h[0]],y=function(t){var e=t[0];return t[1]>=0?Math.acos(e):-Math.acos(e)},v=y(m),b=y(f);this.lineTo(d+m[0]*n,g+m[1]*n),this.arc(d,g,n,v,b)}else this.lineTo(t,e)}}},r.prototype.stroke=function(){\"path\"===this.__currentElement.nodeName&&this.__currentElement.setAttribute(\"paint-order\",\"fill stroke markers\"),this.__applyCurrentDefaultPath(),this.__applyStyleToCurrentElement(\"stroke\")},r.prototype.fill=function(){\"path\"===this.__currentElement.nodeName&&this.__currentElement.setAttribute(\"paint-order\",\"stroke fill markers\"),this.__applyCurrentDefaultPath(),this.__applyStyleToCurrentElement(\"fill\")},r.prototype.rect=function(t,e,r,i){\"path\"!==this.__currentElement.nodeName&&this.beginPath(),this.moveTo(t,e),this.lineTo(t+r,e),this.lineTo(t+r,e+i),this.lineTo(t,e+i),this.lineTo(t,e),this.closePath()},r.prototype.fillRect=function(t,e,r,i){var n;n=this.__createElement(\"rect\",{x:t,y:e,width:r,height:i},!0),this.__closestGroupOrSvg().appendChild(n),this.__currentElement=n,this.__applyStyleToCurrentElement(\"fill\")},r.prototype.strokeRect=function(t,e,r,i){var n;n=this.__createElement(\"rect\",{x:t,y:e,width:r,height:i},!0),this.__closestGroupOrSvg().appendChild(n),this.__currentElement=n,this.__applyStyleToCurrentElement(\"stroke\")},r.prototype.__clearCanvas=function(){for(var t=this.__closestGroupOrSvg().getAttribute(\"transform\"),e=this.__root.childNodes[1],r=e.childNodes,i=r.length-1;i>=0;i--)r[i]&&e.removeChild(r[i]);this.__currentElement=e,this.__groupStack=[],t&&this.__addTransform(t)},r.prototype.clearRect=function(t,e,r,i){if(0!==t||0!==e||r!==this.width||i!==this.height){var n,s=this.__closestGroupOrSvg();n=this.__createElement(\"rect\",{x:t,y:e,width:r,height:i,fill:\"#FFFFFF\"},!0),s.appendChild(n)}else this.__clearCanvas()},r.prototype.createLinearGradient=function(t,e,r,n){var s=this.__createElement(\"linearGradient\",{id:o(this.__ids),x1:t+\"px\",x2:r+\"px\",y1:e+\"px\",y2:n+\"px\",gradientUnits:\"userSpaceOnUse\"},!1);return this.__defs.appendChild(s),new i(s,this)},r.prototype.createRadialGradient=function(t,e,r,n,s,a){var h=this.__createElement(\"radialGradient\",{id:o(this.__ids),cx:n+\"px\",cy:s+\"px\",r:a+\"px\",fx:t+\"px\",fy:e+\"px\",gradientUnits:\"userSpaceOnUse\"},!1);return this.__defs.appendChild(h),new i(h,this)},r.prototype.__parseFont=function(){var t=/^\\s*(?=(?:(?:[-a-z]+\\s*){0,2}(italic|oblique))?)(?=(?:(?:[-a-z]+\\s*){0,2}(small-caps))?)(?=(?:(?:[-a-z]+\\s*){0,2}(bold(?:er)?|lighter|[1-9]00))?)(?:(?:normal|\\1|\\2|\\3)\\s*){0,3}((?:xx?-)?(?:small|large)|medium|smaller|larger|[.\\d]+(?:\\%|in|[cem]m|ex|p[ctx]))(?:\\s*\\/\\s*(normal|[.\\d]+(?:\\%|in|[cem]m|ex|p[ctx])))?\\s*([-,\\'\\\"\\sa-z0-9]+?)\\s*$/i.exec(this.font),e={style:t[1]||\"normal\",size:t[4]||\"10px\",family:t[6]||\"sans-serif\",weight:t[3]||\"normal\",decoration:t[2]||\"normal\",href:null};return\"underline\"===this.__fontUnderline&&(e.decoration=\"underline\"),this.__fontHref&&(e.href=this.__fontHref),e},r.prototype.__wrapTextLink=function(t,e){if(t.href){var r=this.__createElement(\"a\");return r.setAttributeNS(\"http://www.w3.org/1999/xlink\",\"xlink:href\",t.href),r.appendChild(e),r}return e},r.prototype.__applyText=function(t,e,r,i){var n,s,a=this.__parseFont(),o=this.__closestGroupOrSvg(),l=this.__createElement(\"text\",{\"font-family\":a.family,\"font-size\":a.size,\"font-style\":a.style,\"font-weight\":a.weight,\"text-decoration\":a.decoration,x:e,y:r,\"text-anchor\":(n=this.textAlign,s={left:\"start\",right:\"end\",center:\"middle\",start:\"start\",end:\"end\"},s[n]||s.start),\"dominant-baseline\":h(this.textBaseline)},!0);l.appendChild(this.__document.createTextNode(t)),this.__currentElement=l,this.__applyStyleToCurrentElement(i),o.appendChild(this.__wrapTextLink(a,l))},r.prototype.fillText=function(t,e,r){this.__applyText(t,e,r,\"fill\")},r.prototype.strokeText=function(t,e,r){this.__applyText(t,e,r,\"stroke\")},r.prototype.measureText=function(t){return this.__ctx.font=this.font,this.__ctx.measureText(t)},r.prototype.arc=function(t,e,r,i,n,s){if(i!==n){(i%=2*Math.PI)===(n%=2*Math.PI)&&(n=(n+2*Math.PI-.001*(s?-1:1))%(2*Math.PI));var o=t+r*Math.cos(n),h=e+r*Math.sin(n),l=t+r*Math.cos(i),c=e+r*Math.sin(i),p=s?0:1,_=0,u=n-i;u<0&&(u+=2*Math.PI),_=s?u>Math.PI?0:1:u>Math.PI?1:0,this.lineTo(l,c),this.__addPathCommand(a(\"A {rx} {ry} {xAxisRotation} {largeArcFlag} {sweepFlag} {endX} {endY}\",{rx:r,ry:r,xAxisRotation:0,largeArcFlag:_,sweepFlag:p,endX:o,endY:h})),this.__currentPosition={x:o,y:h}}},r.prototype.clip=function(){var t=this.__closestGroupOrSvg(),e=this.__createElement(\"clipPath\"),r=o(this.__ids),i=this.__createElement(\"g\");this.__applyCurrentDefaultPath(),t.removeChild(this.__currentElement),e.setAttribute(\"id\",r),e.appendChild(this.__currentElement),this.__defs.appendChild(e),t.setAttribute(\"clip-path\",a(\"url(#{id})\",{id:r})),t.appendChild(i),this.__currentElement=i},r.prototype.drawImage=function(){var t,e,i,n,s,a,o,h,l,c,p,_,u,d,g=Array.prototype.slice.call(arguments),m=g[0],f=0,y=0;if(3===g.length)t=g[1],e=g[2],i=s=m.width,n=a=m.height;else if(5===g.length)t=g[1],e=g[2],i=g[3],n=g[4],s=m.width,a=m.height;else{if(9!==g.length)throw new Error(\"Inavlid number of arguments passed to drawImage: \"+arguments.length);f=g[1],y=g[2],s=g[3],a=g[4],t=g[5],e=g[6],i=g[7],n=g[8]}o=this.__closestGroupOrSvg(),this.__currentElement;var v=\"translate(\"+t+\", \"+e+\")\";if(m instanceof r){if((h=m.getSvg().cloneNode(!0)).childNodes&&h.childNodes.length>1){for(l=h.childNodes[0];l.childNodes.length;)d=l.childNodes[0].getAttribute(\"id\"),this.__ids[d]=d,this.__defs.appendChild(l.childNodes[0]);if(c=h.childNodes[1]){var b,w=c.getAttribute(\"transform\");b=w?w+\" \"+v:v,c.setAttribute(\"transform\",b),o.appendChild(c)}}}else\"IMG\"===m.nodeName?((p=this.__createElement(\"image\")).setAttribute(\"width\",i),p.setAttribute(\"height\",n),p.setAttribute(\"preserveAspectRatio\",\"none\"),(f||y||s!==m.width||a!==m.height)&&((_=this.__document.createElement(\"canvas\")).width=i,_.height=n,(u=_.getContext(\"2d\")).drawImage(m,f,y,s,a,0,0,i,n),m=_),p.setAttribute(\"transform\",v),p.setAttributeNS(\"http://www.w3.org/1999/xlink\",\"xlink:href\",\"CANVAS\"===m.nodeName?m.toDataURL():m.getAttribute(\"src\")),o.appendChild(p)):\"CANVAS\"===m.nodeName&&((p=this.__createElement(\"image\")).setAttribute(\"width\",i),p.setAttribute(\"height\",n),p.setAttribute(\"preserveAspectRatio\",\"none\"),(_=this.__document.createElement(\"canvas\")).width=i,_.height=n,(u=_.getContext(\"2d\")).imageSmoothingEnabled=!1,u.mozImageSmoothingEnabled=!1,u.oImageSmoothingEnabled=!1,u.webkitImageSmoothingEnabled=!1,u.drawImage(m,f,y,s,a,0,0,i,n),m=_,p.setAttribute(\"transform\",v),p.setAttributeNS(\"http://www.w3.org/1999/xlink\",\"xlink:href\",m.toDataURL()),o.appendChild(p))},r.prototype.createPattern=function(t,e){var i,s=this.__document.createElementNS(\"http://www.w3.org/2000/svg\",\"pattern\"),a=o(this.__ids);return s.setAttribute(\"id\",a),s.setAttribute(\"width\",t.width),s.setAttribute(\"height\",t.height),\"CANVAS\"===t.nodeName||\"IMG\"===t.nodeName?((i=this.__document.createElementNS(\"http://www.w3.org/2000/svg\",\"image\")).setAttribute(\"width\",t.width),i.setAttribute(\"height\",t.height),i.setAttributeNS(\"http://www.w3.org/1999/xlink\",\"xlink:href\",\"CANVAS\"===t.nodeName?t.toDataURL():t.getAttribute(\"src\")),s.appendChild(i),this.__defs.appendChild(s)):t instanceof r&&(s.appendChild(t.__root.childNodes[1]),this.__defs.appendChild(s)),new n(s,this)},r.prototype.setLineDash=function(t){t&&t.length>0?this.lineDash=t.join(\",\"):this.lineDash=null},r.prototype.drawFocusRing=function(){},r.prototype.createImageData=function(){},r.prototype.getImageData=function(){},r.prototype.putImageData=function(){},r.prototype.globalCompositeOperation=function(){},r.prototype.setTransform=function(){},\"object\"==typeof window&&(window.C2S=r),\"object\"==typeof e&&\"object\"==typeof e.exports&&(e.exports=r)}()},\n", + " function _(e,t,a){var r=e(113),n=e(279),s=e(215),i=e(224),_=e(225),o=e(280),c=e(184),g=function(e){function t(t,a,r,n,s,i){void 0===s&&(s={}),void 0===i&&(i={});var _=e.call(this)||this;return _.x_scale=t,_.y_scale=a,_.x_range=r,_.y_range=n,_.extra_x_ranges=s,_.extra_y_ranges=i,_._configure_scales(),_}return r.__extends(t,e),t.prototype.map_to_screen=function(e,t,a,r){return void 0===a&&(a=\"default\"),void 0===r&&(r=\"default\"),[this.xscales[a].v_compute(e),this.yscales[r].v_compute(t)]},t.prototype._get_ranges=function(e,t){var a={};if(a.default=e,null!=t)for(var r in t)a[r]=t[r];return a},t.prototype._get_scales=function(e,t,a){var r={};for(var g in t){var l=t[g];if(l instanceof o.DataRange1d||l instanceof _.Range1d){if(!(e instanceof i.LogScale||e instanceof s.LinearScale))throw new Error(\"Range \"+l.type+\" is incompatible is Scale \"+e.type);if(e instanceof n.CategoricalScale)throw new Error(\"Range \"+l.type+\" is incompatible is Scale \"+e.type)}if(l instanceof c.FactorRange&&!(e instanceof n.CategoricalScale))throw new Error(\"Range \"+l.type+\" is incompatible is Scale \"+e.type);e instanceof i.LogScale&&l instanceof o.DataRange1d&&(l.scale_hint=\"log\");var f=e.clone();f.setv({source_range:l,target_range:a}),r[g]=f}return r},t.prototype._configure_frame_ranges=function(){this._h_target=new _.Range1d({start:this._left.value,end:this._right.value}),this._v_target=new _.Range1d({start:this._bottom.value,end:this._top.value})},t.prototype._configure_scales=function(){this._configure_frame_ranges(),this._x_ranges=this._get_ranges(this.x_range,this.extra_x_ranges),this._y_ranges=this._get_ranges(this.y_range,this.extra_y_ranges),this._xscales=this._get_scales(this.x_scale,this._x_ranges,this._h_target),this._yscales=this._get_scales(this.y_scale,this._y_ranges,this._v_target)},t.prototype._update_scales=function(){for(var e in this._configure_frame_ranges(),this._xscales){this._xscales[e].target_range=this._h_target}for(var e in this._yscales){this._yscales[e].target_range=this._v_target}},t.prototype._set_geometry=function(t,a){e.prototype._set_geometry.call(this,t,a),this._update_scales()},Object.defineProperty(t.prototype,\"x_ranges\",{get:function(){return this._x_ranges},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"y_ranges\",{get:function(){return this._y_ranges},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"xscales\",{get:function(){return this._xscales},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"yscales\",{get:function(){return this._yscales},enumerable:!0,configurable:!0}),t}(e(282).LayoutItem);a.CartesianFrame=g,g.__name__=\"CartesianFrame\"},\n", + " function _(t,e,c){var n=t(113),o=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.prototype.compute=function(e){return t.prototype.compute.call(this,this.source_range.synthetic(e))},e.prototype.v_compute=function(e){return t.prototype.v_compute.call(this,this.source_range.v_synthetic(e))},e}(t(215).LinearScale);c.CategoricalScale=o,o.__name__=\"CategoricalScale\"},\n", + " function _(t,i,n){var e=t(113),a=t(281),r=t(175),s=t(167),o=t(121),l=t(181),_=t(110),d=function(t){function i(i){var n=t.call(this,i)||this;return n._plot_bounds={},n.have_updated_interactively=!1,n}return e.__extends(i,t),i.init_DataRange1d=function(){this.define({start:[o.Number],end:[o.Number],range_padding:[o.Number,.1],range_padding_units:[o.PaddingUnits,\"percent\"],flipped:[o.Boolean,!1],follow:[o.StartEnd],follow_interval:[o.Number],default_span:[o.Number,2],only_visible:[o.Boolean,!1]}),this.internal({scale_hint:[o.String,\"auto\"]})},i.prototype.initialize=function(){t.prototype.initialize.call(this),this._initial_start=this.start,this._initial_end=this.end,this._initial_range_padding=this.range_padding,this._initial_range_padding_units=this.range_padding_units,this._initial_follow=this.follow,this._initial_follow_interval=this.follow_interval,this._initial_default_span=this.default_span},Object.defineProperty(i.prototype,\"min\",{get:function(){return Math.min(this.start,this.end)},enumerable:!0,configurable:!0}),Object.defineProperty(i.prototype,\"max\",{get:function(){return Math.max(this.start,this.end)},enumerable:!0,configurable:!0}),i.prototype.computed_renderers=function(){var t=this.names,i=this.renderers;if(0==i.length)for(var n=0,e=this.plots;n0&&(i=i.filter(function(i){return _.includes(t,i.name)})),s.logger.debug(\"computed \"+i.length+\" renderers for DataRange1d \"+this.id);for(var o=0,l=i;ou&&(\"start\"==this.follow?a=e+h*u:\"end\"==this.follow&&(e=a-h*u)),[e,a]},i.prototype.update=function(t,i,n,e){if(!this.have_updated_interactively){var a=this.computed_renderers(),r=this._compute_plot_bounds(a,t);null!=e&&(r=this.adjust_bounds_for_aspect(r,e)),this._plot_bounds[n]=r;var s=this._compute_min_max(this._plot_bounds,i),o=s[0],l=s[1],_=this._compute_range(o,l),d=_[0],h=_[1];null!=this._initial_start&&(\"log\"==this.scale_hint?this._initial_start>0&&(d=this._initial_start):d=this._initial_start),null!=this._initial_end&&(\"log\"==this.scale_hint?this._initial_end>0&&(h=this._initial_end):h=this._initial_end);var u=[this.start,this.end],p=u[0],g=u[1];if(d!=p||h!=g){var f={};d!=p&&(f.start=d),h!=g&&(f.end=h),this.setv(f)}\"auto\"==this.bounds&&this.setv({bounds:[d,h]},{silent:!0}),this.change.emit()}},i.prototype.reset=function(){this.have_updated_interactively=!1,this.setv({range_padding:this._initial_range_padding,range_padding_units:this._initial_range_padding_units,follow:this._initial_follow,follow_interval:this._initial_follow_interval,default_span:this._initial_default_span},{silent:!0}),this.change.emit()},i}(a.DataRange);n.DataRange1d=d,d.__name__=\"DataRange1d\",d.init_DataRange1d()},\n", + " function _(n,a,e){var t=n(113),i=n(185),r=n(121),_=function(n){function a(a){return n.call(this,a)||this}return t.__extends(a,n),a.init_DataRange=function(){this.define({names:[r.Array,[]],renderers:[r.Array,[]]})},a}(i.Range);e.DataRange=_,_.__name__=\"DataRange\",_.init_DataRange()},\n", + " function _(a,o,t){var r=a(283);t.Sizeable=r.Sizeable;var e=a(284);t.Layoutable=e.Layoutable,t.LayoutItem=e.LayoutItem;var n=a(285);t.HStack=n.HStack,t.VStack=n.VStack,t.AnchorLayout=n.AnchorLayout;var c=a(286);t.Grid=c.Grid,t.Row=c.Row,t.Column=c.Column;var i=a(287);t.ContentBox=i.ContentBox,t.VariadicBox=i.VariadicBox},\n", + " function _(t,h,i){var e=Math.min,n=Math.max,o=function(){function t(t){void 0===t&&(t={}),this.width=null!=t.width?t.width:0,this.height=null!=t.height?t.height:0}return t.prototype.bounded_to=function(h){var i=h.width,e=h.height;return new t({width:this.width==1/0&&null!=i?i:this.width,height:this.height==1/0&&null!=e?e:this.height})},t.prototype.expanded_to=function(h){var i=h.width,e=h.height;return new t({width:i!=1/0?n(this.width,i):this.width,height:e!=1/0?n(this.height,e):this.height})},t.prototype.expand_to=function(t){var h=t.width,i=t.height;this.width=n(this.width,h),this.height=n(this.height,i)},t.prototype.narrowed_to=function(h){var i=h.width,n=h.height;return new t({width:e(this.width,i),height:e(this.height,n)})},t.prototype.narrow_to=function(t){var h=t.width,i=t.height;this.width=e(this.width,h),this.height=e(this.height,i)},t.prototype.grow_by=function(h){var i=h.left,e=h.right,n=h.top,o=h.bottom;return new t({width:this.width+i+e,height:this.height+n+o})},t.prototype.shrink_by=function(h){var i=h.left,e=h.right,o=h.top,r=h.bottom;return new t({width:n(this.width-i-e,0),height:n(this.height-o-r,0)})},t.prototype.map=function(h,i){return new t({width:h(this.width),height:(null!=i?i:h)(this.height)})},t}();i.Sizeable=o,o.__name__=\"Sizeable\"},\n", + " function _(i,t,e){var h=i(113),n=i(283),r=i(181),s=Math.min,o=Math.max,g=Math.round,u=function(){function i(){this._bbox=new r.BBox,this._inner_bbox=new r.BBox;var i=this;this._top={get value(){return i.bbox.top}},this._left={get value(){return i.bbox.left}},this._width={get value(){return i.bbox.width}},this._height={get value(){return i.bbox.height}},this._right={get value(){return i.bbox.right}},this._bottom={get value(){return i.bbox.bottom}},this._hcenter={get value(){return i.bbox.hcenter}},this._vcenter={get value(){return i.bbox.vcenter}}}return Object.defineProperty(i.prototype,\"bbox\",{get:function(){return this._bbox},enumerable:!0,configurable:!0}),Object.defineProperty(i.prototype,\"inner_bbox\",{get:function(){return this._inner_bbox},enumerable:!0,configurable:!0}),Object.defineProperty(i.prototype,\"sizing\",{get:function(){return this._sizing},enumerable:!0,configurable:!0}),i.prototype.set_sizing=function(i){var t=i.width_policy||\"fit\",e=i.width,h=null!=i.min_width?i.min_width:0,n=null!=i.max_width?i.max_width:1/0,r=i.height_policy||\"fit\",s=i.height,o=null!=i.min_height?i.min_height:0,g=null!=i.max_height?i.max_height:1/0,u=i.aspect,a=i.margin||{top:0,right:0,bottom:0,left:0},l=!1!==i.visible,_=i.halign||\"start\",d=i.valign||\"start\";this._sizing={width_policy:t,min_width:h,width:e,max_width:n,height_policy:r,min_height:o,height:s,max_height:g,aspect:u,margin:a,visible:l,halign:_,valign:d,size:{width:e,height:s},min_size:{width:h,height:o},max_size:{width:n,height:g}},this._init()},i.prototype._init=function(){},i.prototype._set_geometry=function(i,t){this._bbox=i,this._inner_bbox=t},i.prototype.set_geometry=function(i,t){this._set_geometry(i,t||i)},i.prototype.is_width_expanding=function(){return\"max\"==this.sizing.width_policy},i.prototype.is_height_expanding=function(){return\"max\"==this.sizing.height_policy},i.prototype.apply_aspect=function(i,t){var e=t.width,h=t.height,n=this.sizing.aspect;if(null!=n){var r=this.sizing,s=r.width_policy,o=r.height_policy;if(\"fixed\"!=s&&\"fixed\"!=o)if(s==o){var u=e,a=g(e/n),l=g(h*n),_=h;Math.abs(i.width-u)+Math.abs(i.height-a)<=Math.abs(i.width-l)+Math.abs(i.height-_)?(e=u,h=a):(e=l,h=_)}else!function(i,t){var e={max:4,fit:3,min:2,fixed:1};return e[i]>e[t]}(s,o)?e=g(h*n):h=g(e/n);else\"fixed\"==s?h=g(e/n):\"fixed\"==o&&(e=g(h*n))}return{width:e,height:h}},i.prototype.measure=function(i){var t=this;if(!this.sizing.visible)return{width:0,height:0};var e=function(i){return\"fixed\"==t.sizing.width_policy&&null!=t.sizing.width?t.sizing.width:i},h=function(i){return\"fixed\"==t.sizing.height_policy&&null!=t.sizing.height?t.sizing.height:i},r=new n.Sizeable(i).shrink_by(this.sizing.margin).map(e,h),s=this._measure(r),o=this.clip_size(s),g=e(o.width),u=h(o.height),a=this.apply_aspect(r,{width:g,height:u});return Object.assign(Object.assign({},s),a)},i.prototype.compute=function(i){void 0===i&&(i={});var t=this.measure({width:null!=i.width&&this.is_width_expanding()?i.width:1/0,height:null!=i.height&&this.is_height_expanding()?i.height:1/0}),e=t.width,h=t.height,n=new r.BBox({left:0,top:0,width:e,height:h}),s=void 0;if(null!=t.inner){var o=t.inner,g=o.left,u=o.top,a=o.right,l=o.bottom;s=new r.BBox({left:g,top:u,right:e-a,bottom:h-l})}this.set_geometry(n,s)},Object.defineProperty(i.prototype,\"xview\",{get:function(){return this.bbox.xview},enumerable:!0,configurable:!0}),Object.defineProperty(i.prototype,\"yview\",{get:function(){return this.bbox.yview},enumerable:!0,configurable:!0}),i.prototype.clip_width=function(i){return o(this.sizing.min_width,s(i,this.sizing.max_width))},i.prototype.clip_height=function(i){return o(this.sizing.min_height,s(i,this.sizing.max_height))},i.prototype.clip_size=function(i){var t=i.width,e=i.height;return{width:this.clip_width(t),height:this.clip_height(e)}},i}();e.Layoutable=u,u.__name__=\"Layoutable\";var a=function(i){function t(){return null!==i&&i.apply(this,arguments)||this}return h.__extends(t,i),t.prototype._measure=function(i){var t,e,h=this.sizing,n=h.width_policy,r=h.height_policy;if(i.width==1/0)t=null!=this.sizing.width?this.sizing.width:0;else if(\"fixed\"==n)t=null!=this.sizing.width?this.sizing.width:0;else if(\"min\"==n)t=null!=this.sizing.width?s(i.width,this.sizing.width):0;else if(\"fit\"==n)t=null!=this.sizing.width?s(i.width,this.sizing.width):i.width;else{if(\"max\"!=n)throw new Error(\"unrechable\");t=null!=this.sizing.width?o(i.width,this.sizing.width):i.width}if(i.height==1/0)e=null!=this.sizing.height?this.sizing.height:0;else if(\"fixed\"==r)e=null!=this.sizing.height?this.sizing.height:0;else if(\"min\"==r)e=null!=this.sizing.height?s(i.height,this.sizing.height):0;else if(\"fit\"==r)e=null!=this.sizing.height?s(i.height,this.sizing.height):i.height;else{if(\"max\"!=r)throw new Error(\"unrechable\");e=null!=this.sizing.height?o(i.height,this.sizing.height):i.height}return{width:t,height:e}},t}(u);e.LayoutItem=a,a.__name__=\"LayoutItem\";var l=function(i){function t(){return null!==i&&i.apply(this,arguments)||this}return h.__extends(t,i),t.prototype._measure=function(i){var t=this,e=this._content_size(),h=i.bounded_to(this.sizing.size).bounded_to(e);return{width:function(){switch(t.sizing.width_policy){case\"fixed\":return null!=t.sizing.width?t.sizing.width:e.width;case\"min\":return e.width;case\"fit\":return h.width;case\"max\":return Math.max(e.width,h.width);default:throw new Error(\"unexpected\")}}(),height:function(){switch(t.sizing.height_policy){case\"fixed\":return null!=t.sizing.height?t.sizing.height:e.height;case\"min\":return e.height;case\"fit\":return h.height;case\"max\":return Math.max(e.height,h.height);default:throw new Error(\"unexpected\")}}()}},t}(u);e.ContentLayoutable=l,l.__name__=\"ContentLayoutable\"},\n", + " function _(t,e,r){var h=t(113),o=t(284),i=t(181),n=function(t){function e(){var e=t.apply(this,arguments)||this;return e.children=[],e}return h.__extends(e,t),e}(o.Layoutable);r.Stack=n,n.__name__=\"Stack\";var a=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return h.__extends(e,t),e.prototype._measure=function(t){for(var e=0,r=0,h=0,o=this.children;h0)for(var A=l(j.height/O.length),M=0,P=O;M0)for(var S=l(j.width/C.length),E=0,G=C;E0)for(g=0;gy?y:m,_--}}}u=\"fixed\"==this.sizing.width_policy&&null!=this.sizing.width?this.sizing.width:t.width!=1/0&&this.is_width_expanding()?t.width:f.size.width;for(var v=0,x=0;x0)for(x=0;xj?j:m,_--}}}var O=this._measure_cells(function(t,i){return{width:f.col_widths[i],height:f.row_heights[t]}}),B=O.row_heights,A=O.col_widths,M=O.size_hints;return{size:this._measure_totals(B,A),row_heights:B,col_widths:A,size_hints:M}},i.prototype._measure=function(t){return this._measure_grid(t).size},i.prototype._set_geometry=function(i,e){t.prototype._set_geometry.call(this,i,e);for(var n=this._state,r=n.nrows,o=n.ncols,s=n.rspacing,h=n.cspacing,u=this._measure_grid(i),p=u.row_heights,g=u.col_widths,_=u.size_hints,d=this._state.rows.map(function(t,i){return Object.assign(Object.assign({},t),{top:0,height:p[i],get bottom(){return this.top+this.height}})}),w=this._state.cols.map(function(t,i){return Object.assign(Object.assign({},t),{left:0,width:g[i],get right(){return this.left+this.width}})}),y=_.map(function(t,i){return Object.assign(Object.assign({},i),{outer:new a.BBox,inner:new a.BBox})}),m=0,v=this.absolute?i.top:0;m0?a.every(e,s.isBoolean)?(e.length!==n.get_length()&&r.logger.warn(\"BooleanFilter \"+this.id+\": length of booleans doesn't match data source\"),a.range(0,e.length).filter(function(n){return!0===e[n]})):(r.logger.warn(\"BooleanFilter \"+this.id+\": booleans should be array of booleans, defaulting to no filtering\"),null):(null!=e&&0==e.length?r.logger.warn(\"BooleanFilter \"+this.id+\": booleans is empty, defaulting to no filtering\"):r.logger.warn(\"BooleanFilter \"+this.id+\": booleans was not set, defaulting to no filtering\"),null)},e}(l.Filter);o.BooleanFilter=g,g.__name__=\"BooleanFilter\",g.init_BooleanFilter()},\n", + " function _(t,n,e){var i=t(113),r=t(166),l=t(121),o=t(109),a=t(110),f=t(167),u=function(t){function n(n){return t.call(this,n)||this}return i.__extends(n,t),n.init_Filter=function(){this.define({filter:[l.Array,null]})},n.prototype.compute_indices=function(t){var n=this.filter;return null!=n&&n.length>=0?o.isArrayOf(n,o.isBoolean)?a.range(0,n.length).filter(function(t){return!0===n[t]}):o.isArrayOf(n,o.isInteger)?n:(f.logger.warn(\"Filter \"+this.id+\": filter should either be array of only booleans or only integers, defaulting to no filtering\"),null):(f.logger.warn(\"Filter \"+this.id+\": filter was not set to be an array, defaulting to no filtering\"),null)},n}(r.Model);e.Filter=u,u.__name__=\"Filter\",u.init_Filter()},\n", + " function _(e,t,r){var i=e(113),n=e(294),s=e(121),o=e(125),u=e(127),c=function(t){function r(e){return t.call(this,e)||this}return i.__extends(r,t),r.init_CustomJSFilter=function(){this.define({args:[s.Any,{}],code:[s.String,\"\"],use_strict:[s.Boolean,!1]})},Object.defineProperty(r.prototype,\"names\",{get:function(){return o.keys(this.args)},enumerable:!0,configurable:!0}),Object.defineProperty(r.prototype,\"values\",{get:function(){return o.values(this.args)},enumerable:!0,configurable:!0}),Object.defineProperty(r.prototype,\"func\",{get:function(){var e=this.use_strict?u.use_strict(this.code):this.code;return new(Function.bind.apply(Function,i.__spreadArrays([void 0],this.names,[\"source\",\"require\",\"exports\",e])))},enumerable:!0,configurable:!0}),r.prototype.compute_indices=function(r){return this.filter=this.func.apply(this,i.__spreadArrays(this.values,[r,e,{}])),t.prototype.compute_indices.call(this,r)},r}(n.Filter);r.CustomJSFilter=c,c.__name__=\"CustomJSFilter\",c.init_CustomJSFilter()},\n", + " function _(n,i,t){var r=n(113),e=n(294),u=n(121),o=n(167),l=n(110),c=function(n){function i(i){var t=n.call(this,i)||this;return t.indices=null,t}return r.__extends(i,n),i.init_GroupFilter=function(){this.define({column_name:[u.String],group:[u.String]})},i.prototype.compute_indices=function(n){var i=this,t=n.get_column(this.column_name);return null==t?(o.logger.warn(\"group filter: groupby column not found in data source\"),null):(this.indices=l.range(0,n.get_length()||0).filter(function(n){return t[n]===i.group}),0===this.indices.length&&o.logger.warn(\"group filter: group '\"+this.group+\"' did not match any values in column '\"+this.column_name+\"'\"),this.indices)},i}(e.Filter);t.GroupFilter=c,c.__name__=\"GroupFilter\",c.init_GroupFilter()},\n", + " function _(i,n,e){var t=i(113),r=i(294),l=i(121),s=i(167),d=i(109),o=i(110),u=function(i){function n(n){return i.call(this,n)||this}return t.__extends(n,i),n.init_IndexFilter=function(){this.define({indices:[l.Array,null]})},n.prototype.compute_indices=function(i){return null!=this.indices&&this.indices.length>=0?o.every(this.indices,d.isInteger)?this.indices:(s.logger.warn(\"IndexFilter \"+this.id+\": indices should be array of integers, defaulting to no filtering\"),null):(s.logger.warn(\"IndexFilter \"+this.id+\": indices was not set, defaulting to no filtering\"),null)},n}(r.Filter);e.IndexFilter=u,u.__name__=\"IndexFilter\",u.init_IndexFilter()},\n", + " function _(r,t,a){var e=r(208);a.BasicTickFormatter=e.BasicTickFormatter;var c=r(247);a.CategoricalTickFormatter=c.CategoricalTickFormatter;var i=r(251);a.DatetimeTickFormatter=i.DatetimeTickFormatter;var o=r(299);a.FuncTickFormatter=o.FuncTickFormatter;var m=r(264);a.LogTickFormatter=m.LogTickFormatter;var F=r(267);a.MercatorTickFormatter=F.MercatorTickFormatter;var k=r(300);a.NumeralTickFormatter=k.NumeralTickFormatter;var T=r(301);a.PrintfTickFormatter=T.PrintfTickFormatter;var v=r(209);a.TickFormatter=v.TickFormatter},\n", + " function _(t,e,r){var n=t(113),i=t(209),o=t(121),c=t(125),u=t(127),a=function(e){function r(t){return e.call(this,t)||this}return n.__extends(r,e),r.init_FuncTickFormatter=function(){this.define({args:[o.Any,{}],code:[o.String,\"\"],use_strict:[o.Boolean,!1]})},Object.defineProperty(r.prototype,\"names\",{get:function(){return c.keys(this.args)},enumerable:!0,configurable:!0}),Object.defineProperty(r.prototype,\"values\",{get:function(){return c.values(this.args)},enumerable:!0,configurable:!0}),r.prototype._make_func=function(){var t=this.use_strict?u.use_strict(this.code):this.code;return new(Function.bind.apply(Function,n.__spreadArrays([void 0,\"tick\",\"index\",\"ticks\"],this.names,[\"require\",\"exports\",t])))},r.prototype.doFormat=function(e,r){var i=this,o=this._make_func().bind({});return e.map(function(e,r,c){return o.apply(void 0,n.__spreadArrays([e,r,c],i.values,[t,{}]))})},r}(i.TickFormatter);r.FuncTickFormatter=a,a.__name__=\"FuncTickFormatter\",a.init_FuncTickFormatter()},\n", + " function _(n,r,t){var e=n(113),o=n(255),i=n(209),a=n(121),u=function(n){function r(r){return n.call(this,r)||this}return e.__extends(r,n),r.init_NumeralTickFormatter=function(){this.define({format:[a.String,\"0,0\"],language:[a.String,\"en\"],rounding:[a.RoundingFunction,\"round\"]})},Object.defineProperty(r.prototype,\"_rounding_fn\",{get:function(){switch(this.rounding){case\"round\":case\"nearest\":return Math.round;case\"floor\":case\"rounddown\":return Math.floor;case\"ceil\":case\"roundup\":return Math.ceil}},enumerable:!0,configurable:!0}),r.prototype.doFormat=function(n,r){var t=this.format,e=this.language,i=this._rounding_fn;return n.map(function(n){return o.format(n,t,e,i)})},r}(i.TickFormatter);t.NumeralTickFormatter=u,u.__name__=\"NumeralTickFormatter\",u.init_NumeralTickFormatter()},\n", + " function _(t,r,n){var i=t(113),o=t(209),e=t(253),f=t(121),a=function(t){function r(r){return t.call(this,r)||this}return i.__extends(r,t),r.init_PrintfTickFormatter=function(){this.define({format:[f.String,\"%s\"]})},r.prototype.doFormat=function(t,r){var n=this;return t.map(function(t){return e.sprintf(n.format,t)})},r}(o.TickFormatter);n.PrintfTickFormatter=a,a.__name__=\"PrintfTickFormatter\",a.init_PrintfTickFormatter()},\n", + " function _(a,e,r){var v=a(303);r.AnnularWedge=v.AnnularWedge;var l=a(304);r.Annulus=l.Annulus;var t=a(305);r.Arc=t.Arc;var i=a(306);r.Bezier=i.Bezier;var n=a(307);r.Circle=n.Circle;var u=a(308);r.CenterRotatable=u.CenterRotatable;var g=a(309);r.Ellipse=g.Ellipse;var c=a(310);r.EllipseOval=c.EllipseOval;var A=a(182);r.Glyph=A.Glyph;var p=a(188);r.HArea=p.HArea;var s=a(311);r.HBar=s.HBar;var R=a(313);r.HexTile=R.HexTile;var d=a(314);r.Image=d.Image;var h=a(316);r.ImageRGBA=h.ImageRGBA;var m=a(317);r.ImageURL=m.ImageURL;var y=a(177);r.Line=y.Line;var B=a(319);r.MultiLine=B.MultiLine;var o=a(320);r.MultiPolygons=o.MultiPolygons;var G=a(321);r.Oval=G.Oval;var H=a(187);r.Patch=H.Patch;var I=a(322);r.Patches=I.Patches;var L=a(323);r.Quad=L.Quad;var P=a(324);r.Quadratic=P.Quadratic;var x=a(325);r.Ray=x.Ray;var C=a(326);r.Rect=C.Rect;var E=a(327);r.Segment=E.Segment;var M=a(328);r.Step=M.Step;var O=a(329);r.Text=O.Text;var Q=a(190);r.VArea=Q.VArea;var S=a(330);r.VBar=S.VBar;var T=a(331);r.Wedge=T.Wedge;var V=a(178);r.XYGlyph=V.XYGlyph},\n", + " function _(t,e,i){var r=t(113),s=t(178),n=t(186),a=t(183),_=t(121),h=t(111),o=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(e,t),e.prototype._map_data=function(){\"data\"==this.model.properties.inner_radius.units?this.sinner_radius=this.sdist(this.renderer.xscale,this._x,this._inner_radius):this.sinner_radius=this._inner_radius,\"data\"==this.model.properties.outer_radius.units?this.souter_radius=this.sdist(this.renderer.xscale,this._x,this._outer_radius):this.souter_radius=this._outer_radius,this._angle=new Float32Array(this._start_angle.length);for(var t=0,e=this._start_angle.length;t=A&&v.push([m,z])}for(var S=this.model.properties.direction.value(),D=[],V=0,b=v;V=M&&v.push([m,g])}return a.create_hit_test_result_from_hits(v)},r.prototype.draw_legend_for_index=function(i,r,t){var s=r.x0,e=r.y0,a=r.x1,n=r.y1,u=t+1,_=new Array(u);_[t]=(s+a)/2;var h=new Array(u);h[t]=(e+n)/2;var o=.5*Math.min(Math.abs(a-s),Math.abs(n-e)),d=new Array(u);d[t]=.4*o;var l=new Array(u);l[t]=.8*o,this._render(i,[t],{sx:_,sy:h,sinner_radius:d,souter_radius:l})},r}(e.XYGlyphView);t.AnnulusView=_,_.__name__=\"AnnulusView\";var h=function(i){function r(r){return i.call(this,r)||this}return s.__extends(r,i),r.init_Annulus=function(){this.prototype.default_view=_,this.mixins([\"line\",\"fill\"]),this.define({inner_radius:[n.DistanceSpec],outer_radius:[n.DistanceSpec]})},r}(e.XYGlyph);t.Annulus=h,h.__name__=\"Annulus\",h.init_Annulus()},\n", + " function _(i,e,t){var n=i(113),s=i(178),r=i(186),a=i(121),_=function(i){function e(){return null!==i&&i.apply(this,arguments)||this}return n.__extends(e,i),e.prototype._map_data=function(){\"data\"==this.model.properties.radius.units?this.sradius=this.sdist(this.renderer.xscale,this._x,this._radius):this.sradius=this._radius},e.prototype._render=function(i,e,t){var n=t.sx,s=t.sy,r=t.sradius,a=t._start_angle,_=t._end_angle;if(this.visuals.line.doit)for(var o=this.model.properties.direction.value(),c=0,l=e;c1?(p[e]=d,x[e]=d/o):(p[e]=d*o,x[e]=d),this._render(t,[e],{sx:_,sy:l,sw:p,sh:x,_angle:[0]})},i.prototype._bounds=function(t){var i=t.x0,e=t.x1,s=t.y0,h=t.y1;return{x0:i-this.max_w2,x1:e+this.max_w2,y0:s-this.max_h2,y1:h+this.max_h2}},i}(h.CenterRotatableView);e.EllipseOvalView=a,a.__name__=\"EllipseOvalView\";var n=function(t){function i(i){return t.call(this,i)||this}return s.__extends(i,t),i}(h.CenterRotatable);e.EllipseOval=n,n.__name__=\"EllipseOval\"},\n", + " function _(t,i,e){var s=t(113),h=t(312),r=t(121),n=function(t){function i(){return null!==t&&t.apply(this,arguments)||this}return s.__extends(i,t),i.prototype.scenterx=function(t){return(this.sleft[t]+this.sright[t])/2},i.prototype.scentery=function(t){return this.sy[t]},i.prototype._index_data=function(){return this._index_box(this._y.length)},i.prototype._lrtb=function(t){return[Math.min(this._left[t],this._right[t]),Math.max(this._left[t],this._right[t]),this._y[t]+.5*this._height[t],this._y[t]-.5*this._height[t]]},i.prototype._map_data=function(){this.sy=this.renderer.yscale.v_compute(this._y),this.sh=this.sdist(this.renderer.yscale,this._y,this._height,\"center\"),this.sleft=this.renderer.xscale.v_compute(this._left),this.sright=this.renderer.xscale.v_compute(this._right);var t=this.sy.length;this.stop=new Float64Array(t),this.sbottom=new Float64Array(t);for(var i=0;i0){i=this._image[t];var n=this._image_shape[t];this._height[t]=n[0],this._width[t]=n[1]}else{var r=this._image[t];i=s.concat(r),this._height[t]=r.length,this._width[t]=r[0].length}var _=e.v_compute(i);this._set_image_data_from_buffer(t,_)}},t.prototype._render=function(e,t,a){var i=a.image_data,n=a.sx,r=a.sy,_=a.sw,s=a.sh,o=e.getImageSmoothingEnabled();e.setImageSmoothingEnabled(!1),e.globalAlpha=this.model.global_alpha;for(var h=0,l=t;h0){i=this._image[t].buffer;var n=this._image_shape[t];this._height[t]=n[0],this._width[t]=n[1]}else{var h=this._image[t],s=r.concat(h);i=new ArrayBuffer(4*s.length);for(var _=new Uint32Array(i),l=0,o=s.length;l0&&(_[l]=u)}return h.indices=o.keys(_).map(function(t){return parseInt(t,10)}),h.multiline_indices=_,h},e.prototype.get_interpolation_hit=function(t,e,i){var n=[this._xs[t][e],this._ys[t][e],this._xs[t][e+1],this._ys[t][e+1]],s=n[0],r=n[1],o=n[2],h=n[3];return a.line_interpolation(this.renderer,i,s,r,o,h)},e.prototype.draw_legend_for_index=function(t,e,i){a.generic_line_legend(this.visuals,t,e,i)},e.prototype.scenterx=function(){throw new Error(\"not implemented\")},e.prototype.scentery=function(){throw new Error(\"not implemented\")},e}(l.GlyphView);i.MultiLineView=u,u.__name__=\"MultiLineView\";var p=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.init_MultiLine=function(){this.prototype.default_view=u,this.coords([[\"xs\",\"ys\"]]),this.mixins([\"line\"])},e}(l.Glyph);i.MultiLine=p,p.__name__=\"MultiLine\",p.init_MultiLine()},\n", + " function _(t,i,e){var n=t(113),r=t(179),s=t(182),o=t(186),h=t(110),a=t(114),l=t(183),_=t(109),u=function(t){function i(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(i,t),i.prototype._index_data=function(){for(var t=[],i=0,e=this._xs.length;i1)for(var o=1,a=this._xs[i][n].length;o1){for(var c=!1,x=1;x0;){var r=_.find_last_index(s,function(t){return h.isStrictNaN(t)}),o=void 0;r>=0?o=s.splice(r):(o=s,s=[]);var a=o.filter(function(t){return!h.isStrictNaN(t)});e[i].push(a)}}return e},e.prototype._index_data=function(){for(var t=this._build_discontinuous_object(this._xs),e=this._build_discontinuous_object(this._ys),i=[],n=0,r=this._xs.length;n=0,m=i-this.sy1[n]<=this.sh[n]&&i-this.sy1[n]>=0;m&&w&&p.push(n)}var M=a.create_empty_hit_test_result();return M.indices=p,M},s.prototype._map_dist_corner_for_data_side_length=function(t,s,i){for(var e=t.length,h=new Float64Array(e),r=new Float64Array(e),a=0;a1&&(e.stroke(),d=!1)}d?(e.lineTo(b,m),e.lineTo(g,w)):(e.beginPath(),e.moveTo(_[v],u[v]),d=!0),f=v}e.lineTo(_[h-1],u[h-1]),e.stroke()}},t.prototype.draw_legend_for_index=function(e,t,i){r.generic_line_legend(this.visuals,e,t,i)},t}(o.XYGlyphView);i.StepView=a,a.__name__=\"StepView\";var l=function(e){function t(t){return e.call(this,t)||this}return n.__extends(t,e),t.init_Step=function(){this.prototype.default_view=a,this.mixins([\"line\"]),this.define({mode:[s.StepMode,\"before\"]})},t}(o.XYGlyph);i.Step=l,l.__name__=\"Step\",l.init_Step()},\n", + " function _(t,e,s){var i=t(113),n=t(178),r=t(183),_=t(121),o=t(226),h=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return i.__extends(e,t),e.prototype._rotate_point=function(t,e,s,i,n){return[(t-s)*Math.cos(n)-(e-i)*Math.sin(n)+s,(t-s)*Math.sin(n)+(e-i)*Math.cos(n)+i]},e.prototype._text_bounds=function(t,e,s,i){return[[t,t+s,t+s,t,t],[e,e,e-i,e-i,e]]},e.prototype._render=function(t,e,s){var i=s.sx,n=s.sy,r=s._x_offset,_=s._y_offset,h=s._angle,a=s._text;this._sys=[],this._sxs=[];for(var u=0,l=e;uo[1]&&(n=o[1]);else{i=o[0],n=o[1];for(var _=0,s=this.plot_view.axis_views;_0||v>0)return{width:y>0?y:void 0,height:v>0?v:void 0}}return{}})},i.prototype.serializable_state=function(){return Object.assign(Object.assign({},t.prototype.serializable_state.call(this)),{bbox:this.layout.bbox.box,children:this.child_views.map(function(t){return t.serializable_state()})})},i}(_.DOMView);e.LayoutDOMView=d,d.__name__=\"LayoutDOMView\";var c=function(t){function i(i){return t.call(this,i)||this}return o.__extends(i,t),i.init_LayoutDOM=function(){this.define({width:[h.Number,null],height:[h.Number,null],min_width:[h.Number,null],min_height:[h.Number,null],max_width:[h.Number,null],max_height:[h.Number,null],margin:[h.Any,[0,0,0,0]],width_policy:[h.Any,\"auto\"],height_policy:[h.Any,\"auto\"],aspect_ratio:[h.Any,null],sizing_mode:[h.SizingMode,null],visible:[h.Boolean,!0],disabled:[h.Boolean,!1],align:[h.Any,\"start\"],background:[h.Color,null],css_classes:[h.Array,[]]})},i}(n.Model);e.LayoutDOM=c,c.__name__=\"LayoutDOM\",c.init_LayoutDOM()},\n", + " function _(t,n,i){var o=t(113),u=t(338),e=t(286),s=t(121),l=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return o.__extends(n,t),n.prototype._update_layout=function(){var t=this.child_views.map(function(t){return t.layout});this.layout=new e.Column(t),this.layout.rows=this.model.rows,this.layout.spacing=[this.model.spacing,0],this.layout.set_sizing(this.box_sizing())},n}(u.BoxView);i.ColumnView=l,l.__name__=\"ColumnView\";var _=function(t){function n(n){return t.call(this,n)||this}return o.__extends(n,t),n.init_Column=function(){this.prototype.default_view=l,this.define({rows:[s.Any,\"auto\"]})},n}(u.Box);i.Column=_,_.__name__=\"Column\",_.init_Column()},\n", + " function _(t,i,n){var o=t(113),e=t(339),r=t(286),s=t(121),l=function(t){function i(){return null!==t&&t.apply(this,arguments)||this}return o.__extends(i,t),i.prototype.connect_signals=function(){var i=this;t.prototype.connect_signals.call(this),this.connect(this.model.properties.children.change,function(){return i.rebuild()})},Object.defineProperty(i.prototype,\"child_models\",{get:function(){return this.model.children.map(function(t){return t[0]})},enumerable:!0,configurable:!0}),i.prototype._update_layout=function(){this.layout=new r.Grid,this.layout.rows=this.model.rows,this.layout.cols=this.model.cols,this.layout.spacing=this.model.spacing;for(var t=0,i=this.model.children;tr?(this.wrapper_el.style.maxWidth=r-a.width+\"px\",l.display(this.scroll_el)):(this.wrapper_el.style.maxWidth=\"\",l.undisplay(this.scroll_el))}else{var n=this.header.bbox.height;s.height>n?(this.wrapper_el.style.maxHeight=n-a.height+\"px\",l.display(this.scroll_el)):(this.wrapper_el.style.maxHeight=\"\",l.undisplay(this.scroll_el))}for(var h=this.child_views,o=0,c=h;oi-1&&(t.model.active=i-1)}}),s.appendChild(n)}return s});this.headers_el=l.div({class:[d.bk_headers]},n),this.wrapper_el=l.div({class:d.bk_headers_wrapper},this.headers_el);var h=l.div({class:[_.bk_btn,_.bk_btn_default],disabled:\"\"},l.div({class:[u.bk_caret,c.bk_left]})),o=l.div({class:[_.bk_btn,_.bk_btn_default]},l.div({class:[u.bk_caret,c.bk_right]})),p=0,b=function(e){return function(){var i=t.model.tabs.length;0==(p=\"left\"==e?Math.max(p-1,0):Math.min(p+1,i-1))?h.setAttribute(\"disabled\",\"\"):h.removeAttribute(\"disabled\"),p==i-1?o.setAttribute(\"disabled\",\"\"):o.removeAttribute(\"disabled\");var a=l.children(t.headers_el).slice(0,p).map(function(e){return e.getBoundingClientRect()});if(s){var n=-r.sum(a.map(function(e){return e.width}));t.headers_el.style.left=n+\"px\"}else{var c=-r.sum(a.map(function(e){return e.height}));t.headers_el.style.top=c+\"px\"}}};h.addEventListener(\"click\",b(\"left\")),o.addEventListener(\"click\",b(\"right\")),this.scroll_el=l.div({class:_.bk_btn_group},h,o),this.header_el=l.div({class:[d.bk_tabs_header,c.bk_side(a)]},this.scroll_el,this.wrapper_el),this.el.appendChild(this.header_el)},t.prototype.change_active=function(e){e!=this.model.active&&(this.model.active=e,null!=this.model.callback&&this.model.callback.execute(this.model))},t.prototype.on_active_change=function(){for(var e=this.model.active,t=l.children(this.headers_el),i=0,a=t;i .bk-btn {\\n flex-grow: 0;\\n -webkit-flex-grow: 0;\\n height: auto;\\n padding: 4px 4px;\\n}\\n.bk-root .bk-tabs-header .bk-headers-wrapper {\\n flex-grow: 1;\\n -webkit-flex-grow: 1;\\n overflow: hidden;\\n color: #666666;\\n}\\n.bk-root .bk-tabs-header.bk-above .bk-headers-wrapper {\\n border-bottom: 1px solid #e6e6e6;\\n}\\n.bk-root .bk-tabs-header.bk-right .bk-headers-wrapper {\\n border-left: 1px solid #e6e6e6;\\n}\\n.bk-root .bk-tabs-header.bk-below .bk-headers-wrapper {\\n border-top: 1px solid #e6e6e6;\\n}\\n.bk-root .bk-tabs-header.bk-left .bk-headers-wrapper {\\n border-right: 1px solid #e6e6e6;\\n}\\n.bk-root .bk-tabs-header.bk-above,\\n.bk-root .bk-tabs-header.bk-below {\\n flex-direction: row;\\n -webkit-flex-direction: row;\\n}\\n.bk-root .bk-tabs-header.bk-above .bk-headers,\\n.bk-root .bk-tabs-header.bk-below .bk-headers {\\n flex-direction: row;\\n -webkit-flex-direction: row;\\n}\\n.bk-root .bk-tabs-header.bk-left,\\n.bk-root .bk-tabs-header.bk-right {\\n flex-direction: column;\\n -webkit-flex-direction: column;\\n}\\n.bk-root .bk-tabs-header.bk-left .bk-headers,\\n.bk-root .bk-tabs-header.bk-right .bk-headers {\\n flex-direction: column;\\n -webkit-flex-direction: column;\\n}\\n.bk-root .bk-tabs-header .bk-headers {\\n position: relative;\\n display: flex;\\n display: -webkit-flex;\\n flex-wrap: nowrap;\\n -webkit-flex-wrap: nowrap;\\n align-items: center;\\n -webkit-align-items: center;\\n}\\n.bk-root .bk-tabs-header .bk-tab {\\n padding: 4px 8px;\\n border: solid transparent;\\n white-space: nowrap;\\n cursor: pointer;\\n}\\n.bk-root .bk-tabs-header .bk-tab:hover {\\n background-color: #f2f2f2;\\n}\\n.bk-root .bk-tabs-header .bk-tab.bk-active {\\n color: #4d4d4d;\\n background-color: white;\\n border-color: #e6e6e6;\\n}\\n.bk-root .bk-tabs-header .bk-tab .bk-close {\\n margin-left: 10px;\\n}\\n.bk-root .bk-tabs-header.bk-above .bk-tab {\\n border-width: 3px 1px 0px 1px;\\n border-radius: 4px 4px 0 0;\\n}\\n.bk-root .bk-tabs-header.bk-right .bk-tab {\\n border-width: 1px 3px 1px 0px;\\n border-radius: 0 4px 4px 0;\\n}\\n.bk-root .bk-tabs-header.bk-below .bk-tab {\\n border-width: 0px 1px 3px 1px;\\n border-radius: 0 0 4px 4px;\\n}\\n.bk-root .bk-tabs-header.bk-left .bk-tab {\\n border-width: 1px 0px 1px 3px;\\n border-radius: 4px 0 0 4px;\\n}\\n.bk-root .bk-close {\\n display: inline-block;\\n width: 10px;\\n height: 10px;\\n vertical-align: middle;\\n background-image: url(\\'data:image/svg+xml;utf8,\\\\\\n \\\\\\n \\\\\\n \\\\\\n \\');\\n}\\n.bk-root .bk-close:hover {\\n background-image: url(\\'data:image/svg+xml;utf8,\\\\\\n \\\\\\n \\\\\\n \\\\\\n \\');\\n}\\n'),n.bk_tabs_header=\"bk-tabs-header\",n.bk_headers_wrapper=\"bk-headers-wrapper\",n.bk_headers=\"bk-headers\",n.bk_tab=\"bk-tab\",n.bk_close=\"bk-close\"},\n", + " function _(n,b,o){n(164),n(163).styles.append(\".bk-root .bk-btn {\\n height: 100%;\\n display: inline-block;\\n text-align: center;\\n vertical-align: middle;\\n white-space: nowrap;\\n cursor: pointer;\\n padding: 6px 12px;\\n font-size: 12px;\\n border: 1px solid transparent;\\n border-radius: 4px;\\n outline: 0;\\n user-select: none;\\n -ms-user-select: none;\\n -moz-user-select: none;\\n -webkit-user-select: none;\\n}\\n.bk-root .bk-btn:hover,\\n.bk-root .bk-btn:focus {\\n text-decoration: none;\\n}\\n.bk-root .bk-btn:active,\\n.bk-root .bk-btn.bk-active {\\n background-image: none;\\n box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);\\n}\\n.bk-root .bk-btn[disabled] {\\n cursor: not-allowed;\\n pointer-events: none;\\n opacity: 0.65;\\n box-shadow: none;\\n}\\n.bk-root .bk-btn-default {\\n color: #333;\\n background-color: #fff;\\n border-color: #ccc;\\n}\\n.bk-root .bk-btn-default:hover {\\n background-color: #f5f5f5;\\n border-color: #b8b8b8;\\n}\\n.bk-root .bk-btn-default.bk-active {\\n background-color: #ebebeb;\\n border-color: #adadad;\\n}\\n.bk-root .bk-btn-default[disabled],\\n.bk-root .bk-btn-default[disabled]:hover,\\n.bk-root .bk-btn-default[disabled]:focus,\\n.bk-root .bk-btn-default[disabled]:active,\\n.bk-root .bk-btn-default[disabled].bk-active {\\n background-color: #e6e6e6;\\n border-color: #ccc;\\n}\\n.bk-root .bk-btn-primary {\\n color: #fff;\\n background-color: #428bca;\\n border-color: #357ebd;\\n}\\n.bk-root .bk-btn-primary:hover {\\n background-color: #3681c1;\\n border-color: #2c699e;\\n}\\n.bk-root .bk-btn-primary.bk-active {\\n background-color: #3276b1;\\n border-color: #285e8e;\\n}\\n.bk-root .bk-btn-primary[disabled],\\n.bk-root .bk-btn-primary[disabled]:hover,\\n.bk-root .bk-btn-primary[disabled]:focus,\\n.bk-root .bk-btn-primary[disabled]:active,\\n.bk-root .bk-btn-primary[disabled].bk-active {\\n background-color: #506f89;\\n border-color: #357ebd;\\n}\\n.bk-root .bk-btn-success {\\n color: #fff;\\n background-color: #5cb85c;\\n border-color: #4cae4c;\\n}\\n.bk-root .bk-btn-success:hover {\\n background-color: #4eb24e;\\n border-color: #409240;\\n}\\n.bk-root .bk-btn-success.bk-active {\\n background-color: #47a447;\\n border-color: #398439;\\n}\\n.bk-root .bk-btn-success[disabled],\\n.bk-root .bk-btn-success[disabled]:hover,\\n.bk-root .bk-btn-success[disabled]:focus,\\n.bk-root .bk-btn-success[disabled]:active,\\n.bk-root .bk-btn-success[disabled].bk-active {\\n background-color: #667b66;\\n border-color: #4cae4c;\\n}\\n.bk-root .bk-btn-warning {\\n color: #fff;\\n background-color: #f0ad4e;\\n border-color: #eea236;\\n}\\n.bk-root .bk-btn-warning:hover {\\n background-color: #eea43b;\\n border-color: #e89014;\\n}\\n.bk-root .bk-btn-warning.bk-active {\\n background-color: #ed9c28;\\n border-color: #d58512;\\n}\\n.bk-root .bk-btn-warning[disabled],\\n.bk-root .bk-btn-warning[disabled]:hover,\\n.bk-root .bk-btn-warning[disabled]:focus,\\n.bk-root .bk-btn-warning[disabled]:active,\\n.bk-root .bk-btn-warning[disabled].bk-active {\\n background-color: #c89143;\\n border-color: #eea236;\\n}\\n.bk-root .bk-btn-danger {\\n color: #fff;\\n background-color: #d9534f;\\n border-color: #d43f3a;\\n}\\n.bk-root .bk-btn-danger:hover {\\n background-color: #d5433e;\\n border-color: #bd2d29;\\n}\\n.bk-root .bk-btn-danger.bk-active {\\n background-color: #d2322d;\\n border-color: #ac2925;\\n}\\n.bk-root .bk-btn-danger[disabled],\\n.bk-root .bk-btn-danger[disabled]:hover,\\n.bk-root .bk-btn-danger[disabled]:focus,\\n.bk-root .bk-btn-danger[disabled]:active,\\n.bk-root .bk-btn-danger[disabled].bk-active {\\n background-color: #a55350;\\n border-color: #d43f3a;\\n}\\n.bk-root .bk-btn-group {\\n height: 100%;\\n display: flex;\\n display: -webkit-flex;\\n flex-wrap: nowrap;\\n -webkit-flex-wrap: nowrap;\\n align-items: center;\\n -webkit-align-items: center;\\n flex-direction: row;\\n -webkit-flex-direction: row;\\n}\\n.bk-root .bk-btn-group > .bk-btn {\\n flex-grow: 1;\\n -webkit-flex-grow: 1;\\n}\\n.bk-root .bk-btn-group > .bk-btn + .bk-btn {\\n margin-left: -1px;\\n}\\n.bk-root .bk-btn-group > .bk-btn:first-child:not(:last-child) {\\n border-bottom-right-radius: 0;\\n border-top-right-radius: 0;\\n}\\n.bk-root .bk-btn-group > .bk-btn:not(:first-child):last-child {\\n border-bottom-left-radius: 0;\\n border-top-left-radius: 0;\\n}\\n.bk-root .bk-btn-group > .bk-btn:not(:first-child):not(:last-child) {\\n border-radius: 0;\\n}\\n.bk-root .bk-btn-group .bk-dropdown-toggle {\\n flex: 0 0 0;\\n -webkit-flex: 0 0 0;\\n padding: 6px 6px;\\n}\\n\"),o.bk_btn=\"bk-btn\",o.bk_btn_group=\"bk-btn-group\",o.bk_btn_default=\"bk-btn-default\",o.bk_btn_primary=\"bk-btn-primary\",o.bk_btn_success=\"bk-btn-success\",o.bk_btn_warning=\"bk-btn-warning\",o.bk_btn_danger=\"bk-btn-danger\",o.bk_btn_type=function(n){switch(n){case\"default\":return o.bk_btn_default;case\"primary\":return o.bk_btn_primary;case\"success\":return o.bk_btn_success;case\"warning\":return o.bk_btn_warning;case\"danger\":return o.bk_btn_danger}},o.bk_dropdown_toggle=\"bk-dropdown-toggle\"},\n", + " function _(n,o,r){n(164),n(163).styles.append(\".bk-root .bk-menu {\\n position: absolute;\\n left: 0;\\n width: 100%;\\n z-index: 100;\\n cursor: pointer;\\n font-size: 12px;\\n background-color: #fff;\\n border: 1px solid #ccc;\\n border-radius: 4px;\\n box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175);\\n}\\n.bk-root .bk-menu.bk-above {\\n bottom: 100%;\\n}\\n.bk-root .bk-menu.bk-below {\\n top: 100%;\\n}\\n.bk-root .bk-menu > .bk-divider {\\n height: 1px;\\n margin: 7.5px 0;\\n overflow: hidden;\\n background-color: #e5e5e5;\\n}\\n.bk-root .bk-menu > :not(.bk-divider) {\\n padding: 6px 12px;\\n}\\n.bk-root .bk-menu > :not(.bk-divider):hover,\\n.bk-root .bk-menu > :not(.bk-divider).bk-active {\\n background-color: #e6e6e6;\\n}\\n.bk-root .bk-caret {\\n display: inline-block;\\n vertical-align: middle;\\n width: 0;\\n height: 0;\\n margin: 0 5px;\\n}\\n.bk-root .bk-caret.bk-down {\\n border-top: 4px solid;\\n}\\n.bk-root .bk-caret.bk-up {\\n border-bottom: 4px solid;\\n}\\n.bk-root .bk-caret.bk-down,\\n.bk-root .bk-caret.bk-up {\\n border-right: 4px solid transparent;\\n border-left: 4px solid transparent;\\n}\\n.bk-root .bk-caret.bk-left {\\n border-right: 4px solid;\\n}\\n.bk-root .bk-caret.bk-right {\\n border-left: 4px solid;\\n}\\n.bk-root .bk-caret.bk-left,\\n.bk-root .bk-caret.bk-right {\\n border-top: 4px solid transparent;\\n border-bottom: 4px solid transparent;\\n}\\n\"),r.bk_menu=\"bk-menu\",r.bk_caret=\"bk-caret\",r.bk_divider=\"bk-divider\"},\n", + " function _(t,i,n){var e=t(113),o=t(340),_=function(t){function i(){return null!==t&&t.apply(this,arguments)||this}return e.__extends(i,t),i}(o.ColumnView);n.WidgetBoxView=_,_.__name__=\"WidgetBoxView\";var u=function(t){function i(i){return t.call(this,i)||this}return e.__extends(i,t),i.init_WidgetBox=function(){this.prototype.default_view=_},i}(o.Column);n.WidgetBox=u,u.__name__=\"WidgetBox\",u.init_WidgetBox()},\n", + " function _(r,a,o){var p=r(351);o.CategoricalColorMapper=p.CategoricalColorMapper;var e=r(353);o.CategoricalMarkerMapper=e.CategoricalMarkerMapper;var C=r(354);o.CategoricalPatternMapper=C.CategoricalPatternMapper;var l=r(211);o.ContinuousColorMapper=l.ContinuousColorMapper;var M=r(212);o.ColorMapper=M.ColorMapper;var t=r(210);o.LinearColorMapper=t.LinearColorMapper;var i=r(355);o.LogColorMapper=i.LogColorMapper},\n", + " function _(t,r,o){var a=t(113),e=t(352),n=t(212),i=t(121),c=function(t){function r(r){return t.call(this,r)||this}return a.__extends(r,t),r.init_CategoricalColorMapper=function(){this.define({factors:[i.Array],start:[i.Number,0],end:[i.Number]})},r.prototype._v_compute=function(t,r,o,a){var n=a.nan_color;e.cat_v_compute(t,this.factors,o,r,this.start,this.end,n)},r}(n.ColorMapper);o.CategoricalColorMapper=c,c.__name__=\"CategoricalColorMapper\",c.init_CategoricalColorMapper()},\n", + " function _(n,t,e){var i=n(114),l=n(109);function r(n,t){if(n.length!=t.length)return!1;for(var e=0,i=n.length;e=e.length?c:e[g],u[a]=d},v=0,_=n.length;v<_;v++)a(v)}},\n", + " function _(r,e,t){var a=r(113),i=r(352),n=r(213),c=r(121),u=function(r){function e(e){return r.call(this,e)||this}return a.__extends(e,r),e.init_CategoricalMarkerMapper=function(){this.define({factors:[c.Array],markers:[c.Array],start:[c.Number,0],end:[c.Number],default_value:[c.MarkerType,\"circle\"]})},e.prototype.v_compute=function(r){var e=new Array(r.length);return i.cat_v_compute(r,this.factors,this.markers,e,this.start,this.end,this.default_value),e},e}(n.Mapper);t.CategoricalMarkerMapper=u,u.__name__=\"CategoricalMarkerMapper\",u.init_CategoricalMarkerMapper()},\n", + " function _(t,e,a){var r=t(113),n=t(352),i=t(213),p=t(121),c=function(t){function e(e){return t.call(this,e)||this}return r.__extends(e,t),e.init_CategoricalPatternMapper=function(){this.define({factors:[p.Array],patterns:[p.Array],start:[p.Number,0],end:[p.Number],default_value:[p.HatchPatternType,\" \"]})},e.prototype.v_compute=function(t){var e=new Array(t.length);return n.cat_v_compute(t,this.factors,this.patterns,e,this.start,this.end,this.default_value),e},e}(i.Mapper);a.CategoricalPatternMapper=c,c.__name__=\"CategoricalPatternMapper\",c.init_CategoricalPatternMapper()},\n", + " function _(o,l,n){var t=o(113),e=o(211),r=o(114),i=null!=Math.log1p?Math.log1p:function(o){return Math.log(1+o)},h=function(o){function l(l){return o.call(this,l)||this}return t.__extends(l,o),l.prototype._v_compute=function(o,l,n,t){for(var e=t.nan_color,h=t.low_color,a=t.high_color,u=n.length,s=null!=this.low?this.low:r.min(o),_=null!=this.high?this.high:r.max(o),f=u/(i(_)-i(s)),g=n.length-1,p=0,c=o.length;p_)l[p]=null!=a?a:n[g];else if(M!=_)if(Mg&&(m=g),l[p]=n[m]}else l[p]=n[g]}},l}(e.ContinuousColorMapper);n.LogColorMapper=h,h.__name__=\"LogColorMapper\"},\n", + " function _(r,a,t){!function(r){for(var a in r)t.hasOwnProperty(a)||(t[a]=r[a])}(r(357));var n=r(358);t.Marker=n.Marker;var e=r(359);t.Scatter=e.Scatter},\n", + " function _(e,t,o){var i=e(113),r=e(358),n=Math.sqrt(3);function s(e,t){e.moveTo(-t,t),e.lineTo(t,-t),e.moveTo(-t,-t),e.lineTo(t,t)}function c(e,t){e.moveTo(0,t),e.lineTo(0,-t),e.moveTo(-t,0),e.lineTo(t,0)}function l(e,t){e.moveTo(0,t),e.lineTo(t/1.5,0),e.lineTo(0,-t),e.lineTo(-t/1.5,0),e.closePath()}function a(e,t){var o=t*n,i=o/3;e.moveTo(-t,i),e.lineTo(t,i),e.lineTo(0,i-o),e.closePath()}function u(e,t,o,i,r){var n=.65*o;c(e,o),s(e,n),i.doit&&(i.set_vectorize(e,t),e.stroke())}function v(e,t,o,i,r){e.arc(0,0,o,0,2*Math.PI,!1),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),c(e,o),e.stroke())}function _(e,t,o,i,r){e.arc(0,0,o,0,2*Math.PI,!1),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),s(e,o),e.stroke())}function d(e,t,o,i,r){c(e,o),i.doit&&(i.set_vectorize(e,t),e.stroke())}function f(e,t,o,i,r){l(e,o),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),e.stroke())}function T(e,t,o,i,r){l(e,o),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),c(e,o),e.stroke())}function z(e,t,o,i,r){!function(e,t){var o=t/2,i=n*o;e.moveTo(t,0),e.lineTo(o,-i),e.lineTo(-o,-i),e.lineTo(-t,0),e.lineTo(-o,i),e.lineTo(o,i),e.closePath()}(e,o),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),e.stroke())}function k(e,t,o,i,r){e.rotate(Math.PI),a(e,o),e.rotate(-Math.PI),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),e.stroke())}function h(e,t,o,i,r){var n=2*o;e.rect(-o,-o,n,n),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),e.stroke())}function m(e,t,o,i,r){var n=2*o;e.rect(-o,-o,n,n),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),c(e,o),e.stroke())}function C(e,t,o,i,r){var n=2*o;e.rect(-o,-o,n,n),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),s(e,o),e.stroke())}function q(e,t,o,i,r){a(e,o),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),e.stroke())}function p(e,t,o,i,r){!function(e,t){e.moveTo(-t,0),e.lineTo(t,0)}(e,o),i.doit&&(i.set_vectorize(e,t),e.stroke())}function x(e,t,o,i,r){s(e,o),i.doit&&(i.set_vectorize(e,t),e.stroke())}function M(e,t){var o,n=function(e){function o(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(o,e),o.initClass=function(){this.prototype._render_one=t},o}(r.MarkerView);n.initClass();var s=((o=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(t,e),t.initClass=function(){this.prototype.default_view=n},t}(r.Marker)).__name__=e,o);return s.initClass(),s}o.Asterisk=M(\"Asterisk\",u),o.CircleCross=M(\"CircleCross\",v),o.CircleX=M(\"CircleX\",_),o.Cross=M(\"Cross\",d),o.Dash=M(\"Dash\",p),o.Diamond=M(\"Diamond\",f),o.DiamondCross=M(\"DiamondCross\",T),o.Hex=M(\"Hex\",z),o.InvertedTriangle=M(\"InvertedTriangle\",k),o.Square=M(\"Square\",h),o.SquareCross=M(\"SquareCross\",m),o.SquareX=M(\"SquareX\",C),o.Triangle=M(\"Triangle\",q),o.X=M(\"X\",x),o.marker_funcs={asterisk:u,circle:function(e,t,o,i,r){e.arc(0,0,o,0,2*Math.PI,!1),r.doit&&(r.set_vectorize(e,t),e.fill()),i.doit&&(i.set_vectorize(e,t),e.stroke())},circle_cross:v,circle_x:_,cross:d,diamond:f,diamond_cross:T,hex:z,inverted_triangle:k,square:h,square_cross:m,square_x:C,triangle:q,dash:p,x:x}},\n", + " function _(e,t,r){var i=e(113),s=e(178),n=e(183),a=e(121),_=e(110),h=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(t,e),t.prototype._render=function(e,t,r){for(var i=r.sx,s=r.sy,n=r._size,a=r._angle,_=0,h=t;_#grayscale\\\");\\n /* Firefox 10+, Firefox on Android */\\n filter: gray;\\n /* IE6-9 */\\n -webkit-filter: grayscale(100%);\\n /* Chrome 19+, Safari 6+, Safari 6+ iOS */\\n}\\n.bk-root .bk-logo-small {\\n width: 20px;\\n height: 20px;\\n background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAABx0RVh0U29mdHdhcmUAQWRvYmUgRmlyZXdvcmtzIENTNui8sowAAAOkSURBVDiNjZRtaJVlGMd/1/08zzln5zjP1LWcU9N0NkN8m2CYjpgQYQXqSs0I84OLIC0hkEKoPtiH3gmKoiJDU7QpLgoLjLIQCpEsNJ1vqUOdO7ppbuec5+V+rj4ctwzd8IIbbi6u+8f1539dt3A78eXC7QizUF7gyV1fD1Yqg4JWz84yffhm0qkFqBogB9rM8tZdtwVsPUhWhGcFJngGeWrPzHm5oaMmkfEg1usvLFyc8jLRqDOMru7AyC8saQr7GG7f5fvDeH7Ej8CM66nIF+8yngt6HWaKh7k49Soy9nXurCi1o3qUbS3zWfrYeQDTB/Qj6kX6Ybhw4B+bOYoLKCC9H3Nu/leUTZ1JdRWkkn2ldcCamzrcf47KKXdAJllSlxAOkRgyHsGC/zRday5Qld9DyoM4/q/rUoy/CXh3jzOu3bHUVZeU+DEn8FInkPBFlu3+nW3Nw0mk6vCDiWg8CeJaxEwuHS3+z5RgY+YBR6V1Z1nxSOfoaPa4LASWxxdNp+VWTk7+4vzaou8v8PN+xo+KY2xsw6une2frhw05CTYOmQvsEhjhWjn0bmXPjpE1+kplmmkP3suftwTubK9Vq22qKmrBhpY4jvd5afdRA3wGjFAgcnTK2s4hY0/GPNIb0nErGMCRxWOOX64Z8RAC4oCXdklmEvcL8o0BfkNK4lUg9HTl+oPlQxdNo3Mg4Nv175e/1LDGzZen30MEjRUtmXSfiTVu1kK8W4txyV6BMKlbgk3lMwYCiusNy9fVfvvwMxv8Ynl6vxoByANLTWplvuj/nF9m2+PDtt1eiHPBr1oIfhCChQMBw6Aw0UulqTKZdfVvfG7VcfIqLG9bcldL/+pdWTLxLUy8Qq38heUIjh4XlzZxzQm19lLFlr8vdQ97rjZVOLf8nclzckbcD4wxXMidpX30sFd37Fv/GtwwhzhxGVAprjbg0gCAEeIgwCZyTV2Z1REEW8O4py0wsjeloKoMr6iCY6dP92H6Vw/oTyICIthibxjm/DfN9lVz8IqtqKYLUXfoKVMVQVVJOElGjrnnUt9T9wbgp8AyYKaGlqingHZU/uG2NTZSVqwHQTWkx9hxjkpWDaCg6Ckj5qebgBVbT3V3NNXMSiWSDdGV3hrtzla7J+duwPOToIg42ChPQOQjspnSlp1V+Gjdged7+8UN5CRAV7a5EdFNwCjEaBR27b3W890TE7g24NAP/mMDXRWrGoFPQI9ls/MWO2dWFAar/xcOIImbbpA3zgAAAABJRU5ErkJggg==);\\n}\\n.bk-root .bk-logo-notebook {\\n display: inline-block;\\n vertical-align: middle;\\n margin-right: 5px;\\n}\\n\"),g.bk_logo=\"bk-logo\",g.bk_logo_notebook=\"bk-logo-notebook\",g.bk_logo_small=\"bk-logo-small\",g.bk_grey=\"bk-grey\"},\n", + " function _(t,e,i){var n=t(113),s=this&&this.__rest||function(t,e){var i={};for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&e.indexOf(n)<0&&(i[n]=t[n]);if(null!=t&&\"function\"==typeof Object.getOwnPropertySymbols){var s=0;for(n=Object.getOwnPropertySymbols(t);s=0},i.prototype.can_redo=function(){return this.state.index=h.end&&(s=!0,h.end=d,(e||i)&&(h.start=d+c)),null!=p&&p<=h.start&&(s=!0,h.start=p,(e||i)&&(h.end=p-c))):(null!=d&&d>=h.start&&(s=!0,h.start=d,(e||i)&&(h.end=d+c)),null!=p&&p<=h.end&&(s=!0,h.end=p,(e||i)&&(h.start=p-c)))}}if(!(i&&s&&n))for(var v=0,g=t;v0&&_0&&_>n&&(l=(n-h)/(_-h)),l=Math.max(0,Math.min(1,l))}return l},i.prototype.update_range=function(t,e,i,n){void 0===e&&(e=!1),void 0===i&&(i=!1),void 0===n&&(n=!0),this.pause();var s=this.frame,r=s.x_ranges,a=s.y_ranges;if(null==t){for(var o in r){(h=r[o]).reset()}for(var o in a){(h=a[o]).reset()}this.update_dataranges()}else{var l=[];for(var o in r){var h=r[o];l.push([h,t.xrs[o]])}for(var o in a){h=a[o];l.push([h,t.yrs[o]])}i&&this._update_ranges_together(l),this._update_ranges_individually(l,e,i,n)}this.unpause()},i.prototype.reset_range=function(){this.update_range(null)},i.prototype._invalidate_layout=function(){var t=this;(function(){for(var e=0,i=t.model.side_panels;e=0&&it.model.lod_timeout&&e.interactive_stop(t.model),t.request_paint()},this.model.lod_timeout):e.interactive_stop(this.model)}for(var n in this.renderer_views){var s=this.renderer_views[n];if(null==this.range_update_timestamp||s instanceof l.GlyphRendererView&&s.set_data_timestamp>this.range_update_timestamp){this.update_dataranges();break}}var r=this.canvas_view.ctx,a=this.canvas.pixel_ratio;r.save(),r.scale(a,a),r.translate(.5,.5);var o=[this.frame._left.value,this.frame._top.value,this.frame._width.value,this.frame._height.value];if(this._map_hook(r,o),this._paint_empty(r,o),this.prepare_webgl(a,o),this.clear_webgl(),this.visuals.outline_line.doit){r.save(),this.visuals.outline_line.set_value(r);var h=o[0],_=o[1],u=o[2],d=o[3];h+u==this.layout._width.value&&(u-=1),_+d==this.layout._height.value&&(d-=1),r.strokeRect(h,_,u,d),r.restore()}this._paint_levels(r,[\"image\",\"underlay\",\"glyph\"],o,!0),this._paint_levels(r,[\"annotation\"],o,!1),this._paint_levels(r,[\"overlay\"],o,!1),null==this._initial_state_info.range&&this.set_initial_range(),r.restore()}},i.prototype._paint_levels=function(t,e,i,n){for(var s=0,r=e;s=0;i--)(_=t[i])&&(s=(o<3?_(s):o>3?_(n,e,s):_(n,e))||s);return o>3&&s&&Object.defineProperty(n,e,s),s};function o(t){return function(n){n.prototype.event_name=t}}var s=function(){function t(){}return t.prototype.to_json=function(){return{event_name:this.event_name,event_values:this._to_json()}},t.prototype._to_json=function(){var t=this.origin;return{model_id:null!=t?t.id:null}},t}();e.BokehEvent=s,s.__name__=\"BokehEvent\";var i=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(s);i.__name__=\"ButtonClick\",i=_([o(\"button_click\")],i),e.ButtonClick=i;var a=function(t){function n(n){var e=t.call(this)||this;return e.item=n,e}return r.__extends(n,t),n.prototype._to_json=function(){var n=this.item;return Object.assign(Object.assign({},t.prototype._to_json.call(this)),{item:n})},n}(s);a.__name__=\"MenuItemClick\",a=_([o(\"menu_item_click\")],a),e.MenuItemClick=a;var u=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(s);e.UIEvent=u,u.__name__=\"UIEvent\";var l=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(u);l.__name__=\"LODStart\",l=_([o(\"lodstart\")],l),e.LODStart=l;var c=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(u);c.__name__=\"LODEnd\",c=_([o(\"lodend\")],c),e.LODEnd=c;var p=function(t){function n(n,e){var r=t.call(this)||this;return r.geometry=n,r.final=e,r}return r.__extends(n,t),n.prototype._to_json=function(){var n=this.geometry,e=this.final;return Object.assign(Object.assign({},t.prototype._to_json.call(this)),{geometry:n,final:e})},n}(u);p.__name__=\"SelectionGeometry\",p=_([o(\"selectiongeometry\")],p),e.SelectionGeometry=p;var h=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(u);h.__name__=\"Reset\",h=_([o(\"reset\")],h),e.Reset=h;var f=function(t){function n(n,e,r,_){var o=t.call(this)||this;return o.sx=n,o.sy=e,o.x=r,o.y=_,o}return r.__extends(n,t),n.prototype._to_json=function(){var n=this.sx,e=this.sy,r=this.x,_=this.y;return Object.assign(Object.assign({},t.prototype._to_json.call(this)),{sx:n,sy:e,x:r,y:_})},n}(u);e.PointEvent=f,f.__name__=\"PointEvent\";var y=function(t){function n(n,e,r,_,o,s){var i=t.call(this,n,e,r,_)||this;return i.sx=n,i.sy=e,i.x=r,i.y=_,i.delta_x=o,i.delta_y=s,i}return r.__extends(n,t),n.prototype._to_json=function(){var n=this.delta_x,e=this.delta_y;return Object.assign(Object.assign({},t.prototype._to_json.call(this)),{delta_x:n,delta_y:e})},n}(f);y.__name__=\"Pan\",y=_([o(\"pan\")],y),e.Pan=y;var v=function(t){function n(n,e,r,_,o){var s=t.call(this,n,e,r,_)||this;return s.sx=n,s.sy=e,s.x=r,s.y=_,s.scale=o,s}return r.__extends(n,t),n.prototype._to_json=function(){var n=this.scale;return Object.assign(Object.assign({},t.prototype._to_json.call(this)),{scale:n})},n}(f);v.__name__=\"Pinch\",v=_([o(\"pinch\")],v),e.Pinch=v;var d=function(t){function n(n,e,r,_,o){var s=t.call(this,n,e,r,_)||this;return s.sx=n,s.sy=e,s.x=r,s.y=_,s.rotation=o,s}return r.__extends(n,t),n.prototype._to_json=function(){var n=this.rotation;return Object.assign(Object.assign({},t.prototype._to_json.call(this)),{rotation:n})},n}(f);d.__name__=\"Rotate\",d=_([o(\"rotate\")],d),e.Rotate=d;var m=function(t){function n(n,e,r,_,o){var s=t.call(this,n,e,r,_)||this;return s.sx=n,s.sy=e,s.x=r,s.y=_,s.delta=o,s}return r.__extends(n,t),n.prototype._to_json=function(){var n=this.delta;return Object.assign(Object.assign({},t.prototype._to_json.call(this)),{delta:n})},n}(f);m.__name__=\"MouseWheel\",m=_([o(\"wheel\")],m),e.MouseWheel=m;var x=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);x.__name__=\"MouseMove\",x=_([o(\"mousemove\")],x),e.MouseMove=x;var j=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);j.__name__=\"MouseEnter\",j=_([o(\"mouseenter\")],j),e.MouseEnter=j;var g=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);g.__name__=\"MouseLeave\",g=_([o(\"mouseleave\")],g),e.MouseLeave=g;var b=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);b.__name__=\"Tap\",b=_([o(\"tap\")],b),e.Tap=b;var O=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);O.__name__=\"DoubleTap\",O=_([o(\"doubletap\")],O),e.DoubleTap=O;var P=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);P.__name__=\"Press\",P=_([o(\"press\")],P),e.Press=P;var E=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);E.__name__=\"PressUp\",E=_([o(\"pressup\")],E),e.PressUp=E;var M=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);M.__name__=\"PanStart\",M=_([o(\"panstart\")],M),e.PanStart=M;var R=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);R.__name__=\"PanEnd\",R=_([o(\"panend\")],R),e.PanEnd=R;var S=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);S.__name__=\"PinchStart\",S=_([o(\"pinchstart\")],S),e.PinchStart=S;var k=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);k.__name__=\"PinchEnd\",k=_([o(\"pinchend\")],k),e.PinchEnd=k;var D=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);D.__name__=\"RotateStart\",D=_([o(\"rotatestart\")],D),e.RotateStart=D;var L=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(n,t),n}(f);L.__name__=\"RotateEnd\",L=_([o(\"rotateend\")],L),e.RotateEnd=L},\n", + " function _(n,e,i){var o=(\"undefined\"!=typeof window?window.requestAnimationFrame:void 0)||(\"undefined\"!=typeof window?window.webkitRequestAnimationFrame:void 0)||(\"undefined\"!=typeof window?window.mozRequestAnimationFrame:void 0)||(\"undefined\"!=typeof window?window.msRequestAnimationFrame:void 0)||function(n){return n(Date.now()),-1};i.throttle=function(n,e){var i=null,t=0,u=!1,d=function(){t=Date.now(),i=null,u=!1,n()};return function(){var n=Date.now(),w=e-(n-t);w<=0&&!u?(null!=i&&clearTimeout(i),u=!0,o(d)):i||u||(i=setTimeout(function(){return o(d)},w))}}},\n", + " function _(e,t,i){var l=e(113),r=e(283),a=e(284),o=e(109),n=Math.PI/2,h=\"left\",s=\"center\",d={above:{parallel:0,normal:-n,horizontal:0,vertical:-n},below:{parallel:0,normal:n,horizontal:0,vertical:n},left:{parallel:-n,normal:0,horizontal:0,vertical:-n},right:{parallel:n,normal:0,horizontal:0,vertical:n}},c={above:{justified:\"top\",parallel:\"alphabetic\",normal:\"middle\",horizontal:\"alphabetic\",vertical:\"middle\"},below:{justified:\"bottom\",parallel:\"hanging\",normal:\"middle\",horizontal:\"hanging\",vertical:\"middle\"},left:{justified:\"top\",parallel:\"alphabetic\",normal:\"middle\",horizontal:\"middle\",vertical:\"alphabetic\"},right:{justified:\"top\",parallel:\"alphabetic\",normal:\"middle\",horizontal:\"middle\",vertical:\"alphabetic\"}},p={above:{justified:s,parallel:s,normal:h,horizontal:s,vertical:h},below:{justified:s,parallel:s,normal:h,horizontal:s,vertical:h},left:{justified:s,parallel:s,normal:\"right\",horizontal:\"right\",vertical:s},right:{justified:s,parallel:s,normal:h,horizontal:h,vertical:s}},b={above:\"right\",below:h,left:\"right\",right:h},_={above:h,below:\"right\",left:\"right\",right:h},m=function(e){function t(t,i){var l=e.call(this)||this;switch(l.side=t,l.obj=i,l.side){case\"above\":l._dim=0,l._normals=[0,-1];break;case\"below\":l._dim=0,l._normals=[0,1];break;case\"left\":l._dim=1,l._normals=[-1,0];break;case\"right\":l._dim=1,l._normals=[1,0];break;default:throw new Error(\"unreachable\")}return l.is_horizontal?l.set_sizing({width_policy:\"max\",height_policy:\"fixed\"}):l.set_sizing({width_policy:\"fixed\",height_policy:\"max\"}),l}return l.__extends(t,e),t.prototype._content_size=function(){return new r.Sizeable(this.get_oriented_size())},t.prototype.get_oriented_size=function(){var e=this.obj.get_size(),t=e.width,i=e.height;return!this.obj.rotate||this.is_horizontal?{width:t,height:i}:{width:i,height:t}},t.prototype.has_size_changed=function(){var e=this.get_oriented_size(),t=e.width,i=e.height;return this.is_horizontal?this.bbox.height!=i:this.bbox.width!=t},Object.defineProperty(t.prototype,\"dimension\",{get:function(){return this._dim},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"normals\",{get:function(){return this._normals},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"is_horizontal\",{get:function(){return 0==this._dim},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"is_vertical\",{get:function(){return 1==this._dim},enumerable:!0,configurable:!0}),t.prototype.apply_label_text_heuristics=function(e,t){var i,l,r=this.side;o.isString(t)?(i=c[r][t],l=p[r][t]):0===t?(i=\"whatever\",l=\"whatever\"):t<0?(i=\"middle\",l=b[r]):(i=\"middle\",l=_[r]),e.textBaseline=i,e.textAlign=l},t.prototype.get_label_angle_heuristic=function(e){return d[this.side][e]},t}(a.ContentLayoutable);i.SidePanel=m,m.__name__=\"SidePanel\"},\n", + " function _(t,e,n){var i=t(380),r=t(116),s=t(167),o=t(163),a=t(381),_=t(110),h=t(125),p=t(109),c=t(197),u=t(376),l=function(){function t(t,e,n){var s=this;this.plot_view=t,this.toolbar=e,this.hit_area=n,this.pan_start=new r.Signal(this,\"pan:start\"),this.pan=new r.Signal(this,\"pan\"),this.pan_end=new r.Signal(this,\"pan:end\"),this.pinch_start=new r.Signal(this,\"pinch:start\"),this.pinch=new r.Signal(this,\"pinch\"),this.pinch_end=new r.Signal(this,\"pinch:end\"),this.rotate_start=new r.Signal(this,\"rotate:start\"),this.rotate=new r.Signal(this,\"rotate\"),this.rotate_end=new r.Signal(this,\"rotate:end\"),this.tap=new r.Signal(this,\"tap\"),this.doubletap=new r.Signal(this,\"doubletap\"),this.press=new r.Signal(this,\"press\"),this.pressup=new r.Signal(this,\"pressup\"),this.move_enter=new r.Signal(this,\"move:enter\"),this.move=new r.Signal(this,\"move\"),this.move_exit=new r.Signal(this,\"move:exit\"),this.scroll=new r.Signal(this,\"scroll\"),this.keydown=new r.Signal(this,\"keydown\"),this.keyup=new r.Signal(this,\"keyup\"),this.hammer=new i(this.hit_area,{touchAction:\"auto\"}),this._configure_hammerjs(),this.hit_area.addEventListener(\"mousemove\",function(t){return s._mouse_move(t)}),this.hit_area.addEventListener(\"mouseenter\",function(t){return s._mouse_enter(t)}),this.hit_area.addEventListener(\"mouseleave\",function(t){return s._mouse_exit(t)}),this.hit_area.addEventListener(\"wheel\",function(t){return s._mouse_wheel(t)}),document.addEventListener(\"keydown\",this),document.addEventListener(\"keyup\",this)}return t.prototype.destroy=function(){this.hammer.destroy(),document.removeEventListener(\"keydown\",this),document.removeEventListener(\"keyup\",this)},t.prototype.handleEvent=function(t){\"keydown\"==t.type?this._key_down(t):\"keyup\"==t.type&&this._key_up(t)},t.prototype._configure_hammerjs=function(){var t=this;this.hammer.get(\"doubletap\").recognizeWith(\"tap\"),this.hammer.get(\"tap\").requireFailure(\"doubletap\"),this.hammer.get(\"doubletap\").dropRequireFailure(\"tap\"),this.hammer.on(\"doubletap\",function(e){return t._doubletap(e)}),this.hammer.on(\"tap\",function(e){return t._tap(e)}),this.hammer.on(\"press\",function(e){return t._press(e)}),this.hammer.on(\"pressup\",function(e){return t._pressup(e)}),this.hammer.get(\"pan\").set({direction:i.DIRECTION_ALL}),this.hammer.on(\"panstart\",function(e){return t._pan_start(e)}),this.hammer.on(\"pan\",function(e){return t._pan(e)}),this.hammer.on(\"panend\",function(e){return t._pan_end(e)}),this.hammer.get(\"pinch\").set({enable:!0}),this.hammer.on(\"pinchstart\",function(e){return t._pinch_start(e)}),this.hammer.on(\"pinch\",function(e){return t._pinch(e)}),this.hammer.on(\"pinchend\",function(e){return t._pinch_end(e)}),this.hammer.get(\"rotate\").set({enable:!0}),this.hammer.on(\"rotatestart\",function(e){return t._rotate_start(e)}),this.hammer.on(\"rotate\",function(e){return t._rotate(e)}),this.hammer.on(\"rotateend\",function(e){return t._rotate_end(e)})},t.prototype.register_tool=function(t){var e=this,n=t.model.event_type;null!=n&&(p.isString(n)?this._register_tool(t,n):n.forEach(function(n,i){return e._register_tool(t,n,i<1)}))},t.prototype._register_tool=function(t,e,n){void 0===n&&(n=!0);var i=t,r=i.model.id,o=function(t){return function(e){e.id==r&&t(e.e)}},a=function(t){return function(e){t(e.e)}};switch(e){case\"pan\":null!=i._pan_start&&i.connect(this.pan_start,o(i._pan_start.bind(i))),null!=i._pan&&i.connect(this.pan,o(i._pan.bind(i))),null!=i._pan_end&&i.connect(this.pan_end,o(i._pan_end.bind(i)));break;case\"pinch\":null!=i._pinch_start&&i.connect(this.pinch_start,o(i._pinch_start.bind(i))),null!=i._pinch&&i.connect(this.pinch,o(i._pinch.bind(i))),null!=i._pinch_end&&i.connect(this.pinch_end,o(i._pinch_end.bind(i)));break;case\"rotate\":null!=i._rotate_start&&i.connect(this.rotate_start,o(i._rotate_start.bind(i))),null!=i._rotate&&i.connect(this.rotate,o(i._rotate.bind(i))),null!=i._rotate_end&&i.connect(this.rotate_end,o(i._rotate_end.bind(i)));break;case\"move\":null!=i._move_enter&&i.connect(this.move_enter,o(i._move_enter.bind(i))),null!=i._move&&i.connect(this.move,o(i._move.bind(i))),null!=i._move_exit&&i.connect(this.move_exit,o(i._move_exit.bind(i)));break;case\"tap\":null!=i._tap&&i.connect(this.tap,o(i._tap.bind(i)));break;case\"press\":null!=i._press&&i.connect(this.press,o(i._press.bind(i))),null!=i._pressup&&i.connect(this.pressup,o(i._pressup.bind(i)));break;case\"scroll\":null!=i._scroll&&i.connect(this.scroll,o(i._scroll.bind(i)));break;default:throw new Error(\"unsupported event_type: \"+e)}n&&(null!=i._doubletap&&i.connect(this.doubletap,a(i._doubletap.bind(i))),null!=i._keydown&&i.connect(this.keydown,a(i._keydown.bind(i))),null!=i._keyup&&i.connect(this.keyup,a(i._keyup.bind(i))),c.is_mobile&&null!=i._scroll&&\"pinch\"==e&&(s.logger.debug(\"Registering scroll on touch screen\"),i.connect(this.scroll,o(i._scroll.bind(i)))))},t.prototype._hit_test_renderers=function(t,e){for(var n=this.plot_view.get_renderer_views(),i=0,r=_.reversed(n);i\\s*\\(/gm,\"{anonymous}()@\"):\"Unknown Stack Trace\",s=t.console&&(t.console.warn||t.console.log);return s&&s.call(t.console,r,n),e.apply(this,arguments)}}s=\"function\"!=typeof Object.assign?function(t){if(t===r||null===t)throw new TypeError(\"Cannot convert undefined or null to object\");for(var e=Object(t),i=1;i-1}function b(t){return t.trim().split(/\\s+/g)}function P(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var n=0;ni[e]}):n.sort()),n}function w(t,e){for(var i,n,s=e[0].toUpperCase()+e.slice(1),a=0;a1&&!i.firstMultiple?i.firstMultiple=Q(e):1===s&&(i.firstMultiple=!1);var o=i.firstInput,a=i.firstMultiple,h=a?a.center:o.center,u=e.center=tt(n);e.timeStamp=l(),e.deltaTime=e.timeStamp-o.timeStamp,e.angle=rt(h,u),e.distance=nt(h,u),function(t,e){var i=e.center,n=t.offsetDelta||{},r=t.prevDelta||{},s=t.prevInput||{};e.eventType!==Y&&s.eventType!==W||(r=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},n=t.offsetDelta={x:i.x,y:i.y});e.deltaX=r.x+(i.x-n.x),e.deltaY=r.y+(i.y-n.y)}(i,e),e.offsetDirection=it(e.deltaX,e.deltaY);var p=et(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=p.x,e.overallVelocityY=p.y,e.overallVelocity=c(p.x)>c(p.y)?p.x:p.y,e.scale=a?(f=a.pointers,v=n,nt(v[0],v[1],$)/nt(f[0],f[1],$)):1,e.rotation=a?function(t,e){return rt(e[1],e[0],$)+rt(t[1],t[0],$)}(a.pointers,n):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,function(t,e){var i,n,s,o,a=t.lastInterval||e,h=e.timeStamp-a.timeStamp;if(e.eventType!=q&&(h>X||a.velocity===r)){var u=e.deltaX-a.deltaX,l=e.deltaY-a.deltaY,p=et(h,u,l);n=p.x,s=p.y,i=c(p.x)>c(p.y)?p.x:p.y,o=it(u,l),t.lastInterval=e}else i=a.velocity,n=a.velocityX,s=a.velocityY,o=a.direction;e.velocity=i,e.velocityX=n,e.velocityY=s,e.direction=o}(i,e);var f,v;var d=t.element;C(e.srcEvent.target,d)&&(d=e.srcEvent.target);e.target=d}(t,i),t.emit(\"hammer.input\",i),t.recognize(i),t.session.prevInput=i}function Q(t){for(var e=[],i=0;i=c(e)?t<0?H:L:e<0?U:V}function nt(t,e,i){i||(i=B);var n=e[i[0]]-t[i[0]],r=e[i[1]]-t[i[1]];return Math.sqrt(n*n+r*r)}function rt(t,e,i){i||(i=B);var n=e[i[0]]-t[i[0]],r=e[i[1]]-t[i[1]];return 180*Math.atan2(r,n)/Math.PI}J.prototype={handler:function(){},init:function(){this.evEl&&A(this.element,this.evEl,this.domHandler),this.evTarget&&A(this.target,this.evTarget,this.domHandler),this.evWin&&A(R(this.element),this.evWin,this.domHandler)},destroy:function(){this.evEl&&_(this.element,this.evEl,this.domHandler),this.evTarget&&_(this.target,this.evTarget,this.domHandler),this.evWin&&_(R(this.element),this.evWin,this.domHandler)}};var st={mousedown:Y,mousemove:F,mouseup:W},ot=\"mousedown\",at=\"mousemove mouseup\";function ht(){this.evEl=ot,this.evWin=at,this.pressed=!1,J.apply(this,arguments)}T(ht,J,{handler:function(t){var e=st[t.type];e&Y&&0===t.button&&(this.pressed=!0),e&F&&1!==t.which&&(e=W),this.pressed&&(e&W&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:\"mouse\",srcEvent:t}))}});var ut={pointerdown:Y,pointermove:F,pointerup:W,pointercancel:q,pointerout:q},ct={2:\"touch\",3:\"pen\",4:\"mouse\",5:\"kinect\"},lt=\"pointerdown\",pt=\"pointermove pointerup pointercancel\";function ft(){this.evEl=lt,this.evWin=pt,J.apply(this,arguments),this.store=this.manager.session.pointerEvents=[]}t.MSPointerEvent&&!t.PointerEvent&&(lt=\"MSPointerDown\",pt=\"MSPointerMove MSPointerUp MSPointerCancel\"),T(ft,J,{handler:function(t){var e=this.store,i=!1,n=t.type.toLowerCase().replace(\"ms\",\"\"),r=ut[n],s=ct[t.pointerType]||t.pointerType,o=\"touch\"==s,a=P(e,t.pointerId,\"pointerId\");r&Y&&(0===t.button||o)?a<0&&(e.push(t),a=e.length-1):r&(W|q)&&(i=!0),a<0||(e[a]=t,this.callback(this.manager,r,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))}});var vt={touchstart:Y,touchmove:F,touchend:W,touchcancel:q},dt=\"touchstart\",mt=\"touchstart touchmove touchend touchcancel\";function gt(){this.evTarget=dt,this.evWin=mt,this.started=!1,J.apply(this,arguments)}T(gt,J,{handler:function(t){var e=vt[t.type];if(e===Y&&(this.started=!0),this.started){var i=function(t,e){var i=D(t.touches),n=D(t.changedTouches);e&(W|q)&&(i=x(i.concat(n),\"identifier\",!0));return[i,n]}.call(this,t,e);e&(W|q)&&i[0].length-i[1].length==0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:\"touch\",srcEvent:t})}}});var Tt={touchstart:Y,touchmove:F,touchend:W,touchcancel:q},yt=\"touchstart touchmove touchend touchcancel\";function Et(){this.evTarget=yt,this.targetIds={},J.apply(this,arguments)}T(Et,J,{handler:function(t){var e=Tt[t.type],i=function(t,e){var i=D(t.touches),n=this.targetIds;if(e&(Y|F)&&1===i.length)return n[i[0].identifier]=!0,[i,i];var r,s,o=D(t.changedTouches),a=[],h=this.target;if(s=i.filter(function(t){return C(t.target,h)}),e===Y)for(r=0;r-1&&n.splice(t,1)},It)}}T(_t,J,{handler:function(t,e,i){var n=\"touch\"==i.pointerType,r=\"mouse\"==i.pointerType;if(!(r&&i.sourceCapabilities&&i.sourceCapabilities.firesTouchEvents)){if(n)(function(t,e){t&Y?(this.primaryTouch=e.changedPointers[0].identifier,Ct.call(this,e)):t&(W|q)&&Ct.call(this,e)}).call(this,e,i);else if(r&&function(t){for(var e=t.srcEvent.clientX,i=t.srcEvent.clientY,n=0;n-1&&this.requireFail.splice(e,1),this},hasRequireFailures:function(){return this.requireFail.length>0},canRecognizeWith:function(t){return!!this.simultaneous[t.id]},emit:function(t){var e=this,i=this.state;function n(i){e.manager.emit(i,t)}i=Yt&&n(e.options.event+kt(i))},tryEmit:function(t){if(this.canEmit())return this.emit(t);this.state=32},canEmit:function(){for(var t=0;te.threshold&&r&e.direction},attrTest:function(t){return Ut.prototype.attrTest.call(this,t)&&(this.state&Nt||!(this.state&Nt)&&this.directionTest(t))},emit:function(t){this.pX=t.deltaX,this.pY=t.deltaY;var e=Ht(t.direction);e&&(t.additionalEvent=this.options.event+e),this._super.emit.call(this,t)}}),T(jt,Ut,{defaults:{event:\"pinch\",threshold:0,pointers:2},getTouchAction:function(){return[xt]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.scale-1)>this.options.threshold||this.state&Nt)},emit:function(t){if(1!==t.scale){var e=t.scale<1?\"in\":\"out\";t.additionalEvent=this.options.event+e}this._super.emit.call(this,t)}}),T(Gt,qt,{defaults:{event:\"press\",pointers:1,time:251,threshold:9},getTouchAction:function(){return[Pt]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,n=t.distancee.time;if(this._input=t,!n||!i||t.eventType&(W|q)&&!r)this.reset();else if(t.eventType&Y)this.reset(),this._timer=p(function(){this.state=Ft,this.tryEmit()},e.time,this);else if(t.eventType&W)return Ft;return 32},reset:function(){clearTimeout(this._timer)},emit:function(t){this.state===Ft&&(t&&t.eventType&W?this.manager.emit(this.options.event+\"up\",t):(this._input.timeStamp=l(),this.manager.emit(this.options.event,this._input)))}}),T(Zt,Ut,{defaults:{event:\"rotate\",threshold:0,pointers:2},getTouchAction:function(){return[xt]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.rotation)>this.options.threshold||this.state&Nt)}}),T(Bt,Ut,{defaults:{event:\"swipe\",threshold:10,velocity:.3,direction:j|G,pointers:1},getTouchAction:function(){return Vt.prototype.getTouchAction.call(this)},attrTest:function(t){var e,i=this.options.direction;return i&(j|G)?e=t.overallVelocity:i&j?e=t.overallVelocityX:i&G&&(e=t.overallVelocityY),this._super.attrTest.call(this,t)&&i&t.offsetDirection&&t.distance>this.options.threshold&&t.maxPointers==this.options.pointers&&c(e)>this.options.velocity&&t.eventType&W},emit:function(t){var e=Ht(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)}}),T($t,qt,{defaults:{event:\"tap\",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},getTouchAction:function(){return[Dt]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,n=t.distance=2){this.map.setZoom(n);var s=this._get_projected_bounds(),a=s[0];s[1]-a<0&&this.map.setZoom(i)}this.unpause()}this._set_bokeh_ranges()},e.prototype._build_map=function(){var t=this,e=google.maps;this.map_types={satellite:e.MapTypeId.SATELLITE,terrain:e.MapTypeId.TERRAIN,roadmap:e.MapTypeId.ROADMAP,hybrid:e.MapTypeId.HYBRID};var o=this.model.map_options,i={center:new e.LatLng(o.lat,o.lng),zoom:o.zoom,disableDefaultUI:!0,mapTypeId:this.map_types[o.map_type],scaleControl:o.scale_control,tilt:o.tilt};null!=o.styles&&(i.styles=JSON.parse(o.styles)),this.map=new e.Map(this.canvas_view.map_el,i),e.event.addListener(this.map,\"idle\",function(){return t._set_bokeh_ranges()}),e.event.addListener(this.map,\"bounds_changed\",function(){return t._set_bokeh_ranges()}),e.event.addListenerOnce(this.map,\"tilesloaded\",function(){return t._render_finished()}),this.connect(this.model.properties.map_options.change,function(){return t._update_options()}),this.connect(this.model.map_options.properties.styles.change,function(){return t._update_styles()}),this.connect(this.model.map_options.properties.lat.change,function(){return t._update_center(\"lat\")}),this.connect(this.model.map_options.properties.lng.change,function(){return t._update_center(\"lng\")}),this.connect(this.model.map_options.properties.zoom.change,function(){return t._update_zoom()}),this.connect(this.model.map_options.properties.map_type.change,function(){return t._update_map_type()}),this.connect(this.model.map_options.properties.scale_control.change,function(){return t._update_scale_control()}),this.connect(this.model.map_options.properties.tilt.change,function(){return t._update_tilt()})},e.prototype._render_finished=function(){this._tiles_loaded=!0,this.notify_finished()},e.prototype.has_finished=function(){return t.prototype.has_finished.call(this)&&!0===this._tiles_loaded},e.prototype._get_latlon_bounds=function(){var t=this.map.getBounds(),e=t.getNorthEast(),o=t.getSouthWest();return[o.lng(),e.lng(),o.lat(),e.lat()]},e.prototype._get_projected_bounds=function(){var t=this._get_latlon_bounds(),e=t[0],o=t[1],i=t[2],n=t[3],a=s.wgs84_mercator.forward([e,i]),p=a[0],l=a[1],_=s.wgs84_mercator.forward([o,n]);return[p,_[0],l,_[1]]},e.prototype._set_bokeh_ranges=function(){var t=this._get_projected_bounds(),e=t[0],o=t[1],i=t[2],n=t[3];this.frame.x_range.setv({start:e,end:o}),this.frame.y_range.setv({start:i,end:n})},e.prototype._update_center=function(t){var e=this.map.getCenter().toJSON();e[t]=this.model.map_options[t],this.map.setCenter(e),this._set_bokeh_ranges()},e.prototype._update_map_type=function(){this.map.setOptions({mapTypeId:this.map_types[this.model.map_options.map_type]})},e.prototype._update_scale_control=function(){this.map.setOptions({scaleControl:this.model.map_options.scale_control})},e.prototype._update_tilt=function(){this.map.setOptions({tilt:this.model.map_options.tilt})},e.prototype._update_options=function(){this._update_styles(),this._update_center(\"lat\"),this._update_center(\"lng\"),this._update_zoom(),this._update_map_type()},e.prototype._update_styles=function(){this.map.setOptions({styles:JSON.parse(this.model.map_options.styles)})},e.prototype._update_zoom=function(){this.map.setOptions({zoom:this.model.map_options.zoom}),this._set_bokeh_ranges()},e.prototype._map_hook=function(t,e){var o=e[0],i=e[1],n=e[2],s=e[3];this.canvas_view.map_el.style.top=i+\"px\",this.canvas_view.map_el.style.left=o+\"px\",this.canvas_view.map_el.style.width=n+\"px\",this.canvas_view.map_el.style.height=s+\"px\",null==this.map&&\"undefined\"!=typeof google&&null!=google.maps&&this._build_map()},e.prototype._paint_empty=function(t,e){var o=this.layout._width.value,i=this.layout._height.value,n=e[0],s=e[1],a=e[2],p=e[3];t.clearRect(0,0,o,i),t.beginPath(),t.moveTo(0,0),t.lineTo(0,i),t.lineTo(o,i),t.lineTo(o,0),t.lineTo(0,0),t.moveTo(n,s),t.lineTo(n+a,s),t.lineTo(n+a,s+p),t.lineTo(n,s+p),t.lineTo(n,s),t.closePath(),null!=this.model.border_fill_color&&(t.fillStyle=this.model.border_fill_color,t.fill())},e}(a.PlotView);o.GMapPlotView=l,l.__name__=\"GMapPlotView\"},\n", + " function _(a,n,e){var g=a(281);e.DataRange=g.DataRange;var R=a(280);e.DataRange1d=R.DataRange1d;var r=a(184);e.FactorRange=r.FactorRange;var t=a(185);e.Range=t.Range;var v=a(225);e.Range1d=v.Range1d},\n", + " function _(e,r,d){var n=e(175);d.GlyphRenderer=n.GlyphRenderer;var R=e(192);d.GraphRenderer=R.GraphRenderer;var a=e(244);d.GuideRenderer=a.GuideRenderer;var G=e(160);d.Renderer=G.Renderer},\n", + " function _(a,e,c){var l=a(279);c.CategoricalScale=l.CategoricalScale;var r=a(215);c.LinearScale=r.LinearScale;var S=a(224);c.LogScale=S.LogScale;var i=a(216);c.Scale=i.Scale},\n", + " function _(n,o,e){!function(n){for(var o in n)e.hasOwnProperty(o)||(e[o]=n[o])}(n(195));var i=n(173);e.Selection=i.Selection},\n", + " function _(a,e,r){var o=a(388);r.ServerSentDataSource=o.ServerSentDataSource;var S=a(390);r.AjaxDataSource=S.AjaxDataSource;var t=a(170);r.ColumnDataSource=t.ColumnDataSource;var u=a(171);r.ColumnarDataSource=u.ColumnarDataSource;var D=a(191);r.CDSView=D.CDSView;var c=a(172);r.DataSource=c.DataSource;var v=a(392);r.GeoJSONDataSource=v.GeoJSONDataSource;var n=a(391);r.RemoteDataSource=n.RemoteDataSource},\n", + " function _(t,e,i){var a=t(113),n=function(t){function e(e){var i=t.call(this,e)||this;return i.initialized=!1,i}return a.__extends(e,t),e.prototype.destroy=function(){t.prototype.destroy.call(this)},e.prototype.setup=function(){var t=this;this.initialized||(this.initialized=!0,new EventSource(this.data_url).onmessage=function(e){t.load_data(JSON.parse(e.data),t.mode,t.max_size)})},e}(t(389).WebDataSource);i.ServerSentDataSource=n,n.__name__=\"ServerSentDataSource\"},\n", + " function _(t,a,e){var i=t(113),n=t(170),r=t(121),o=function(t){function a(a){return t.call(this,a)||this}return i.__extends(a,t),a.prototype.get_column=function(t){var a=this.data[t];return null!=a?a:[]},a.prototype.initialize=function(){t.prototype.initialize.call(this),this.setup()},a.prototype.load_data=function(t,a,e){var i,n=this.adapter;switch(i=null!=n?n.execute(this,{response:t}):t,a){case\"replace\":this.data=i;break;case\"append\":for(var r=this.data,o=0,c=this.columns();o1&&a.logger.warn(\"Bokeh does not support Polygons with holes in, only exterior ring used.\");var h=e.coordinates[0];for(c=0;c1&&a.logger.warn(\"Bokeh does not support Polygons with holes in, only exterior ring used.\"),d.push(w[0])}for(_=d.reduce(o),c=0;c<_.length;c++){var v=_[c];i=v[0],s=v[1],u=v[2];t.xs[r][c]=i,t.ys[r][c]=s,t.zs[r][c]=l(u)}break;default:throw new Error(\"Invalid GeoJSON geometry type: \"+e.type)}},t.prototype.geojson_to_column_data=function(){var e,t=JSON.parse(this.geojson);switch(t.type){case\"GeometryCollection\":if(null==t.geometries)throw new Error(\"No geometries found in GeometryCollection\");if(0===t.geometries.length)throw new Error(\"geojson.geometries must have one or more items\");e=t.geometries;break;case\"FeatureCollection\":if(null==t.features)throw new Error(\"No features found in FeaturesCollection\");if(0==t.features.length)throw new Error(\"geojson.features must have one or more items\");e=t.features;break;default:throw new Error(\"Bokeh only supports type GeometryCollection and FeatureCollection at top level\")}for(var r=0,o=0,n=e;o=Math.pow(2,i)))&&!(e<0||e>=Math.pow(2,i))},e.prototype.parent_by_tile_xyz=function(t,e,i){var o=this.tile_xyz_to_quadkey(t,e,i),r=o.substring(0,o.length-1);return this.quadkey_to_tile_xyz(r)},e.prototype.get_resolution=function(t){return this._computed_initial_resolution()/Math.pow(2,t)},e.prototype.get_resolution_by_extent=function(t,e,i){return[(t[2]-t[0])/i,(t[3]-t[1])/e]},e.prototype.get_level_by_extent=function(t,e,i){for(var o=(t[2]-t[0])/i,r=(t[3]-t[1])/e,n=Math.max(o,r),_=0,s=0,u=this._resolutions;su[s]){if(0==_)return 0;if(_>0)return _-1}_+=1}return _-1},e.prototype.get_closest_level_by_extent=function(t,e,i){var o=(t[2]-t[0])/i,r=(t[3]-t[1])/e,n=Math.max(o,r),_=this._resolutions.reduce(function(t,e){return Math.abs(e-n)h?(a=_-r,l*=p):(a*=h,l=s-n)}var y=(a-(_-r))/2,c=(l-(s-n))/2;return[r-y,n-c,_+y,s+c]},e.prototype.tms_to_wmts=function(t,e,i){return[t,Math.pow(2,i)-1-e,i]},e.prototype.wmts_to_tms=function(t,e,i){return[t,Math.pow(2,i)-1-e,i]},e.prototype.pixels_to_meters=function(t,e,i){var o=this.get_resolution(i);return[t*o-this.x_origin_offset,e*o-this.y_origin_offset]},e.prototype.meters_to_pixels=function(t,e,i){var o=this.get_resolution(i);return[(t+this.x_origin_offset)/o,(e+this.y_origin_offset)/o]},e.prototype.pixels_to_tile=function(t,e){var i=Math.ceil(t/this.tile_size);return[i=0===i?i:i-1,Math.max(Math.ceil(e/this.tile_size)-1,0)]},e.prototype.pixels_to_raster=function(t,e,i){return[t,(this.tile_size<=a;c--)for(var f=u;f<=p;f++)this.is_valid_tile(f,c,e)&&y.push([f,c,e,this.get_tile_meter_bounds(f,c,e)]);return this.sort_tiles_from_center(y,[u,a,p,h]),y},e.prototype.quadkey_to_tile_xyz=function(t){for(var e=0,i=0,o=t.length,r=o;r>0;r--){var n=1<0;r--){var n=1<0;)if(s=s.substring(0,s.length-1),t=(r=this.quadkey_to_tile_xyz(s))[0],e=r[1],i=r[2],t=(n=this.denormalize_xyz(t,e,i,_))[0],e=n[1],i=n[2],this.tiles.has(this.tile_xyz_to_key(t,e,i)))return[t,e,i];return[0,0,0]},e.prototype.normalize_xyz=function(t,e,i){if(this.wrap_around){var o=Math.pow(2,i);return[(t%o+o)%o,e,i]}return[t,e,i]},e.prototype.denormalize_xyz=function(t,e,i,o){return[t+o*Math.pow(2,i),e,i]},e.prototype.denormalize_meters=function(t,e,i,o){return[t+2*o*Math.PI*6378137,e]},e.prototype.calculate_world_x_by_tile_xyz=function(t,e,i){return Math.floor(t/Math.pow(2,i))},e}(r.TileSource);i.MercatorTileSource=u,u.__name__=\"MercatorTileSource\",u.init_MercatorTileSource()},\n", + " function _(t,e,r){var i=t(113),n=t(166),o=t(121),a=function(t){function e(e){return t.call(this,e)||this}return i.__extends(e,t),e.init_TileSource=function(){this.define({url:[o.String,\"\"],tile_size:[o.Number,256],max_zoom:[o.Number,30],min_zoom:[o.Number,0],extra_url_vars:[o.Any,{}],attribution:[o.String,\"\"],x_origin_offset:[o.Number],y_origin_offset:[o.Number],initial_resolution:[o.Number]})},e.prototype.initialize=function(){t.prototype.initialize.call(this),this.tiles=new Map,this._normalize_case()},e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this),this.connect(this.change,function(){return e._clear_cache()})},e.prototype.string_lookup_replace=function(t,e){var r=t;for(var i in e){var n=e[i];r=r.replace(\"{\"+i+\"}\",n)}return r},e.prototype._normalize_case=function(){var t=this.url.replace(\"{x}\",\"{X}\").replace(\"{y}\",\"{Y}\").replace(\"{z}\",\"{Z}\").replace(\"{q}\",\"{Q}\").replace(\"{xmin}\",\"{XMIN}\").replace(\"{ymin}\",\"{YMIN}\").replace(\"{xmax}\",\"{XMAX}\").replace(\"{ymax}\",\"{YMAX}\");this.url=t},e.prototype._clear_cache=function(){this.tiles=new Map},e.prototype.tile_xyz_to_key=function(t,e,r){return t+\":\"+e+\":\"+r},e.prototype.key_to_tile_xyz=function(t){var e=t.split(\":\").map(function(t){return parseInt(t)});return[e[0],e[1],e[2]]},e.prototype.sort_tiles_from_center=function(t,e){var r=e[0],i=e[1],n=e[2],o=e[3],a=(n-r)/2+r,c=(o-i)/2+i;t.sort(function(t,e){return Math.sqrt(Math.pow(a-t[0],2)+Math.pow(c-t[1],2))-Math.sqrt(Math.pow(a-e[0],2)+Math.pow(c-e[1],2))})},e.prototype.get_image_url=function(t,e,r){return this.string_lookup_replace(this.url,this.extra_url_vars).replace(\"{X}\",t.toString()).replace(\"{Y}\",e.toString()).replace(\"{Z}\",r.toString())},e}(n.Model);r.TileSource=a,a.__name__=\"TileSource\",a.init_TileSource()},\n", + " function _(r,e,t){var n=r(132);function o(r,e){return n.wgs84_mercator.forward([r,e])}function _(r,e){return n.wgs84_mercator.inverse([r,e])}t.geographic_to_meters=o,t.meters_to_geographic=_,t.geographic_extent_to_meters=function(r){var e=r[0],t=r[1],n=r[2],_=r[3],c=o(e,t),a=c[0],g=c[1],i=o(n,_);return[a,g,i[0],i[1]]},t.meters_extent_to_geographic=function(r){var e=r[0],t=r[1],n=r[2],o=r[3],c=_(e,t),a=c[0],g=c[1],i=_(n,o);return[a,g,i[0],i[1]]}},\n", + " function _(t,e,r){var _=t(113),i=function(t){function e(e){return t.call(this,e)||this}return _.__extends(e,t),e.prototype.get_image_url=function(t,e,r){var _=this.string_lookup_replace(this.url,this.extra_url_vars),i=this.tms_to_wmts(t,e,r),u=i[0],n=i[1],o=i[2],l=this.tile_xyz_to_quadkey(u,n,o);return _.replace(\"{Q}\",l)},e}(t(397).MercatorTileSource);r.QUADKEYTileSource=i,i.__name__=\"QUADKEYTileSource\"},\n", + " function _(e,t,i){var n=e(113),a=e(402),r=e(176),_=e(225),s=e(163),o=e(121),l=e(318),h=e(110),u=e(109),p=e(174),d=e(170),c=e(403),m=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return n.__extends(t,e),t.prototype.initialize=function(){this._tiles=[],e.prototype.initialize.call(this)},t.prototype.connect_signals=function(){var t=this;e.prototype.connect_signals.call(this),this.connect(this.model.change,function(){return t.request_render()}),this.connect(this.model.tile_source.change,function(){return t.request_render()})},t.prototype.get_extent=function(){return[this.x_range.start,this.y_range.start,this.x_range.end,this.y_range.end]},Object.defineProperty(t.prototype,\"map_plot\",{get:function(){return this.plot_model},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"map_canvas\",{get:function(){return this.plot_view.canvas_view.ctx},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"map_frame\",{get:function(){return this.plot_view.frame},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"x_range\",{get:function(){return this.map_plot.x_range},enumerable:!0,configurable:!0}),Object.defineProperty(t.prototype,\"y_range\",{get:function(){return this.map_plot.y_range},enumerable:!0,configurable:!0}),t.prototype._set_data=function(){this.extent=this.get_extent(),this._last_height=void 0,this._last_width=void 0},t.prototype._update_attribution=function(){null!=this.attribution_el&&s.removeElement(this.attribution_el);var e=this.model.tile_source.attribution;if(u.isString(e)&&e.length>0){var t=this.plot_view,i=t.layout,n=t.frame,a=i._width.value-n._right.value,r=i._height.value-n._bottom.value,_=n._width.value;this.attribution_el=s.div({class:c.bk_tile_attribution,style:{position:\"absolute\",right:a+\"px\",bottom:r+\"px\",\"max-width\":_-4+\"px\",padding:\"2px\",\"background-color\":\"rgba(255,255,255,0.5)\",\"font-size\":\"7pt\",\"line-height\":\"1.05\",\"white-space\":\"nowrap\",overflow:\"hidden\",\"text-overflow\":\"ellipsis\"}}),this.plot_view.canvas_view.events_el.appendChild(this.attribution_el),this.attribution_el.innerHTML=e,this.attribution_el.title=this.attribution_el.textContent.replace(/\\s*\\n\\s*/g,\" \")}},t.prototype._map_data=function(){this.initial_extent=this.get_extent();var e=this.model.tile_source.get_level_by_extent(this.initial_extent,this.map_frame._height.value,this.map_frame._width.value),t=this.model.tile_source.snap_to_zoom_level(this.initial_extent,this.map_frame._height.value,this.map_frame._width.value,e);this.x_range.start=t[0],this.y_range.start=t[1],this.x_range.end=t[2],this.y_range.end=t[3],this.x_range instanceof _.Range1d&&(this.x_range.reset_start=t[0],this.x_range.reset_end=t[2]),this.y_range instanceof _.Range1d&&(this.y_range.reset_start=t[1],this.y_range.reset_end=t[3]),this._update_attribution()},t.prototype._create_tile=function(e,t,i,n,a){var r=this;void 0===a&&(a=!1);var _=this.model.tile_source.normalize_xyz(e,t,i),s=_[0],o=_[1],h=_[2],u={img:void 0,tile_coords:[e,t,i],normalized_coords:[s,o,h],quadkey:this.model.tile_source.tile_xyz_to_quadkey(e,t,i),cache_key:this.model.tile_source.tile_xyz_to_key(e,t,i),bounds:n,loaded:!1,finished:!1,x_coord:n[0],y_coord:n[3]},p=this.model.tile_source.get_image_url(s,o,h);new l.ImageLoader(p,{loaded:function(e){Object.assign(u,{img:e,loaded:!0}),a?(u.finished=!0,r.notify_finished()):r.request_render()},failed:function(){u.finished=!0}}),this.model.tile_source.tiles.set(u.cache_key,u),this._tiles.push(u)},t.prototype._enforce_aspect_ratio=function(){if(this._last_height!==this.map_frame._height.value||this._last_width!==this.map_frame._width.value){var e=this.get_extent(),t=this.model.tile_source.get_level_by_extent(e,this.map_frame._height.value,this.map_frame._width.value),i=this.model.tile_source.snap_to_zoom_level(e,this.map_frame._height.value,this.map_frame._width.value,t);this.x_range.setv({start:i[0],end:i[2]}),this.y_range.setv({start:i[1],end:i[3]}),this.extent=i,this._last_height=this.map_frame._height.value,this._last_width=this.map_frame._width.value}},t.prototype.has_finished=function(){if(!e.prototype.has_finished.call(this))return!1;if(0===this._tiles.length)return!1;for(var t=0,i=this._tiles;tn&&(a=this.extent,o=n,l=!0),l&&(this.x_range.setv({x_range:{start:a[0],end:a[2]}}),this.y_range.setv({start:a[1],end:a[3]}),this.extent=a),this.extent=a;for(var u=t.get_tiles_by_extent(a,o),p=[],d=[],c=[],m=[],f=0,g=u;f0&&(u=u.filter(function(n){return t.includes(e,n.name)})),u}},\n", + " function _(t,o,e){var n=t(113),i=t(370),a=t(201),r=t(121),s=t(373),_=function(t){function o(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(o,t),o.prototype._match_aspect=function(t,o,e){var n,i,a,r,s=e.bbox.aspect,_=e.bbox.h_range.end,l=e.bbox.h_range.start,u=e.bbox.v_range.end,p=e.bbox.v_range.start,h=Math.abs(t[0]-o[0]),c=Math.abs(t[1]-o[1]),m=0==c?0:h/c,v=(m>=s?[1,m/s]:[s/m,1])[0];return t[0]<=o[0]?(n=t[0],(i=t[0]+h*v)>_&&(i=_)):(i=t[0],(n=t[0]-h*v)u&&(a=u)):(a=t[1],(r=t[1]-h/s)o.end)&&(this.v_axis_only=!0),(es.end)&&(this.h_axis_only=!0)}null!=this.model.document&&this.model.document.interactive_start(this.plot_model)},n.prototype._pan=function(t){this._update(t.deltaX,t.deltaY),null!=this.model.document&&this.model.document.interactive_start(this.plot_model)},n.prototype._pan_end=function(t){this.h_axis_only=!1,this.v_axis_only=!1,null!=this.pan_info&&this.plot_view.push_state(\"pan\",{range:this.pan_info})},n.prototype._update=function(t,n){var e,i,o,s,a,r,_=this.plot_view.frame,l=t-this.last_dx,h=n-this.last_dy,d=_.bbox.h_range,p=d.start-l,u=d.end-l,c=_.bbox.v_range,f=c.start-h,v=c.end-h,y=this.model.dimensions;\"width\"!=y&&\"both\"!=y||this.v_axis_only?(e=d.start,i=d.end,o=0):(e=p,i=u,o=-l),\"height\"!=y&&\"both\"!=y||this.h_axis_only?(s=c.start,a=c.end,r=0):(s=f,a=v,r=-h),this.last_dx=t,this.last_dy=n;var m=_.xscales,b=_.yscales,x={};for(var g in m){var w=m[g].r_invert(e,i),P=w[0],T=w[1];x[g]={start:P,end:T}}var k={};for(var g in b){var V=b[g].r_invert(s,a);P=V[0],T=V[1];k[g]={start:P,end:T}}this.pan_info={xrs:x,yrs:k,sdx:o,sdy:r},this.plot_view.update_range(this.pan_info,!0)},n}(o.GestureToolView);e.PanToolView=r,r.__name__=\"PanToolView\";var _=function(t){function n(n){var e=t.call(this,n)||this;return e.tool_name=\"Pan\",e.event_type=\"pan\",e.default_order=10,e}return i.__extends(n,t),n.init_PanTool=function(){this.prototype.default_view=r,this.define({dimensions:[s.Dimensions,\"both\"]})},Object.defineProperty(n.prototype,\"tooltip\",{get:function(){return this._get_dim_tooltip(\"Pan\",this.dimensions)},enumerable:!0,configurable:!0}),Object.defineProperty(n.prototype,\"icon\",{get:function(){switch(this.dimensions){case\"both\":return a.bk_tool_icon_pan;case\"width\":return a.bk_tool_icon_xpan;case\"height\":return a.bk_tool_icon_ypan}},enumerable:!0,configurable:!0}),n}(o.GestureTool);e.PanTool=_,_.__name__=\"PanTool\",_.init_PanTool()},\n", + " function _(t,e,o){var l=t(113),i=t(426),a=t(233),n=t(163),s=t(121),c=t(110),_=t(373),r=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return l.__extends(e,t),e.prototype.initialize=function(){t.prototype.initialize.call(this),this.data={sx:[],sy:[]}},e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this),this.connect(this.model.properties.active.change,function(){return e._active_change()})},e.prototype._active_change=function(){this.model.active||this._clear_data()},e.prototype._keyup=function(t){t.keyCode==n.Keys.Enter&&this._clear_data()},e.prototype._doubletap=function(t){var e=t.shiftKey;this._do_select(this.data.sx,this.data.sy,!0,e),this.plot_view.push_state(\"poly_select\",{selection:this.plot_view.get_selection()}),this._clear_data()},e.prototype._clear_data=function(){this.data={sx:[],sy:[]},this.model.overlay.update({xs:[],ys:[]})},e.prototype._tap=function(t){var e=t.sx,o=t.sy;this.plot_view.frame.bbox.contains(e,o)&&(this.data.sx.push(e),this.data.sy.push(o),this.model.overlay.update({xs:c.copy(this.data.sx),ys:c.copy(this.data.sy)}))},e.prototype._do_select=function(t,e,o,l){var i={type:\"poly\",sx:t,sy:e};this._select(i,o,l)},e.prototype._emit_callback=function(t){var e=this.computed_renderers[0],o=this.plot_view.frame,l=o.xscales[e.x_range_name],i=o.yscales[e.y_range_name],a=l.v_invert(t.sx),n=i.v_invert(t.sy),s=Object.assign({x:a,y:n},t);null!=this.model.callback&&this.model.callback.execute(this.model,{geometry:s})},e}(i.SelectToolView);o.PolySelectToolView=r,r.__name__=\"PolySelectToolView\";var y=function(){return new a.PolyAnnotation({level:\"overlay\",xs_units:\"screen\",ys_units:\"screen\",fill_color:{value:\"lightgrey\"},fill_alpha:{value:.5},line_color:{value:\"black\"},line_alpha:{value:1},line_width:{value:2},line_dash:{value:[4,4]}})},p=function(t){function e(e){var o=t.call(this,e)||this;return o.tool_name=\"Poly Select\",o.icon=_.bk_tool_icon_polygon_select,o.event_type=\"tap\",o.default_order=11,o}return l.__extends(e,t),e.init_PolySelectTool=function(){this.prototype.default_view=r,this.define({callback:[s.Any],overlay:[s.Instance,y]})},e}(i.SelectTool);o.PolySelectTool=p,p.__name__=\"PolySelectTool\",p.init_PolySelectTool()},\n", + " function _(t,e,i){var n=t(113),s=t(201),r=t(167),l=t(121),a=t(370),o=t(373);function _(t){switch(t){case 1:return 2;case 2:return 1;case 4:return 5;case 5:return 4;default:return t}}function h(t,e,i,n){if(null==e)return!1;var s=i.compute(e);return Math.abs(t-s)s.right)&&(r=!1)}if(null!=s.bottom&&null!=s.top){var a=n.invert(e);(as.top)&&(r=!1)}return r}function d(t,e,i){var n=0;return t>=i.start&&t<=i.end&&(n+=1),e>=i.start&&e<=i.end&&(n+=1),n}function c(t,e,i,n){var s=e.compute(t),r=e.invert(s+i);return r>=n.start&&r<=n.end?r:t}function y(t,e,i){return t>e.start?(e.end=t,i):(e.end=e.start,e.start=t,_(i))}function f(t,e,i){return t=h&&(t.start=o,t.end=_)}i.flip_side=_,i.is_near=h,i.is_inside=u,i.sides_inside=d,i.compute_value=c,i.update_range_end_side=y,i.update_range_start_side=f,i.update_range=g;var v=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(e,t),e.prototype.initialize=function(){t.prototype.initialize.call(this),this.side=0,this.model.update_overlay_from_ranges()},e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this),null!=this.model.x_range&&this.connect(this.model.x_range.change,function(){return e.model.update_overlay_from_ranges()}),null!=this.model.y_range&&this.connect(this.model.y_range.change,function(){return e.model.update_overlay_from_ranges()})},e.prototype._pan_start=function(t){this.last_dx=0,this.last_dy=0;var e=this.model.x_range,i=this.model.y_range,n=this.plot_view.frame,r=n.xscales.default,l=n.yscales.default,a=this.model.overlay,o=a.left,_=a.right,d=a.top,c=a.bottom,y=this.model.overlay.properties.line_width.value()+s.EDGE_TOLERANCE;null!=e&&this.model.x_interaction&&(h(t.sx,o,r,y)?this.side=1:h(t.sx,_,r,y)?this.side=2:u(t.sx,t.sy,r,l,a)&&(this.side=3)),null!=i&&this.model.y_interaction&&(0==this.side&&h(t.sy,c,l,y)&&(this.side=4),0==this.side&&h(t.sy,d,l,y)?this.side=5:u(t.sx,t.sy,r,l,this.model.overlay)&&(3==this.side?this.side=7:this.side=6))},e.prototype._pan=function(t){var e=this.plot_view.frame,i=t.deltaX-this.last_dx,n=t.deltaY-this.last_dy,s=this.model.x_range,r=this.model.y_range,l=e.xscales.default,a=e.yscales.default;if(null!=s)if(3==this.side||7==this.side)g(s,l,i,e.x_range);else if(1==this.side){var o=c(s.start,l,i,e.x_range);this.side=f(o,s,this.side)}else if(2==this.side){var _=c(s.end,l,i,e.x_range);this.side=y(_,s,this.side)}if(null!=r)if(6==this.side||7==this.side)g(r,a,n,e.y_range);else if(4==this.side){o=c(r.start,a,n,e.y_range);this.side=f(o,r,this.side)}else if(5==this.side){_=c(r.end,a,n,e.y_range);this.side=y(_,r,this.side)}this.last_dx=t.deltaX,this.last_dy=t.deltaY},e.prototype._pan_end=function(t){this.side=0},e}(a.GestureToolView);i.RangeToolView=v,v.__name__=\"RangeToolView\";var p=function(){return new s.BoxAnnotation({level:\"overlay\",render_mode:\"canvas\",fill_color:\"lightgrey\",fill_alpha:{value:.5},line_color:{value:\"black\"},line_alpha:{value:1},line_width:{value:.5},line_dash:[2,2]})},m=function(t){function e(e){var i=t.call(this,e)||this;return i.tool_name=\"Range Tool\",i.icon=o.bk_tool_icon_range,i.event_type=\"pan\",i.default_order=1,i}return n.__extends(e,t),e.init_RangeTool=function(){this.prototype.default_view=v,this.define({x_range:[l.Instance,null],x_interaction:[l.Boolean,!0],y_range:[l.Instance,null],y_interaction:[l.Boolean,!0],overlay:[l.Instance,p]})},e.prototype.initialize=function(){t.prototype.initialize.call(this),this.overlay.in_cursor=\"grab\",this.overlay.ew_cursor=null!=this.x_range&&this.x_interaction?\"ew-resize\":null,this.overlay.ns_cursor=null!=this.y_range&&this.y_interaction?\"ns-resize\":null},e.prototype.update_overlay_from_ranges=function(){null==this.x_range&&null==this.y_range&&(this.overlay.left=null,this.overlay.right=null,this.overlay.bottom=null,this.overlay.top=null,r.logger.warn(\"RangeTool not configured with any Ranges.\")),null==this.x_range?(this.overlay.left=null,this.overlay.right=null):(this.overlay.left=this.x_range.start,this.overlay.right=this.x_range.end),null==this.y_range?(this.overlay.bottom=null,this.overlay.top=null):(this.overlay.bottom=this.y_range.start,this.overlay.top=this.y_range.end)},e}(a.GestureTool);i.RangeTool=m,m.__name__=\"RangeTool\",m.init_RangeTool()},\n", + " function _(e,t,i){var s=e(113),n=e(426),o=e(121),a=e(373),r=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return s.__extends(t,e),t.prototype._tap=function(e){var t={type:\"point\",sx:e.sx,sy:e.sy},i=e.shiftKey;this._select(t,!0,i)},t.prototype._select=function(e,t,i){var s=this,n=this.model.callback;if(\"select\"==this.model.behavior){var o=this._computed_renderers_by_data_source();for(var a in o){var r=o[a],_=r[0].get_selection_manager(),l=r.map(function(e){return s.plot_view.renderer_views[e.id]});if(_.select(l,e,t,i)&&null!=n){var c=(y=this.plot_view.frame).xscales[r[0].x_range_name],p=y.yscales[r[0].y_range_name],v=c.invert(e.sx),u=p.invert(e.sy),h={geometries:Object.assign(Object.assign({},e),{x:v,y:u}),source:_.source};n.execute(this.model,h)}}this._emit_selection_event(e),this.plot_view.push_state(\"tap\",{selection:this.plot_view.get_selection()})}else for(var m=0,f=this.computed_renderers;m.9?t=.9:t<-.9&&(t=-.9),this._update_ranges(t)},t.prototype._update_ranges=function(e){var t,n,o,r,i=this.plot_view.frame,a=i.bbox.h_range,s=i.bbox.v_range,l=[a.start,a.end],_=l[0],h=l[1],d=[s.start,s.end],u=d[0],p=d[1];switch(this.model.dimension){case\"height\":var c=Math.abs(p-u);t=_,n=h,o=u-c*e,r=p-c*e;break;case\"width\":var v=Math.abs(h-_);t=_-v*e,n=h-v*e,o=u,r=p;break;default:throw new Error(\"this shouldn't have happened\")}var f=i.xscales,m=i.yscales,w={};for(var b in f){var g=f[b].r_invert(t,n),y=g[0],P=g[1];w[b]={start:y,end:P}}var T={};for(var b in m){var W=m[b].r_invert(o,r);y=W[0],P=W[1];T[b]={start:y,end:P}}var x={xrs:w,yrs:T,factor:e};this.plot_view.push_state(\"wheel_pan\",{range:x}),this.plot_view.update_range(x,!1,!0),null!=this.model.document&&this.model.document.interactive_start(this.plot_model)},t}(r.GestureToolView);n.WheelPanToolView=s,s.__name__=\"WheelPanToolView\";var l=function(e){function t(t){var n=e.call(this,t)||this;return n.tool_name=\"Wheel Pan\",n.icon=a.bk_tool_icon_wheel_pan,n.event_type=\"scroll\",n.default_order=12,n}return o.__extends(t,e),t.init_WheelPanTool=function(){this.prototype.default_view=s,this.define({dimension:[i.Dimension,\"width\"]}),this.internal({speed:[i.Number,.001]})},Object.defineProperty(t.prototype,\"tooltip\",{get:function(){return this._get_dim_tooltip(this.tool_name,this.dimension)},enumerable:!0,configurable:!0}),t}(r.GestureTool);n.WheelPanTool=l,l.__name__=\"WheelPanTool\",l.init_WheelPanTool()},\n", + " function _(e,o,t){var i=e(113),n=e(370),l=e(416),s=e(121),_=e(197),r=e(373),a=function(e){function o(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(o,e),o.prototype._pinch=function(e){var o,t=e.sx,i=e.sy,n=e.scale;o=n>=1?20*(n-1):-20/n,this._scroll({type:\"wheel\",sx:t,sy:i,delta:o})},o.prototype._scroll=function(e){var o=this.plot_view.frame,t=o.bbox.h_range,i=o.bbox.v_range,n=e.sx,s=e.sy,_=this.model.dimensions,r=(\"width\"==_||\"both\"==_)&&t.start=0){var v=d.match(/\\$color(\\[.*\\])?:(\\w*)/),y=v[1],x=void 0===y?\"\":y,g=v[2],b=e.get_column(g);if(null==b){var w=_.span({},g+\" unknown\");m.appendChild(w);continue}var k=x.indexOf(\"hex\")>=0,T=x.indexOf(\"swatch\")>=0,H=u.isNumber(t)?b[t]:null;if(null==H){var C=_.span({},\"(null)\");m.appendChild(C);continue}k&&(H=h.color2hex(H));var G=_.span({},H);m.appendChild(G),T&&(G=_.span({class:f.bk_tooltip_color_block,style:{backgroundColor:H}},\" \"),m.appendChild(G))}else{(G=_.span()).innerHTML=c.replace_placeholders(d.replace(\"$~\",\"$data_\"),e,t,this.model.formatters,n),m.appendChild(G)}}return o},t}(o.InspectToolView);n.HoverToolView=b,b.__name__=\"HoverToolView\";var w=function(e){function t(t){var n=e.call(this,t)||this;return n.tool_name=\"Hover\",n.icon=y.bk_tool_icon_hover,n}return i.__extends(t,e),t.init_HoverTool=function(){this.prototype.default_view=b,this.define({tooltips:[p.Any,[[\"index\",\"$index\"],[\"data (x, y)\",\"($x, $y)\"],[\"screen (x, y)\",\"($sx, $sy)\"]]],formatters:[p.Any,{}],renderers:[p.Any,\"auto\"],names:[p.Array,[]],mode:[p.HoverMode,\"mouse\"],point_policy:[p.PointPolicy,\"snap_to_data\"],line_policy:[p.LinePolicy,\"nearest\"],show_arrow:[p.Boolean,!0],anchor:[p.Anchor,\"center\"],attachment:[p.TooltipAttachment,\"horizontal\"],callback:[p.Any]})},t}(o.InspectTool);n.HoverTool=w,w.__name__=\"HoverTool\",w.init_HoverTool()},\n", + " function _(t,e,o){var n=t(113),i=t(121),r=t(116),c=t(166),l=t(364),u=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.init_ToolProxy=function(){this.define({tools:[i.Array,[]],active:[i.Boolean,!1],disabled:[i.Boolean,!1]})},Object.defineProperty(e.prototype,\"button_view\",{get:function(){return this.tools[0].button_view},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"event_type\",{get:function(){return this.tools[0].event_type},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"tooltip\",{get:function(){return this.tools[0].tooltip},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"tool_name\",{get:function(){return this.tools[0].tool_name},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"icon\",{get:function(){return this.tools[0].computed_icon},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"computed_icon\",{get:function(){return this.icon},enumerable:!0,configurable:!0}),Object.defineProperty(e.prototype,\"toggleable\",{get:function(){var t=this.tools[0];return t instanceof l.InspectTool&&t.toggleable},enumerable:!0,configurable:!0}),e.prototype.initialize=function(){t.prototype.initialize.call(this),this.do=new r.Signal0(this,\"do\")},e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this),this.connect(this.do,function(){return e.doit()}),this.connect(this.properties.active.change,function(){return e.set_active()})},e.prototype.doit=function(){for(var t=0,e=this.tools;t0)if(\"multi\"==u)for(var w=0,T=z;w0&&this.actions.push(x(z))}for(var m in this.inspectors=[],i){(z=i[m]).length>0&&this.inspectors.push(x(z,!0))}for(var V in this.gestures){0!=(_=this.gestures[V]).tools.length&&(_.tools=r.sort_by(_.tools,function(t){return t.default_order}),\"pinch\"!=V&&\"scroll\"!=V&&\"multi\"!=V&&(_.tools[0].active=!0))}},o}(s.ToolbarBase);i.ProxyToolbar=p,p.__name__=\"ProxyToolbar\";var c=function(t){function o(){return null!==t&&t.apply(this,arguments)||this}return e.__extends(o,t),o.prototype.initialize=function(){this.model.toolbar.toolbar_location=this.model.toolbar_location,t.prototype.initialize.call(this)},Object.defineProperty(o.prototype,\"child_models\",{get:function(){return[this.model.toolbar]},enumerable:!0,configurable:!0}),o.prototype._update_layout=function(){this.layout=new h.ContentBox(this.child_views[0].el),this.model.toolbar.horizontal?this.layout.set_sizing({width_policy:\"fit\",min_width:100,height_policy:\"fixed\"}):this.layout.set_sizing({width_policy:\"fixed\",height_policy:\"fit\",min_height:100})},o}(a.LayoutDOMView);i.ToolbarBoxView=c,c.__name__=\"ToolbarBoxView\";var u=function(t){function o(o){return t.call(this,o)||this}return e.__extends(o,t),o.init_ToolbarBox=function(){this.prototype.default_view=c,this.define({toolbar:[n.Instance],toolbar_location:[n.Location,\"right\"]})},o}(a.LayoutDOM);i.ToolbarBox=u,u.__name__=\"ToolbarBox\",u.init_ToolbarBox()},\n", + " function _(e,n,t){var d=e(106),i=e(163),o=e(442);t.index={},t.add_document_standalone=function(e,n,a,l){void 0===a&&(a={}),void 0===l&&(l=!1);var r={};function v(e){var d;e.id in a?d=a[e.id]:n.classList.contains(o.BOKEH_ROOT)?d=n:(d=i.div({class:o.BOKEH_ROOT}),n.appendChild(d));var l=function(e){var n=new e.default_view({model:e,parent:null});return t.index[e.id]=n,n}(e);l.renderTo(d),r[e.id]=l}for(var c=0,u=e.roots();c\");if(\"SCRIPT\"==r.tagName){var t=n.div({class:o.BOKEH_ROOT});n.replaceWith(r,t),r=t}return r}o.BOKEH_ROOT=t.bk_root,o._resolve_element=function(e){var r=e.elementid;return null!=r?l(r):document.body},o._resolve_root_elements=function(e){var r={};if(null!=e.roots)for(var o in e.roots)r[o]=l(e.roots[o]);return r}},\n", + " function _(n,o,t){var e=n(444),r=n(167),a=n(441);t._get_ws_url=function(n,o){var t,e=\"ws:\";return\"https:\"==window.location.protocol&&(e=\"wss:\"),null!=o?(t=document.createElement(\"a\")).href=o:t=window.location,null!=n?\"/\"==n&&(n=\"\"):n=t.pathname.replace(/\\/+$/,\"\"),e+\"//\"+t.host+n+\"/ws\"};var i={};t.add_document_from_session=function(n,o,t,s,u){void 0===s&&(s={}),void 0===u&&(u=!1);var c=window.location.search.substr(1);return function(n,o,t){n in i||(i[n]={});var r=i[n];return o in r||(r[o]=e.pull_session(n,o,t)),r[o]}(n,o,c).then(function(n){return a.add_document_standalone(n.document,t,s,u)},function(n){throw r.logger.error(\"Failed to load Bokeh session \"+o+\": \"+n),n})}},\n", + " function _(e,n,o){var t=e(167),s=e(106),r=e(445),i=e(446),c=e(447);o.DEFAULT_SERVER_WEBSOCKET_URL=\"ws://localhost:5006/ws\",o.DEFAULT_SESSION_ID=\"default\";var l=0,_=function(){function e(e,n,s,r,c){void 0===e&&(e=o.DEFAULT_SERVER_WEBSOCKET_URL),void 0===n&&(n=o.DEFAULT_SESSION_ID),void 0===s&&(s=null),void 0===r&&(r=null),void 0===c&&(c=null),this.url=e,this.id=n,this.args_string=s,this._on_have_session_hook=r,this._on_closed_permanently_hook=c,this._number=l++,this.socket=null,this.session=null,this.closed_permanently=!1,this._current_handler=null,this._pending_ack=null,this._pending_replies={},this._pending_messages=[],this._receiver=new i.Receiver,t.logger.debug(\"Creating websocket \"+this._number+\" to '\"+this.url+\"' session '\"+this.id+\"'\")}return e.prototype.connect=function(){var e=this;if(this.closed_permanently)return Promise.reject(new Error(\"Cannot connect() a closed ClientConnection\"));if(null!=this.socket)return Promise.reject(new Error(\"Already connected\"));this._pending_replies={},this._current_handler=null;try{var n=this.url+\"?bokeh-protocol-version=1.0&bokeh-session-id=\"+this.id;return null!=this.args_string&&this.args_string.length>0&&(n+=\"&\"+this.args_string),this.socket=new WebSocket(n),new Promise(function(n,o){e.socket.binaryType=\"arraybuffer\",e.socket.onopen=function(){return e._on_open(n,o)},e.socket.onmessage=function(n){return e._on_message(n)},e.socket.onclose=function(n){return e._on_close(n)},e.socket.onerror=function(){return e._on_error(o)}})}catch(e){return t.logger.error(\"websocket creation failed to url: \"+this.url),t.logger.error(\" - \"+e),Promise.reject(e)}},e.prototype.close=function(){this.closed_permanently||(t.logger.debug(\"Permanently closing websocket connection \"+this._number),this.closed_permanently=!0,null!=this.socket&&this.socket.close(1e3,\"close method called on ClientConnection \"+this._number),this.session._connection_closed(),null!=this._on_closed_permanently_hook&&(this._on_closed_permanently_hook(),this._on_closed_permanently_hook=null))},e.prototype._schedule_reconnect=function(e){var n=this;setTimeout(function(){n.closed_permanently||t.logger.info(\"Websocket connection \"+n._number+\" disconnected, will not attempt to reconnect\")},e)},e.prototype.send=function(e){if(null==this.socket)throw new Error(\"not connected so cannot send \"+e);e.send(this.socket)},e.prototype.send_with_reply=function(e){var n=this;return new Promise(function(o,t){n._pending_replies[e.msgid()]=[o,t],n.send(e)}).then(function(e){if(\"ERROR\"===e.msgtype())throw new Error(\"Error reply \"+e.content.text);return e},function(e){throw e})},e.prototype._pull_doc_json=function(){var e=r.Message.create(\"PULL-DOC-REQ\",{});return this.send_with_reply(e).then(function(e){if(!(\"doc\"in e.content))throw new Error(\"No 'doc' field in PULL-DOC-REPLY\");return e.content.doc},function(e){throw e})},e.prototype._repull_session_doc=function(){var e=this;null==this.session?t.logger.debug(\"Pulling session for first time\"):t.logger.debug(\"Repulling session\"),this._pull_doc_json().then(function(n){if(null==e.session)if(e.closed_permanently)t.logger.debug(\"Got new document after connection was already closed\");else{var o=s.Document.from_json(n),i=s.Document._compute_patch_since_json(n,o);if(i.events.length>0){t.logger.debug(\"Sending \"+i.events.length+\" changes from model construction back to server\");var l=r.Message.create(\"PATCH-DOC\",{},i);e.send(l)}e.session=new c.ClientSession(e,o,e.id);for(var _=0,h=e._pending_messages;_0)throw new Error(\"BokehJS only supports receiving buffers, not sending\");var t=JSON.stringify(this.header),r=JSON.stringify(this.metadata),n=JSON.stringify(this.content);e.send(t),e.send(r),e.send(n)},e.prototype.msgid=function(){return this.header.msgid},e.prototype.msgtype=function(){return this.header.msgtype},e.prototype.reqid=function(){return this.header.reqid},e.prototype.problem=function(){return\"msgid\"in this.header?\"msgtype\"in this.header?null:\"No msgtype in header\":\"No msgid in header\"},e}();r.Message=s,s.__name__=\"Message\"},\n", + " function _(t,e,s){var r=t(445),_=function(){function t(){this.message=null,this._partial=null,this._fragments=[],this._buf_header=null,this._current_consumer=this._HEADER}return t.prototype.consume=function(t){this._current_consumer(t)},t.prototype._HEADER=function(t){this._assume_text(t),this.message=null,this._partial=null,this._fragments=[t],this._buf_header=null,this._current_consumer=this._METADATA},t.prototype._METADATA=function(t){this._assume_text(t),this._fragments.push(t),this._current_consumer=this._CONTENT},t.prototype._CONTENT=function(t){this._assume_text(t),this._fragments.push(t);var e=this._fragments.slice(0,3),s=e[0],_=e[1],i=e[2];this._partial=r.Message.assemble(s,_,i),this._check_complete()},t.prototype._BUFFER_HEADER=function(t){this._assume_text(t),this._buf_header=t,this._current_consumer=this._BUFFER_PAYLOAD},t.prototype._BUFFER_PAYLOAD=function(t){this._assume_binary(t),this._partial.assemble_buffer(this._buf_header,t),this._check_complete()},t.prototype._assume_text=function(t){if(t instanceof ArrayBuffer)throw new Error(\"Expected text fragment but received binary fragment\")},t.prototype._assume_binary=function(t){if(!(t instanceof ArrayBuffer))throw new Error(\"Expected binary fragment but received text fragment\")},t.prototype._check_complete=function(){this._partial.complete()?(this.message=this._partial,this._current_consumer=this._HEADER):this._current_consumer=this._BUFFER_HEADER},t}();s.Receiver=_,_.__name__=\"Receiver\"},\n", + " function _(e,t,n){var o=e(106),i=e(445),r=e(167),s=function(){function e(e,t,n){var o=this;this._connection=e,this.document=t,this.id=n,this._document_listener=function(e){return o._document_changed(e)},this.document.on_change(this._document_listener),this.event_manager=this.document.event_manager,this.event_manager.session=this}return e.prototype.handle=function(e){var t=e.msgtype();\"PATCH-DOC\"===t?this._handle_patch(e):\"OK\"===t?this._handle_ok(e):\"ERROR\"===t?this._handle_error(e):r.logger.debug(\"Doing nothing with message \"+e.msgtype())},e.prototype.close=function(){this._connection.close()},e.prototype.send_event=function(e){var t=i.Message.create(\"EVENT\",{},JSON.stringify(e.to_json()));this._connection.send(t)},e.prototype._connection_closed=function(){this.document.remove_on_change(this._document_listener)},e.prototype.request_server_info=function(){var e=i.Message.create(\"SERVER-INFO-REQ\",{});return this._connection.send_with_reply(e).then(function(e){return e.content})},e.prototype.force_roundtrip=function(){return this.request_server_info().then(function(e){})},e.prototype._document_changed=function(e){if(e.setter_id!==this.id&&(!(e instanceof o.ModelChangedEvent)||e.attr in e.model.serializable_attributes())){var t=i.Message.create(\"PATCH-DOC\",{},this.document.create_json_patch([e]));this._connection.send(t)}},e.prototype._handle_patch=function(e){this.document.apply_json_patch(e.content,e.buffers,this.id)},e.prototype._handle_ok=function(e){r.logger.trace(\"Unhandled OK reply to \"+e.reqid())},e.prototype._handle_error=function(e){r.logger.error(\"Unhandled ERROR reply to \"+e.reqid()+\": \"+e.content.text)},e}();n.ClientSession=s,s.__name__=\"ClientSession\"},\n", + " function _(e,o,t){var n=e(106),r=e(446),s=e(167),i=e(125),a=e(441),l=e(442);function c(e,o){o.buffers.length>0?e.consume(o.buffers[0].buffer):e.consume(o.content.data);var t=e.message;null!=t&&this.apply_json_patch(t.content,t.buffers)}function g(e,o){if(\"undefined\"!=typeof Jupyter&&null!=Jupyter.notebook.kernel){s.logger.info(\"Registering Jupyter comms for target \"+e);var n=Jupyter.notebook.kernel.comm_manager;try{n.register_target(e,function(t){s.logger.info(\"Registering Jupyter comms for target \"+e);var n=new r.Receiver;t.on_msg(c.bind(o,n))})}catch(e){s.logger.warn(\"Jupyter comms failed to register. push_notebook() will not function. (exception reported: \"+e+\")\")}}else if(o.roots()[0].id in t.kernels){s.logger.info(\"Registering JupyterLab comms for target \"+e);var i=t.kernels[o.roots()[0].id];try{i.registerCommTarget(e,function(t){s.logger.info(\"Registering JupyterLab comms for target \"+e);var n=new r.Receiver;t.onMsg=c.bind(o,n)})}catch(e){s.logger.warn(\"Jupyter comms failed to register. push_notebook() will not function. (exception reported: \"+e+\")\")}}else console.warn(\"Jupyter notebooks comms not available. push_notebook() will not function. If running JupyterLab ensure the latest @bokeh/jupyter_bokeh extension is installed. In an exported notebook this warning is expected.\")}e(374),e(449),t.kernels={},t.embed_items_notebook=function(e,o){if(1!=i.size(e))throw new Error(\"embed_items_notebook expects exactly one document in docs_json\");for(var t=n.Document.from_json(i.values(e)[0]),r=0,s=o;r0&&(this.model.value=this.menu.children[this._hover_index].textContent,this.input_el.focus(),this._hide_menu())},t.prototype._update_completions=function(e){s.empty(this.menu);for(var t=0,n=e;t0&&this.menu.children[0].classList.add(r.bk_active)},t.prototype._show_menu=function(){var e=this;if(!this._open){this._open=!0,this._hover_index=0,this._last_value=this.model.value,s.display(this.menu);var t=function(n){var i=n.target;i instanceof HTMLElement&&!e.el.contains(i)&&(document.removeEventListener(\"click\",t),e._hide_menu())};document.addEventListener(\"click\",t)}},t.prototype._hide_menu=function(){this._open&&(this._open=!1,s.undisplay(this.menu))},t.prototype._menu_click=function(e){e.target!=e.currentTarget&&e.target instanceof Element&&(this.model.value=e.target.textContent,this.input_el.focus(),this._hide_menu())},t.prototype._menu_hover=function(e){if(e.target!=e.currentTarget&&e.target instanceof Element){var t=0;for(t=0;t0&&(this.menu.children[this._hover_index].classList.remove(r.bk_active),this._hover_index=u.clamp(e,0,t-1),this.menu.children[this._hover_index].classList.add(r.bk_active))},t.prototype._keydown=function(e){},t.prototype._keyup=function(e){switch(e.keyCode){case s.Keys.Enter:this.change_input();break;case s.Keys.Esc:this._hide_menu();break;case s.Keys.Up:this._bump_hover(this._hover_index-1);break;case s.Keys.Down:this._bump_hover(this._hover_index+1);break;default:var t=this.input_el.value;if(t.length *:not(:first-child) {\\n margin-left: 5px;\\n}\\n.bk-root .bk-input-group input[type=\"checkbox\"] + span,\\n.bk-root .bk-input-group input[type=\"radio\"] + span {\\n position: relative;\\n top: -2px;\\n margin-left: 3px;\\n}\\n'),t.bk_input=\"bk-input\",t.bk_input_group=\"bk-input-group\"},\n", + " 482: function _(t,n,i){var e=t(113),o=t(474),u=t(376),c=t(121),r=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return e.__extends(n,t),n.prototype.click=function(){this.model.clicks=this.model.clicks+1,this.model.trigger_event(new u.ButtonClick),t.prototype.click.call(this)},n}(o.AbstractButtonView);i.ButtonView=r,r.__name__=\"ButtonView\";var l=function(t){function n(n){return t.call(this,n)||this}return e.__extends(n,t),n.init_Button=function(){this.prototype.default_view=r,this.define({clicks:[c.Number,0]}),this.override({label:\"Button\"})},n}(o.AbstractButton);i.Button=l,l.__name__=\"Button\",l.init_Button()},\n", + " 483: function _(t,e,o){var n=t(113),i=t(484),u=t(163),c=t(117),r=t(121),a=t(240),h=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(e,t),Object.defineProperty(e.prototype,\"active\",{get:function(){return new c.Set(this.model.active)},enumerable:!0,configurable:!0}),e.prototype.change_active=function(t){var e=this.active;e.toggle(t),this.model.active=e.values,null!=this.model.callback&&this.model.callback.execute(this.model)},e.prototype._update_active=function(){var t=this.active;this._buttons.forEach(function(e,o){u.classes(e).toggle(a.bk_active,t.has(o))})},e}(i.ButtonGroupView);o.CheckboxButtonGroupView=h,h.__name__=\"CheckboxButtonGroupView\";var l=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.init_CheckboxButtonGroup=function(){this.prototype.default_view=h,this.define({active:[r.Array,[]]})},e}(i.ButtonGroup);o.CheckboxButtonGroup=l,l.__name__=\"CheckboxButtonGroup\",l.init_CheckboxButtonGroup()},\n", + " 484: function _(t,n,e){var o=t(113),i=t(475),r=t(163),u=t(121),a=t(347),s=function(t){function n(){return null!==t&&t.apply(this,arguments)||this}return o.__extends(n,t),n.prototype.connect_signals=function(){var n=this;t.prototype.connect_signals.call(this);var e=this.model.properties;this.on_change(e.button_type,function(){return n.render()}),this.on_change(e.labels,function(){return n.render()}),this.on_change(e.active,function(){return n._update_active()})},n.prototype.render=function(){var n=this;t.prototype.render.call(this),this._buttons=this.model.labels.map(function(t,e){var o=r.div({class:[a.bk_btn,a.bk_btn_type(n.model.button_type)],disabled:n.model.disabled},t);return o.addEventListener(\"click\",function(){return n.change_active(e)}),o}),this._update_active();var e=r.div({class:a.bk_btn_group},this._buttons);this.el.appendChild(e)},n}(i.ControlView);e.ButtonGroupView=s,s.__name__=\"ButtonGroupView\";var _=function(t){function n(n){return t.call(this,n)||this}return o.__extends(n,t),n.init_ButtonGroup=function(){this.define({labels:[u.Array,[]],button_type:[u.ButtonType,\"default\"],callback:[u.Any]})},n}(i.Control);e.ButtonGroup=_,_.__name__=\"ButtonGroup\",_.init_ButtonGroup()},\n", + " 485: function _(e,t,n){var i=e(113),l=e(486),o=e(163),a=e(110),r=e(117),c=e(121),u=e(240),h=e(481),p=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(t,e),t.prototype.render=function(){var t=this;e.prototype.render.call(this);var n=o.div({class:[h.bk_input_group,this.model.inline?u.bk_inline:null]});this.el.appendChild(n);for(var i=this.model,l=i.active,r=i.labels,c=function(e){var i=o.input({type:\"checkbox\",value:\"\"+e});i.addEventListener(\"change\",function(){return t.change_active(e)}),p.model.disabled&&(i.disabled=!0),a.includes(l,e)&&(i.checked=!0);var c=o.label({},i,o.span({},r[e]));n.appendChild(c)},p=this,s=0;sn||this._o.position.indexOf(\"right\")>-1&&a-e+t.offsetWidth>0)&&(a=a-e+t.offsetWidth),(this._o.reposition&&r+i>o+s||this._o.position.indexOf(\"top\")>-1&&r-i-t.offsetHeight>0)&&(r=r-i-t.offsetHeight),this.el.style.left=a+\"px\",this.el.style.top=r+\"px\"}};var d=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(e,t),e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this),this.connect(this.model.change,function(){return e.render()})},e.prototype.render=function(){var e=this;null!=this._picker&&this._picker.destroy(),t.prototype.render.call(this),this.input_el=s.input({type:\"text\",class:r.bk_input,disabled:this.model.disabled}),this.group_el.appendChild(this.input_el),this._picker=new a({field:this.input_el,defaultDate:this._unlocal_date(new Date(this.model.value)),setDefaultDate:!0,minDate:null!=this.model.min_date?this._unlocal_date(new Date(this.model.min_date)):void 0,maxDate:null!=this.model.max_date?this._unlocal_date(new Date(this.model.max_date)):void 0,onSelect:function(t){return e._on_select(t)}}),this._root_element.appendChild(this._picker.el)},e.prototype._unlocal_date=function(t){var e=6e4*t.getTimezoneOffset();t.setTime(t.getTime()-e);var i=t.toISOString().substr(0,10).split(\"-\");return new Date(Number(i[0]),Number(i[1])-1,Number(i[2]))},e.prototype._on_select=function(t){this.model.value=t.toDateString(),this.change_input()},e}(o.InputWidgetView);i.DatePickerView=d,d.__name__=\"DatePickerView\";var h=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.init_DatePicker=function(){this.prototype.default_view=d,this.define({value:[l.Any,(new Date).toDateString()],min_date:[l.Any],max_date:[l.Any]})},e}(o.InputWidget);i.DatePicker=h,h.__name__=\"DatePicker\",h.init_DatePicker()},\n", + " 489: function _(e,t,n){var a=function(e,t,n,a){e.addEventListener(t,n,!!a)},i=function(e,t,n,a){e.removeEventListener(t,n,!!a)},s=function(e,t){return-1!==(\" \"+e.className+\" \").indexOf(\" \"+t+\" \")},o=function(e,t){s(e,t)||(e.className=\"\"===e.className?t:e.className+\" \"+t)},r=function(e,t){var n;e.className=(n=(\" \"+e.className+\" \").replace(\" \"+t+\" \",\" \")).trim?n.trim():n.replace(/^\\s+|\\s+$/g,\"\")},l=function(e){return/Array/.test(Object.prototype.toString.call(e))},h=function(e){return/Date/.test(Object.prototype.toString.call(e))&&!isNaN(e.getTime())},d=function(e){var t=e.getDay();return 0===t||6===t},u=function(e){\n", + " // solution lifted from date.js (MIT license): https://github.com/datejs/Datejs\n", + " return e%4==0&&e%100!=0||e%400==0},c=function(e,t){return[31,u(e)?29:28,31,30,31,30,31,31,30,31,30,31][t]},f=function(e){h(e)&&e.setHours(0,0,0,0)},g=function(e,t){return e.getTime()===t.getTime()},m=function(e,t,n){var a,i;for(a in t)(i=void 0!==e[a])&&\"object\"==typeof t[a]&&null!==t[a]&&void 0===t[a].nodeName?h(t[a])?n&&(e[a]=new Date(t[a].getTime())):l(t[a])?n&&(e[a]=t[a].slice(0)):e[a]=m({},t[a],n):!n&&i||(e[a]=t[a]);return e},p=function(e,t,n){var a;document.createEvent?((a=document.createEvent(\"HTMLEvents\")).initEvent(t,!0,!1),a=m(a,n),e.dispatchEvent(a)):document.createEventObject&&(a=document.createEventObject(),a=m(a,n),e.fireEvent(\"on\"+t,a))},y=function(e){return e.month<0&&(e.year-=Math.ceil(Math.abs(e.month)/12),e.month+=12),e.month>11&&(e.year+=Math.floor(Math.abs(e.month)/12),e.month-=12),e},D={field:null,bound:void 0,ariaLabel:\"Use the arrow keys to pick a date\",position:\"bottom left\",reposition:!0,format:\"YYYY-MM-DD\",toString:null,parse:null,defaultDate:null,setDefaultDate:!1,firstDay:0,formatStrict:!1,minDate:null,maxDate:null,yearRange:10,showWeekNumber:!1,pickWholeWeek:!1,minYear:0,maxYear:9999,minMonth:void 0,maxMonth:void 0,startRange:null,endRange:null,isRTL:!1,yearSuffix:\"\",showMonthAfterYear:!1,showDaysInNextAndPreviousMonths:!1,enableSelectionDaysInNextAndPreviousMonths:!1,numberOfMonths:1,mainCalendar:\"left\",container:void 0,blurFieldOnSelect:!0,i18n:{previousMonth:\"Previous Month\",nextMonth:\"Next Month\",months:[\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"],weekdays:[\"Sunday\",\"Monday\",\"Tuesday\",\"Wednesday\",\"Thursday\",\"Friday\",\"Saturday\"],weekdaysShort:[\"Sun\",\"Mon\",\"Tue\",\"Wed\",\"Thu\",\"Fri\",\"Sat\"]},theme:null,events:[],onSelect:null,onOpen:null,onClose:null,onDraw:null,keyboardInput:!0},b=function(e,t,n){for(t+=e.firstDay;t>=7;)t-=7;return n?e.i18n.weekdaysShort[t]:e.i18n.weekdays[t]},_=function(e){var t=[],n=\"false\";if(e.isEmpty){if(!e.showDaysInNextAndPreviousMonths)return'';t.push(\"is-outside-current-month\"),e.enableSelectionDaysInNextAndPreviousMonths||t.push(\"is-selection-disabled\")}return e.isDisabled&&t.push(\"is-disabled\"),e.isToday&&t.push(\"is-today\"),e.isSelected&&(t.push(\"is-selected\"),n=\"true\"),e.hasEvent&&t.push(\"has-event\"),e.isInRange&&t.push(\"is-inrange\"),e.isStartRange&&t.push(\"is-startrange\"),e.isEndRange&&t.push(\"is-endrange\"),'\"},v=function(e,t,n){return''+function(e){e.setHours(0,0,0,0);var t=e.getDate(),n=e.getDay(),a=function(e){return(e+7-1)%7};e.setDate(t+3-a(n));var i=new Date(e.getFullYear(),0,4),s=(e.getTime()-i.getTime())/864e5;return 1+Math.round((s-3+a(i.getDay()))/7)}(new Date(n,t,e))+\"\"},w=function(e,t,n,a){return''+(t?e.reverse():e).join(\"\")+\"\"},k=function(e,t,n,a,i,s){var o,r,h,d,u,c=e._o,f=n===c.minYear,g=n===c.maxYear,m='
',p=!0,y=!0;for(h=[],o=0;o<12;o++)h.push('\");for(d='
'+c.i18n.months[a]+'
\",l(c.yearRange)?(o=c.yearRange[0],r=c.yearRange[1]+1):(o=n-c.yearRange,r=1+n+c.yearRange),h=[];o=c.minYear&&h.push('\");return u='
'+n+c.yearSuffix+'
\",c.showMonthAfterYear?m+=u+d:m+=d+u,f&&(0===a||c.minMonth>=a)&&(p=!1),g&&(11===a||c.maxMonth<=a)&&(y=!1),0===t&&(m+='\"),t===e._o.numberOfMonths-1&&(m+='\"),m+\"
\"},M=function(e,t,n){return''+function(e){var t,n=[];for(e.showWeekNumber&&n.push(\"\"),t=0;t<7;t++)n.push('\");return\"\"+(e.isRTL?n.reverse():n).join(\"\")+\"\"}(e)+(\"\"+t.join(\"\")+\"\")+\"
'+b(e,t,!0)+\"
\"},x=function(e){var t=this,n=t.config(e);t._onMouseDown=function(e){if(t._v){var a=(e=e||window.event).target||e.srcElement;if(a)if(s(a,\"is-disabled\")||(!s(a,\"pika-button\")||s(a,\"is-empty\")||s(a.parentNode,\"is-disabled\")?s(a,\"pika-prev\")?t.prevMonth():s(a,\"pika-next\")&&t.nextMonth():(t.setDate(new Date(a.getAttribute(\"data-pika-year\"),a.getAttribute(\"data-pika-month\"),a.getAttribute(\"data-pika-day\"))),n.bound&&setTimeout(function(){t.hide(),n.blurFieldOnSelect&&n.field&&n.field.blur()},100))),s(a,\"pika-select\"))t._c=!0;else{if(!e.preventDefault)return e.returnValue=!1,!1;e.preventDefault()}}},t._onChange=function(e){var n=(e=e||window.event).target||e.srcElement;n&&(s(n,\"pika-select-month\")?t.gotoMonth(n.value):s(n,\"pika-select-year\")&&t.gotoYear(n.value))},t._onKeyChange=function(e){if(e=e||window.event,t.isVisible())switch(e.keyCode){case 13:case 27:n.field&&n.field.blur();break;case 37:t.adjustDate(\"subtract\",1);break;case 38:t.adjustDate(\"subtract\",7);break;case 39:t.adjustDate(\"add\",1);break;case 40:t.adjustDate(\"add\",7);break;case 8:case 46:t.setDate(null)}},t._parseFieldValue=function(){return n.parse?n.parse(n.field.value,n.format):new Date(Date.parse(n.field.value))},t._onInputChange=function(e){var n;e.firedBy!==t&&(n=t._parseFieldValue(),h(n)&&t.setDate(n),t._v||t.show())},t._onInputFocus=function(){t.show()},t._onInputClick=function(){t.show()},t._onInputBlur=function(){var e=document.activeElement;do{if(s(e,\"pika-single\"))return}while(e=e.parentNode);t._c||(t._b=setTimeout(function(){t.hide()},50)),t._c=!1},t._onClick=function(e){var a=(e=e||window.event).target||e.srcElement,i=a;if(a){do{if(s(i,\"pika-single\")||i===n.trigger)return}while(i=i.parentNode);t._v&&a!==n.trigger&&i!==n.trigger&&t.hide()}},t.el=document.createElement(\"div\"),t.el.className=\"pika-single\"+(n.isRTL?\" is-rtl\":\"\")+(n.theme?\" \"+n.theme:\"\"),a(t.el,\"mousedown\",t._onMouseDown,!0),a(t.el,\"touchend\",t._onMouseDown,!0),a(t.el,\"change\",t._onChange),n.keyboardInput&&a(document,\"keydown\",t._onKeyChange),n.field&&(n.container?n.container.appendChild(t.el):n.bound?document.body.appendChild(t.el):n.field.parentNode.insertBefore(t.el,n.field.nextSibling),a(n.field,\"change\",t._onInputChange),n.defaultDate||(n.defaultDate=t._parseFieldValue(),n.setDefaultDate=!0));var i=n.defaultDate;h(i)?n.setDefaultDate?t.setDate(i,!0):t.gotoDate(i):t.gotoDate(new Date),n.bound?(this.hide(),t.el.className+=\" is-bound\",a(n.trigger,\"click\",t._onInputClick),a(n.trigger,\"focus\",t._onInputFocus),a(n.trigger,\"blur\",t._onInputBlur)):this.show()};x.prototype={config:function(e){this._o||(this._o=m({},D,!0));var t=m(this._o,e,!0);t.isRTL=!!t.isRTL,t.field=t.field&&t.field.nodeName?t.field:null,t.theme=\"string\"==typeof t.theme&&t.theme?t.theme:null,t.bound=!!(void 0!==t.bound?t.field&&t.bound:t.field),t.trigger=t.trigger&&t.trigger.nodeName?t.trigger:t.field,t.disableWeekends=!!t.disableWeekends,t.disableDayFn=\"function\"==typeof t.disableDayFn?t.disableDayFn:null;var n=parseInt(t.numberOfMonths,10)||1;if(t.numberOfMonths=n>4?4:n,h(t.minDate)||(t.minDate=!1),h(t.maxDate)||(t.maxDate=!1),t.minDate&&t.maxDate&&t.maxDate100&&(t.yearRange=100);return t},toString:function(e){return e=e||this._o.format,h(this._d)?this._o.toString?this._o.toString(this._d,e):this._d.toDateString():\"\"},getDate:function(){return h(this._d)?new Date(this._d.getTime()):null},setDate:function(e,t){if(!e)return this._d=null,this._o.field&&(this._o.field.value=\"\",p(this._o.field,\"change\",{firedBy:this})),this.draw();if(\"string\"==typeof e&&(e=new Date(Date.parse(e))),h(e)){var n=this._o.minDate,a=this._o.maxDate;h(n)&&ea&&(e=a),this._d=new Date(e.getTime()),f(this._d),this.gotoDate(this._d),this._o.field&&(this._o.field.value=this.toString(),p(this._o.field,\"change\",{firedBy:this})),t||\"function\"!=typeof this._o.onSelect||this._o.onSelect.call(this,this.getDate())}},clear:function(){this.setDate(null)},gotoDate:function(e){var t=!0;if(h(e)){if(this.calendars){var n=new Date(this.calendars[0].year,this.calendars[0].month,1),a=new Date(this.calendars[this.calendars.length-1].year,this.calendars[this.calendars.length-1].month,1),i=e.getTime();a.setMonth(a.getMonth()+1),a.setDate(a.getDate()-1),t=i=i&&(this._y=i,!isNaN(o)&&this._m>o&&(this._m=o));for(var l=0;l\";this.el.innerHTML=r,n.bound&&\"hidden\"!==n.field.type&&setTimeout(function(){n.trigger.focus()},1),\"function\"==typeof this._o.onDraw&&this._o.onDraw(this),n.bound&&n.field.setAttribute(\"aria-label\",n.ariaLabel)}},adjustPosition:function(){var e,t,n,a,i,s,l,h,d,u,c,f;if(!this._o.container){if(this.el.style.position=\"absolute\",t=e=this._o.trigger,n=this.el.offsetWidth,a=this.el.offsetHeight,i=window.innerWidth||document.documentElement.clientWidth,s=window.innerHeight||document.documentElement.clientHeight,l=window.pageYOffset||document.body.scrollTop||document.documentElement.scrollTop,c=!0,f=!0,\"function\"==typeof e.getBoundingClientRect)h=(u=e.getBoundingClientRect()).left+window.pageXOffset,d=u.bottom+window.pageYOffset;else for(h=t.offsetLeft,d=t.offsetTop+t.offsetHeight;t=t.offsetParent;)h+=t.offsetLeft,d+=t.offsetTop;(this._o.reposition&&h+n>i||this._o.position.indexOf(\"right\")>-1&&h-n+e.offsetWidth>0)&&(h=h-n+e.offsetWidth,c=!1),(this._o.reposition&&d+a>s+l||this._o.position.indexOf(\"top\")>-1&&d-a-e.offsetHeight>0)&&(d=d-a-e.offsetHeight,f=!1),this.el.style.left=h+\"px\",this.el.style.top=d+\"px\",o(this.el,c?\"left-aligned\":\"right-aligned\"),o(this.el,f?\"bottom-aligned\":\"top-aligned\"),r(this.el,c?\"right-aligned\":\"left-aligned\"),r(this.el,f?\"top-aligned\":\"bottom-aligned\")}},render:function(e,t,n){var a=this._o,i=new Date,s=c(e,t),o=new Date(e,t,1).getDay(),r=[],l=[];f(i),a.firstDay>0&&(o-=a.firstDay)<0&&(o+=7);for(var u=0===t?11:t-1,m=11===t?0:t+1,p=0===t?e-1:e,y=11===t?e+1:e,D=c(p,u),b=s+o,k=b;k>7;)k-=7;b+=7-k;for(var x=!1,R=0,N=0;R=s+o,O=R-o+1,E=t,j=e,F=a.startRange&&g(a.startRange,S),W=a.endRange&&g(a.endRange,S),A=a.startRange&&a.endRange&&a.startRangea.maxDate||a.disableWeekends&&d(S)||a.disableDayFn&&a.disableDayFn(S),isEmpty:Y,isStartRange:F,isEndRange:W,isInRange:A,showDaysInNextAndPreviousMonths:a.showDaysInNextAndPreviousMonths,enableSelectionDaysInNextAndPreviousMonths:a.enableSelectionDaysInNextAndPreviousMonths};a.pickWholeWeek&&T&&(x=!0),l.push(_(L)),7==++N&&(a.showWeekNumber&&l.unshift(v(R-o,t,e)),r.push(w(l,a.isRTL,a.pickWholeWeek,x)),l=[],N=0,x=!1)}return M(a,r,n)},isVisible:function(){return this._v},show:function(){this.isVisible()||(this._v=!0,this.draw(),r(this.el,\"is-hidden\"),this._o.bound&&(a(document,\"click\",this._onClick),this.adjustPosition()),\"function\"==typeof this._o.onOpen&&this._o.onOpen.call(this))},hide:function(){var e=this._v;!1!==e&&(this._o.bound&&i(document,\"click\",this._onClick),this.el.style.position=\"static\",this.el.style.left=\"auto\",this.el.style.top=\"auto\",o(this.el,\"is-hidden\"),this._v=!1,void 0!==e&&\"function\"==typeof this._o.onClose&&this._o.onClose.call(this))},destroy:function(){var e=this._o;this.hide(),i(this.el,\"mousedown\",this._onMouseDown,!0),i(this.el,\"touchend\",this._onMouseDown,!0),i(this.el,\"change\",this._onChange),e.keyboardInput&&i(document,\"keydown\",this._onKeyChange),e.field&&(i(e.field,\"change\",this._onInputChange),e.bound&&(i(e.trigger,\"click\",this._onInputClick),i(e.trigger,\"focus\",this._onInputFocus),i(e.trigger,\"blur\",this._onInputBlur))),this.el.parentNode&&this.el.parentNode.removeChild(this.el)}},t.exports=x},\n", + " 490: function _(n,o,t){n(164),n(163).styles.append('.bk-root {\\n @charset \"UTF-8\";\\n /*!\\n * Pikaday\\n * Copyright © 2014 David Bushell | BSD & MIT license | https://dbushell.com/\\n */\\n /*\\nclear child float (pika-lendar), using the famous micro clearfix hack\\nhttp://nicolasgallagher.com/micro-clearfix-hack/\\n*/\\n /* styling for abbr */\\n}\\n.bk-root .pika-single {\\n z-index: 9999;\\n display: block;\\n position: relative;\\n color: #333;\\n background: #fff;\\n border: 1px solid #ccc;\\n border-bottom-color: #bbb;\\n font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\\n}\\n.bk-root .pika-single:before,\\n.bk-root .pika-single:after {\\n content: \" \";\\n display: table;\\n}\\n.bk-root .pika-single:after {\\n clear: both;\\n}\\n.bk-root .pika-single.is-hidden {\\n display: none;\\n}\\n.bk-root .pika-single.is-bound {\\n position: absolute;\\n box-shadow: 0 5px 15px -5px rgba(0, 0, 0, 0.5);\\n}\\n.bk-root .pika-lendar {\\n float: left;\\n width: 240px;\\n margin: 8px;\\n}\\n.bk-root .pika-title {\\n position: relative;\\n text-align: center;\\n}\\n.bk-root .pika-label {\\n display: inline-block;\\n position: relative;\\n z-index: 9999;\\n overflow: hidden;\\n margin: 0;\\n padding: 5px 3px;\\n font-size: 14px;\\n line-height: 20px;\\n font-weight: bold;\\n background-color: #fff;\\n}\\n.bk-root .pika-title select {\\n cursor: pointer;\\n position: absolute;\\n z-index: 9998;\\n margin: 0;\\n left: 0;\\n top: 5px;\\n opacity: 0;\\n}\\n.bk-root .pika-prev,\\n.bk-root .pika-next {\\n display: block;\\n cursor: pointer;\\n position: relative;\\n outline: none;\\n border: 0;\\n padding: 0;\\n width: 20px;\\n height: 30px;\\n /* hide text using text-indent trick, using width value (it\\'s enough) */\\n text-indent: 20px;\\n white-space: nowrap;\\n overflow: hidden;\\n background-color: transparent;\\n background-position: center center;\\n background-repeat: no-repeat;\\n background-size: 75% 75%;\\n opacity: 0.5;\\n}\\n.bk-root .pika-prev:hover,\\n.bk-root .pika-next:hover {\\n opacity: 1;\\n}\\n.bk-root .pika-prev,\\n.bk-root .is-rtl .pika-next {\\n float: left;\\n background-image: url(\\'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAeCAYAAAAsEj5rAAAAUklEQVR42u3VMQoAIBADQf8Pgj+OD9hG2CtONJB2ymQkKe0HbwAP0xucDiQWARITIDEBEnMgMQ8S8+AqBIl6kKgHiXqQqAeJepBo/z38J/U0uAHlaBkBl9I4GwAAAABJRU5ErkJggg==\\');\\n}\\n.bk-root .pika-next,\\n.bk-root .is-rtl .pika-prev {\\n float: right;\\n background-image: url(\\'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAeCAYAAAAsEj5rAAAAU0lEQVR42u3VOwoAMAgE0dwfAnNjU26bYkBCFGwfiL9VVWoO+BJ4Gf3gtsEKKoFBNTCoCAYVwaAiGNQGMUHMkjGbgjk2mIONuXo0nC8XnCf1JXgArVIZAQh5TKYAAAAASUVORK5CYII=\\');\\n}\\n.bk-root .pika-prev.is-disabled,\\n.bk-root .pika-next.is-disabled {\\n cursor: default;\\n opacity: 0.2;\\n}\\n.bk-root .pika-select {\\n display: inline-block;\\n}\\n.bk-root .pika-table {\\n width: 100%;\\n border-collapse: collapse;\\n border-spacing: 0;\\n border: 0;\\n}\\n.bk-root .pika-table th,\\n.bk-root .pika-table td {\\n width: 14.28571429%;\\n padding: 0;\\n}\\n.bk-root .pika-table th {\\n color: #999;\\n font-size: 12px;\\n line-height: 25px;\\n font-weight: bold;\\n text-align: center;\\n}\\n.bk-root .pika-button {\\n cursor: pointer;\\n display: block;\\n box-sizing: border-box;\\n -moz-box-sizing: border-box;\\n outline: none;\\n border: 0;\\n margin: 0;\\n width: 100%;\\n padding: 5px;\\n color: #666;\\n font-size: 12px;\\n line-height: 15px;\\n text-align: right;\\n background: #f5f5f5;\\n}\\n.bk-root .pika-week {\\n font-size: 11px;\\n color: #999;\\n}\\n.bk-root .is-today .pika-button {\\n color: #33aaff;\\n font-weight: bold;\\n}\\n.bk-root .is-selected .pika-button,\\n.bk-root .has-event .pika-button {\\n color: #fff;\\n font-weight: bold;\\n background: #33aaff;\\n box-shadow: inset 0 1px 3px #178fe5;\\n border-radius: 3px;\\n}\\n.bk-root .has-event .pika-button {\\n background: #005da9;\\n box-shadow: inset 0 1px 3px #0076c9;\\n}\\n.bk-root .is-disabled .pika-button,\\n.bk-root .is-inrange .pika-button {\\n background: #D5E9F7;\\n}\\n.bk-root .is-startrange .pika-button {\\n color: #fff;\\n background: #6CB31D;\\n box-shadow: none;\\n border-radius: 3px;\\n}\\n.bk-root .is-endrange .pika-button {\\n color: #fff;\\n background: #33aaff;\\n box-shadow: none;\\n border-radius: 3px;\\n}\\n.bk-root .is-disabled .pika-button {\\n pointer-events: none;\\n cursor: default;\\n color: #999;\\n opacity: 0.3;\\n}\\n.bk-root .is-outside-current-month .pika-button {\\n color: #999;\\n opacity: 0.3;\\n}\\n.bk-root .is-selection-disabled {\\n pointer-events: none;\\n cursor: default;\\n}\\n.bk-root .pika-button:hover,\\n.bk-root .pika-row.pick-whole-week:hover .pika-button {\\n color: #fff;\\n background: #ff8000;\\n box-shadow: none;\\n border-radius: 3px;\\n}\\n.bk-root .pika-table abbr {\\n border-bottom: none;\\n cursor: help;\\n}\\n')},\n", + " 491: function _(e,t,n){var r=e(113),i=e(252),a=e(492),_=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return r.__extends(t,e),t}(a.AbstractRangeSliderView);n.DateRangeSliderView=_,_.__name__=\"DateRangeSliderView\";var o=function(e){function t(t){var n=e.call(this,t)||this;return n.behaviour=\"drag\",n.connected=[!1,!0,!1],n}return r.__extends(t,e),t.init_DateRangeSlider=function(){this.prototype.default_view=_,this.override({format:\"%d %b %Y\"})},t.prototype._formatter=function(e,t){return i(e,t)},t}(a.AbstractSlider);n.DateRangeSlider=o,o.__name__=\"DateRangeSlider\",o.init_DateRangeSlider()},\n", + " 492: function _(t,e,i){var l=t(113),r=t(493),n=t(121),o=t(163),s=t(110),a=t(119),c=t(475),d=t(494),h=\"bk-noUi-\",_=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return l.__extends(e,t),Object.defineProperty(e.prototype,\"noUiSlider\",{get:function(){return this.slider_el.noUiSlider},enumerable:!0,configurable:!0}),e.prototype.initialize=function(){t.prototype.initialize.call(this),this._init_callback()},e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this);var i=this.model.properties,l=i.callback,r=i.callback_policy,n=i.callback_throttle;this.on_change([l,r,n],function(){return e._init_callback()});var o=this.model.properties,s=o.start,a=o.end,c=o.value,d=o.step,h=o.title;this.on_change([s,a,c,d],function(){var t=e._calc_to(),i=t.start,l=t.end,r=t.value,n=t.step;e.noUiSlider.updateOptions({range:{min:i,max:l},start:r,step:n})});var _=this.model.properties.bar_color;this.on_change(_,function(){e._set_bar_color()}),this.on_change([c,h],function(){return e._update_title()})},e.prototype._init_callback=function(){var t=this,e=this.model.callback,i=function(){null!=e&&e.execute(t.model),t.model.value_throttled=t.model.value};switch(this.model.callback_policy){case\"continuous\":this.callback_wrapper=i;break;case\"throttle\":this.callback_wrapper=a.throttle(i,this.model.callback_throttle);break;default:this.callback_wrapper=void 0}},e.prototype._update_title=function(){var t=this;o.empty(this.title_el);var e=null==this.model.title||0==this.model.title.length&&!this.model.show_value;if(this.title_el.style.display=e?\"none\":\"\",!e&&(0!=this.model.title.length&&(this.title_el.textContent=this.model.title+\": \"),this.model.show_value)){var i=this._calc_to().value.map(function(e){return t.model.pretty(e)}).join(\" .. \");this.title_el.appendChild(o.span({class:d.bk_slider_value},i))}},e.prototype._set_bar_color=function(){this.model.disabled||(this.slider_el.querySelector(\".bk-noUi-connect\").style.backgroundColor=this.model.bar_color)},e.prototype._keypress_handle=function(t,e){void 0===e&&(e=0);var i=this._calc_to(),l=i.start,r=i.value,n=i.end,o=i.step,s=2==r.length,a=l,c=n;switch(s&&0==e?c=r[1]:s&&1==e&&(a=r[0]),t.which){case 37:r[e]=Math.max(r[e]-o,a);break;case 39:r[e]=Math.min(r[e]+o,c);break;default:return}s?(this.model.value=r,this.model.properties.value.change.emit()):this.model.value=r[0],this.noUiSlider.set(r),null!=this.callback_wrapper&&this.callback_wrapper()},e.prototype.render=function(){var e=this;t.prototype.render.call(this);var i,l=this._calc_to(),n=l.start,a=l.end,c=l.value,_=l.step;if(this.model.tooltips){var u={to:function(t){return e.model.pretty(t)}};i=s.repeat(u,c.length)}else i=!1;if(null==this.slider_el){this.slider_el=o.div(),r.create(this.slider_el,{cssPrefix:h,range:{min:n,max:a},start:c,step:_,behaviour:this.model.behaviour,connect:this.model.connected,tooltips:i,orientation:this.model.orientation,direction:this.model.direction}),this.noUiSlider.on(\"slide\",function(t,i,l){return e._slide(l)}),this.noUiSlider.on(\"change\",function(t,i,l){return e._change(l)}),this._set_keypress_handles();var p=function(t,l){i&&(e.slider_el.querySelectorAll(\".bk-noUi-handle\")[t].querySelector(\".bk-noUi-tooltip\").style.display=l?\"block\":\"\")};this.noUiSlider.on(\"start\",function(t,e){return p(e,!0)}),this.noUiSlider.on(\"end\",function(t,e){return p(e,!1)})}else this.noUiSlider.updateOptions({range:{min:n,max:a},start:c,step:_});this._set_bar_color(),this.model.disabled?this.slider_el.setAttribute(\"disabled\",\"true\"):this.slider_el.removeAttribute(\"disabled\"),this.title_el=o.div({class:d.bk_slider_title}),this._update_title(),this.group_el=o.div({class:d.bk_input_group},this.title_el,this.slider_el),this.el.appendChild(this.group_el)},e.prototype._slide=function(t){this.model.value=this._calc_from(t),null!=this.callback_wrapper&&this.callback_wrapper()},e.prototype._change=function(t){switch(this.model.value=this._calc_from(t),this.model.value_throttled=this.model.value,this.model.callback_policy){case\"mouseup\":case\"throttle\":null!=this.model.callback&&this.model.callback.execute(this.model)}},e}(c.ControlView);_.__name__=\"AbstractBaseSliderView\";var u=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return l.__extends(e,t),e.prototype._calc_to=function(){return{start:this.model.start,end:this.model.end,value:[this.model.value],step:this.model.step}},e.prototype._calc_from=function(t){var e=t[0];return Number.isInteger(this.model.start)&&Number.isInteger(this.model.end)&&Number.isInteger(this.model.step)?Math.round(e):e},e.prototype._set_keypress_handles=function(){var t=this,e=this.slider_el.querySelector(\".bk-noUi-handle\");e.setAttribute(\"tabindex\",\"0\"),e.addEventListener(\"keydown\",function(e){return t._keypress_handle(e)})},e}(_);i.AbstractSliderView=u,u.__name__=\"AbstractSliderView\";var p=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return l.__extends(e,t),e.prototype._calc_to=function(){return{start:this.model.start,end:this.model.end,value:this.model.value,step:this.model.step}},e.prototype._calc_from=function(t){return t},e.prototype._set_keypress_handles=function(){var t=this,e=this.slider_el.querySelector(\".bk-noUi-handle-lower\"),i=this.slider_el.querySelector(\".bk-noUi-handle-upper\");e.setAttribute(\"tabindex\",\"0\"),e.addEventListener(\"keydown\",function(e){return t._keypress_handle(e,0)}),i.setAttribute(\"tabindex\",\"1\"),i.addEventListener(\"keydown\",function(e){return t._keypress_handle(e,1)})},e}(_);i.AbstractRangeSliderView=p,p.__name__=\"AbstractRangeSliderView\";var m=function(t){function e(e){var i=t.call(this,e)||this;return i.connected=!1,i}return l.__extends(e,t),e.init_AbstractSlider=function(){this.define({title:[n.String,\"\"],show_value:[n.Boolean,!0],start:[n.Any],end:[n.Any],value:[n.Any],value_throttled:[n.Any],step:[n.Number,1],format:[n.String],direction:[n.Any,\"ltr\"],tooltips:[n.Boolean,!0],callback:[n.Any],callback_throttle:[n.Number,200],callback_policy:[n.SliderCallbackPolicy,\"throttle\"],bar_color:[n.Color,\"#e6e6e6\"]})},e.prototype._formatter=function(t,e){return\"\"+t},e.prototype.pretty=function(t){return this._formatter(t,this.format)},e}(c.Control);i.AbstractSlider=m,m.__name__=\"AbstractSlider\",m.init_AbstractSlider()},\n", + " 493: function _(t,e,r){\n", + " /*! nouislider - 10.1.0 - 2017-07-28 17:11:18 */var n;n=function(){\"use strict\";var t=\"10.1.0\";function e(t){t.preventDefault()}function r(t){return\"number\"==typeof t&&!isNaN(t)&&isFinite(t)}function n(t,e,r){r>0&&(s(t,e),setTimeout(function(){a(t,e)},r))}function i(t){return Array.isArray(t)?t:[t]}function o(t){var e=(t=String(t)).split(\".\");return e.length>1?e[1].length:0}function s(t,e){t.classList?t.classList.add(e):t.className+=\" \"+e}function a(t,e){t.classList?t.classList.remove(e):t.className=t.className.replace(new RegExp(\"(^|\\\\b)\"+e.split(\" \").join(\"|\")+\"(\\\\b|$)\",\"gi\"),\" \")}function l(t){var e=void 0!==window.pageXOffset,r=\"CSS1Compat\"===(t.compatMode||\"\");return{x:e?window.pageXOffset:r?t.documentElement.scrollLeft:t.body.scrollLeft,y:e?window.pageYOffset:r?t.documentElement.scrollTop:t.body.scrollTop}}function u(t,e){return 100/(e-t)}function c(t,e){return 100*e/(t[1]-t[0])}function p(t,e){for(var r=1;t>=e[r];)r+=1;return r}function f(t,e,r){if(r>=t.slice(-1)[0])return 100;var n,i,o,s,a=p(r,t);return n=t[a-1],i=t[a],o=e[a-1],s=e[a],o+function(t,e){return c(t,t[0]<0?e+Math.abs(t[0]):e-t[0])}([n,i],r)/u(o,s)}function d(t,e,r,n){if(100===n)return n;var i,o,s=p(n,t);return r?n-(i=t[s-1])>((o=t[s])-i)/2?o:i:e[s-1]?t[s-1]+function(t,e){return Math.round(t/e)*e}(n-t[s-1],e[s-1]):n}function h(e,n,i){var o;if(\"number\"==typeof n&&(n=[n]),\"[object Array]\"!==Object.prototype.toString.call(n))throw new Error(\"noUiSlider (\"+t+\"): 'range' contains invalid value.\");if(!r(o=\"min\"===e?0:\"max\"===e?100:parseFloat(e))||!r(n[0]))throw new Error(\"noUiSlider (\"+t+\"): 'range' value isn't numeric.\");i.xPct.push(o),i.xVal.push(n[0]),o?i.xSteps.push(!isNaN(n[1])&&n[1]):isNaN(n[1])||(i.xSteps[0]=n[1]),i.xHighestCompleteStep.push(0)}function m(t,e,r){if(!e)return!0;r.xSteps[t]=c([r.xVal[t],r.xVal[t+1]],e)/u(r.xPct[t],r.xPct[t+1]);var n=(r.xVal[t+1]-r.xVal[t])/r.xNumSteps[t],i=Math.ceil(Number(n.toFixed(3))-1),o=r.xVal[t]+r.xNumSteps[t]*i;r.xHighestCompleteStep[t]=o}function g(t,e,r){this.xPct=[],this.xVal=[],this.xSteps=[r||!1],this.xNumSteps=[!1],this.xHighestCompleteStep=[],this.snap=e;var n,i=[];for(n in t)t.hasOwnProperty(n)&&i.push([t[n],n]);for(i.length&&\"object\"==typeof i[0][0]?i.sort(function(t,e){return t[0][0]-e[0][0]}):i.sort(function(t,e){return t[0]-e[0]}),n=0;n=100)return t.slice(-1)[0];var n,i=p(r,e);return function(t,e){return e*(t[1]-t[0])/100+t[0]}([t[i-1],t[i]],(r-(n=e[i-1]))*u(n,e[i]))}(this.xVal,this.xPct,t)},g.prototype.getStep=function(t){return t=d(this.xPct,this.xSteps,this.snap,t)},g.prototype.getNearbySteps=function(t){var e=p(t,this.xPct);return{stepBefore:{startValue:this.xVal[e-2],step:this.xNumSteps[e-2],highestStep:this.xHighestCompleteStep[e-2]},thisStep:{startValue:this.xVal[e-1],step:this.xNumSteps[e-1],highestStep:this.xHighestCompleteStep[e-1]},stepAfter:{startValue:this.xVal[e-0],step:this.xNumSteps[e-0],highestStep:this.xHighestCompleteStep[e-0]}}},g.prototype.countStepDecimals=function(){var t=this.xNumSteps.map(o);return Math.max.apply(null,t)},g.prototype.convert=function(t){return this.getStep(this.toStepping(t))};var v={to:function(t){return void 0!==t&&t.toFixed(2)},from:Number};function b(e){if(function(t){return\"object\"==typeof t&&\"function\"==typeof t.to&&\"function\"==typeof t.from}(e))return!0;throw new Error(\"noUiSlider (\"+t+\"): 'format' requires 'to' and 'from' methods.\")}function S(e,n){if(!r(n))throw new Error(\"noUiSlider (\"+t+\"): 'step' is not numeric.\");e.singleStep=n}function w(e,r){if(\"object\"!=typeof r||Array.isArray(r))throw new Error(\"noUiSlider (\"+t+\"): 'range' is not an object.\");if(void 0===r.min||void 0===r.max)throw new Error(\"noUiSlider (\"+t+\"): Missing 'min' or 'max' in 'range'.\");if(r.min===r.max)throw new Error(\"noUiSlider (\"+t+\"): 'range' 'min' and 'max' cannot be equal.\");e.spectrum=new g(r,e.snap,e.singleStep)}function x(e,r){if(r=i(r),!Array.isArray(r)||!r.length)throw new Error(\"noUiSlider (\"+t+\"): 'start' option is incorrect.\");e.handles=r.length,e.start=r}function y(e,r){if(e.snap=r,\"boolean\"!=typeof r)throw new Error(\"noUiSlider (\"+t+\"): 'snap' option must be a boolean.\")}function E(e,r){if(e.animate=r,\"boolean\"!=typeof r)throw new Error(\"noUiSlider (\"+t+\"): 'animate' option must be a boolean.\")}function C(e,r){if(e.animationDuration=r,\"number\"!=typeof r)throw new Error(\"noUiSlider (\"+t+\"): 'animationDuration' option must be a number.\")}function N(e,r){var n,i=[!1];if(\"lower\"===r?r=[!0,!1]:\"upper\"===r&&(r=[!1,!0]),!0===r||!1===r){for(n=1;n=50)throw new Error(\"noUiSlider (\"+t+\"): 'padding' option must be less than half the range.\")}}function O(e,r){switch(r){case\"ltr\":e.dir=0;break;case\"rtl\":e.dir=1;break;default:throw new Error(\"noUiSlider (\"+t+\"): 'direction' option was not recognized.\")}}function k(e,r){if(\"string\"!=typeof r)throw new Error(\"noUiSlider (\"+t+\"): 'behaviour' must be a string containing options.\");var n=r.indexOf(\"tap\")>=0,i=r.indexOf(\"drag\")>=0,o=r.indexOf(\"fixed\")>=0,s=r.indexOf(\"snap\")>=0,a=r.indexOf(\"hover\")>=0;if(o){if(2!==e.handles)throw new Error(\"noUiSlider (\"+t+\"): 'fixed' behaviour must be used with 2 handles\");P(e,e.start[1]-e.start[0])}e.events={tap:n||s,drag:i,fixed:o,snap:s,hover:a}}function V(e,r){if(e.multitouch=r,\"boolean\"!=typeof r)throw new Error(\"noUiSlider (\"+t+\"): 'multitouch' option must be a boolean.\")}function F(e,r){if(!1!==r)if(!0===r){e.tooltips=[];for(var n=0;n-1?1:\"steps\"===e?2:0,!o&&a&&(h=0),c===S&&l||(i[f.toFixed(5)]=[c,h]),u=f}}),i}(n,r,o),a=e.format||{to:Math.round};return h=S.appendChild(F(s,i,a))}function j(){var t=c.getBoundingClientRect(),e=\"offset\"+[\"Width\",\"Height\"][o.ort];return 0===o.ort?t.width||c[e]:t.height||c[e]}function H(t,e,r,n){var i=function(i){return!S.hasAttribute(\"disabled\")&&(s=S,a=o.cssClasses.tap,(s.classList?!s.classList.contains(a):!new RegExp(\"\\\\b\"+a+\"\\\\b\").test(s.className))&&(!!(i=function(t,e,r){var n,i,s=0===t.type.indexOf(\"touch\"),a=0===t.type.indexOf(\"mouse\"),u=0===t.type.indexOf(\"pointer\");0===t.type.indexOf(\"MSPointer\")&&(u=!0);if(s&&o.multitouch){var c=function(t){return t.target===r||r.contains(t.target)};if(\"touchstart\"===t.type){var p=Array.prototype.filter.call(t.touches,c);if(p.length>1)return!1;n=p[0].pageX,i=p[0].pageY}else{var f=Array.prototype.find.call(t.changedTouches,c);if(!f)return!1;n=f.pageX,i=f.pageY}}else if(s){if(t.touches.length>1)return!1;n=t.changedTouches[0].pageX,i=t.changedTouches[0].pageY}e=e||l(U),(a||u)&&(n=t.clientX+e.x,i=t.clientY+e.y);return t.pageOffset=e,t.points=[n,i],t.cursor=a||u,t}(i,n.pageOffset,n.target||e))&&(!(t===v.start&&void 0!==i.buttons&&i.buttons>1)&&((!n.hover||!i.buttons)&&(b||i.preventDefault(),i.calcPoint=i.points[o.ort],void r(i,n))))));var s,a},s=[];return t.split(\" \").forEach(function(t){e.addEventListener(t,i,!!b&&{passive:!0}),s.push([t,i])}),s}function D(t){var e,r,n,i,s,a,u=100*(t-(e=c,r=o.ort,n=e.getBoundingClientRect(),i=e.ownerDocument,s=i.documentElement,a=l(i),/webkit.*Chrome.*Mobile/i.test(navigator.userAgent)&&(a.x=0),r?n.top+a.y-s.clientTop:n.left+a.x-s.clientLeft))/j();return o.dir?100-u:u}function T(t,e,r,n){var i=r.slice(),o=[!t,t],s=[t,!t];n=n.slice(),t&&n.reverse(),n.length>1?n.forEach(function(t,r){var n=$(i,t,i[t]+e,o[r],s[r],!1);!1===n?e=0:(e=n-i[t],i[t]=n)}):o=s=[!0];var a=!1;n.forEach(function(t,n){a=K(t,r[t]+e,o[n],s[n])||a}),a&&n.forEach(function(t){R(\"update\",t),R(\"slide\",t)})}function R(t,e,r){Object.keys(N).forEach(function(n){var i=n.split(\".\")[0];t===i&&N[n].forEach(function(t){t.call(d,C.map(o.format.to),e,C.slice(),r||!1,w.slice())})})}function X(t,e){\"mouseout\"===t.type&&\"HTML\"===t.target.nodeName&&null===t.relatedTarget&&Y(t,e)}function B(t,e){if(-1===navigator.appVersion.indexOf(\"MSIE 9\")&&0===t.buttons&&0!==e.buttonsProperty)return Y(t,e);var r=(o.dir?-1:1)*(t.calcPoint-e.startCalcPoint);T(r>0,100*r/e.baseSize,e.locations,e.handleNumbers)}function Y(t,r){r.handle&&(a(r.handle,o.cssClasses.active),y-=1),r.listeners.forEach(function(t){P.removeEventListener(t[0],t[1])}),0===y&&(a(S,o.cssClasses.drag),J(),t.cursor&&(A.style.cursor=\"\",A.removeEventListener(\"selectstart\",e))),r.handleNumbers.forEach(function(t){R(\"change\",t),R(\"set\",t),R(\"end\",t)})}function _(t,r){var n;if(1===r.handleNumbers.length){var i=p[r.handleNumbers[0]];if(i.hasAttribute(\"disabled\"))return!1;n=i.children[0],y+=1,s(n,o.cssClasses.active)}t.stopPropagation();var a=[],l=H(v.move,P,B,{target:t.target,handle:n,listeners:a,startCalcPoint:t.calcPoint,baseSize:j(),pageOffset:t.pageOffset,handleNumbers:r.handleNumbers,buttonsProperty:t.buttons,locations:w.slice()}),u=H(v.end,P,Y,{target:t.target,handle:n,listeners:a,handleNumbers:r.handleNumbers}),c=H(\"mouseout\",P,X,{target:t.target,handle:n,listeners:a,handleNumbers:r.handleNumbers});a.push.apply(a,l.concat(u,c)),t.cursor&&(A.style.cursor=getComputedStyle(t.target).cursor,p.length>1&&s(S,o.cssClasses.drag),A.addEventListener(\"selectstart\",e,!1)),r.handleNumbers.forEach(function(t){R(\"start\",t)})}function I(t){t.stopPropagation();var e=D(t.calcPoint),r=function(t){var e=100,r=!1;return p.forEach(function(n,i){if(!n.hasAttribute(\"disabled\")){var o=Math.abs(w[i]-t);o1&&(n&&e>0&&(r=Math.max(r,t[e-1]+o.margin)),i&&e1&&o.limit&&(n&&e>0&&(r=Math.min(r,t[e-1]+o.limit)),i&&e50?-1:1,r=3+(p.length+e*t);p[t].childNodes[0].style.zIndex=r})}function K(t,e,r,n){return!1!==(e=$(w,t,e,r,n,!1))&&(function(t,e){w[t]=e,C[t]=E.fromStepping(e);var r=function(){p[t].style[o.style]=G(e),Q(t),Q(t+1)};window.requestAnimationFrame&&o.useRequestAnimationFrame?window.requestAnimationFrame(r):r()}(t,e),!0)}function Q(t){if(f[t]){var e=0,r=100;0!==t&&(e=w[t-1]),t!==f.length-1&&(r=w[t]),f[t].style[o.style]=G(e),f[t].style[o.styleOposite]=G(100-r)}}function Z(t,e){null!==t&&!1!==t&&(\"number\"==typeof t&&(t=String(t)),!1===(t=o.format.from(t))||isNaN(t)||K(e,E.toStepping(t),!1,!1))}function tt(t,e){var r=i(t),s=void 0===w[0];e=void 0===e||!!e,r.forEach(Z),o.animate&&!s&&n(S,o.cssClasses.tap,o.animationDuration),x.forEach(function(t){K(t,w[t],!0,!1)}),J(),x.forEach(function(t){R(\"update\",t),null!==r[t]&&e&&R(\"set\",t)})}function et(){var t=C.map(o.format.to);return 1===t.length?t[0]:t}function rt(t,e){N[t]=N[t]||[],N[t].push(e),\"update\"===t.split(\".\")[0]&&p.forEach(function(t,e){R(\"update\",e)})}if(S.noUiSlider)throw new Error(\"noUiSlider (\"+t+\"): Slider was already initialized.\");return function(t){s(t,o.cssClasses.target),0===o.dir?s(t,o.cssClasses.ltr):s(t,o.cssClasses.rtl),0===o.ort?s(t,o.cssClasses.horizontal):s(t,o.cssClasses.vertical),c=M(t,o.cssClasses.base)}(S),function(t,e){p=[],(f=[]).push(k(e,t[0]));for(var r=0;rr.stepAfter.startValue&&(i=r.stepAfter.startValue-n),o=n>r.thisStep.startValue?r.thisStep.step:!1!==r.stepBefore.step&&n-r.stepBefore.highestStep,100===t?i=null:0===t&&(o=null);var s=E.countStepDecimals();return null!==i&&!1!==i&&(i=Number(i.toFixed(s))),null!==o&&!1!==o&&(o=Number(o.toFixed(s))),[o,i]})},on:rt,off:function(t){var e=t&&t.split(\".\")[0],r=e&&t.substring(e.length);Object.keys(N).forEach(function(t){var n=t.split(\".\")[0],i=t.substring(n.length);e&&e!==n||r&&r!==i||delete N[t]})},get:et,set:tt,reset:function(t){tt(o.start,t)},__moveHandles:function(t,e,r){T(t,e,w,r)},options:u,updateOptions:function(t,e){var r=et(),n=[\"margin\",\"limit\",\"padding\",\"range\",\"animate\",\"snap\",\"step\",\"format\"];n.forEach(function(e){void 0!==t[e]&&(u[e]=t[e])});var i=q(u);n.forEach(function(e){void 0!==t[e]&&(o[e]=i[e])}),E=i.spectrum,o.margin=i.margin,o.limit=i.limit,o.padding=i.padding,o.pips&&z(o.pips),w=[],tt(t.start||r,e)},target:S,removePips:L,pips:z},(m=o.events).fixed||p.forEach(function(t,e){H(v.start,t.children[0],_,{handleNumbers:[e]})}),m.tap&&H(v.start,c,I,{}),m.hover&&H(v.move,c,W,{hover:!0}),m.drag&&f.forEach(function(t,e){if(!1!==t&&0!==e&&e!==f.length-1){var r=p[e-1],n=p[e],i=[t];s(t,o.cssClasses.draggable),m.fixed&&(i.push(r.children[0]),i.push(n.children[0])),i.forEach(function(t){H(v.start,t,_,{handles:[r,n],handleNumbers:[e-1,e]})})}}),tt(o.start),o.pips&&z(o.pips),o.tooltips&&(g=p.map(V),rt(\"update\",function(t,e,r){if(g[e]){var n=t[e];!0!==o.tooltips[e]&&(n=o.tooltips[e].to(r[e])),g[e].innerHTML=n}})),rt(\"update\",function(t,e,r,n,i){x.forEach(function(t){var e=p[t],n=$(w,t,0,!0,!0,!0),s=$(w,t,100,!0,!0,!0),a=i[t],l=o.ariaFormat.to(r[t]);e.children[0].setAttribute(\"aria-valuemin\",n.toFixed(1)),e.children[0].setAttribute(\"aria-valuemax\",s.toFixed(1)),e.children[0].setAttribute(\"aria-valuenow\",a.toFixed(1)),e.children[0].setAttribute(\"aria-valuetext\",l)})}),d}return{version:t,create:function(e,r){if(!e||!e.nodeName)throw new Error(\"noUiSlider (\"+t+\"): create requires a single element, got: \"+e);var n=T(e,q(r),r);return e.noUiSlider=n,n}}},\"function\"==typeof define&&define.amd?define([],n):\"object\"==typeof r?e.exports=n():window.noUiSlider=n()},\n", + " 494: function _(e,t,i){e(164),e(495),e(163).styles.append(\".bk-root .bk-slider-title {\\n white-space: nowrap;\\n}\\n.bk-root .bk-slider-value {\\n font-weight: 600;\\n}\\n\"),i.bk_slider_value=\"bk-slider-value\",i.bk_slider_title=\"bk-slider-title\",i.bk_input_group=\"bk-input-group\"},\n", + " 495: function _(n,o,t){n(164),n(163).styles.append('.bk-root {\\n /* Functional styling;\\n * These styles are required for noUiSlider to function.\\n * You don\\'t need to change these rules to apply your design.\\n */\\n /* Painting and performance;\\n * Browsers can paint handles in their own layer.\\n */\\n /* Slider size and handle placement;\\n */\\n /* Styling;\\n */\\n /* Handles and cursors;\\n */\\n /* Handle stripes;\\n */\\n /* Disabled state;\\n */\\n /* Base;\\n *\\n */\\n /* Values;\\n *\\n */\\n /* Markings;\\n *\\n */\\n /* Horizontal layout;\\n *\\n */\\n /* Vertical layout;\\n *\\n */\\n}\\n.bk-root .bk-noUi-target,\\n.bk-root .bk-noUi-target * {\\n -webkit-touch-callout: none;\\n -webkit-tap-highlight-color: rgba(0, 0, 0, 0);\\n -webkit-user-select: none;\\n -ms-touch-action: none;\\n touch-action: none;\\n -ms-user-select: none;\\n -moz-user-select: none;\\n user-select: none;\\n -moz-box-sizing: border-box;\\n box-sizing: border-box;\\n}\\n.bk-root .bk-noUi-target {\\n position: relative;\\n direction: ltr;\\n}\\n.bk-root .bk-noUi-base {\\n width: 100%;\\n height: 100%;\\n position: relative;\\n z-index: 1;\\n /* Fix 401 */\\n}\\n.bk-root .bk-noUi-connect {\\n position: absolute;\\n right: 0;\\n top: 0;\\n left: 0;\\n bottom: 0;\\n}\\n.bk-root .bk-noUi-origin {\\n position: absolute;\\n height: 0;\\n width: 0;\\n}\\n.bk-root .bk-noUi-handle {\\n position: relative;\\n z-index: 1;\\n}\\n.bk-root .bk-noUi-state-tap .bk-noUi-connect,\\n.bk-root .bk-noUi-state-tap .bk-noUi-origin {\\n -webkit-transition: top 0.3s, right 0.3s, bottom 0.3s, left 0.3s;\\n transition: top 0.3s, right 0.3s, bottom 0.3s, left 0.3s;\\n}\\n.bk-root .bk-noUi-state-drag * {\\n cursor: inherit !important;\\n}\\n.bk-root .bk-noUi-base,\\n.bk-root .bk-noUi-handle {\\n -webkit-transform: translate3d(0, 0, 0);\\n transform: translate3d(0, 0, 0);\\n}\\n.bk-root .bk-noUi-horizontal {\\n height: 18px;\\n}\\n.bk-root .bk-noUi-horizontal .bk-noUi-handle {\\n width: 34px;\\n height: 28px;\\n left: -17px;\\n top: -6px;\\n}\\n.bk-root .bk-noUi-vertical {\\n width: 18px;\\n}\\n.bk-root .bk-noUi-vertical .bk-noUi-handle {\\n width: 28px;\\n height: 34px;\\n left: -6px;\\n top: -17px;\\n}\\n.bk-root .bk-noUi-target {\\n background: #FAFAFA;\\n border-radius: 4px;\\n border: 1px solid #D3D3D3;\\n box-shadow: inset 0 1px 1px #F0F0F0, 0 3px 6px -5px #BBB;\\n}\\n.bk-root .bk-noUi-connect {\\n background: #3FB8AF;\\n border-radius: 4px;\\n box-shadow: inset 0 0 3px rgba(51, 51, 51, 0.45);\\n -webkit-transition: background 450ms;\\n transition: background 450ms;\\n}\\n.bk-root .bk-noUi-draggable {\\n cursor: ew-resize;\\n}\\n.bk-root .bk-noUi-vertical .bk-noUi-draggable {\\n cursor: ns-resize;\\n}\\n.bk-root .bk-noUi-handle {\\n border: 1px solid #D9D9D9;\\n border-radius: 3px;\\n background: #FFF;\\n cursor: default;\\n box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #EBEBEB, 0 3px 6px -3px #BBB;\\n}\\n.bk-root .bk-noUi-active {\\n box-shadow: inset 0 0 1px #FFF, inset 0 1px 7px #DDD, 0 3px 6px -3px #BBB;\\n}\\n.bk-root .bk-noUi-handle:before,\\n.bk-root .bk-noUi-handle:after {\\n content: \"\";\\n display: block;\\n position: absolute;\\n height: 14px;\\n width: 1px;\\n background: #E8E7E6;\\n left: 14px;\\n top: 6px;\\n}\\n.bk-root .bk-noUi-handle:after {\\n left: 17px;\\n}\\n.bk-root .bk-noUi-vertical .bk-noUi-handle:before,\\n.bk-root .bk-noUi-vertical .bk-noUi-handle:after {\\n width: 14px;\\n height: 1px;\\n left: 6px;\\n top: 14px;\\n}\\n.bk-root .bk-noUi-vertical .bk-noUi-handle:after {\\n top: 17px;\\n}\\n.bk-root [disabled] .bk-noUi-connect {\\n background: #B8B8B8;\\n}\\n.bk-root [disabled].bk-noUi-target,\\n.bk-root [disabled].bk-noUi-handle,\\n.bk-root [disabled] .bk-noUi-handle {\\n cursor: not-allowed;\\n}\\n.bk-root .bk-noUi-pips,\\n.bk-root .bk-noUi-pips * {\\n -moz-box-sizing: border-box;\\n box-sizing: border-box;\\n}\\n.bk-root .bk-noUi-pips {\\n position: absolute;\\n color: #999;\\n}\\n.bk-root .bk-noUi-value {\\n position: absolute;\\n white-space: nowrap;\\n text-align: center;\\n}\\n.bk-root .bk-noUi-value-sub {\\n color: #ccc;\\n font-size: 10px;\\n}\\n.bk-root .bk-noUi-marker {\\n position: absolute;\\n background: #CCC;\\n}\\n.bk-root .bk-noUi-marker-sub {\\n background: #AAA;\\n}\\n.bk-root .bk-noUi-marker-large {\\n background: #AAA;\\n}\\n.bk-root .bk-noUi-pips-horizontal {\\n padding: 10px 0;\\n height: 80px;\\n top: 100%;\\n left: 0;\\n width: 100%;\\n}\\n.bk-root .bk-noUi-value-horizontal {\\n -webkit-transform: translate3d(-50%, 50%, 0);\\n transform: translate3d(-50%, 50%, 0);\\n}\\n.bk-root .bk-noUi-marker-horizontal.bk-noUi-marker {\\n margin-left: -1px;\\n width: 2px;\\n height: 5px;\\n}\\n.bk-root .bk-noUi-marker-horizontal.bk-noUi-marker-sub {\\n height: 10px;\\n}\\n.bk-root .bk-noUi-marker-horizontal.bk-noUi-marker-large {\\n height: 15px;\\n}\\n.bk-root .bk-noUi-pips-vertical {\\n padding: 0 10px;\\n height: 100%;\\n top: 0;\\n left: 100%;\\n}\\n.bk-root .bk-noUi-value-vertical {\\n -webkit-transform: translate3d(0, 50%, 0);\\n transform: translate3d(0, 50%, 0);\\n padding-left: 25px;\\n}\\n.bk-root .bk-noUi-marker-vertical.bk-noUi-marker {\\n width: 5px;\\n height: 2px;\\n margin-top: -1px;\\n}\\n.bk-root .bk-noUi-marker-vertical.bk-noUi-marker-sub {\\n width: 10px;\\n}\\n.bk-root .bk-noUi-marker-vertical.bk-noUi-marker-large {\\n width: 15px;\\n}\\n.bk-root .bk-noUi-tooltip {\\n display: block;\\n position: absolute;\\n border: 1px solid #D9D9D9;\\n border-radius: 3px;\\n background: #fff;\\n color: #000;\\n padding: 5px;\\n text-align: center;\\n white-space: nowrap;\\n}\\n.bk-root .bk-noUi-horizontal .bk-noUi-tooltip {\\n -webkit-transform: translate(-50%, 0);\\n transform: translate(-50%, 0);\\n left: 50%;\\n bottom: 120%;\\n}\\n.bk-root .bk-noUi-vertical .bk-noUi-tooltip {\\n -webkit-transform: translate(0, -50%);\\n transform: translate(0, -50%);\\n top: 50%;\\n right: 120%;\\n}\\n.bk-root .bk-noUi-handle {\\n cursor: grab;\\n cursor: -webkit-grab;\\n}\\n.bk-root .bk-noUi-handle.bk-noUi-active {\\n cursor: grabbing;\\n cursor: -webkit-grabbing;\\n}\\n.bk-root .bk-noUi-tooltip {\\n display: none;\\n white-space: nowrap;\\n}\\n.bk-root .bk-noUi-handle:hover .bk-noUi-tooltip {\\n display: block;\\n}\\n.bk-root .bk-noUi-horizontal {\\n width: 100%;\\n height: 10px;\\n}\\n.bk-root .bk-noUi-horizontal.bk-noUi-target {\\n margin: 5px 0px;\\n}\\n.bk-root .bk-noUi-horizontal .bk-noUi-handle {\\n width: 14px;\\n height: 18px;\\n left: -7px;\\n top: -5px;\\n}\\n.bk-root .bk-noUi-vertical {\\n width: 10px;\\n height: 100%;\\n}\\n.bk-root .bk-noUi-vertical.bk-noUi-target {\\n margin: 0px 5px;\\n}\\n.bk-root .bk-noUi-vertical .bk-noUi-handle {\\n width: 18px;\\n height: 14px;\\n left: -5px;\\n top: -7px;\\n}\\n.bk-root .bk-noUi-handle:after,\\n.bk-root .bk-noUi-handle:before {\\n display: none;\\n}\\n.bk-root .bk-noUi-connect {\\n box-shadow: none;\\n}\\n')},\n", + " 496: function _(t,e,i){var r=t(113),n=t(252),a=t(492),_=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return r.__extends(e,t),e}(a.AbstractSliderView);i.DateSliderView=_,_.__name__=\"DateSliderView\";var o=function(t){function e(e){var i=t.call(this,e)||this;return i.behaviour=\"tap\",i.connected=[!0,!1],i}return r.__extends(e,t),e.init_DateSlider=function(){this.prototype.default_view=_,this.override({format:\"%d %b %Y\"})},e.prototype._formatter=function(t,e){return n(t,e)},e}(a.AbstractSlider);i.DateSlider=o,o.__name__=\"DateSlider\",o.init_DateSlider()},\n", + " 497: function _(t,e,i){var n=t(113),r=t(498),_=t(121),o=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(e,t),e.prototype.render=function(){t.prototype.render.call(this),this.model.render_as_text?this.markup_el.textContent=this.model.text:this.markup_el.innerHTML=this.model.text},e}(r.MarkupView);i.DivView=o,o.__name__=\"DivView\";var u=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.init_Div=function(){this.prototype.default_view=o,this.define({render_as_text:[_.Boolean,!1]})},e}(r.Markup);i.Div=u,u.__name__=\"Div\",u.init_Div()},\n", + " 498: function _(t,i,n){var e=t(113),s=t(282),o=t(163),r=t(121),a=t(534),l=t(499),u=function(t){function i(){return null!==t&&t.apply(this,arguments)||this}return e.__extends(i,t),i.prototype.connect_signals=function(){var i=this;t.prototype.connect_signals.call(this),this.connect(this.model.change,function(){i.render(),i.root.compute_layout()})},i.prototype._update_layout=function(){this.layout=new s.VariadicBox(this.el),this.layout.set_sizing(this.box_sizing())},i.prototype.render=function(){t.prototype.render.call(this);var i=Object.assign(Object.assign({},this.model.style),{display:\"inline-block\"});this.markup_el=o.div({class:l.bk_clearfix,style:i}),this.el.appendChild(this.markup_el)},i}(a.WidgetView);n.MarkupView=u,u.__name__=\"MarkupView\";var c=function(t){function i(i){return t.call(this,i)||this}return e.__extends(i,t),i.init_Markup=function(){this.define({text:[r.String,\"\"],style:[r.Any,{}]})},i}(a.Widget);n.Markup=c,c.__name__=\"Markup\",c.init_Markup()},\n", + " 499: function _(e,n,r){e(164),e(163).styles.append('.bk-root .bk-clearfix:before,\\n.bk-root .bk-clearfix:after {\\n content: \"\";\\n display: table;\\n}\\n.bk-root .bk-clearfix:after {\\n clear: both;\\n}\\n'),r.bk_clearfix=\"bk-clearfix\"},\n", + " 500: function _(e,t,i){var n=e(113),o=e(474),l=e(376),s=e(163),r=e(121),u=e(109),d=e(240),a=e(347),c=e(348),_=function(e){function t(){var t=e.apply(this,arguments)||this;return t._open=!1,t}return n.__extends(t,e),t.prototype.render=function(){var t=this;e.prototype.render.call(this);var i=s.div({class:[c.bk_caret,d.bk_down]});if(this.model.is_split){var n=this._render_button(i);n.classList.add(a.bk_dropdown_toggle),n.addEventListener(\"click\",function(){return t._toggle_menu()}),this.group_el.appendChild(n)}else this.button_el.appendChild(i);var o=this.model.menu.map(function(e,i){if(null==e)return s.div({class:c.bk_divider});var n=u.isString(e)?e:e[0],o=s.div({},n);return o.addEventListener(\"click\",function(){return t._item_click(i)}),o});this.menu=s.div({class:[c.bk_menu,d.bk_below]},o),this.el.appendChild(this.menu),s.undisplay(this.menu)},t.prototype._show_menu=function(){var e=this;if(!this._open){this._open=!0,s.display(this.menu);var t=function(i){var n=i.target;n instanceof HTMLElement&&!e.el.contains(n)&&(document.removeEventListener(\"click\",t),e._hide_menu())};document.addEventListener(\"click\",t)}},t.prototype._hide_menu=function(){this._open&&(this._open=!1,s.undisplay(this.menu))},t.prototype._toggle_menu=function(){this._open?this._hide_menu():this._show_menu()},t.prototype.click=function(){this.model.is_split?(this._hide_menu(),this.model.trigger_event(new l.ButtonClick),this.model.value=this.model.default_value,null!=this.model.callback&&this.model.callback.execute(this.model),e.prototype.click.call(this)):this._toggle_menu()},t.prototype._item_click=function(e){this._hide_menu();var t=this.model.menu[e];if(null!=t){var i=u.isString(t)?t:t[1];u.isString(i)?(this.model.trigger_event(new l.MenuItemClick(i)),this.model.value=i,null!=this.model.callback&&this.model.callback.execute(this.model)):(i.execute(this.model,{index:e}),null!=this.model.callback&&this.model.callback.execute(this.model))}},t}(o.AbstractButtonView);i.DropdownView=_,_.__name__=\"DropdownView\";var h=function(e){function t(t){return e.call(this,t)||this}return n.__extends(t,e),t.init_Dropdown=function(){this.prototype.default_view=_,this.define({split:[r.Boolean,!1],menu:[r.Array,[]],value:[r.String],default_value:[r.String]}),this.override({label:\"Dropdown\"})},Object.defineProperty(t.prototype,\"is_split\",{get:function(){return this.split||null!=this.default_value},enumerable:!0,configurable:!0}),t}(o.AbstractButton);i.Dropdown=h,h.__name__=\"Dropdown\",h.init_Dropdown()},\n", + " 501: function _(t,e,i){var n=t(113),l=t(121),o=t(534),a=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return n.__extends(e,t),e.prototype.connect_signals=function(){var e=this;t.prototype.connect_signals.call(this),this.connect(this.model.change,function(){return e.render()}),this.connect(this.model.properties.width.change,function(){return e.render()})},e.prototype.render=function(){var t=this;this.dialogEl||(this.dialogEl=document.createElement(\"input\"),this.dialogEl.type=\"file\",this.dialogEl.multiple=!1,null!=this.model.accept&&\"\"!=this.model.accept&&(this.dialogEl.accept=this.model.accept),this.dialogEl.style.width=\"{this.model.width}px\",this.dialogEl.onchange=function(e){return t.load_file(e)},this.el.appendChild(this.dialogEl))},e.prototype.load_file=function(t){var e=this,i=new FileReader;this.model.filename=t.target.files[0].name,i.onload=function(t){return e.file(t)},i.readAsDataURL(t.target.files[0])},e.prototype.file=function(t){var e=t.target.result.split(\",\"),i=e[1],n=e[0].split(\":\")[1].split(\";\")[0];this.model.value=i,this.model.mime_type=n},e}(o.WidgetView);i.FileInputView=a,a.__name__=\"FileInputView\";var r=function(t){function e(e){return t.call(this,e)||this}return n.__extends(e,t),e.init_FileInput=function(){this.prototype.default_view=a,this.define({value:[l.String,\"\"],mime_type:[l.String,\"\"],filename:[l.String,\"\"],accept:[l.String,\"\"]})},e}(o.Widget);i.FileInput=r,r.__name__=\"FileInput\",r.init_FileInput()},\n", + " 502: function _(e,t,n){var i=e(113),r=e(163),l=e(109),o=e(117),s=e(121),c=e(480),u=e(481),h=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return i.__extends(t,e),t.prototype.connect_signals=function(){var t=this;e.prototype.connect_signals.call(this),this.connect(this.model.properties.value.change,function(){return t.render_selection()}),this.connect(this.model.properties.options.change,function(){return t.render()}),this.connect(this.model.properties.name.change,function(){return t.render()}),this.connect(this.model.properties.title.change,function(){return t.render()}),this.connect(this.model.properties.size.change,function(){return t.render()}),this.connect(this.model.properties.disabled.change,function(){return t.render()})},t.prototype.render=function(){var t=this;e.prototype.render.call(this);var n=this.model.options.map(function(e){var t,n;return l.isString(e)?t=n=e:(t=e[0],n=e[1]),r.option({value:t},n)});this.select_el=r.select({multiple:!0,class:u.bk_input,name:this.model.name,disabled:this.model.disabled},n),this.select_el.addEventListener(\"change\",function(){return t.change_input()}),this.group_el.appendChild(this.select_el),this.render_selection()},t.prototype.render_selection=function(){for(var e=new o.Set(this.model.value),t=0,n=Array.from(this.el.querySelectorAll(\"option\"));tu?d:-d;if(0!=h)return h}return 0})},e}();i.TableDataProvider=b,b.__name__=\"TableDataProvider\";var v=function(e){function t(){var t=e.apply(this,arguments)||this;return t._in_selection_update=!1,t._warned_not_reorderable=!1,t}return n.__extends(t,e),t.prototype.connect_signals=function(){var t=this;e.prototype.connect_signals.call(this),this.connect(this.model.change,function(){return t.render()}),this.connect(this.model.source.streaming,function(){return t.updateGrid()}),this.connect(this.model.source.patching,function(){return t.updateGrid()}),this.connect(this.model.source.change,function(){return t.updateGrid()}),this.connect(this.model.source.properties.data.change,function(){return t.updateGrid()}),this.connect(this.model.source.selected.change,function(){return t.updateSelection()}),this.connect(this.model.source.selected.properties.indices.change,function(){return t.updateSelection()})},t.prototype._update_layout=function(){this.layout=new p.LayoutItem,this.layout.set_sizing(this.box_sizing())},t.prototype.update_position=function(){e.prototype.update_position.call(this),this.grid.resizeCanvas()},t.prototype.updateGrid=function(){var e=this;if(this.model.view.compute_indices(),this.data.constructor(this.model.source,this.model.view),this.model.sortable){var t=this.grid.getColumns(),i=this.grid.getSortColumns().map(function(i){return{sortCol:{field:t[e.grid.getColumnIndex(i.columnId)].field},sortAsc:i.sortAsc}});this.data.sort(i)}this.grid.invalidate(),this.grid.render()},t.prototype.updateSelection=function(){var e=this;if(!this._in_selection_update){var t=this.model.source.selected.indices.map(function(t){return e.data.index.indexOf(t)}).sort();this._in_selection_update=!0,this.grid.setSelectedRows(t),this._in_selection_update=!1;var i=this.grid.getViewport(),n=this.model.get_scroll_index(i,t);null!=n&&this.grid.scrollRowToTop(n)}},t.prototype.newIndexColumn=function(){return{id:d.uniqueId(),name:this.model.index_header,field:i.DTINDEX_NAME,width:this.model.index_width,behavior:\"select\",cannotTriggerInsert:!0,resizable:!1,selectable:!1,sortable:!0,cssClass:g.bk_cell_index,headerCssClass:g.bk_header_index}},t.prototype.css_classes=function(){return e.prototype.css_classes.call(this).concat(g.bk_data_table)},t.prototype.render=function(){var e,t=this,i=this.model.columns.map(function(e){return e.toColumn()});if(\"checkbox\"==this.model.selectable&&(e=new r({cssClass:g.bk_cell_select}),i.unshift(e.getColumnDefinition())),null!=this.model.index_position){var n=this.model.index_position,a=this.newIndexColumn();-1==n?i.push(a):n<-1?i.splice(n+1,0,a):i.splice(n,0,a)}var d=this.model.reorderable;!d||\"undefined\"!=typeof $&&null!=$.fn&&null!=$.fn.sortable||(this._warned_not_reorderable||(_.logger.warn(\"jquery-ui is required to enable DataTable.reorderable\"),this._warned_not_reorderable=!0),d=!1);var u={enableCellNavigation:!1!==this.model.selectable,enableColumnReorder:d,forceFitColumns:this.model.fit_columns,multiColumnSort:this.model.sortable,editable:this.model.editable,autoEdit:!1,rowHeight:this.model.row_height};if(this.data=new b(this.model.source,this.model.view),this.grid=new l.Grid(this.el,this.data,i,u),this.grid.onSort.subscribe(function(e,n){t.model.sortable&&(i=n.sortCols,t.data.sort(i),t.grid.invalidate(),t.updateSelection(),t.grid.render(),t.model.header_row||t._hide_header(),t.model.update_sort_columns(i))}),!1!==this.model.selectable){this.grid.setSelectionModel(new o({selectActiveRow:null==e})),null!=e&&this.grid.registerPlugin(e);var h={dataItemColumnValueExtractor:function(e,t){var i=e[t.field];return c.isString(i)&&(i=i.replace(/\\n/g,\"\\\\n\")),i},includeHeaderWhenCopying:!1};this.grid.registerPlugin(new s(h)),this.grid.onSelectedRowsChanged.subscribe(function(e,i){t._in_selection_update||(t.model.source.selected.indices=i.rows.map(function(e){return t.data.index[e]}))}),this.updateSelection(),this.model.header_row||this._hide_header()}},t.prototype._hide_header=function(){for(var e=0,t=Array.from(this.el.querySelectorAll(\".slick-header-columns\"));e=0&&l0&&t-1 in e)}b.fn=b.prototype={jquery:\"3.4.1\",constructor:b,length:0,toArray:function(){return o.call(this)},get:function(e){return null==e?o.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=b.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return b.each(this,e)},map:function(e){return this.pushStack(b.map(this,function(t,n){return e.call(t,n,t)}))},slice:function(){return this.pushStack(o.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(n>=0&&n+~]|\"+M+\")\"+M+\"*\"),U=new RegExp(M+\"|>\"),X=new RegExp($),V=new RegExp(\"^\"+I+\"$\"),G={ID:new RegExp(\"^#(\"+I+\")\"),CLASS:new RegExp(\"^\\\\.(\"+I+\")\"),TAG:new RegExp(\"^(\"+I+\"|[*])\"),ATTR:new RegExp(\"^\"+W),PSEUDO:new RegExp(\"^\"+$),CHILD:new RegExp(\"^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\\\(\"+M+\"*(even|odd|(([+-]|)(\\\\d*)n|)\"+M+\"*(?:([+-]|)\"+M+\"*(\\\\d+)|))\"+M+\"*\\\\)|)\",\"i\"),bool:new RegExp(\"^(?:\"+R+\")$\",\"i\"),needsContext:new RegExp(\"^\"+M+\"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\\\(\"+M+\"*((?:-\\\\d)?\\\\d*)\"+M+\"*\\\\)|)(?=[^-]|$)\",\"i\")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\\d$/i,K=/^[^{]+\\{\\s*\\[native \\w/,Z=/^(?:#([\\w-]+)|(\\w+)|\\.([\\w-]+))$/,ee=/[+~]/,te=new RegExp(\"\\\\\\\\([\\\\da-f]{1,6}\"+M+\"?|(\"+M+\")|.)\",\"ig\"),ne=function(e,t,n){var r=\"0x\"+t-65536;return r!=r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},re=/([\\0-\\x1f\\x7f]|^-?\\d)|^-$|[^\\0-\\x1f\\x7f-\\uFFFF\\w-]/g,ie=function(e,t){return t?\"\\0\"===e?\"�\":e.slice(0,-1)+\"\\\\\"+e.charCodeAt(e.length-1).toString(16)+\" \":\"\\\\\"+e},oe=function(){p()},ae=be(function(e){return!0===e.disabled&&\"fieldset\"===e.nodeName.toLowerCase()},{dir:\"parentNode\",next:\"legend\"});try{H.apply(j=O.call(w.childNodes),w.childNodes),j[w.childNodes.length].nodeType}catch(e){H={apply:j.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){for(var n=e.length,r=0;e[n++]=t[r++];);e.length=n-1}}}function se(e,t,r,i){var o,s,l,c,f,h,y,m=t&&t.ownerDocument,T=t?t.nodeType:9;if(r=r||[],\"string\"!=typeof e||!e||1!==T&&9!==T&&11!==T)return r;if(!i&&((t?t.ownerDocument||t:w)!==d&&p(t),t=t||d,g)){if(11!==T&&(f=Z.exec(e)))if(o=f[1]){if(9===T){if(!(l=t.getElementById(o)))return r;if(l.id===o)return r.push(l),r}else if(m&&(l=m.getElementById(o))&&x(t,l)&&l.id===o)return r.push(l),r}else{if(f[2])return H.apply(r,t.getElementsByTagName(e)),r;if((o=f[3])&&n.getElementsByClassName&&t.getElementsByClassName)return H.apply(r,t.getElementsByClassName(o)),r}if(n.qsa&&!N[e+\" \"]&&(!v||!v.test(e))&&(1!==T||\"object\"!==t.nodeName.toLowerCase())){if(y=e,m=t,1===T&&U.test(e)){for((c=t.getAttribute(\"id\"))?c=c.replace(re,ie):t.setAttribute(\"id\",c=b),s=(h=a(e)).length;s--;)h[s]=\"#\"+c+\" \"+xe(h[s]);y=h.join(\",\"),m=ee.test(e)&&ye(t.parentNode)||t}try{return H.apply(r,m.querySelectorAll(y)),r}catch(t){N(e,!0)}finally{c===b&&t.removeAttribute(\"id\")}}}return u(e.replace(B,\"$1\"),t,r,i)}function ue(){var e=[];return function t(n,i){return e.push(n+\" \")>r.cacheLength&&delete t[e.shift()],t[n+\" \"]=i}}function le(e){return e[b]=!0,e}function ce(e){var t=d.createElement(\"fieldset\");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){for(var n=e.split(\"|\"),i=n.length;i--;)r.attrHandle[n[i]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)for(;n=n.nextSibling;)if(n===t)return-1;return e?1:-1}function de(e){return function(t){return\"input\"===t.nodeName.toLowerCase()&&t.type===e}}function he(e){return function(t){var n=t.nodeName.toLowerCase();return(\"input\"===n||\"button\"===n)&&t.type===e}}function ge(e){return function(t){return\"form\"in t?t.parentNode&&!1===t.disabled?\"label\"in t?\"label\"in t.parentNode?t.parentNode.disabled===e:t.disabled===e:t.isDisabled===e||t.isDisabled!==!e&&ae(t)===e:t.disabled===e:\"label\"in t&&t.disabled===e}}function ve(e){return le(function(t){return t=+t,le(function(n,r){for(var i,o=e([],n.length,t),a=o.length;a--;)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}function ye(e){return e&&void 0!==e.getElementsByTagName&&e}for(t in n=se.support={},o=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||\"HTML\")},p=se.setDocument=function(e){var t,i,a=e?e.ownerDocument||e:w;return a!==d&&9===a.nodeType&&a.documentElement?(h=(d=a).documentElement,g=!o(d),w!==d&&(i=d.defaultView)&&i.top!==i&&(i.addEventListener?i.addEventListener(\"unload\",oe,!1):i.attachEvent&&i.attachEvent(\"onunload\",oe)),n.attributes=ce(function(e){return e.className=\"i\",!e.getAttribute(\"className\")}),n.getElementsByTagName=ce(function(e){return e.appendChild(d.createComment(\"\")),!e.getElementsByTagName(\"*\").length}),n.getElementsByClassName=K.test(d.getElementsByClassName),n.getById=ce(function(e){return h.appendChild(e).id=b,!d.getElementsByName||!d.getElementsByName(b).length}),n.getById?(r.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute(\"id\")===t}},r.find.ID=function(e,t){if(void 0!==t.getElementById&&g){var n=t.getElementById(e);return n?[n]:[]}}):(r.filter.ID=function(e){var t=e.replace(te,ne);return function(e){var n=void 0!==e.getAttributeNode&&e.getAttributeNode(\"id\");return n&&n.value===t}},r.find.ID=function(e,t){if(void 0!==t.getElementById&&g){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode(\"id\"))&&n.value===e)return[o];for(i=t.getElementsByName(e),r=0;o=i[r++];)if((n=o.getAttributeNode(\"id\"))&&n.value===e)return[o]}return[]}}),r.find.TAG=n.getElementsByTagName?function(e,t){return void 0!==t.getElementsByTagName?t.getElementsByTagName(e):n.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if(\"*\"===e){for(;n=o[i++];)1===n.nodeType&&r.push(n);return r}return o},r.find.CLASS=n.getElementsByClassName&&function(e,t){if(void 0!==t.getElementsByClassName&&g)return t.getElementsByClassName(e)},y=[],v=[],(n.qsa=K.test(d.querySelectorAll))&&(ce(function(e){h.appendChild(e).innerHTML=\"\",e.querySelectorAll(\"[msallowcapture^='']\").length&&v.push(\"[*^$]=\"+M+\"*(?:''|\\\"\\\")\"),e.querySelectorAll(\"[selected]\").length||v.push(\"\\\\[\"+M+\"*(?:value|\"+R+\")\"),e.querySelectorAll(\"[id~=\"+b+\"-]\").length||v.push(\"~=\"),e.querySelectorAll(\":checked\").length||v.push(\":checked\"),e.querySelectorAll(\"a#\"+b+\"+*\").length||v.push(\".#.+[+~]\")}),ce(function(e){e.innerHTML=\"\";var t=d.createElement(\"input\");t.setAttribute(\"type\",\"hidden\"),e.appendChild(t).setAttribute(\"name\",\"D\"),e.querySelectorAll(\"[name=d]\").length&&v.push(\"name\"+M+\"*[*^$|!~]?=\"),2!==e.querySelectorAll(\":enabled\").length&&v.push(\":enabled\",\":disabled\"),h.appendChild(e).disabled=!0,2!==e.querySelectorAll(\":disabled\").length&&v.push(\":enabled\",\":disabled\"),e.querySelectorAll(\"*,:x\"),v.push(\",.*:\")})),(n.matchesSelector=K.test(m=h.matches||h.webkitMatchesSelector||h.mozMatchesSelector||h.oMatchesSelector||h.msMatchesSelector))&&ce(function(e){n.disconnectedMatch=m.call(e,\"*\"),m.call(e,\"[s!='']:x\"),y.push(\"!=\",$)}),v=v.length&&new RegExp(v.join(\"|\")),y=y.length&&new RegExp(y.join(\"|\")),t=K.test(h.compareDocumentPosition),x=t||K.test(h.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)for(;t=t.parentNode;)if(t===e)return!0;return!1},A=t?function(e,t){if(e===t)return f=!0,0;var r=!e.compareDocumentPosition-!t.compareDocumentPosition;return r||(1&(r=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!n.sortDetached&&t.compareDocumentPosition(e)===r?e===d||e.ownerDocument===w&&x(w,e)?-1:t===d||t.ownerDocument===w&&x(w,t)?1:c?P(c,e)-P(c,t):0:4&r?-1:1)}:function(e,t){if(e===t)return f=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===d?-1:t===d?1:i?-1:o?1:c?P(c,e)-P(c,t):0;if(i===o)return pe(e,t);for(n=e;n=n.parentNode;)a.unshift(n);for(n=t;n=n.parentNode;)s.unshift(n);for(;a[r]===s[r];)r++;return r?pe(a[r],s[r]):a[r]===w?-1:s[r]===w?1:0},d):d},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if((e.ownerDocument||e)!==d&&p(e),n.matchesSelector&&g&&!N[t+\" \"]&&(!y||!y.test(t))&&(!v||!v.test(t)))try{var r=m.call(e,t);if(r||n.disconnectedMatch||e.document&&11!==e.document.nodeType)return r}catch(e){N(t,!0)}return se(t,d,null,[e]).length>0},se.contains=function(e,t){return(e.ownerDocument||e)!==d&&p(e),x(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!==d&&p(e);var i=r.attrHandle[t.toLowerCase()],o=i&&D.call(r.attrHandle,t.toLowerCase())?i(e,t,!g):void 0;return void 0!==o?o:n.attributes||!g?e.getAttribute(t):(o=e.getAttributeNode(t))&&o.specified?o.value:null},se.escape=function(e){return(e+\"\").replace(re,ie)},se.error=function(e){throw new Error(\"Syntax error, unrecognized expression: \"+e)},se.uniqueSort=function(e){var t,r=[],i=0,o=0;if(f=!n.detectDuplicates,c=!n.sortStable&&e.slice(0),e.sort(A),f){for(;t=e[o++];)t===e[o]&&(i=r.push(o));for(;i--;)e.splice(r[i],1)}return c=null,e},i=se.getText=function(e){var t,n=\"\",r=0,o=e.nodeType;if(o){if(1===o||9===o||11===o){if(\"string\"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=i(e)}else if(3===o||4===o)return e.nodeValue}else for(;t=e[r++];)n+=i(t);return n},(r=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{\">\":{dir:\"parentNode\",first:!0},\" \":{dir:\"parentNode\"},\"+\":{dir:\"previousSibling\",first:!0},\"~\":{dir:\"previousSibling\"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||\"\").replace(te,ne),\"~=\"===e[2]&&(e[3]=\" \"+e[3]+\" \"),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),\"nth\"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*(\"even\"===e[3]||\"odd\"===e[3])),e[5]=+(e[7]+e[8]||\"odd\"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||\"\":n&&X.test(n)&&(t=a(n,!0))&&(t=n.indexOf(\")\",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return\"*\"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=E[e+\" \"];return t||(t=new RegExp(\"(^|\"+M+\")\"+e+\"(\"+M+\"|$)\"))&&E(e,function(e){return t.test(\"string\"==typeof e.className&&e.className||void 0!==e.getAttribute&&e.getAttribute(\"class\")||\"\")})},ATTR:function(e,t,n){return function(r){var i=se.attr(r,e);return null==i?\"!=\"===t:!t||(i+=\"\",\"=\"===t?i===n:\"!=\"===t?i!==n:\"^=\"===t?n&&0===i.indexOf(n):\"*=\"===t?n&&i.indexOf(n)>-1:\"$=\"===t?n&&i.slice(-n.length)===n:\"~=\"===t?(\" \"+i.replace(F,\" \")+\" \").indexOf(n)>-1:\"|=\"===t&&(i===n||i.slice(0,n.length+1)===n+\"-\"))}},CHILD:function(e,t,n,r,i){var o=\"nth\"!==e.slice(0,3),a=\"last\"!==e.slice(-4),s=\"of-type\"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,u){var l,c,f,p,d,h,g=o!==a?\"nextSibling\":\"previousSibling\",v=t.parentNode,y=s&&t.nodeName.toLowerCase(),m=!u&&!s,x=!1;if(v){if(o){for(;g;){for(p=t;p=p[g];)if(s?p.nodeName.toLowerCase()===y:1===p.nodeType)return!1;h=g=\"only\"===e&&!h&&\"nextSibling\"}return!0}if(h=[a?v.firstChild:v.lastChild],a&&m){for(x=(d=(l=(c=(f=(p=v)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1])&&l[2],p=d&&v.childNodes[d];p=++d&&p&&p[g]||(x=d=0)||h.pop();)if(1===p.nodeType&&++x&&p===t){c[e]=[T,d,x];break}}else if(m&&(x=d=(l=(c=(f=(p=t)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1]),!1===x)for(;(p=++d&&p&&p[g]||(x=d=0)||h.pop())&&((s?p.nodeName.toLowerCase()!==y:1!==p.nodeType)||!++x||(m&&((c=(f=p[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]=[T,x]),p!==t)););return(x-=i)===r||x%r==0&&x/r>=0}}},PSEUDO:function(e,t){var n,i=r.pseudos[e]||r.setFilters[e.toLowerCase()]||se.error(\"unsupported pseudo: \"+e);return i[b]?i(t):i.length>1?(n=[e,e,\"\",t],r.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,n){for(var r,o=i(e,t),a=o.length;a--;)e[r=P(e,o[a])]=!(n[r]=o[a])}):function(e){return i(e,0,n)}):i}},pseudos:{not:le(function(e){var t=[],n=[],r=s(e.replace(B,\"$1\"));return r[b]?le(function(e,t,n,i){for(var o,a=r(e,null,i,[]),s=e.length;s--;)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),t[0]=null,!n.pop()}}),has:le(function(e){return function(t){return se(e,t).length>0}}),contains:le(function(e){return e=e.replace(te,ne),function(t){return(t.textContent||i(t)).indexOf(e)>-1}}),lang:le(function(e){return V.test(e||\"\")||se.error(\"unsupported lang: \"+e),e=e.replace(te,ne).toLowerCase(),function(t){var n;do{if(n=g?t.lang:t.getAttribute(\"xml:lang\")||t.getAttribute(\"lang\"))return(n=n.toLowerCase())===e||0===n.indexOf(e+\"-\")}while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===h},focus:function(e){return e===d.activeElement&&(!d.hasFocus||d.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return\"input\"===t&&!!e.checked||\"option\"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!r.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return\"input\"===t&&\"button\"===e.type||\"button\"===t},text:function(e){var t;return\"input\"===e.nodeName.toLowerCase()&&\"text\"===e.type&&(null==(t=e.getAttribute(\"type\"))||\"text\"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;nt?t:n;--r>=0;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r1?function(t,n,r){for(var i=e.length;i--;)if(!e[i](t,n,r))return!1;return!0}:e[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s-1&&(o[l]=!(a[l]=f))}}else y=Te(y===a?y.splice(h,y.length):y),i?i(null,a,y,u):H.apply(a,y)})}function Ee(e){for(var t,n,i,o=e.length,a=r.relative[e[0].type],s=a||r.relative[\" \"],u=a?1:0,c=be(function(e){return e===t},s,!0),f=be(function(e){return P(t,e)>-1},s,!0),p=[function(e,n,r){var i=!a&&(r||n!==l)||((t=n).nodeType?c(e,n,r):f(e,n,r));return t=null,i}];u1&&we(p),u>1&&xe(e.slice(0,u-1).concat({value:\" \"===e[u-2].type?\"*\":\"\"})).replace(B,\"$1\"),n,u0,i=e.length>0,o=function(o,a,s,u,c){var f,h,v,y=0,m=\"0\",x=o&&[],b=[],w=l,C=o||i&&r.find.TAG(\"*\",c),E=T+=null==w?1:Math.random()||.1,k=C.length;for(c&&(l=a===d||a||c);m!==k&&null!=(f=C[m]);m++){if(i&&f){for(h=0,a||f.ownerDocument===d||(p(f),s=!g);v=e[h++];)if(v(f,a||d,s)){u.push(f);break}c&&(T=E)}n&&((f=!v&&f)&&y--,o&&x.push(f))}if(y+=m,n&&m!==y){for(h=0;v=t[h++];)v(x,b,a,s);if(o){if(y>0)for(;m--;)x[m]||b[m]||(b[m]=q.call(u));b=Te(b)}H.apply(u,b),c&&!o&&b.length>0&&y+t.length>1&&se.uniqueSort(u)}return c&&(T=E,l=w),x};return n?le(o):o}(o,i))).selector=e}return s},u=se.select=function(e,t,n,i){var o,u,l,c,f,p=\"function\"==typeof e&&e,d=!i&&a(e=p.selector||e);if(n=n||[],1===d.length){if((u=d[0]=d[0].slice(0)).length>2&&\"ID\"===(l=u[0]).type&&9===t.nodeType&&g&&r.relative[u[1].type]){if(!(t=(r.find.ID(l.matches[0].replace(te,ne),t)||[])[0]))return n;p&&(t=t.parentNode),e=e.slice(u.shift().value.length)}for(o=G.needsContext.test(e)?0:u.length;o--&&(l=u[o],!r.relative[c=l.type]);)if((f=r.find[c])&&(i=f(l.matches[0].replace(te,ne),ee.test(u[0].type)&&ye(t.parentNode)||t))){if(u.splice(o,1),!(e=i.length&&xe(u)))return H.apply(n,i),n;break}}return(p||s(e,d))(i,t,!g,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},n.sortStable=b.split(\"\").sort(A).join(\"\")===b,n.detectDuplicates=!!f,p(),n.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(d.createElement(\"fieldset\"))}),ce(function(e){return e.innerHTML=\"\",\"#\"===e.firstChild.getAttribute(\"href\")})||fe(\"type|href|height|width\",function(e,t,n){if(!n)return e.getAttribute(t,\"type\"===t.toLowerCase()?1:2)}),n.attributes&&ce(function(e){return e.innerHTML=\"\",e.firstChild.setAttribute(\"value\",\"\"),\"\"===e.firstChild.getAttribute(\"value\")})||fe(\"value\",function(e,t,n){if(!n&&\"input\"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute(\"disabled\")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(e);b.find=C,b.expr=C.selectors,b.expr[\":\"]=b.expr.pseudos,b.uniqueSort=b.unique=C.uniqueSort,b.text=C.getText,b.isXMLDoc=C.isXML,b.contains=C.contains,b.escapeSelector=C.escape;var E=function(e,t,n){for(var r=[],i=void 0!==n;(e=e[t])&&9!==e.nodeType;)if(1===e.nodeType){if(i&&b(e).is(n))break;r.push(e)}return r},k=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},S=b.expr.match.needsContext;function N(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var A=/^<([a-z][^\\/\\0>:\\x20\\t\\r\\n\\f]*)[\\x20\\t\\r\\n\\f]*\\/?>(?:<\\/\\1>|)$/i;function D(e,t,n){return g(t)?b.grep(e,function(e,r){return!!t.call(e,r,e)!==n}):t.nodeType?b.grep(e,function(e){return e===t!==n}):\"string\"!=typeof t?b.grep(e,function(e){return u.call(t,e)>-1!==n}):b.filter(t,e,n)}b.filter=function(e,t,n){var r=t[0];return n&&(e=\":not(\"+e+\")\"),1===t.length&&1===r.nodeType?b.find.matchesSelector(r,e)?[r]:[]:b.find.matches(e,b.grep(t,function(e){return 1===e.nodeType}))},b.fn.extend({find:function(e){var t,n,r=this.length,i=this;if(\"string\"!=typeof e)return this.pushStack(b(e).filter(function(){for(t=0;t1?b.uniqueSort(n):n},filter:function(e){return this.pushStack(D(this,e||[],!1))},not:function(e){return this.pushStack(D(this,e||[],!0))},is:function(e){return!!D(this,\"string\"==typeof e&&S.test(e)?b(e):e||[],!1).length}});var j,q=/^(?:\\s*(<[\\w\\W]+>)[^>]*|#([\\w-]+))$/;(b.fn.init=function(e,t,n){var i,o;if(!e)return this;if(n=n||j,\"string\"==typeof e){if(!(i=\"<\"===e[0]&&\">\"===e[e.length-1]&&e.length>=3?[null,e,null]:q.exec(e))||!i[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(i[1]){if(t=t instanceof b?t[0]:t,b.merge(this,b.parseHTML(i[1],t&&t.nodeType?t.ownerDocument||t:r,!0)),A.test(i[1])&&b.isPlainObject(t))for(i in t)g(this[i])?this[i](t[i]):this.attr(i,t[i]);return this}return(o=r.getElementById(i[2]))&&(this[0]=o,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):g(e)?void 0!==n.ready?n.ready(e):e(b):b.makeArray(e,this)}).prototype=b.fn,j=b(r);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){for(;(e=e[t])&&1!==e.nodeType;);return e}b.fn.extend({has:function(e){var t=b(e,this),n=t.length;return this.filter(function(){for(var e=0;e-1:1===n.nodeType&&b.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(o.length>1?b.uniqueSort(o):o)},index:function(e){return e?\"string\"==typeof e?u.call(b(e),this[0]):u.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(b.uniqueSort(b.merge(this.get(),b(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),b.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return E(e,\"parentNode\")},parentsUntil:function(e,t,n){return E(e,\"parentNode\",n)},next:function(e){return O(e,\"nextSibling\")},prev:function(e){return O(e,\"previousSibling\")},nextAll:function(e){return E(e,\"nextSibling\")},prevAll:function(e){return E(e,\"previousSibling\")},nextUntil:function(e,t,n){return E(e,\"nextSibling\",n)},prevUntil:function(e,t,n){return E(e,\"previousSibling\",n)},siblings:function(e){return k((e.parentNode||{}).firstChild,e)},children:function(e){return k(e.firstChild)},contents:function(e){return void 0!==e.contentDocument?e.contentDocument:(N(e,\"template\")&&(e=e.content||e),b.merge([],e.childNodes))}},function(e,t){b.fn[e]=function(n,r){var i=b.map(this,t,n);return\"Until\"!==e.slice(-5)&&(r=n),r&&\"string\"==typeof r&&(i=b.filter(r,i)),this.length>1&&(H[e]||b.uniqueSort(i),L.test(e)&&i.reverse()),this.pushStack(i)}});var P=/[^\\x20\\t\\r\\n\\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&g(i=e.promise)?i.call(e).done(t).fail(n):e&&g(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}b.Callbacks=function(e){e=\"string\"==typeof e?function(e){var t={};return b.each(e.match(P)||[],function(e,n){t[n]=!0}),t}(e):b.extend({},e);var t,n,r,i,o=[],a=[],s=-1,u=function(){for(i=i||e.once,r=t=!0;a.length;s=-1)for(n=a.shift();++s-1;)o.splice(n,1),n<=s&&s--}),this},has:function(e){return e?b.inArray(e,o)>-1:o.length>0},empty:function(){return o&&(o=[]),this},disable:function(){return i=a=[],o=n=\"\",this},disabled:function(){return!o},lock:function(){return i=a=[],n||t||(o=n=\"\"),this},locked:function(){return!!i},fireWith:function(e,n){return i||(n=[e,(n=n||[]).slice?n.slice():n],a.push(n),t||u()),this},fire:function(){return l.fireWith(this,arguments),this},fired:function(){return!!r}};return l},b.extend({Deferred:function(t){var n=[[\"notify\",\"progress\",b.Callbacks(\"memory\"),b.Callbacks(\"memory\"),2],[\"resolve\",\"done\",b.Callbacks(\"once memory\"),b.Callbacks(\"once memory\"),0,\"resolved\"],[\"reject\",\"fail\",b.Callbacks(\"once memory\"),b.Callbacks(\"once memory\"),1,\"rejected\"]],r=\"pending\",i={state:function(){return r},always:function(){return o.done(arguments).fail(arguments),this},catch:function(e){return i.then(null,e)},pipe:function(){var e=arguments;return b.Deferred(function(t){b.each(n,function(n,r){var i=g(e[r[4]])&&e[r[4]];o[r[1]](function(){var e=i&&i.apply(this,arguments);e&&g(e.promise)?e.promise().progress(t.notify).done(t.resolve).fail(t.reject):t[r[0]+\"With\"](this,i?[e]:arguments)})}),e=null}).promise()},then:function(t,r,i){var o=0;function a(t,n,r,i){return function(){var s=this,u=arguments,l=function(){var e,l;if(!(t=o&&(r!==M&&(s=void 0,u=[e]),n.rejectWith(s,u))}};t?c():(b.Deferred.getStackHook&&(c.stackTrace=b.Deferred.getStackHook()),e.setTimeout(c))}}return b.Deferred(function(e){n[0][3].add(a(0,e,g(i)?i:R,e.notifyWith)),n[1][3].add(a(0,e,g(t)?t:R)),n[2][3].add(a(0,e,g(r)?r:M))}).promise()},promise:function(e){return null!=e?b.extend(e,i):i}},o={};return b.each(n,function(e,t){var a=t[2],s=t[5];i[t[1]]=a.add,s&&a.add(function(){r=s},n[3-e][2].disable,n[3-e][3].disable,n[0][2].lock,n[0][3].lock),a.add(t[3].fire),o[t[0]]=function(){return o[t[0]+\"With\"](this===o?void 0:this,arguments),this},o[t[0]+\"With\"]=a.fireWith}),i.promise(o),t&&t.call(o,o),o},when:function(e){var t=arguments.length,n=t,r=Array(n),i=o.call(arguments),a=b.Deferred(),s=function(e){return function(n){r[e]=this,i[e]=arguments.length>1?o.call(arguments):n,--t||a.resolveWith(r,i)}};if(t<=1&&(I(e,a.done(s(n)).resolve,a.reject,!t),\"pending\"===a.state()||g(i[n]&&i[n].then)))return a.then();for(;n--;)I(i[n],s(n),a.reject);return a.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;b.Deferred.exceptionHook=function(t,n){e.console&&e.console.warn&&t&&W.test(t.name)&&e.console.warn(\"jQuery.Deferred exception: \"+t.message,t.stack,n)},b.readyException=function(t){e.setTimeout(function(){throw t})};var $=b.Deferred();function F(){r.removeEventListener(\"DOMContentLoaded\",F),e.removeEventListener(\"load\",F),b.ready()}b.fn.ready=function(e){return $.then(e).catch(function(e){b.readyException(e)}),this},b.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--b.readyWait:b.isReady)||(b.isReady=!0,!0!==e&&--b.readyWait>0||$.resolveWith(r,[b]))}}),b.ready.then=$.then,\"complete\"===r.readyState||\"loading\"!==r.readyState&&!r.documentElement.doScroll?e.setTimeout(b.ready):(r.addEventListener(\"DOMContentLoaded\",F),e.addEventListener(\"load\",F));var B=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if(\"object\"===x(n))for(s in i=!0,n)B(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,g(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(b(e),n)})),t))for(;s1,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),b.extend({queue:function(e,t,n){var r;if(e)return t=(t||\"fx\")+\"queue\",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,b.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||\"fx\";var n=b.queue(e,t),r=n.length,i=n.shift(),o=b._queueHooks(e,t);\"inprogress\"===i&&(i=n.shift(),r--),i&&(\"fx\"===t&&n.unshift(\"inprogress\"),delete o.stop,i.call(e,function(){b.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+\"queueHooks\";return Y.get(e,n)||Y.access(e,n,{empty:b.Callbacks(\"once memory\").add(function(){Y.remove(e,[t+\"queue\",n])})})}}),b.fn.extend({queue:function(e,t){var n=2;return\"string\"!=typeof e&&(t=e,e=\"fx\",n--),arguments.length\\x20\\t\\r\\n\\f]*)/i,he=/^$|^module$|\\/(?:java|ecma)script/i,ge={option:[1,\"\"],thead:[1,\"\",\"
\"],col:[2,\"\",\"
\"],tr:[2,\"\",\"
\"],td:[3,\"\",\"
\"],_default:[0,\"\",\"\"]};function ve(e,t){var n;return n=void 0!==e.getElementsByTagName?e.getElementsByTagName(t||\"*\"):void 0!==e.querySelectorAll?e.querySelectorAll(t||\"*\"):[],void 0===t||t&&N(e,t)?b.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n-1)i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),\"script\"),l&&ye(a),n)for(c=0;o=a[c++];)he.test(o.type||\"\")&&n.push(o);return f}me=r.createDocumentFragment().appendChild(r.createElement(\"div\")),(xe=r.createElement(\"input\")).setAttribute(\"type\",\"radio\"),xe.setAttribute(\"checked\",\"checked\"),xe.setAttribute(\"name\",\"t\"),me.appendChild(xe),h.checkClone=me.cloneNode(!0).cloneNode(!0).lastChild.checked,me.innerHTML=\"\",h.noCloneChecked=!!me.cloneNode(!0).lastChild.defaultValue;var Te=/^key/,Ce=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ee=/^([^.]*)(?:\\.(.+)|)/;function ke(){return!0}function Se(){return!1}function Ne(e,t){return e===function(){try{return r.activeElement}catch(e){}}()==(\"focus\"===t)}function Ae(e,t,n,r,i,o){var a,s;if(\"object\"==typeof t){for(s in\"string\"!=typeof n&&(r=r||n,n=void 0),t)Ae(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&(\"string\"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Se;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return b().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=b.guid++)),e.each(function(){b.event.add(this,t,i,r,n)})}function De(e,t,n){n?(Y.set(e,t,!1),b.event.add(e,t,{namespace:!1,handler:function(e){var r,i,a=Y.get(this,t);if(1&e.isTrigger&&this[t]){if(a.length)(b.event.special[t]||{}).delegateType&&e.stopPropagation();else if(a=o.call(arguments),Y.set(this,t,a),r=n(this,t),this[t](),a!==(i=Y.get(this,t))||r?Y.set(this,t,!1):i={},a!==i)return e.stopImmediatePropagation(),e.preventDefault(),i.value}else a.length&&(Y.set(this,t,{value:b.event.trigger(b.extend(a[0],b.Event.prototype),a.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,t)&&b.event.add(e,t,ke)}b.event={global:{},add:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(e);if(v)for(n.handler&&(n=(o=n).handler,i=o.selector),i&&b.find.matchesSelector(re,i),n.guid||(n.guid=b.guid++),(u=v.events)||(u=v.events={}),(a=v.handle)||(a=v.handle=function(t){return void 0!==b&&b.event.triggered!==t.type?b.event.dispatch.apply(e,arguments):void 0}),l=(t=(t||\"\").match(P)||[\"\"]).length;l--;)d=g=(s=Ee.exec(t[l])||[])[1],h=(s[2]||\"\").split(\".\").sort(),d&&(f=b.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=b.event.special[d]||{},c=b.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&b.expr.match.needsContext.test(i),namespace:h.join(\".\")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(e,r,h,a)||e.addEventListener&&e.addEventListener(d,a)),f.add&&(f.add.call(e,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),b.event.global[d]=!0)},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){for(l=(t=(t||\"\").match(P)||[\"\"]).length;l--;)if(d=g=(s=Ee.exec(t[l])||[])[1],h=(s[2]||\"\").split(\".\").sort(),d){for(f=b.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp(\"(^|\\\\.)\"+h.join(\"\\\\.(?:.*\\\\.|)\")+\"(\\\\.|$)\"),a=o=p.length;o--;)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&(\"**\"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||b.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)b.event.remove(e,d+t[l],n,r,!0);b.isEmptyObject(u)&&Y.remove(e,\"handle events\")}},dispatch:function(e){var t,n,r,i,o,a,s=b.event.fix(e),u=new Array(arguments.length),l=(Y.get(this,\"events\")||{})[s.type]||[],c=b.event.special[s.type]||{};for(u[0]=s,t=1;t=1))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&(\"click\"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n-1:b.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u\\x20\\t\\r\\n\\f]*)[^>]*)\\/>/gi,qe=/\\s*$/g;function Oe(e,t){return N(e,\"table\")&&N(11!==t.nodeType?t:t.firstChild,\"tr\")&&b(e).children(\"tbody\")[0]||e}function Pe(e){return e.type=(null!==e.getAttribute(\"type\"))+\"/\"+e.type,e}function Re(e){return\"true/\"===(e.type||\"\").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute(\"type\"),e}function Me(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(Y.hasData(e)&&(o=Y.access(e),a=Y.set(t,o),l=o.events))for(i in delete a.handle,a.events={},l)for(n=0,r=l[i].length;n1&&\"string\"==typeof v&&!h.checkClone&&Le.test(v))return e.each(function(i){var o=e.eq(i);y&&(t[0]=v.call(this,i,o.html())),Ie(o,t,n,r)});if(p&&(o=(i=we(t,e[0].ownerDocument,!1,e,r)).firstChild,1===i.childNodes.length&&(i=o),o||r)){for(u=(s=b.map(ve(i,\"script\"),Pe)).length;f\")},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(h.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||b.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r0&&ye(a,!f&&ve(e,\"script\")),c},cleanData:function(e){for(var t,n,r,i=b.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?b.event.remove(n,r):b.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),b.fn.extend({detach:function(e){return We(this,e,!0)},remove:function(e){return We(this,e)},text:function(e){return B(this,function(e){return void 0===e?b.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Ie(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Oe(this,e).appendChild(e)})},prepend:function(){return Ie(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Oe(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Ie(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Ie(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(b.cleanData(ve(e,!1)),e.textContent=\"\");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return b.clone(this,e,t)})},html:function(e){return B(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if(\"string\"==typeof e&&!qe.test(e)&&!ge[(de.exec(e)||[\"\",\"\"])[1].toLowerCase()]){e=b.htmlPrefilter(e);try{for(;n=0&&(u+=Math.max(0,Math.ceil(e[\"offset\"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function tt(e,t,n){var r=Fe(e),i=(!h.boxSizingReliable()||n)&&\"border-box\"===b.css(e,\"boxSizing\",!1,r),o=i,a=_e(e,t,r),s=\"offset\"+t[0].toUpperCase()+t.slice(1);if($e.test(a)){if(!n)return a;a=\"auto\"}return(!h.boxSizingReliable()&&i||\"auto\"===a||!parseFloat(a)&&\"inline\"===b.css(e,\"display\",!1,r))&&e.getClientRects().length&&(i=\"border-box\"===b.css(e,\"boxSizing\",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+et(e,t,n||(i?\"border\":\"content\"),o,r,a)+\"px\"}function nt(e,t,n,r,i){return new nt.prototype.init(e,t,n,r,i)}b.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=_e(e,\"opacity\");return\"\"===n?\"1\":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Qe.test(t),l=e.style;if(u||(t=Ge(s)),a=b.cssHooks[t]||b.cssHooks[s],void 0===n)return a&&\"get\"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];\"string\"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=ue(e,t,i),o=\"number\"),null!=n&&n==n&&(\"number\"!==o||u||(n+=i&&i[3]||(b.cssNumber[s]?\"\":\"px\")),h.clearCloneStyle||\"\"!==n||0!==t.indexOf(\"background\")||(l[t]=\"inherit\"),a&&\"set\"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Qe.test(t)||(t=Ge(s)),(a=b.cssHooks[t]||b.cssHooks[s])&&\"get\"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=_e(e,t,r)),\"normal\"===i&&t in Ke&&(i=Ke[t]),\"\"===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),b.each([\"height\",\"width\"],function(e,t){b.cssHooks[t]={get:function(e,n,r){if(n)return!Ye.test(b.css(e,\"display\"))||e.getClientRects().length&&e.getBoundingClientRect().width?tt(e,t,r):se(e,Je,function(){return tt(e,t,r)})},set:function(e,n,r){var i,o=Fe(e),a=!h.scrollboxSize()&&\"absolute\"===o.position,s=(a||r)&&\"border-box\"===b.css(e,\"boxSizing\",!1,o),u=r?et(e,t,r,s,o):0;return s&&a&&(u-=Math.ceil(e[\"offset\"+t[0].toUpperCase()+t.slice(1)]-parseFloat(o[t])-et(e,t,\"border\",!1,o)-.5)),u&&(i=te.exec(n))&&\"px\"!==(i[3]||\"px\")&&(e.style[t]=n,n=b.css(e,t)),Ze(0,n,u)}}}),b.cssHooks.marginLeft=ze(h.reliableMarginLeft,function(e,t){if(t)return(parseFloat(_e(e,\"marginLeft\"))||e.getBoundingClientRect().left-se(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+\"px\"}),b.each({margin:\"\",padding:\"\",border:\"Width\"},function(e,t){b.cssHooks[e+t]={expand:function(n){for(var r=0,i={},o=\"string\"==typeof n?n.split(\" \"):[n];r<4;r++)i[e+ne[r]+t]=o[r]||o[r-2]||o[0];return i}},\"margin\"!==e&&(b.cssHooks[e+t].set=Ze)}),b.fn.extend({css:function(e,t){return B(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Fe(e),i=t.length;a1)}}),b.Tween=nt,nt.prototype={constructor:nt,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||b.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(b.cssNumber[n]?\"\":\"px\")},cur:function(){var e=nt.propHooks[this.prop];return e&&e.get?e.get(this):nt.propHooks._default.get(this)},run:function(e){var t,n=nt.propHooks[this.prop];return this.options.duration?this.pos=t=b.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):nt.propHooks._default.set(this),this}},nt.prototype.init.prototype=nt.prototype,nt.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=b.css(e.elem,e.prop,\"\"))&&\"auto\"!==t?t:0},set:function(e){b.fx.step[e.prop]?b.fx.step[e.prop](e):1!==e.elem.nodeType||!b.cssHooks[e.prop]&&null==e.elem.style[Ge(e.prop)]?e.elem[e.prop]=e.now:b.style(e.elem,e.prop,e.now+e.unit)}}},nt.propHooks.scrollTop=nt.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},b.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:\"swing\"},b.fx=nt.prototype.init,b.fx.step={};var rt,it,ot=/^(?:toggle|show|hide)$/,at=/queueHooks$/;function st(){it&&(!1===r.hidden&&e.requestAnimationFrame?e.requestAnimationFrame(st):e.setTimeout(st,b.fx.interval),b.fx.tick())}function ut(){return e.setTimeout(function(){rt=void 0}),rt=Date.now()}function lt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i[\"margin\"+(n=ne[r])]=i[\"padding\"+n]=e;return t&&(i.opacity=i.width=e),i}function ct(e,t,n){for(var r,i=(ft.tweeners[t]||[]).concat(ft.tweeners[\"*\"]),o=0,a=i.length;o1)},removeAttr:function(e){return this.each(function(){b.removeAttr(this,e)})}}),b.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return void 0===e.getAttribute?b.prop(e,t,n):(1===o&&b.isXMLDoc(e)||(i=b.attrHooks[t.toLowerCase()]||(b.expr.match.bool.test(t)?pt:void 0)),void 0!==n?null===n?void b.removeAttr(e,t):i&&\"set\"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+\"\"),n):i&&\"get\"in i&&null!==(r=i.get(e,t))?r:null==(r=b.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!h.radioValue&&\"radio\"===t&&N(e,\"input\")){var n=e.value;return e.setAttribute(\"type\",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)for(;n=i[r++];)e.removeAttribute(n)}}),pt={set:function(e,t,n){return!1===t?b.removeAttr(e,n):e.setAttribute(n,n),n}},b.each(b.expr.match.bool.source.match(/\\w+/g),function(e,t){var n=dt[t]||b.find.attr;dt[t]=function(e,t,r){var i,o,a=t.toLowerCase();return r||(o=dt[a],dt[a]=i,i=null!=n(e,t,r)?a:null,dt[a]=o),i}});var ht=/^(?:input|select|textarea|button)$/i,gt=/^(?:a|area)$/i;function vt(e){return(e.match(P)||[]).join(\" \")}function yt(e){return e.getAttribute&&e.getAttribute(\"class\")||\"\"}function mt(e){return Array.isArray(e)?e:\"string\"==typeof e&&e.match(P)||[]}b.fn.extend({prop:function(e,t){return B(this,b.prop,e,t,arguments.length>1)},removeProp:function(e){return this.each(function(){delete this[b.propFix[e]||e]})}}),b.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&b.isXMLDoc(e)||(t=b.propFix[t]||t,i=b.propHooks[t]),void 0!==n?i&&\"set\"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&\"get\"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=b.find.attr(e,\"tabindex\");return t?parseInt(t,10):ht.test(e.nodeName)||gt.test(e.nodeName)&&e.href?0:-1}}},propFix:{for:\"htmlFor\",class:\"className\"}}),h.optSelected||(b.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),b.each([\"tabIndex\",\"readOnly\",\"maxLength\",\"cellSpacing\",\"cellPadding\",\"rowSpan\",\"colSpan\",\"useMap\",\"frameBorder\",\"contentEditable\"],function(){b.propFix[this.toLowerCase()]=this}),b.fn.extend({addClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){b(this).addClass(e.call(this,t,yt(this)))});if((t=mt(e)).length)for(;n=this[u++];)if(i=yt(n),r=1===n.nodeType&&\" \"+vt(i)+\" \"){for(a=0;o=t[a++];)r.indexOf(\" \"+o+\" \")<0&&(r+=o+\" \");i!==(s=vt(r))&&n.setAttribute(\"class\",s)}return this},removeClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){b(this).removeClass(e.call(this,t,yt(this)))});if(!arguments.length)return this.attr(\"class\",\"\");if((t=mt(e)).length)for(;n=this[u++];)if(i=yt(n),r=1===n.nodeType&&\" \"+vt(i)+\" \"){for(a=0;o=t[a++];)for(;r.indexOf(\" \"+o+\" \")>-1;)r=r.replace(\" \"+o+\" \",\" \");i!==(s=vt(r))&&n.setAttribute(\"class\",s)}return this},toggleClass:function(e,t){var n=typeof e,r=\"string\"===n||Array.isArray(e);return\"boolean\"==typeof t&&r?t?this.addClass(e):this.removeClass(e):g(e)?this.each(function(n){b(this).toggleClass(e.call(this,n,yt(this),t),t)}):this.each(function(){var t,i,o,a;if(r)for(i=0,o=b(this),a=mt(e);t=a[i++];)o.hasClass(t)?o.removeClass(t):o.addClass(t);else void 0!==e&&\"boolean\"!==n||((t=yt(this))&&Y.set(this,\"__className__\",t),this.setAttribute&&this.setAttribute(\"class\",t||!1===e?\"\":Y.get(this,\"__className__\")||\"\"))})},hasClass:function(e){var t,n,r=0;for(t=\" \"+e+\" \";n=this[r++];)if(1===n.nodeType&&(\" \"+vt(yt(n))+\" \").indexOf(t)>-1)return!0;return!1}});var xt=/\\r/g;b.fn.extend({val:function(e){var t,n,r,i=this[0];return arguments.length?(r=g(e),this.each(function(n){var i;1===this.nodeType&&(null==(i=r?e.call(this,n,b(this).val()):e)?i=\"\":\"number\"==typeof i?i+=\"\":Array.isArray(i)&&(i=b.map(i,function(e){return null==e?\"\":e+\"\"})),(t=b.valHooks[this.type]||b.valHooks[this.nodeName.toLowerCase()])&&\"set\"in t&&void 0!==t.set(this,i,\"value\")||(this.value=i))})):i?(t=b.valHooks[i.type]||b.valHooks[i.nodeName.toLowerCase()])&&\"get\"in t&&void 0!==(n=t.get(i,\"value\"))?n:\"string\"==typeof(n=i.value)?n.replace(xt,\"\"):null==n?\"\":n:void 0}}),b.extend({valHooks:{option:{get:function(e){var t=b.find.attr(e,\"value\");return null!=t?t:vt(b.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a=\"select-one\"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r-1)&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),b.each([\"radio\",\"checkbox\"],function(){b.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=b.inArray(b(e).val(),t)>-1}},h.checkOn||(b.valHooks[this].get=function(e){return null===e.getAttribute(\"value\")?\"on\":e.value})}),h.focusin=\"onfocusin\"in e;var bt=/^(?:focusinfocus|focusoutblur)$/,wt=function(e){e.stopPropagation()};b.extend(b.event,{trigger:function(t,n,i,o){var a,s,u,l,c,p,d,h,y=[i||r],m=f.call(t,\"type\")?t.type:t,x=f.call(t,\"namespace\")?t.namespace.split(\".\"):[];if(s=h=u=i=i||r,3!==i.nodeType&&8!==i.nodeType&&!bt.test(m+b.event.triggered)&&(m.indexOf(\".\")>-1&&(x=m.split(\".\"),m=x.shift(),x.sort()),c=m.indexOf(\":\")<0&&\"on\"+m,(t=t[b.expando]?t:new b.Event(m,\"object\"==typeof t&&t)).isTrigger=o?2:3,t.namespace=x.join(\".\"),t.rnamespace=t.namespace?new RegExp(\"(^|\\\\.)\"+x.join(\"\\\\.(?:.*\\\\.|)\")+\"(\\\\.|$)\"):null,t.result=void 0,t.target||(t.target=i),n=null==n?[t]:b.makeArray(n,[t]),d=b.event.special[m]||{},o||!d.trigger||!1!==d.trigger.apply(i,n))){if(!o&&!d.noBubble&&!v(i)){for(l=d.delegateType||m,bt.test(l+m)||(s=s.parentNode);s;s=s.parentNode)y.push(s),u=s;u===(i.ownerDocument||r)&&y.push(u.defaultView||u.parentWindow||e)}for(a=0;(s=y[a++])&&!t.isPropagationStopped();)h=s,t.type=a>1?l:d.bindType||m,(p=(Y.get(s,\"events\")||{})[t.type]&&Y.get(s,\"handle\"))&&p.apply(s,n),(p=c&&s[c])&&p.apply&&V(s)&&(t.result=p.apply(s,n),!1===t.result&&t.preventDefault());return t.type=m,o||t.isDefaultPrevented()||d._default&&!1!==d._default.apply(y.pop(),n)||!V(i)||c&&g(i[m])&&!v(i)&&((u=i[c])&&(i[c]=null),b.event.triggered=m,t.isPropagationStopped()&&h.addEventListener(m,wt),i[m](),t.isPropagationStopped()&&h.removeEventListener(m,wt),b.event.triggered=void 0,u&&(i[c]=u)),t.result}},simulate:function(e,t,n){var r=b.extend(new b.Event,n,{type:e,isSimulated:!0});b.event.trigger(r,null,t)}}),b.fn.extend({trigger:function(e,t){return this.each(function(){b.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return b.event.trigger(e,t,n,!0)}}),h.focusin||b.each({focus:\"focusin\",blur:\"focusout\"},function(e,t){var n=function(e){b.event.simulate(t,e.target,b.event.fix(e))};b.event.special[t]={setup:function(){var r=this.ownerDocument||this,i=Y.access(r,t);i||r.addEventListener(e,n,!0),Y.access(r,t,(i||0)+1)},teardown:function(){var r=this.ownerDocument||this,i=Y.access(r,t)-1;i?Y.access(r,t,i):(r.removeEventListener(e,n,!0),Y.remove(r,t))}}});var Tt=e.location,Ct=Date.now(),Et=/\\?/;b.parseXML=function(t){var n;if(!t||\"string\"!=typeof t)return null;try{n=(new e.DOMParser).parseFromString(t,\"text/xml\")}catch(e){n=void 0}return n&&!n.getElementsByTagName(\"parsererror\").length||b.error(\"Invalid XML: \"+t),n};var kt=/\\[\\]$/,St=/\\r?\\n/g,Nt=/^(?:submit|button|image|reset|file)$/i,At=/^(?:input|select|textarea|keygen)/i;function Dt(e,t,n,r){var i;if(Array.isArray(t))b.each(t,function(t,i){n||kt.test(e)?r(e,i):Dt(e+\"[\"+(\"object\"==typeof i&&null!=i?t:\"\")+\"]\",i,n,r)});else if(n||\"object\"!==x(t))r(e,t);else for(i in t)Dt(e+\"[\"+i+\"]\",t[i],n,r)}b.param=function(e,t){var n,r=[],i=function(e,t){var n=g(t)?t():t;r[r.length]=encodeURIComponent(e)+\"=\"+encodeURIComponent(null==n?\"\":n)};if(null==e)return\"\";if(Array.isArray(e)||e.jquery&&!b.isPlainObject(e))b.each(e,function(){i(this.name,this.value)});else for(n in e)Dt(n,e[n],t,i);return r.join(\"&\")},b.fn.extend({serialize:function(){return b.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=b.prop(this,\"elements\");return e?b.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!b(this).is(\":disabled\")&&At.test(this.nodeName)&&!Nt.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=b(this).val();return null==n?null:Array.isArray(n)?b.map(n,function(e){return{name:t.name,value:e.replace(St,\"\\r\\n\")}}):{name:t.name,value:n.replace(St,\"\\r\\n\")}}).get()}});var jt=/%20/g,qt=/#.*$/,Lt=/([?&])_=[^&]*/,Ht=/^(.*?):[ \\t]*([^\\r\\n]*)$/gm,Ot=/^(?:GET|HEAD)$/,Pt=/^\\/\\//,Rt={},Mt={},It=\"*/\".concat(\"*\"),Wt=r.createElement(\"a\");function $t(e){return function(t,n){\"string\"!=typeof t&&(n=t,t=\"*\");var r,i=0,o=t.toLowerCase().match(P)||[];if(g(n))for(;r=o[i++];)\"+\"===r[0]?(r=r.slice(1)||\"*\",(e[r]=e[r]||[]).unshift(n)):(e[r]=e[r]||[]).push(n)}}function Ft(e,t,n,r){var i={},o=e===Mt;function a(s){var u;return i[s]=!0,b.each(e[s]||[],function(e,s){var l=s(t,n,r);return\"string\"!=typeof l||o||i[l]?o?!(u=l):void 0:(t.dataTypes.unshift(l),a(l),!1)}),u}return a(t.dataTypes[0])||!i[\"*\"]&&a(\"*\")}function Bt(e,t){var n,r,i=b.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&b.extend(!0,e,r),e}Wt.href=Tt.href,b.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Tt.href,type:\"GET\",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Tt.protocol),global:!0,processData:!0,async:!0,contentType:\"application/x-www-form-urlencoded; charset=UTF-8\",accepts:{\"*\":It,text:\"text/plain\",html:\"text/html\",xml:\"application/xml, text/xml\",json:\"application/json, text/javascript\"},contents:{xml:/\\bxml\\b/,html:/\\bhtml/,json:/\\bjson\\b/},responseFields:{xml:\"responseXML\",text:\"responseText\",json:\"responseJSON\"},converters:{\"* text\":String,\"text html\":!0,\"text json\":JSON.parse,\"text xml\":b.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Bt(Bt(e,b.ajaxSettings),t):Bt(b.ajaxSettings,e)},ajaxPrefilter:$t(Rt),ajaxTransport:$t(Mt),ajax:function(t,n){\"object\"==typeof t&&(n=t,t=void 0),n=n||{};var i,o,a,s,u,l,c,f,p,d,h=b.ajaxSetup({},n),g=h.context||h,v=h.context&&(g.nodeType||g.jquery)?b(g):b.event,y=b.Deferred(),m=b.Callbacks(\"once memory\"),x=h.statusCode||{},w={},T={},C=\"canceled\",E={readyState:0,getResponseHeader:function(e){var t;if(c){if(!s)for(s={};t=Ht.exec(a);)s[t[1].toLowerCase()+\" \"]=(s[t[1].toLowerCase()+\" \"]||[]).concat(t[2]);t=s[e.toLowerCase()+\" \"]}return null==t?null:t.join(\", \")},getAllResponseHeaders:function(){return c?a:null},setRequestHeader:function(e,t){return null==c&&(e=T[e.toLowerCase()]=T[e.toLowerCase()]||e,w[e]=t),this},overrideMimeType:function(e){return null==c&&(h.mimeType=e),this},statusCode:function(e){var t;if(e)if(c)E.always(e[E.status]);else for(t in e)x[t]=[x[t],e[t]];return this},abort:function(e){var t=e||C;return i&&i.abort(t),k(0,t),this}};if(y.promise(E),h.url=((t||h.url||Tt.href)+\"\").replace(Pt,Tt.protocol+\"//\"),h.type=n.method||n.type||h.method||h.type,h.dataTypes=(h.dataType||\"*\").toLowerCase().match(P)||[\"\"],null==h.crossDomain){l=r.createElement(\"a\");try{l.href=h.url,l.href=l.href,h.crossDomain=Wt.protocol+\"//\"+Wt.host!=l.protocol+\"//\"+l.host}catch(e){h.crossDomain=!0}}if(h.data&&h.processData&&\"string\"!=typeof h.data&&(h.data=b.param(h.data,h.traditional)),Ft(Rt,h,n,E),c)return E;for(p in(f=b.event&&h.global)&&0==b.active++&&b.event.trigger(\"ajaxStart\"),h.type=h.type.toUpperCase(),h.hasContent=!Ot.test(h.type),o=h.url.replace(qt,\"\"),h.hasContent?h.data&&h.processData&&0===(h.contentType||\"\").indexOf(\"application/x-www-form-urlencoded\")&&(h.data=h.data.replace(jt,\"+\")):(d=h.url.slice(o.length),h.data&&(h.processData||\"string\"==typeof h.data)&&(o+=(Et.test(o)?\"&\":\"?\")+h.data,delete h.data),!1===h.cache&&(o=o.replace(Lt,\"$1\"),d=(Et.test(o)?\"&\":\"?\")+\"_=\"+Ct+++d),h.url=o+d),h.ifModified&&(b.lastModified[o]&&E.setRequestHeader(\"If-Modified-Since\",b.lastModified[o]),b.etag[o]&&E.setRequestHeader(\"If-None-Match\",b.etag[o])),(h.data&&h.hasContent&&!1!==h.contentType||n.contentType)&&E.setRequestHeader(\"Content-Type\",h.contentType),E.setRequestHeader(\"Accept\",h.dataTypes[0]&&h.accepts[h.dataTypes[0]]?h.accepts[h.dataTypes[0]]+(\"*\"!==h.dataTypes[0]?\", \"+It+\"; q=0.01\":\"\"):h.accepts[\"*\"]),h.headers)E.setRequestHeader(p,h.headers[p]);if(h.beforeSend&&(!1===h.beforeSend.call(g,E,h)||c))return E.abort();if(C=\"abort\",m.add(h.complete),E.done(h.success),E.fail(h.error),i=Ft(Mt,h,n,E)){if(E.readyState=1,f&&v.trigger(\"ajaxSend\",[E,h]),c)return E;h.async&&h.timeout>0&&(u=e.setTimeout(function(){E.abort(\"timeout\")},h.timeout));try{c=!1,i.send(w,k)}catch(e){if(c)throw e;k(-1,e)}}else k(-1,\"No Transport\");function k(t,n,r,s){var l,p,d,w,T,C=n;c||(c=!0,u&&e.clearTimeout(u),i=void 0,a=s||\"\",E.readyState=t>0?4:0,l=t>=200&&t<300||304===t,r&&(w=function(e,t,n){for(var r,i,o,a,s=e.contents,u=e.dataTypes;\"*\"===u[0];)u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader(\"Content-Type\"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+\" \"+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(h,E,r)),w=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];for(o=c.shift();o;)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if(\"*\"===o)o=u;else if(\"*\"!==u&&u!==o){if(!(a=l[u+\" \"+o]||l[\"* \"+o]))for(i in l)if((s=i.split(\" \"))[1]===o&&(a=l[u+\" \"+s[0]]||l[\"* \"+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e.throws)t=a(t);else try{t=a(t)}catch(e){return{state:\"parsererror\",error:a?e:\"No conversion from \"+u+\" to \"+o}}}return{state:\"success\",data:t}}(h,w,E,l),l?(h.ifModified&&((T=E.getResponseHeader(\"Last-Modified\"))&&(b.lastModified[o]=T),(T=E.getResponseHeader(\"etag\"))&&(b.etag[o]=T)),204===t||\"HEAD\"===h.type?C=\"nocontent\":304===t?C=\"notmodified\":(C=w.state,p=w.data,l=!(d=w.error))):(d=C,!t&&C||(C=\"error\",t<0&&(t=0))),E.status=t,E.statusText=(n||C)+\"\",l?y.resolveWith(g,[p,C,E]):y.rejectWith(g,[E,C,d]),E.statusCode(x),x=void 0,f&&v.trigger(l?\"ajaxSuccess\":\"ajaxError\",[E,h,l?p:d]),m.fireWith(g,[E,C]),f&&(v.trigger(\"ajaxComplete\",[E,h]),--b.active||b.event.trigger(\"ajaxStop\")))}return E},getJSON:function(e,t,n){return b.get(e,t,n,\"json\")},getScript:function(e,t){return b.get(e,void 0,t,\"script\")}}),b.each([\"get\",\"post\"],function(e,t){b[t]=function(e,n,r,i){return g(n)&&(i=i||r,r=n,n=void 0),b.ajax(b.extend({url:e,type:t,dataType:i,data:n,success:r},b.isPlainObject(e)&&e))}}),b._evalUrl=function(e,t){return b.ajax({url:e,type:\"GET\",dataType:\"script\",cache:!0,async:!1,global:!1,converters:{\"text script\":function(){}},dataFilter:function(e){b.globalEval(e,t)}})},b.fn.extend({wrapAll:function(e){var t;return this[0]&&(g(e)&&(e=e.call(this[0])),t=b(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){for(var e=this;e.firstElementChild;)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(e){return g(e)?this.each(function(t){b(this).wrapInner(e.call(this,t))}):this.each(function(){var t=b(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=g(e);return this.each(function(n){b(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(e){return this.parent(e).not(\"body\").each(function(){b(this).replaceWith(this.childNodes)}),this}}),b.expr.pseudos.hidden=function(e){return!b.expr.pseudos.visible(e)},b.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},b.ajaxSettings.xhr=function(){try{return new e.XMLHttpRequest}catch(e){}};var _t={0:200,1223:204},zt=b.ajaxSettings.xhr();h.cors=!!zt&&\"withCredentials\"in zt,h.ajax=zt=!!zt,b.ajaxTransport(function(t){var n,r;if(h.cors||zt&&!t.crossDomain)return{send:function(i,o){var a,s=t.xhr();if(s.open(t.type,t.url,t.async,t.username,t.password),t.xhrFields)for(a in t.xhrFields)s[a]=t.xhrFields[a];for(a in t.mimeType&&s.overrideMimeType&&s.overrideMimeType(t.mimeType),t.crossDomain||i[\"X-Requested-With\"]||(i[\"X-Requested-With\"]=\"XMLHttpRequest\"),i)s.setRequestHeader(a,i[a]);n=function(e){return function(){n&&(n=r=s.onload=s.onerror=s.onabort=s.ontimeout=s.onreadystatechange=null,\"abort\"===e?s.abort():\"error\"===e?\"number\"!=typeof s.status?o(0,\"error\"):o(s.status,s.statusText):o(_t[s.status]||s.status,s.statusText,\"text\"!==(s.responseType||\"text\")||\"string\"!=typeof s.responseText?{binary:s.response}:{text:s.responseText},s.getAllResponseHeaders()))}},s.onload=n(),r=s.onerror=s.ontimeout=n(\"error\"),void 0!==s.onabort?s.onabort=r:s.onreadystatechange=function(){4===s.readyState&&e.setTimeout(function(){n&&r()})},n=n(\"abort\");try{s.send(t.hasContent&&t.data||null)}catch(e){if(n)throw e}},abort:function(){n&&n()}}}),b.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),b.ajaxSetup({accepts:{script:\"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript\"},contents:{script:/\\b(?:java|ecma)script\\b/},converters:{\"text script\":function(e){return b.globalEval(e),e}}}),b.ajaxPrefilter(\"script\",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type=\"GET\")}),b.ajaxTransport(\"script\",function(e){var t,n;if(e.crossDomain||e.scriptAttrs)return{send:function(i,o){t=b(\"