GitHub - chulucninh09/llm_serving

This is the repo for quick setup VM for GPU LLM inference

Name		Name	Last commit message	Last commit date
Latest commit History 23 Commits
blis @ b5d5783		blis @ b5d5783
ik_llama @ a903409		ik_llama @ a903409
llama.cpp @ a0ed91a		llama.cpp @ a0ed91a
ragflow @ 1f4a178		ragflow @ 1f4a178
templates		templates
vllm @ 79ed460		vllm @ 79ed460
.gitignore		.gitignore
.gitmodules		.gitmodules
README.md		README.md
build_llama.cpp.sh		build_llama.cpp.sh
compile_vllm.sh		compile_vllm.sh
docker-compose.yaml		docker-compose.yaml
docker_install.sh		docker_install.sh
gpu_undervolt.sh		gpu_undervolt.sh
ik_llama_args.sh		ik_llama_args.sh
install_cuda.sh		install_cuda.sh
install_nvidia_container_toolkit.sh		install_nvidia_container_toolkit.sh
install_vllm.sh		install_vllm.sh
llama.log		llama.log
llama_args.sh		llama_args.sh
lmcache.yaml		lmcache.yaml
parse_args.sh		parse_args.sh
run_ik_llama.sh		run_ik_llama.sh
run_llama.sh		run_llama.sh
run_vllm.sh		run_vllm.sh
run_vllm_embedding.sh		run_vllm_embedding.sh
setup.sh		setup.sh
stress_test_embedding.py		stress_test_embedding.py
stress_test_llm.py		stress_test_llm.py
supported.txt		supported.txt
tmux_monitor.sh		tmux_monitor.sh
vllm_args.sh		vllm_args.sh
vllm_args_embedding.sh		vllm_args_embedding.sh

Navigation Menu