diff --git a/.gitignore b/.gitignore index 9baecb6..5f15105 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /ailia/** /ailia_trial/** /ailia_tokenizer/** +/ailia_llm/** *.onnx *.prototxt CMakeCache.txt @@ -38,6 +39,8 @@ output.png /natural_language_processing/multilingual-e5/multilingual-e5 /natural_language_processing/sentence_transformers/sentence_transformers /natural_language_processing/t5_whisper_medical/t5_whisper_medical +/large_language_model/gemma4/gemma4 +/large_language_model/gemma4/gemma4-multimodal /object_detection/m2det/m2det /object_detection/yolov3-tiny/yolov3-tiny /object_detection/yolox/yolox @@ -68,6 +71,8 @@ output.png /face_recognition/mediapipe_iris/man.jpg /object_detection/yolov3-tiny/couple.jpg /pose_estimation/lightweight-human-pose-estimation/balloon.png +/large_language_model/gemma4/*.gguf +/large_language_model/gemma4/sample_image.jpg # ailia Voice / GPT-SoVITS sample data /audio_processing/gpt-sovits-v2-pro/gpt-sovits-v2-pro diff --git a/.gitmodules b/.gitmodules index 5dcd657..c988b48 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "ailia_tflite"] path = ailia_tflite url = https://github.com/ailia-ai/ailia-tflite-cpp.git +[submodule "ailia_llm"] + path = ailia_llm + url = https://github.com/ailia-ai/ailia-llm-cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index c965c99..b9aef9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,18 +37,19 @@ set(AILIA_SPEECH_PATH ${CMAKE_SOURCE_DIR}/ailia_speech/library) set(AILIA_VOICE_PATH ${CMAKE_SOURCE_DIR}/ailia_voice/library) set(AILIA_TRACKER_PATH ${CMAKE_SOURCE_DIR}/ailia_tracker/library) set(AILIA_TFLITE_PATH ${CMAKE_SOURCE_DIR}/ailia_tflite/library) +set(AILIA_LLM_PATH ${CMAKE_SOURCE_DIR}/ailia_llm/library) -set (INCLUDE_PATH ${AILIA_LIBRARY_PATH}/include ${AILIA_AUDIO_PATH}/include ${AILIA_TOKENIZER_PATH}/include ${AILIA_SPEECH_PATH}/include ${AILIA_VOICE_PATH}/include ${AILIA_TRACKER_PATH}/include ${AILIA_TFLITE_PATH}/include ../../util) +set (INCLUDE_PATH ${AILIA_LIBRARY_PATH}/include ${AILIA_AUDIO_PATH}/include ${AILIA_TOKENIZER_PATH}/include ${AILIA_SPEECH_PATH}/include ${AILIA_VOICE_PATH}/include ${AILIA_TRACKER_PATH}/include ${AILIA_TFLITE_PATH}/include ${AILIA_LLM_PATH}/include ../../util) if(WIN32) - set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/windows/x64 ${AILIA_AUDIO_PATH}/windows/x64 ${AILIA_TOKENIZER_PATH}/windows/x64 ${AILIA_SPEECH_PATH}/windows/x64 ${AILIA_VOICE_PATH}/windows/x64 ${AILIA_TRACKER_PATH}/windows/x64 ${AILIA_TFLITE_PATH}/windows/x64) + set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/windows/x64 ${AILIA_AUDIO_PATH}/windows/x64 ${AILIA_TOKENIZER_PATH}/windows/x64 ${AILIA_SPEECH_PATH}/windows/x64 ${AILIA_VOICE_PATH}/windows/x64 ${AILIA_TRACKER_PATH}/windows/x64 ${AILIA_TFLITE_PATH}/windows/x64 ${AILIA_LLM_PATH}/windows/x64) elseif(APPLE) - set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/mac ${AILIA_AUDIO_PATH}/mac ${AILIA_TOKENIZER_PATH}/mac ${AILIA_SPEECH_PATH}/mac ${AILIA_VOICE_PATH}/mac ${AILIA_TRACKER_PATH}/mac ${AILIA_TFLITE_PATH}/mac) + set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/mac ${AILIA_AUDIO_PATH}/mac ${AILIA_TOKENIZER_PATH}/mac ${AILIA_SPEECH_PATH}/mac ${AILIA_VOICE_PATH}/mac ${AILIA_TRACKER_PATH}/mac ${AILIA_TFLITE_PATH}/mac ${AILIA_LLM_PATH}/mac) elseif(UNIX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") - set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/linux/arm64-v8a ${AILIA_AUDIO_PATH}/linux/arm64-v8a ${AILIA_TOKENIZER_PATH}/linux/arm64-v8a ${AILIA_SPEECH_PATH}/linux/arm64-v8a ${AILIA_VOICE_PATH}/linux/arm64-v8a ${AILIA_TRACKER_PATH}/linux/arm64-v8a ${AILIA_TFLITE_PATH}/linux/arm64-v8a) + set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/linux/arm64-v8a ${AILIA_AUDIO_PATH}/linux/arm64-v8a ${AILIA_TOKENIZER_PATH}/linux/arm64-v8a ${AILIA_SPEECH_PATH}/linux/arm64-v8a ${AILIA_VOICE_PATH}/linux/arm64-v8a ${AILIA_TRACKER_PATH}/linux/arm64-v8a ${AILIA_TFLITE_PATH}/linux/arm64-v8a ${AILIA_LLM_PATH}/linux/arm64-v8a) elseif(UNIX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a") set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/linux/armeabi-v7a ${AILIA_AUDIO_PATH}/linux/armeabi-v7a ${AILIA_TOKENIZER_PATH}/linux/armeabi-v7a ${AILIA_SPEECH_PATH}/linux/armeabi-v7a) else(UNIX) - set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/linux/x64 ${AILIA_AUDIO_PATH}/linux/x64 ${AILIA_TOKENIZER_PATH}/linux/x64 ${AILIA_SPEECH_PATH}/linux/x64 ${AILIA_VOICE_PATH}/linux/x64 ${AILIA_TRACKER_PATH}/linux/x64 ${AILIA_TFLITE_PATH}/linux/x64) + set (LIBRARY_PATH ${AILIA_LIBRARY_PATH}/linux/x64 ${AILIA_AUDIO_PATH}/linux/x64 ${AILIA_TOKENIZER_PATH}/linux/x64 ${AILIA_SPEECH_PATH}/linux/x64 ${AILIA_VOICE_PATH}/linux/x64 ${AILIA_TRACKER_PATH}/linux/x64 ${AILIA_TFLITE_PATH}/linux/x64 ${AILIA_LLM_PATH}/linux/x64) endif(WIN32) # require ailia SDK and ailia Audio (no OpenCV) @@ -95,6 +96,12 @@ add_subdirectory(natural_language_processing/t5_whisper_medical) add_subdirectory(natural_language_processing/multilingual-e5) endif() +# require ailia LLM +if(UNIX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a") +else() +add_subdirectory(large_language_model/gemma4) +endif() + # require ailia Voice if(UNIX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a") else() diff --git a/README.md b/README.md index 90217f9..8b3863d 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,13 @@ cd object_detection/yolox |[multilingual-e5](/natural_language_processing/multilingual-e5) | [multilingual-e5-base](https://huggingface.co/intfloat/multilingual-e5-base) | Pytorch | 1.2.15 and later | [JP](https://medium.com/axinc/multilingual-e5-%E5%A4%9A%E8%A8%80%E8%AA%9E%E3%81%AE%E3%83%86%E3%82%AD%E3%82%B9%E3%83%88%E3%82%92embedding%E3%81%99%E3%82%8B%E6%A9%9F%E6%A2%B0%E5%AD%A6%E7%BF%92%E3%83%A2%E3%83%87%E3%83%AB-71f1dec7c4f0) | |[g2p_en](/natural_language_processing/g2p_en) | [g2p_en](https://github.com/Kyubyong/g2p) | Pytorch | 1.2.14 and later | | +## Large language model + +| Name | Detail | Exported From | Supported Ailia Version | +|:-----------|------------:|:------------:|:------------:| +|[gemma4](/large_language_model/gemma4) | Gemma 4n (E2B) text chat (gguf) via [ailia LLM](https://github.com/ailia-ai/ailia-llm-cpp) | gguf | 1.5.0 and later | | +|[gemma4-multimodal](/large_language_model/gemma4) | Gemma 4n (E2B) multimodal (gguf + mmproj) via [ailia LLM](https://github.com/ailia-ai/ailia-llm-cpp) | gguf | 1.5.0 and later | | + ## Object detection | Name | Detail | Exported From | Supported Ailia Version | diff --git a/ailia_llm b/ailia_llm new file mode 160000 index 0000000..a892f96 --- /dev/null +++ b/ailia_llm @@ -0,0 +1 @@ +Subproject commit a892f96f900ec710b6e5a811332a8492b2dd139b diff --git a/large_language_model/gemma4/CMakeLists.txt b/large_language_model/gemma4/CMakeLists.txt new file mode 100644 index 0000000..c0c8866 --- /dev/null +++ b/large_language_model/gemma4/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.10) + +project(gemma4 CXX) + +set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + +include_directories(${INCLUDE_PATH}) +link_directories(${LIBRARY_PATH}) + +add_executable(gemma4 gemma4.cpp) +target_compile_features(gemma4 PRIVATE cxx_std_11) +target_link_libraries(gemma4 ailia_llm) + +add_executable(gemma4-multimodal gemma4-multimodal.cpp) +target_compile_features(gemma4-multimodal PRIVATE cxx_std_11) +target_link_libraries(gemma4-multimodal ailia_llm) + +set (CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}) +install(TARGETS gemma4 gemma4-multimodal RUNTIME DESTINATION .) diff --git a/large_language_model/gemma4/LICENSE b/large_language_model/gemma4/LICENSE new file mode 100644 index 0000000..8e5d356 --- /dev/null +++ b/large_language_model/gemma4/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Nils Reimers + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and +limitations under the License. diff --git a/large_language_model/gemma4/gemma4-multimodal.bat b/large_language_model/gemma4/gemma4-multimodal.bat new file mode 100644 index 0000000..2261d6e --- /dev/null +++ b/large_language_model/gemma4/gemma4-multimodal.bat @@ -0,0 +1,36 @@ +@echo off +setlocal enabledelayedexpansion +cd %~dp0 + +set MODEL=gemma +set EXE=gemma4-multimodal +set FILE1=gemma-4-E2B-it-Q4_K_M.gguf +set FILE2=gemma-4-E2B-it-mmproj-F16.gguf +set FILE3=sample_image.jpg + +rem download +if not "%1" == "-h" if not "%1" == "--help" ( + if not exist %FILE1% ( + echo Downloading gguf file... ^(save path: %FILE1%^) + curl https://storage.googleapis.com/ailia-models/%MODEL%/%FILE1% -o %FILE1% + ) +) +if not "%1" == "-h" if not "%1" == "--help" ( + if not exist %FILE2% ( + echo Downloading mmproj gguf file... ^(save path: %FILE2%^) + curl https://storage.googleapis.com/ailia-models/%MODEL%/%FILE2% -o %FILE2% + ) +) +if not "%1" == "-h" if not "%1" == "--help" ( + if not exist %FILE3% ( + echo Downloading sample image... ^(save path: %FILE3%^) + curl https://storage.googleapis.com/ailia-models/misc/%FILE3% -o %FILE3% + ) +) + +rem execute +if "%~1" == "" ( + .\%EXE%.exe %FILE1% %FILE2% %FILE3% +) else ( + .\%EXE%.exe %* +) diff --git a/large_language_model/gemma4/gemma4-multimodal.cpp b/large_language_model/gemma4/gemma4-multimodal.cpp new file mode 100644 index 0000000..a1d2d8f --- /dev/null +++ b/large_language_model/gemma4/gemma4-multimodal.cpp @@ -0,0 +1,170 @@ +#include "ailia_llm.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +void DisplayBackend(){ + unsigned int count; + ailiaLLMGetBackendCount(&count); + for (unsigned int i = 0; i < count; i++){ + const char * name; + ailiaLLMGetBackendName(&name, i); + printf("ailia LLM Backend : %d %s\n", i, name); + } +} + +void DisplayContextSize(struct AILIALLM* llm){ + unsigned int context_size; + ailiaLLMGetContextSize(llm, &context_size); + printf("ailia LLM Context size : %d\n", context_size); +} + +void DisplayMultimodalCapabilities(struct AILIALLM* llm){ + unsigned int vision_support, audio_support; + int status = ailiaLLMGetMultimodalCapabilities(llm, &vision_support, &audio_support); + if (status == AILIA_LLM_STATUS_SUCCESS){ + printf("ailia LLM Multimodal Capabilities:\n"); + printf(" Vision Support: %s\n", vision_support ? "Yes" : "No"); + printf(" Audio Support: %s\n", audio_support ? "Yes" : "No"); + } else { + printf("Failed to get multimodal capabilities: %d\n", status); + } +} + +int Generate(struct AILIALLM* llm, std::string &text){ + unsigned int done = 0; + text = ""; + while(true) { + int status = ailiaLLMGenerate(llm, &done); + if (done == 1){ + break; + } + if (status != 0) { + if (status == AILIA_LLM_STATUS_CONTEXT_FULL){ + printf("ailia LLM Context Size Full. Please reduce prompt messages."); + } else { + printf("ailia LLM Error Detected %d\n", status); + } + return -1; + } + + unsigned int size = 0; + ailiaLLMGetDeltaTextSize(llm, &size); + std::vector delta(size); + ailiaLLMGetDeltaText(llm, &delta[0], delta.size()); + text = text + std::string(&delta[0]); + } + return 0; +} + +int main(int argc, char *argv[]){ + const char* model_path = "gemma-4-E2B-it-Q4_K_M.gguf"; + const char* mmproj_path = "gemma-4-E2B-it-mmproj-F16.gguf"; + const char* image_path = "sample_image.jpg"; + + if (argc >= 2) model_path = argv[1]; + if (argc >= 3) mmproj_path = argv[2]; + if (argc >= 4) image_path = argv[3]; + + if (argc == 1){ + printf("Usage : gemma4-multimodal [image_path]\n"); + printf("Using defaults: %s %s %s\n", model_path, mmproj_path, image_path); + } + + DisplayBackend(); + + struct AILIALLM* llm; + int n_ctx = 2048; + + int status = ailiaLLMCreate(&llm); + if (status != AILIA_LLM_STATUS_SUCCESS){ + printf("Failed to create LLM instance: %d\n", status); + return -1; + } + + #ifdef _WIN32 + wchar_t model_path_w[1024]; + MultiByteToWideChar(CP_UTF8, 0, model_path, -1, model_path_w, sizeof(model_path_w)/sizeof(wchar_t)); + status = ailiaLLMOpenModelFileW(llm, model_path_w, n_ctx); + #else + status = ailiaLLMOpenModelFileA(llm, model_path, n_ctx); + #endif + + if (status != AILIA_LLM_STATUS_SUCCESS){ + printf("Failed to load text model: %d\n", status); + ailiaLLMDestroy(llm); + return -1; + } + + #ifdef _WIN32 + wchar_t mmproj_path_w[1024]; + MultiByteToWideChar(CP_UTF8, 0, mmproj_path, -1, mmproj_path_w, sizeof(mmproj_path_w)/sizeof(wchar_t)); + status = ailiaLLMOpenMultimodalProjectorFileW(llm, mmproj_path_w); + #else + status = ailiaLLMOpenMultimodalProjectorFileA(llm, mmproj_path); + #endif + + if (status != AILIA_LLM_STATUS_SUCCESS){ + printf("Failed to load multimodal projector: %d\n", status); + ailiaLLMDestroy(llm); + return -1; + } + + ailiaLLMSetSamplingParams(llm, 40, 0.9, 0.4, 1234); + + DisplayContextSize(llm); + DisplayMultimodalCapabilities(llm); + + std::vector messages; + AILIALLMMultimodalChatMessage message; + + message.role = "system"; + message.content = u8"You are a helpful assistant that can analyze images. Respond in Japanese."; + message.media_data = NULL; + message.media_count = 0; + messages.push_back(message); + + AILIALLMMediaData media_data; + media_data.media_type = "image"; + media_data.file_path = image_path; + media_data.data = NULL; + media_data.data_size = 0; + media_data.width = 0; + media_data.height = 0; + + message.role = "user"; + message.content = u8"この画像について詳しく説明してください: <__image__>"; + message.media_data = &media_data; + message.media_count = 1; + messages.push_back(message); + + printf("ailia LLM Multimodal Query: %s\n", message.content); + printf("ailia LLM Image Path: %s\n", image_path); + + status = ailiaLLMSetMultimodalPrompt(llm, &messages[0], messages.size()); + if (status != AILIA_LLM_STATUS_SUCCESS){ + printf("Failed to set multimodal prompt: %d\n", status); + ailiaLLMDestroy(llm); + return -1; + } + + std::string answer; + int gen_status = Generate(llm, answer); + if (gen_status != 0){ + printf("Failed to generate response\n"); + ailiaLLMDestroy(llm); + return -1; + } + + printf("ailia LLM Multimodal Answer: %s\n", answer.c_str()); + + ailiaLLMDestroy(llm); + + return 0; +} diff --git a/large_language_model/gemma4/gemma4-multimodal.sh b/large_language_model/gemma4/gemma4-multimodal.sh new file mode 100755 index 0000000..d4b2b8b --- /dev/null +++ b/large_language_model/gemma4/gemma4-multimodal.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +MODEL="gemma" +EXE="gemma4-multimodal" +FILE1="gemma-4-E2B-it-Q4_K_M.gguf" +FILE2="gemma-4-E2B-it-mmproj-F16.gguf" +FILE3="sample_image.jpg" + +#download +if [ ! "$1" = "-h" ] && [ ! "$1" = "--help" ]; then + if [ ! -e ${FILE1} ]; then + echo "Downloading gguf file... save path: ${FILE1}" + curl https://storage.googleapis.com/ailia-models/${MODEL}/${FILE1} -o ${FILE1} + fi +fi +if [ ! "$1" = "-h" ] && [ ! "$1" = "--help" ]; then + if [ ! -e ${FILE2} ]; then + echo "Downloading mmproj gguf file... save path: ${FILE2}" + curl https://storage.googleapis.com/ailia-models/${MODEL}/${FILE2} -o ${FILE2} + fi +fi +if [ ! "$1" = "-h" ] && [ ! "$1" = "--help" ]; then + if [ ! -e ${FILE3} ]; then + echo "Downloading sample image... save path: ${FILE3}" + curl https://storage.googleapis.com/ailia-models/misc/${FILE3} -o ${FILE3} + fi +fi + +#execute +if [ $# -eq 0 ]; then + ./${EXE} ${FILE1} ${FILE2} ${FILE3} +else + ./${EXE} $* +fi diff --git a/large_language_model/gemma4/gemma4.bat b/large_language_model/gemma4/gemma4.bat new file mode 100644 index 0000000..5f9c3ea --- /dev/null +++ b/large_language_model/gemma4/gemma4.bat @@ -0,0 +1,22 @@ +@echo off +setlocal enabledelayedexpansion +cd %~dp0 + +set MODEL=gemma +set EXE=gemma4 +set FILE1=gemma-4-E2B-it-Q4_K_M.gguf + +rem download +if not "%1" == "-h" if not "%1" == "--help" ( + if not exist %FILE1% ( + echo Downloading gguf file... ^(save path: %FILE1%^) + curl https://storage.googleapis.com/ailia-models/%MODEL%/%FILE1% -o %FILE1% + ) +) + +rem execute +if "%~1" == "" ( + .\%EXE%.exe -m %FILE1% +) else ( + .\%EXE%.exe %* +) diff --git a/large_language_model/gemma4/gemma4.cpp b/large_language_model/gemma4/gemma4.cpp new file mode 100644 index 0000000..14c9f3a --- /dev/null +++ b/large_language_model/gemma4/gemma4.cpp @@ -0,0 +1,150 @@ +#include "ailia_llm.h" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +void DisplayBackend(){ + unsigned int count; + ailiaLLMGetBackendCount(&count); + for (unsigned int i = 0; i < count; i++){ + const char * name; + ailiaLLMGetBackendName(&name, i); + printf("ailia LLM Backend : %d %s\n", i, name); + } +} + +void DisplayContextSize(struct AILIALLM* llm){ + unsigned int context_size; + ailiaLLMGetContextSize(llm, &context_size); + printf("ailia LLM Context size : %d\n", context_size); +} + +int Generate(struct AILIALLM* llm, std::string &text){ + unsigned int done = 0; + text = ""; + while(true) { + int status = ailiaLLMGenerate(llm, &done); + if (done == 1){ + break; + } + if (status != 0) { + if (status == AILIA_LLM_STATUS_CONTEXT_FULL){ + printf("ailia LLM Context Size Full. Please reduce prompt messages."); + } else { + printf("ailia LLM Error Detected %d\n", status); + } + return -1; + } + + unsigned int size = 0; + ailiaLLMGetDeltaTextSize(llm, &size); + std::vector delta(size); + ailiaLLMGetDeltaText(llm, &delta[0], delta.size()); + text = text + std::string(&delta[0]); + } + return 0; +} + +void printUsage(const char* prog_name) { + std::cerr << "Usage: " << prog_name << " -m MODEL_PATH" << std::endl; + std::cerr << " -m, --model MODEL_PATH path to model file (required)" << std::endl; +} + +int main(int argc, char *argv[]){ + std::string model_path; + bool model_specified = false; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "--model") == 0) { + if (i + 1 < argc) { + model_path = argv[i + 1]; + model_specified = true; + i++; + } else { + std::cerr << "Error: -m/--model requires a model path argument" << std::endl; + printUsage(argv[0]); + return -1; + } + } else { + std::cerr << "Error: Unknown argument: " << argv[i] << std::endl; + printUsage(argv[0]); + return -1; + } + } + + if (!model_specified) { + model_path = "gemma-4-E2B-it-Q4_K_M.gguf"; + } + + DisplayBackend(); + + struct AILIALLM* llm; + int n_ctx = 512; + ailiaLLMCreate(&llm); + + printf("Loading model: %s\n", model_path.c_str()); + #ifdef _WIN32 + int size_needed = MultiByteToWideChar(CP_UTF8, 0, model_path.c_str(), -1, NULL, 0); + std::wstring widestr(size_needed, 0); + MultiByteToWideChar(CP_UTF8, 0, model_path.c_str(), -1, &widestr[0], size_needed); + int status = ailiaLLMOpenModelFileW(llm, widestr.c_str(), n_ctx); + #else + int status = ailiaLLMOpenModelFileA(llm, model_path.c_str(), n_ctx); + #endif + + if (status != 0) { + printf("Failed to load model file: %s (status: %d)\n", model_path.c_str(), status); + ailiaLLMDestroy(llm); + return -1; + } + + ailiaLLMSetSamplingParams(llm, 40, 0.9, 0.4, 1234); + + DisplayContextSize(llm); + + std::vector messages; + AILIALLMChatMessage message; + + message.role = "system"; + message.content = u8"語尾に「くま」をつけて回答してください。"; + messages.push_back(message); + + message.role = "user"; + message.content = u8"こんにちは。"; + messages.push_back(message); + + printf("ailia LLM Query : %s\n", message.content); + + ailiaLLMSetPrompt(llm, &messages[0], messages.size()); + std::string answer; + Generate(llm, answer); + + printf("ailia LLM Answer : %s\n", answer.c_str()); + + message.role = "assistant"; + message.content = answer.c_str(); + messages.push_back(message); + + message.role = "user"; + message.content = u8"さっきの回答を英語に翻訳してください。"; + messages.push_back(message); + + printf("ailia LLM Query : %s\n", message.content); + + ailiaLLMSetPrompt(llm, &messages[0], messages.size()); + std::string answer2; + Generate(llm, answer2); + + printf("ailia LLM Answer : %s\n", answer2.c_str()); + + ailiaLLMDestroy(llm); + + return 0; +} diff --git a/large_language_model/gemma4/gemma4.sh b/large_language_model/gemma4/gemma4.sh new file mode 100755 index 0000000..73da2d2 --- /dev/null +++ b/large_language_model/gemma4/gemma4.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +MODEL="gemma" +EXE="gemma4" +FILE1="gemma-4-E2B-it-Q4_K_M.gguf" + +#download +if [ ! "$1" = "-h" ] && [ ! "$1" = "--help" ]; then + if [ ! -e ${FILE1} ]; then + echo "Downloading gguf file... save path: ${FILE1}" + curl https://storage.googleapis.com/ailia-models/${MODEL}/${FILE1} -o ${FILE1} + fi +fi + +#execute +if [ $# -eq 0 ]; then + ./${EXE} -m ${FILE1} +else + ./${EXE} $* +fi diff --git a/test_all_samples.sh b/test_all_samples.sh index cb4adb6..a926444 100755 --- a/test_all_samples.sh +++ b/test_all_samples.sh @@ -28,7 +28,7 @@ fi mkdir -p "${LOG_DIR}" # Set runtime library search path so the bundled .so files are picked up. -export LD_LIBRARY_PATH="${ROOT_DIR}/ailia/library/linux/x64:${ROOT_DIR}/ailia_audio/library/linux/x64:${ROOT_DIR}/ailia_tokenizer/library/linux/x64:${ROOT_DIR}/ailia_speech/library/linux/x64:${ROOT_DIR}/ailia_voice/library/linux/x64:${ROOT_DIR}/ailia_tracker/library/linux/x64:${ROOT_DIR}/ailia_tflite/library/linux/x64:${LD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="${ROOT_DIR}/ailia/library/linux/x64:${ROOT_DIR}/ailia_audio/library/linux/x64:${ROOT_DIR}/ailia_tokenizer/library/linux/x64:${ROOT_DIR}/ailia_speech/library/linux/x64:${ROOT_DIR}/ailia_voice/library/linux/x64:${ROOT_DIR}/ailia_tracker/library/linux/x64:${ROOT_DIR}/ailia_tflite/library/linux/x64:${ROOT_DIR}/ailia_llm/library/linux/x64:${LD_LIBRARY_PATH:-}" # (sample_dir, sample_binary, args) -- args are appended verbatim SAMPLES=( @@ -52,6 +52,8 @@ SAMPLES=( "natural_language_processing/sentence_transformers|sentence_transformers|" "natural_language_processing/t5_whisper_medical|t5_whisper_medical|" "natural_language_processing/multilingual-e5|multilingual-e5|" + "large_language_model/gemma4|gemma4|" + "large_language_model/gemma4|gemma4-multimodal|" "object_detection/yolov3-tiny|yolov3-tiny|" "object_detection/yolox|yolox|" "object_detection/m2det|m2det|"