Skip to content

Update to llama.cpp and fix deprecations #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tomislaveric wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Update to llama.cpp and fix deprecations #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions .swiftpm/xcode/xcshareddata/xcschemes/LLamaSwift.xcscheme
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "2610"
version = "1.7">
<BuildAction
parallelizeBuildables = "YES"
buildImplicitDependencies = "YES"
buildArchitectures = "Automatic">
<BuildActionEntries>
<BuildActionEntry
buildForTesting = "YES"
buildForRunning = "YES"
buildForProfiling = "YES"
buildForArchiving = "YES"
buildForAnalyzing = "YES">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "LLamaSwift"
BuildableName = "LLamaSwift"
BlueprintName = "LLamaSwift"
ReferencedContainer = "container:">
</BuildableReference>
</BuildActionEntry>
</BuildActionEntries>
</BuildAction>
<TestAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
shouldUseLaunchSchemeArgsEnv = "YES"
shouldAutocreateTestPlan = "YES">
<Testables>
<TestableReference
skipped = "NO">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "llama-cpp-swiftTests"
BuildableName = "llama-cpp-swiftTests"
BlueprintName = "llama-cpp-swiftTests"
ReferencedContainer = "container:">
</BuildableReference>
</TestableReference>
</Testables>
</TestAction>
<LaunchAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
launchStyle = "0"
useCustomWorkingDirectory = "NO"
ignoresPersistentStateOnLaunch = "NO"
debugDocumentVersioning = "YES"
debugServiceExtension = "internal"
allowLocationSimulation = "YES">
</LaunchAction>
<ProfileAction
buildConfiguration = "Release"
shouldUseLaunchSchemeArgsEnv = "YES"
savedToolIdentifier = ""
useCustomWorkingDirectory = "NO"
debugDocumentVersioning = "YES">
<MacroExpansion>
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "LLamaSwift"
BuildableName = "LLamaSwift"
BlueprintName = "LLamaSwift"
ReferencedContainer = "container:">
</BuildableReference>
</MacroExpansion>
</ProfileAction>
<AnalyzeAction
buildConfiguration = "Debug">
</AnalyzeAction>
<ArchiveAction
buildConfiguration = "Release"
revealArchiveInOrganizer = "YES">
</ArchiveAction>
</Scheme>
13 changes: 2 additions & 11 deletions Package.resolved
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 10 additions & 7 deletions Package.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,24 @@ let package = Package(
.visionOS(.v1)
],
products: [
.library(
name: "LLamaSwift",
targets: ["LLamaSwift"]),
.library(name: "LLamaSwift", targets: ["LLamaSwift"]),
],
dependencies: [
.package(url: "https://github.com/ggerganov/llama.cpp", branch: "master"),
.package(url: "https://github.com/apple/swift-log.git", from: "1.6.1"),
],
targets: [
.binaryTarget(
name: "llama",
url: "https://github.com/ggml-org/llama.cpp/releases/download/b7642/llama-b7642-xcframework.zip",
checksum: "59b372b1991470cf6e3b48b49052495ddf5e1f39551b595dabb011a24b08b4fc"
),
.target(
name: "LLamaSwift",
name: "LLamaSwift",
dependencies: [
.product(name: "llama", package: "llama.cpp"),
"llama",
.product(name: "Logging", package: "swift-log"),
]
],
path: "Sources/llama-cpp-swift"
),
.testTarget(
name: "llama-cpp-swiftTests",
Expand Down
9 changes: 5 additions & 4 deletions Sources/llama-cpp-swift/LLama.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ public actor LLama {
// Initialize sampling
let sparams = llama_sampler_chain_default_params()
self.sampling = llama_sampler_chain_init(sparams)
llama_sampler_chain_add(self.sampling, llama_sampler_init_temp(0.8))
llama_sampler_chain_add(self.sampling, llama_sampler_init_softmax())
llama_sampler_chain_add(self.sampling, llama_sampler_init_dist(1234))
llama_sampler_chain_add(self.sampling, llama_sampler_init_top_k(40));
llama_sampler_chain_add(self.sampling, llama_sampler_init_top_p(0.9, 1));
llama_sampler_chain_add(self.sampling, llama_sampler_init_temp (0.4));
llama_sampler_chain_add(self.sampling, llama_sampler_init_dist (1234));
}

deinit {
Expand Down Expand Up @@ -197,7 +198,7 @@ public actor LLama {
var newTokenID: llama_token = 0
newTokenID = llama_sampler_sample(sampling, model.context, batch.n_tokens - 1)

if llama_token_is_eog(model.model, newTokenID) || nCur == nLen {
if llama_vocab_is_eog(model.model, newTokenID) || nCur == nLen {
isDone = true
let newTokenStr = String(
decoding: Data(temporaryInvalidCChars.map { UInt8(bitPattern: $0) }), as: UTF8.self)
Expand Down
8 changes: 4 additions & 4 deletions Sources/llama-cpp-swift/Model.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public final class Model {
logger.debug("Running on simulator, force use n_gpu_layers = 0")
#endif

guard let model = llama_load_model_from_file(modelPath, modelParams) else {
guard let model = llama_model_load_from_file(modelPath, modelParams) else {
llama_backend_free()
throw InitializationError(message: "Failed to load model", code: .failedToLoadModel)
}
Expand All @@ -31,8 +31,8 @@ public final class Model {
ctxParams.n_threads = Int32(nThreads)
ctxParams.n_threads_batch = Int32(nThreads)

guard let context = llama_new_context_with_model(model, ctxParams) else {
llama_free_model(model)
guard let context = llama_init_from_model(model, ctxParams) else {
llama_model_free(model)
llama_backend_free()
throw InitializationError(
message: "Failed to initialize context", code: .failedToInitializeContext)
Expand All @@ -42,7 +42,7 @@ public final class Model {

deinit {
llama_free(context)
llama_free_model(model)
llama_model_free(model)
llama_backend_free()
}
}