From 89a96fdb2bf3c52947403bfbe9b41990dcaeb8f0 Mon Sep 17 00:00:00 2001 From: kchro3 Date: Thu, 7 Sep 2023 22:36:02 -0700 Subject: [PATCH 1/6] Metal support for Swift --- Package.swift | 8 ++++++-- ggml-metal.m | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Package.swift b/Package.swift index 96f52c4f0caad..b86e25570915f 100644 --- a/Package.swift +++ b/Package.swift @@ -4,6 +4,7 @@ import PackageDescription let package = Package( name: "llama", + platforms: [.macOS(.v11)], products: [ .library(name: "llama", targets: ["llama"]), ], @@ -11,16 +12,19 @@ let package = Package( .target( name: "llama", path: ".", - exclude: ["ggml-metal.metal"], sources: [ "ggml.c", "llama.cpp", "ggml-alloc.c", - "k_quants.c" + "k_quants.c", + "ggml-metal.m", ], publicHeadersPath: "spm-headers", cSettings: [ .unsafeFlags(["-Wno-shorten-64-to-32"]), + .unsafeFlags(["-fno-objc-arc"]), + .define("GGML_SWIFT"), + .define("GGML_USE_METAL"), .define("GGML_USE_K_QUANTS"), .define("GGML_USE_ACCELERATE") ], diff --git a/ggml-metal.m b/ggml-metal.m index 7e2355ce6bcc7..cd1308955f31f 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -141,12 +141,22 @@ @implementation GGMLMetalClass ctx->d_queue = dispatch_queue_create("llama.cpp", DISPATCH_QUEUE_CONCURRENT); -#if 0 - // compile from source string and show compile log + MTLCompileOptions* options = [MTLCompileOptions new]; + options.preprocessorMacros = @{ @"QK_K" : @(64) }; + +#ifdef GGML_SWIFT + // load the default.metallib file { NSError * error = nil; - ctx->library = [ctx->device newLibraryWithSource:msl_library_source options:nil error:&error]; + NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; + NSString * llamaBundlePath = [bundle pathForResource:@"llama_llama" ofType:@"bundle"]; + NSBundle * llamaBundle = [NSBundle bundleWithPath:llamaBundlePath]; + NSString * libPath = [llamaBundle pathForResource:@"default" ofType:@"metallib"]; + + // Load the metallib file into a Metal library + ctx->library = [ctx->device newLibraryWithFile:libPath error:&error]; + if (error) { metal_printf("%s: error: %s\n", __func__, [[error description] UTF8String]); return NULL; @@ -171,8 +181,6 @@ @implementation GGMLMetalClass } #ifdef GGML_QKK_64 - MTLCompileOptions* options = [MTLCompileOptions new]; - options.preprocessorMacros = @{ @"QK_K" : @(64) }; ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error]; #else ctx->library = [ctx->device newLibraryWithSource:src options:nil error:&error]; From ce92d754a33d4a35d86b572225ea55714927c42d Mon Sep 17 00:00:00 2001 From: kchro3 Date: Thu, 7 Sep 2023 22:46:56 -0700 Subject: [PATCH 2/6] update --- ggml-metal.m | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index cd1308955f31f..fadc7703d58af 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -141,9 +141,6 @@ @implementation GGMLMetalClass ctx->d_queue = dispatch_queue_create("llama.cpp", DISPATCH_QUEUE_CONCURRENT); - MTLCompileOptions* options = [MTLCompileOptions new]; - options.preprocessorMacros = @{ @"QK_K" : @(64) }; - #ifdef GGML_SWIFT // load the default.metallib file { @@ -181,6 +178,8 @@ @implementation GGMLMetalClass } #ifdef GGML_QKK_64 + MTLCompileOptions* options = [MTLCompileOptions new]; + options.preprocessorMacros = @{ @"QK_K" : @(64) }; ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error]; #else ctx->library = [ctx->device newLibraryWithSource:src options:nil error:&error]; From 99f85e7eb13ee9b2a47659adebd25d1752fb44eb Mon Sep 17 00:00:00 2001 From: kchro3 Date: Fri, 8 Sep 2023 09:26:29 -0700 Subject: [PATCH 3/6] add a toggle for arm/arm64 --- Package.swift | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Package.swift b/Package.swift index b86e25570915f..91a92d327d0cb 100644 --- a/Package.swift +++ b/Package.swift @@ -2,6 +2,20 @@ import PackageDescription +#if arch(arm) || arch(arm64) +let exclude: [String] = [] +let additionalSources: [String] = ["ggml-metal.m"] +let additionalSettings: [CSetting] = [ + .unsafeFlags(["-fno-objc-arc"]), + .define("GGML_SWIFT"), + .define("GGML_USE_METAL") +] +#else +let exclude: [String] = ["ggml-metal.metal"] +let additionalSources: [String] = [] +let additionalSettings: [CSetting] = [] +#endif + let package = Package( name: "llama", platforms: [.macOS(.v11)], @@ -12,26 +26,23 @@ let package = Package( .target( name: "llama", path: ".", + exclude: exclude, sources: [ "ggml.c", "llama.cpp", "ggml-alloc.c", "k_quants.c", - "ggml-metal.m", - ], + ] + additionalSources, publicHeadersPath: "spm-headers", cSettings: [ .unsafeFlags(["-Wno-shorten-64-to-32"]), - .unsafeFlags(["-fno-objc-arc"]), - .define("GGML_SWIFT"), - .define("GGML_USE_METAL"), .define("GGML_USE_K_QUANTS"), .define("GGML_USE_ACCELERATE") - ], + ] + additionalSettings, linkerSettings: [ .linkedFramework("Accelerate") ] - ), + ) ], cxxLanguageStandard: .cxx11 ) From 690c794b32635711bf4712a321dbef951532e5f5 Mon Sep 17 00:00:00 2001 From: kchro3 Date: Fri, 8 Sep 2023 09:48:56 -0700 Subject: [PATCH 4/6] set minimum versions for all platforms --- Package.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 91a92d327d0cb..9f10780f4b037 100644 --- a/Package.swift +++ b/Package.swift @@ -3,6 +3,12 @@ import PackageDescription #if arch(arm) || arch(arm64) +let platforms: [SupportedPlatform]? = [ + .macOS(.v11), + .iOS(.v11), + .watchOS(.v4), + .tvOS(.v11) +] let exclude: [String] = [] let additionalSources: [String] = ["ggml-metal.m"] let additionalSettings: [CSetting] = [ @@ -11,6 +17,7 @@ let additionalSettings: [CSetting] = [ .define("GGML_USE_METAL") ] #else +let platforms: [SupportedPlatform]? = nil let exclude: [String] = ["ggml-metal.metal"] let additionalSources: [String] = [] let additionalSettings: [CSetting] = [] @@ -18,7 +25,7 @@ let additionalSettings: [CSetting] = [] let package = Package( name: "llama", - platforms: [.macOS(.v11)], + platforms: platforms, products: [ .library(name: "llama", targets: ["llama"]), ], From 24013b37b66aa396f27e89d570426912e80e7034 Mon Sep 17 00:00:00 2001 From: kchro3 Date: Fri, 8 Sep 2023 20:27:43 -0700 Subject: [PATCH 5/6] update to use newLibraryWithURL --- ggml-metal.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ggml-metal.m b/ggml-metal.m index fadc7703d58af..d3584aa304380 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -150,9 +150,10 @@ @implementation GGMLMetalClass NSString * llamaBundlePath = [bundle pathForResource:@"llama_llama" ofType:@"bundle"]; NSBundle * llamaBundle = [NSBundle bundleWithPath:llamaBundlePath]; NSString * libPath = [llamaBundle pathForResource:@"default" ofType:@"metallib"]; + NSURL * libURL = [NSURL fileURLWithPath:libPath]; // Load the metallib file into a Metal library - ctx->library = [ctx->device newLibraryWithFile:libPath error:&error]; + ctx->library = [ctx->device newLibraryWithURL:libURL error:&error]; if (error) { metal_printf("%s: error: %s\n", __func__, [[error description] UTF8String]); From d58af4d46f2d9d4e9dd5fe74b194c6c683cdfa89 Mon Sep 17 00:00:00 2001 From: kchro3 <62481661+kchro3@users.noreply.github.com> Date: Sat, 9 Sep 2023 00:34:25 -0700 Subject: [PATCH 6/6] bump version Co-authored-by: Jhen-Jie Hong --- Package.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 9f10780f4b037..fb95ef7ebc59f 100644 --- a/Package.swift +++ b/Package.swift @@ -5,9 +5,9 @@ import PackageDescription #if arch(arm) || arch(arm64) let platforms: [SupportedPlatform]? = [ .macOS(.v11), - .iOS(.v11), + .iOS(.v14), .watchOS(.v4), - .tvOS(.v11) + .tvOS(.v14) ] let exclude: [String] = [] let additionalSources: [String] = ["ggml-metal.m"]