From 01a5cdb7f22c4359ad8750f3117bd2589048c07e Mon Sep 17 00:00:00 2001 From: kang-donghoon-eland Date: Mon, 2 Sep 2024 00:17:56 +0900 Subject: [PATCH] =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PalmHill.BlazorChat.ApiClient.csproj | 2 +- .../Client/PalmHill.BlazorChat.Client.csproj | 10 +++++----- .../Server/PalmHill.BlazorChat.Server.csproj | 4 ++-- .../Server/Properties/launchSettings.json | 2 ++ PalmHill.BlazorChat/Server/appsettings.json | 4 ++-- PalmHill.Llama/LlamaKernel.cs | 7 ++++++- PalmHill.Llama/Models/InjectedModel.cs | 2 ++ PalmHill.Llama/Models/ModelConfig.cs | 2 +- PalmHill.Llama/PalmHill.Llama.csproj | 14 +++++++------- 9 files changed, 28 insertions(+), 19 deletions(-) diff --git a/PalmHill.BlazorChat.ApiClient/PalmHill.BlazorChat.ApiClient.csproj b/PalmHill.BlazorChat.ApiClient/PalmHill.BlazorChat.ApiClient.csproj index 2ab2228..fd54ea8 100644 --- a/PalmHill.BlazorChat.ApiClient/PalmHill.BlazorChat.ApiClient.csproj +++ b/PalmHill.BlazorChat.ApiClient/PalmHill.BlazorChat.ApiClient.csproj @@ -7,7 +7,7 @@ - + diff --git a/PalmHill.BlazorChat/Client/PalmHill.BlazorChat.Client.csproj b/PalmHill.BlazorChat/Client/PalmHill.BlazorChat.Client.csproj index c4b0798..98bb967 100644 --- a/PalmHill.BlazorChat/Client/PalmHill.BlazorChat.Client.csproj +++ b/PalmHill.BlazorChat/Client/PalmHill.BlazorChat.Client.csproj @@ -9,13 +9,13 @@ - - - + + + - + - + diff --git a/PalmHill.BlazorChat/Server/PalmHill.BlazorChat.Server.csproj b/PalmHill.BlazorChat/Server/PalmHill.BlazorChat.Server.csproj index b719597..303a0e8 100644 --- a/PalmHill.BlazorChat/Server/PalmHill.BlazorChat.Server.csproj +++ b/PalmHill.BlazorChat/Server/PalmHill.BlazorChat.Server.csproj @@ -8,8 +8,8 @@ - - + + diff --git a/PalmHill.BlazorChat/Server/Properties/launchSettings.json b/PalmHill.BlazorChat/Server/Properties/launchSettings.json index 4868937..bcafe56 100644 --- a/PalmHill.BlazorChat/Server/Properties/launchSettings.json +++ b/PalmHill.BlazorChat/Server/Properties/launchSettings.json @@ -3,6 +3,7 @@ "http": { "commandName": "Project", "launchBrowser": true, + "launchUrl": "https://localhost:7233", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" }, @@ -13,6 +14,7 @@ "https": { "commandName": "Project", "launchBrowser": true, + "launchUrl": "https://localhost:7233", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" }, diff --git a/PalmHill.BlazorChat/Server/appsettings.json b/PalmHill.BlazorChat/Server/appsettings.json index c6174d1..0855591 100644 --- a/PalmHill.BlazorChat/Server/appsettings.json +++ b/PalmHill.BlazorChat/Server/appsettings.json @@ -7,8 +7,8 @@ }, "AllowedHosts": "*", "InferenceModelConfig": { - "ModelPath": "C:\\models\\capybarahermes-2.5-mistral-7b.Q4_K_M.gguf", - "GpuLayerCount": 40, + "ModelPath": "d:\\llm\\exaone-3.0-7.8B-it-Q4_K_M.gguf", + "GpuLayerCount": 50, "ContextSize": 4096, "Gpu": 0, "AntiPrompts": [ "User:" ] diff --git a/PalmHill.Llama/LlamaKernel.cs b/PalmHill.Llama/LlamaKernel.cs index 56c452f..a63ec0b 100644 --- a/PalmHill.Llama/LlamaKernel.cs +++ b/PalmHill.Llama/LlamaKernel.cs @@ -37,7 +37,12 @@ public LlamaKernel(InjectedModel injectedModel) var embedding = new LLamaEmbedder(injectedModel.Model, injectedModel.EmbeddingParameters); var textEmbeddingGeneration = new LLamaSharpTextEmbeddingGenerator(embedding); km.WithLLamaSharpTextEmbeddingGeneration(textEmbeddingGeneration); - km.Build(); + km.WithCustomTextPartitioningOptions( new Microsoft.KernelMemory.Configuration.TextPartitioningOptions { + MaxTokensPerParagraph = 1, + MaxTokensPerLine = 1, + OverlappingTokens = 0 + }) + .Build(); }) .AddSingleton(); diff --git a/PalmHill.Llama/Models/InjectedModel.cs b/PalmHill.Llama/Models/InjectedModel.cs index 92cbccf..8bcb12f 100644 --- a/PalmHill.Llama/Models/InjectedModel.cs +++ b/PalmHill.Llama/Models/InjectedModel.cs @@ -18,6 +18,8 @@ public ModelParams EmbeddingParameters embeddingParams.ContextSize = ModelParams.ContextSize; embeddingParams.GpuLayerCount = ModelParams.GpuLayerCount; embeddingParams.MainGpu = ModelParams.MainGpu; + embeddingParams.SplitMode = ModelParams.SplitMode; + embeddingParams.TensorSplits = ModelParams.TensorSplits; return embeddingParams; } } diff --git a/PalmHill.Llama/Models/ModelConfig.cs b/PalmHill.Llama/Models/ModelConfig.cs index 51cd734..d7c71c7 100644 --- a/PalmHill.Llama/Models/ModelConfig.cs +++ b/PalmHill.Llama/Models/ModelConfig.cs @@ -5,7 +5,7 @@ public class ModelConfig public string ModelPath { get; set; } = ""; public int GpuLayerCount { get; set; } = 20; public uint ContextSize { get; set; } = 2048; - public int Gpu { get; set; } = 0; + public int Gpu { get; set; } = 1; public List AntiPrompts { get; set; } = []; } } diff --git a/PalmHill.Llama/PalmHill.Llama.csproj b/PalmHill.Llama/PalmHill.Llama.csproj index 74ac2c6..cd4d723 100644 --- a/PalmHill.Llama/PalmHill.Llama.csproj +++ b/PalmHill.Llama/PalmHill.Llama.csproj @@ -7,14 +7,14 @@ - - - - - + + + + + - - + +