Much better prompt building

Bottomly.Tests/LlmBot/LlmBotTests.cs

@@ -114,7 +114,7 @@ public void IsSuccess_LlmTimeoutResponse_ReturnsFalse()
     }
 }
 
-public class LlmBrokerExtensionsTests
+public class LlmClientExtensionsTests
 {
     [Fact]
     public void ToChatContext_IncludesMessageHistoryAndUserInfo()
@@ -132,11 +132,13 @@ public void ToChatContext_IncludesMessageHistoryAndUserInfo()
 
         var result = ctx.ToChatContext();
 
-        result.ShouldContain("alice: hello");
-        result.ShouldContain("bob: world");
-        result.ShouldContain("alice: likes tea");
-        result.ShouldContain("Begin Prompt Context");
-        result.ShouldContain("End Prompt Context");
+        result.ShouldContain("\"message_history\"");
+        result.ShouldContain("\"users\"");
+        result.ShouldContain("alice");
+        result.ShouldContain("hello");
+        result.ShouldContain("bob");
+        result.ShouldContain("world");
+        result.ShouldContain("likes tea");
     }
 
     [Fact]
@@ -148,8 +150,6 @@ public void ToChatPromptMessage_IncludesUsernameAndText()
 
         result.ShouldContain("carol");
         result.ShouldContain("what time is it?");
-        result.ShouldContain("Begin Main Prompt");
-        result.ShouldContain("End Main Prompt");
     }
 }
 
@@ -177,6 +177,8 @@ public void ToArray_ContainsThreeMessages()
 
         array.Length.ShouldBe(3);
         array[0].Role.ShouldBe(ChatRole.System);
+        array[1].Role.ShouldBe(ChatRole.System);
+        array[2].Role.ShouldBe(ChatRole.User);
     }
 
     [Fact]

Bottomly.Tests/Slack/MessageEventHandlers/ConversationMessageHandling/ConversationMessageHandlerTests.cs

@@ -32,6 +32,7 @@ public ConversationMessageHandlerTests()
         _handler = new ConversationMessageHandler(
             _mockLlmBroker.Object,
             _mockSlackBroker.Object,
+            new SlackParser(new Mock<IMemberRepository>().Object),
             _mockApiClient.Object,
             _mockMemberRepo.Object,
             _mockFeatureFlagRepo.Object,

Bottomly.Tests/Slack/MessageEventHandlers/ConversationMessageHandling/MessageContextExtensionsTests.cs

@@ -1,13 +1,18 @@
 using Bottomly.LlmBot;
 using Bottomly.Models;
+using Bottomly.Repositories;
+using Bottomly.Slack;
 using Bottomly.Slack.MessageEventHandlers.ConversationMessageHandling;
+using Moq;
 using Shouldly;
 using SlackNet.Events;
 
 namespace Bottomly.Tests.Slack.MessageEventHandlers.ConversationMessageHandling;
 
 public class MessageContextExtensionsTests
 {
+    private readonly SlackParser _parser = new(new Mock<IMemberRepository>().Object);
+
     [Fact]
     public void CreateFromMember_MapsUsernameAndNote()
     {
@@ -27,24 +32,24 @@ public void CreateFromMember_MapsUsernameAndNote()
     }
 
     [Fact]
-    public void CreateFromSlackMessage_KnownUser_TranslatesUsername()
+    public async Task CreateFromSlackMessage_KnownUser_TranslatesUsername()
     {
         var message = new MessageEvent { User = "U123", Text = "hello there" };
         var memberLookup = new Dictionary<string, string> { ["U123"] = "alice" };
 
-        var inputMessage = BottomlyInputMessage.CreateFromSlackMessage(message, memberLookup);
+        var inputMessage = await BottomlyInputMessage.CreateFromSlackMessage(message, memberLookup, _parser);
 
         inputMessage.Username.ShouldBe("alice");
         inputMessage.Text.ShouldBe("hello there");
     }
 
     [Fact]
-    public void CreateFromSlackMessage_UnknownUser_FallsBackToSlackId()
+    public async Task CreateFromSlackMessage_UnknownUser_FallsBackToSlackId()
     {
         var message = new MessageEvent { User = "U_UNKNOWN", Text = "hey" };
         var memberLookup = new Dictionary<string, string>();
 
-        var inputMessage = BottomlyInputMessage.CreateFromSlackMessage(message, memberLookup);
+        var inputMessage = await BottomlyInputMessage.CreateFromSlackMessage(message, memberLookup, _parser);
 
         inputMessage.Username.ShouldBe("U_UNKNOWN");
         inputMessage.Text.ShouldBe("hey");

Bottomly/Bottomly.csproj

@@ -18,6 +18,7 @@
         <PackageReference Include="Octokit" Version="14.0.0"/>
         <PackageReference Include="SlackNet" Version="0.17.9"/>
         <PackageReference Include="SlackNet.Extensions.DependencyInjection" Version="0.17.9"/>
+        <PackageReference Include="System.Linq.Async" Version="7.0.0"/>
         <PackageReference Include="YamlDotNet" Version="16.3.0"/>
     </ItemGroup>
 

Bottomly/LlmBot/FullPromptContext.cs

@@ -1,4 +1,5 @@
-using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using Microsoft.Extensions.AI;
 
 namespace Bottomly.LlmBot;
@@ -22,35 +23,46 @@ Your character is based on Jeeves from the PG Wodehouse novels. All responses sh
 
     public static FullPromptContext Create(BottomlyInputMessage userPrompt, MessageHistoryContext historyContext)
         => new(
-            new ChatMessage(ChatRole.User, historyContext.ToChatContext()),
+            new ChatMessage(ChatRole.System, historyContext.ToChatContext()),
             new ChatMessage(ChatRole.User, userPrompt.ToChatPromptMessage()));
 
     public ChatMessage[] ToArray() => [SystemPrompt, HistoryContext, PromptingMessage];
 }
 
-public static class LlmBrokerExtensions
+public static class LlmClientExtensions
 {
-    public static string ToChatContext(this MessageHistoryContext historyContext) =>
-        new StringBuilder()
-            .AppendLine("**Begin Prompt Context:**")
-            .AppendLine("_Message History:_")
-            .AppendLine(string.Join("\n", historyContext.MessageHistory.Select(m => m.ToChatContextMessage())))
-            .AppendLine("_User Info:_")
-            .AppendLine(string.Join("\n", historyContext.UserNotes.Select(n => $"{n.Username}: {n.Note}")))
-            .AppendLine("**End Prompt Context**")
-            .ToString();
+    private static readonly JsonSerializerOptions JsonOptions = new()
+    {
+        WriteIndented = false,
+        PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
+    };
+
+    public static string ToChatContext(this MessageHistoryContext historyContext)
+    {
+        var payload = new PromptContextPayload(
+            historyContext.MessageHistory.Select(m => new MessageHistoryEntry(m.Username, m.Text)).ToList(),
+            historyContext.UserNotes.Select(n => new UserInfoEntry(n.Username, n.Note)).ToList()
+        );
+        return JsonSerializer.Serialize(payload, JsonOptions);
+    }
 
     extension(BottomlyInputMessage message)
     {
-        public string ToChatPromptMessage() =>
-            new StringBuilder()
-                .AppendLine("**Begin Main Prompt:**")
-                .AppendLine($"_User to respond to is {message.Username}_")
-                .AppendLine(message.Text)
-                .AppendLine("**End Main Prompt**")
-                .ToString();
-
-        private string ToChatContextMessage() =>
-            $"{message.Username}: {message.Text}";
+        public string ToChatPromptMessage() => $"{message.Username}: {message.Text}";
     }
+
+    private record PromptContextPayload(
+        [property: JsonPropertyName("message_history")]
+        List<MessageHistoryEntry> MessageHistory,
+        [property: JsonPropertyName("users")] List<UserInfoEntry> Users);
+
+    private record MessageHistoryEntry(
+        [property: JsonPropertyName("username")]
+        string Username,
+        [property: JsonPropertyName("text")] string Text);
+
+    private record UserInfoEntry(
+        [property: JsonPropertyName("username")]
+        string Username,
+        [property: JsonPropertyName("note")] string Note);
 }

Bottomly/LlmBot/MessageHistoryContext.cs

@@ -2,14 +2,14 @@ namespace Bottomly.LlmBot;
 
 public record MessageHistoryContext
 {
-    private MessageHistoryContext(List<BottomlyInputMessage> messageHistory, List<BottomlyUserNote> userNotes) =>
+    private MessageHistoryContext(IList<BottomlyInputMessage> messageHistory, IList<BottomlyUserNote> userNotes) =>
         (MessageHistory, UserNotes) = (messageHistory, userNotes);
 
-    public List<BottomlyInputMessage> MessageHistory { get; private init; } = [];
-    public List<BottomlyUserNote> UserNotes { get; private init; } = [];
+    public IList<BottomlyInputMessage> MessageHistory { get; private init; } = [];
+    public IList<BottomlyUserNote> UserNotes { get; private init; } = [];
 
     public static MessageHistoryContext Create(
-        List<BottomlyInputMessage> messageHistory,
-        List<BottomlyUserNote> userNotes) =>
+        IList<BottomlyInputMessage> messageHistory,
+        IList<BottomlyUserNote> userNotes) =>
         new(messageHistory, userNotes);
 }

Bottomly/Slack/MessageEventHandlers/ConversationMessageHandling/ConversationMessageHandler.cs

@@ -8,8 +8,9 @@
 namespace Bottomly.Slack.MessageEventHandlers.ConversationMessageHandling;
 
 public class ConversationMessageHandler(
-    ILlmClient llmMessageBroker,
+    ILlmClient llmClient,
     ISlackMessageBroker slackBroker,
+    SlackParser parser,
     ISlackApiClient apiClient,
     IMemberRepository memberRepository,
     IFeatureFlagRepository featureFlagRepository,
@@ -25,8 +26,7 @@ public bool CanHandle(MessageEvent message)
             return true;
         }
 
-        return _botMemberTask.IsCompletedSuccessfully
-               && _botMemberTask.Result?.SlackId is { } botId
+        return _botMemberTask is { IsCompletedSuccessfully: true, Result.SlackId: { } botId }
                && message.Text.Contains($"<@{botId}>");
     }
 
@@ -46,18 +46,15 @@ public async Task HandleAsync(MessageEvent message)
         var contextMembers = await memberRepository.GetBySlackIdsAsync(contextUsersSlackIds.Union([message.User]));
         var memberLookup = contextMembers.ToDictionary(m => m.SlackId, m => m.Username);
 
-        var contextMessages = history.Messages
-            .OrderBy(h => h.Timestamp)
-            .Select(h => BottomlyInputMessage.CreateFromSlackMessage(h, memberLookup))
-            .ToList();
+        var contextMessages = await history.Messages.ToInputMessagesAsync(memberLookup, parser);
 
         var userNotes = contextMembers.Select(BottomlyUserNote.CreateFromMember).ToList();
 
-        var mainPrompt = BottomlyInputMessage.CreateFromSlackMessage(message, memberLookup);
+        var mainPrompt = await BottomlyInputMessage.CreateFromSlackMessage(message, memberLookup, parser);
 
         var context = MessageHistoryContext.Create(contextMessages, userNotes);
 
-        var response = await llmMessageBroker.Respond(mainPrompt, context);
+        var response = await llmClient.Respond(mainPrompt, context);
 
         var replyToTs = response.IsError() ? message.TsForReply() : null;
 

Bottomly/Slack/MessageEventHandlers/ConversationMessageHandling/MessageContextExtensions.cs

@@ -14,11 +14,27 @@ public static BottomlyUserNote CreateFromMember(Member member) =>
 
     extension(BottomlyInputMessage bottomlyInputMessage)
     {
-        public static BottomlyInputMessage CreateFromSlackMessage(MessageEvent message,
-            IDictionary<string, string> memberLookup)
+        public static async Task<BottomlyInputMessage> CreateFromSlackMessage(MessageEvent message,
+            IDictionary<string, string> memberLookup, SlackParser parser)
         {
             var translatedUserName = memberLookup.TryGetValue(message.User, out var username) ? username : message.User;
-            return BottomlyInputMessage.Create(translatedUserName, message.Text);
+            var parsedText = await parser.ReplaceSlackIdTokensWithUsernamesAsync(message.Text);
+            return BottomlyInputMessage.Create(translatedUserName, parsedText);
+        }
+    }
+
+    extension(IEnumerable<MessageEvent> slackMessages)
+    {
+        public async Task<IList<BottomlyInputMessage>> ToInputMessagesAsync(IDictionary<string, string> memberLookup,
+            SlackParser parser)
+        {
+            List<BottomlyInputMessage> contextMessages = [];
+            foreach (var h in slackMessages.OrderBy(h => h.Timestamp))
+            {
+                contextMessages.Add(await BottomlyInputMessage.CreateFromSlackMessage(h, memberLookup, parser));
+            }
+
+            return contextMessages;
         }
     }
 }