using System.Text.Json; using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NovelSoftware.Application.Agent; using NovelSoftware.Application.Dtos; using NovelSoftware.Application.Services; namespace NovelSoftware.Tests; public class NovelAgentServiceTests : IDisposable { private readonly TestDatabase _db = new(); private readonly TagService _tags; private readonly ProjectService _projects; private readonly CharacterService _characters; private readonly NovelAgentToolset _toolset; public NovelAgentServiceTests() { _tags = new TagService(_db.Context); _projects = new ProjectService(_db.Context); _characters = new CharacterService(_db.Context, _tags); _toolset = new NovelAgentToolset( _projects, _characters, new ChapterService(_db.Context, _tags), new BeatService(_db.Context, _tags), new SceneService(_db.Context), _tags); } private NovelAgentService BuildAgent(ScriptedModelClient model) => new( _db.Context, model, _toolset, Options.Create(new AgentOptions { MaxIterations = 4 }), NullLogger.Instance); [Fact] public async Task A_plain_reply_is_persisted_as_a_conversation() { var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id; var model = new ScriptedModelClient([[new AgentTextBlock("Tell me about the ending.")]]); var agent = BuildAgent(model); var turn = await agent.SendMessageAsync(projectId, new SendAgentMessageRequest("Where do I start?")); turn.Message.Content.Should().Be("Tell me about the ending."); var conversation = await agent.GetConversationAsync(turn.ConversationId); conversation.Messages.Should().HaveCount(2); conversation.Messages[0].Content.Should().Be("Where do I start?"); conversation.Title.Should().Be("Where do I start?"); } [Fact] public async Task Tool_calls_are_executed_against_real_project_data() { var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id; var model = new ScriptedModelClient([ [ToolUse("t1", "create_character", new { name = "Ines", role = "Protagonist" })], [new AgentTextBlock("Added Ines as the protagonist.")] ]); var turn = await BuildAgent(model).SendMessageAsync( projectId, new SendAgentMessageRequest("Add a protagonist called Ines.")); var characters = await _characters.ListAsync(projectId); characters.Should().ContainSingle().Which.Name.Should().Be("Ines"); turn.Message.Content.Should().Be("Added Ines as the protagonist."); turn.Message.ToolCalls.Should().ContainSingle().Which.Name.Should().Be("create_character"); } [Fact] public async Task Every_tool_result_comes_back_in_a_single_user_turn() { var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id; var model = new ScriptedModelClient([ [ ToolUse("t1", "create_character", new { name = "Ines" }), ToolUse("t2", "create_character", new { name = "Mara" }) ], [new AgentTextBlock("Both added.")] ]); await BuildAgent(model).SendMessageAsync(projectId, new SendAgentMessageRequest("Add two characters.")); var secondRequest = model.Transcripts[1]; var resultTurn = secondRequest[^1]; resultTurn.Role.Should().Be("user"); resultTurn.Content.OfType().Should().HaveCount(2); (await _characters.ListAsync(projectId)).Should().HaveCount(2); } [Fact] public async Task A_failing_tool_is_reported_back_rather_than_thrown() { var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id; var model = new ScriptedModelClient([ [ToolUse("t1", "update_character", new { character_id = Guid.NewGuid().ToString(), name = "Ines" })], [new AgentTextBlock("That character does not exist yet — shall I create her?")] ]); var turn = await BuildAgent(model).SendMessageAsync( projectId, new SendAgentMessageRequest("Rename her.")); var errorResult = model.Transcripts[1][^1].Content.OfType().Single(); errorResult.IsError.Should().BeTrue(); errorResult.Content.Should().Contain("was not found"); turn.Message.Content.Should().Contain("does not exist yet"); } [Fact] public async Task Unknown_tools_are_reported_without_breaking_the_loop() { var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id; var model = new ScriptedModelClient([ [ToolUse("t1", "summon_muse", new { })], [new AgentTextBlock("Sorry — I do not have that tool.")] ]); await BuildAgent(model).SendMessageAsync(projectId, new SendAgentMessageRequest("Summon the muse.")); var result = model.Transcripts[1][^1].Content.OfType().Single(); result.IsError.Should().BeTrue(); result.Content.Should().Contain("No such tool"); } [Fact] public async Task The_loop_stops_at_the_iteration_ceiling() { var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id; // A model that only ever asks for more tools would otherwise loop forever. var model = new ScriptedModelClient( Enumerable.Repeat>( [ToolUse("t", "list_characters", new { })], 20).ToList()); var turn = await BuildAgent(model).SendMessageAsync( projectId, new SendAgentMessageRequest("Keep going forever.")); model.Transcripts.Should().HaveCount(4); turn.Message.Content.Should().Contain("tool-call limit"); } [Fact] public async Task Follow_up_messages_continue_the_same_conversation() { var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id; var model = new ScriptedModelClient([ [new AgentTextBlock("First answer.")], [new AgentTextBlock("Second answer.")] ]); var agent = BuildAgent(model); var first = await agent.SendMessageAsync(projectId, new SendAgentMessageRequest("Question one.")); var second = await agent.SendMessageAsync( projectId, new SendAgentMessageRequest("Question two.", first.ConversationId)); second.ConversationId.Should().Be(first.ConversationId); // The second request replays the earlier turns so the model has the history. model.Transcripts[1].Should().HaveCount(3); model.Transcripts[1].Select(m => m.Role).Should().Equal("user", "assistant", "user"); var conversation = await agent.GetConversationAsync(first.ConversationId); conversation.Messages.Should().HaveCount(4); } [Fact] public void Every_tool_declares_an_object_schema_and_a_description() { _toolset.Definitions.Should().NotBeEmpty(); foreach (var tool in _toolset.Definitions) { tool.Description.Should().NotBeNullOrWhiteSpace(); tool.InputSchema.GetProperty("type").GetString().Should().Be("object"); tool.InputSchema.TryGetProperty("properties", out _).Should().BeTrue(); } _toolset.Definitions.Select(t => t.Name).Should().OnlyHaveUniqueItems(); } private static AgentToolUseBlock ToolUse(string id, string name, object input) => new(id, name, JsonSerializer.SerializeToElement(input)); public void Dispose() => _db.Dispose(); } /// /// A model stand-in that returns a fixed script of turns and records every transcript it /// was sent, so tests can assert on what the loop actually put in front of the model. /// internal class ScriptedModelClient(IReadOnlyList> script) : IAgentModelClient { private int _turn; public List> Transcripts { get; } = []; public Task CompleteAsync( string systemPrompt, IReadOnlyList messages, IReadOnlyList tools, CancellationToken ct = default) { Transcripts.Add([.. messages]); var content = _turn < script.Count ? script[_turn] : [new AgentTextBlock("(no more script)")]; _turn++; var stopReason = content.OfType().Any() ? "tool_use" : "end_turn"; return Task.FromResult(new AgentModelResponse(content, stopReason)); } }