novelly/tests/NovelSoftware.Tests/NovelAgentServiceTests.cs

using System.Text.Json;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NovelSoftware.Application.Agent;
using NovelSoftware.Application.Dtos;
using NovelSoftware.Application.Services;

namespace NovelSoftware.Tests;

public class NovelAgentServiceTests : IDisposable
{
    private readonly TestDatabase _db = new();
    private readonly TagService _tags;
    private readonly ProjectService _projects;
    private readonly CharacterService _characters;
    private readonly NovelAgentToolset _toolset;

    public NovelAgentServiceTests()
    {
        _tags = new TagService(_db.Context);
        _projects = new ProjectService(_db.Context);
        _characters = new CharacterService(_db.Context, _tags);
        _toolset = new NovelAgentToolset(
            _projects,
            _characters,
            new ChapterService(_db.Context, _tags),
            new BeatService(_db.Context, _tags),
            new SceneService(_db.Context),
            _tags);
    }

    private NovelAgentService BuildAgent(ScriptedModelClient model) => new(
        _db.Context,
        model,
        _toolset,
        Options.Create(new AgentOptions { MaxIterations = 4 }),
        NullLogger<NovelAgentService>.Instance);

    [Fact]
    public async Task A_plain_reply_is_persisted_as_a_conversation()
    {
        var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
        var model = new ScriptedModelClient([[new AgentTextBlock("Tell me about the ending.")]]);
        var agent = BuildAgent(model);

        var turn = await agent.SendMessageAsync(projectId, new SendAgentMessageRequest("Where do I start?"));

        turn.Message.Content.Should().Be("Tell me about the ending.");

        var conversation = await agent.GetConversationAsync(turn.ConversationId);
        conversation.Messages.Should().HaveCount(2);
        conversation.Messages[0].Content.Should().Be("Where do I start?");
        conversation.Title.Should().Be("Where do I start?");
    }

    [Fact]
    public async Task Tool_calls_are_executed_against_real_project_data()
    {
        var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;

        var model = new ScriptedModelClient([
            [ToolUse("t1", "create_character", new { name = "Ines", role = "Protagonist" })],
            [new AgentTextBlock("Added Ines as the protagonist.")]
        ]);

        var turn = await BuildAgent(model).SendMessageAsync(
            projectId, new SendAgentMessageRequest("Add a protagonist called Ines."));

        var characters = await _characters.ListAsync(projectId);
        characters.Should().ContainSingle().Which.Name.Should().Be("Ines");

        turn.Message.Content.Should().Be("Added Ines as the protagonist.");
        turn.Message.ToolCalls.Should().ContainSingle().Which.Name.Should().Be("create_character");
    }

    [Fact]
    public async Task Every_tool_result_comes_back_in_a_single_user_turn()
    {
        var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;

        var model = new ScriptedModelClient([
            [
                ToolUse("t1", "create_character", new { name = "Ines" }),
                ToolUse("t2", "create_character", new { name = "Mara" })
            ],
            [new AgentTextBlock("Both added.")]
        ]);

        await BuildAgent(model).SendMessageAsync(projectId, new SendAgentMessageRequest("Add two characters."));

        var secondRequest = model.Transcripts[1];
        var resultTurn = secondRequest[^1];

        resultTurn.Role.Should().Be("user");
        resultTurn.Content.OfType<AgentToolResultBlock>().Should().HaveCount(2);
        (await _characters.ListAsync(projectId)).Should().HaveCount(2);
    }

    [Fact]
    public async Task A_failing_tool_is_reported_back_rather_than_thrown()
    {
        var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;

        var model = new ScriptedModelClient([
            [ToolUse("t1", "update_character", new { character_id = Guid.NewGuid().ToString(), name = "Ines" })],
            [new AgentTextBlock("That character does not exist yet — shall I create her?")]
        ]);

        var turn = await BuildAgent(model).SendMessageAsync(
            projectId, new SendAgentMessageRequest("Rename her."));

        var errorResult = model.Transcripts[1][^1].Content.OfType<AgentToolResultBlock>().Single();
        errorResult.IsError.Should().BeTrue();
        errorResult.Content.Should().Contain("was not found");

        turn.Message.Content.Should().Contain("does not exist yet");
    }

    [Fact]
    public async Task Unknown_tools_are_reported_without_breaking_the_loop()
    {
        var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;

        var model = new ScriptedModelClient([
            [ToolUse("t1", "summon_muse", new { })],
            [new AgentTextBlock("Sorry — I do not have that tool.")]
        ]);

        await BuildAgent(model).SendMessageAsync(projectId, new SendAgentMessageRequest("Summon the muse."));

        var result = model.Transcripts[1][^1].Content.OfType<AgentToolResultBlock>().Single();
        result.IsError.Should().BeTrue();
        result.Content.Should().Contain("No such tool");
    }

    [Fact]
    public async Task The_loop_stops_at_the_iteration_ceiling()
    {
        var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;

        // A model that only ever asks for more tools would otherwise loop forever.
        var model = new ScriptedModelClient(
            Enumerable.Repeat<IReadOnlyList<AgentContentBlock>>(
                [ToolUse("t", "list_characters", new { })], 20).ToList());

        var turn = await BuildAgent(model).SendMessageAsync(
            projectId, new SendAgentMessageRequest("Keep going forever."));

        model.Transcripts.Should().HaveCount(4);
        turn.Message.Content.Should().Contain("tool-call limit");
    }

    [Fact]
    public async Task Follow_up_messages_continue_the_same_conversation()
    {
        var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
        var model = new ScriptedModelClient([
            [new AgentTextBlock("First answer.")],
            [new AgentTextBlock("Second answer.")]
        ]);
        var agent = BuildAgent(model);

        var first = await agent.SendMessageAsync(projectId, new SendAgentMessageRequest("Question one."));
        var second = await agent.SendMessageAsync(
            projectId, new SendAgentMessageRequest("Question two.", first.ConversationId));

        second.ConversationId.Should().Be(first.ConversationId);

        // The second request replays the earlier turns so the model has the history.
        model.Transcripts[1].Should().HaveCount(3);
        model.Transcripts[1].Select(m => m.Role).Should().Equal("user", "assistant", "user");

        var conversation = await agent.GetConversationAsync(first.ConversationId);
        conversation.Messages.Should().HaveCount(4);
    }

    [Fact]
    public void Every_tool_declares_an_object_schema_and_a_description()
    {
        _toolset.Definitions.Should().NotBeEmpty();

        foreach (var tool in _toolset.Definitions)
        {
            tool.Description.Should().NotBeNullOrWhiteSpace();
            tool.InputSchema.GetProperty("type").GetString().Should().Be("object");
            tool.InputSchema.TryGetProperty("properties", out _).Should().BeTrue();
        }

        _toolset.Definitions.Select(t => t.Name).Should().OnlyHaveUniqueItems();
    }

    private static AgentToolUseBlock ToolUse(string id, string name, object input) =>
        new(id, name, JsonSerializer.SerializeToElement(input));

    public void Dispose() => _db.Dispose();
}

/// <summary>
/// A model stand-in that returns a fixed script of turns and records every transcript it
/// was sent, so tests can assert on what the loop actually put in front of the model.
/// </summary>
internal sealed class ScriptedModelClient(IReadOnlyList<IReadOnlyList<AgentContentBlock>> script)
    : IAgentModelClient
{
    private int _turn;

    public List<IReadOnlyList<AgentChatMessage>> Transcripts { get; } = [];

    public Task<AgentModelResponse> CompleteAsync(
        string systemPrompt,
        IReadOnlyList<AgentChatMessage> messages,
        IReadOnlyList<AgentToolDefinition> tools,
        CancellationToken ct = default)
    {
        Transcripts.Add([.. messages]);

        var content = _turn < script.Count ? script[_turn] : [new AgentTextBlock("(no more script)")];
        _turn++;

        var stopReason = content.OfType<AgentToolUseBlock>().Any() ? "tool_use" : "end_turn";
        return Task.FromResult(new AgentModelResponse(content, stopReason));
    }
}