wamplerj/novelly
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate the test suite from xUnit + FluentAssertions to NUnit

Browse Source 
All 44 tests, same names, same coverage — verified by diffing the runner's test
list before and after.

The load-bearing change is fixture lifecycle. xUnit builds a new test-class
instance per test, so a `readonly TestDatabase _db = new()` field gave every
test its own database. NUnit reuses one instance for the whole class, so those
field initialisers and constructors would have shared a single database across a
class and let tests read each other's rows. Setup moved into [SetUp]/[TearDown]
via a new ServiceTestFixture base class, which also collapses the per-class
service wiring that was duplicated five times.

Assertions are now Assert.That with the constraint model:

  Should().Be(x)              -> Is.EqualTo(x)
  Should().BeNull()           -> Is.Null
  Should().HaveCount(n)       -> Has.Count.EqualTo(n)
  Should().Equal(a, b)        -> Is.EqualTo(new[] { a, b })
  Should().BeEquivalentTo(..) -> Is.EquivalentTo(..)
  Should().Contain("x")       -> Does.Contain("x")
  Should().OnlyHaveUniqueItems() -> Is.Unique
  ThrowAsync<T>().WithMessage("*m*")
      -> Throws.TypeOf<T>().With.Message.Contains("m")

FluentAssertions' `.Which` chains became plain indexed asserts, grouped in
Assert.Multiple so a failure reports every broken expectation in the case rather
than stopping at the first.

Because a framework migration can quietly produce vacuously-passing tests,
spot-checked four conversions by mutation — breaking the code under an async
Assert.Multiple block, a sync one, a Throws constraint, and a collection
ordering assert. All four failed as they should, confirming the assertions are
live and that NUnit bound the async lambdas to AsyncTestDelegate rather than
silently accepting them as async void.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S56bfZMGe1hnhpWP4CjjNw
This commit is contained in:
James Wampler
2026-08-06 12:11:20 -07:00
co-authored by Claude Opus 5
parent 8394843255
commit 30e0c6926e
9 changed files with 510 additions and 434 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/NovelSoftware.Tests/NovelAgentServiceTests.cs
+85 -73
View File
@@ -1,63 +1,51 @@
using System.Text.Json;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NovelSoftware.Application.Agent;
using NovelSoftware.Application.Dtos;
using NovelSoftware.Application.Services;
namespace NovelSoftware.Tests;
public class NovelAgentServiceTests : IDisposable
[TestFixture]
public class NovelAgentServiceTests : ServiceTestFixture
{
private readonly TestDatabase _db = new();
private readonly TagService _tags;
private readonly ProjectService _projects;
private readonly CharacterService _characters;
private readonly NovelAgentToolset _toolset;
private NovelAgentToolset _toolset = null!;
public NovelAgentServiceTests()
{
_tags = new TagService(_db.Context);
_projects = new ProjectService(_db.Context);
_characters = new CharacterService(_db.Context, _tags);
_toolset = new NovelAgentToolset(
_projects,
_characters,
new ChapterService(_db.Context, _tags),
new BeatService(_db.Context, _tags),
new SceneService(_db.Context),
_tags);
}
protected override void OnSetUp() =>
_toolset = new NovelAgentToolset(Projects, Characters, Chapters, Beats, Scenes, Tags);
private NovelAgentService BuildAgent(ScriptedModelClient model) => new(
_db.Context,
Db.Context,
model,
_toolset,
Options.Create(new AgentOptions { MaxIterations = 4 }),
NullLogger<NovelAgentService>.Instance);
[Fact]
[Test]
public async Task A_plain_reply_is_persisted_as_a_conversation()
{
var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var projectId = (await Projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var model = new ScriptedModelClient([[new AgentTextBlock("Tell me about the ending.")]]);
var agent = BuildAgent(model);
var turn = await agent.SendMessageAsync(projectId, new SendAgentMessageRequest("Where do I start?"));
turn.Message.Content.Should().Be("Tell me about the ending.");
Assert.That(turn.Message.Content, Is.EqualTo("Tell me about the ending."));
var conversation = await agent.GetConversationAsync(turn.ConversationId);
conversation.Messages.Should().HaveCount(2);
conversation.Messages[0].Content.Should().Be("Where do I start?");
conversation.Title.Should().Be("Where do I start?");
Assert.Multiple(() =>
{
Assert.That(conversation.Messages, Has.Count.EqualTo(2));
Assert.That(conversation.Messages[0].Content, Is.EqualTo("Where do I start?"));
Assert.That(conversation.Title, Is.EqualTo("Where do I start?"));
});
}
[Fact]
[Test]
public async Task Tool_calls_are_executed_against_real_project_data()
{
var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var projectId = (await Projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var model = new ScriptedModelClient([
[ToolUse("t1", "create_character", new { name = "Ines", role = "Protagonist" })],
@@ -67,17 +55,22 @@ public class NovelAgentServiceTests : IDisposable
var turn = await BuildAgent(model).SendMessageAsync(
projectId, new SendAgentMessageRequest("Add a protagonist called Ines."));
var characters = await _characters.ListAsync(projectId);
characters.Should().ContainSingle().Which.Name.Should().Be("Ines");
var characters = await Characters.ListAsync(projectId);
turn.Message.Content.Should().Be("Added Ines as the protagonist.");
turn.Message.ToolCalls.Should().ContainSingle().Which.Name.Should().Be("create_character");
Assert.Multiple(() =>
{
Assert.That(characters, Has.Count.EqualTo(1));
Assert.That(characters[0].Name, Is.EqualTo("Ines"));
Assert.That(turn.Message.Content, Is.EqualTo("Added Ines as the protagonist."));
Assert.That(turn.Message.ToolCalls, Has.Count.EqualTo(1));
Assert.That(turn.Message.ToolCalls[0].Name, Is.EqualTo("create_character"));
});
}
[Fact]
[Test]
public async Task Every_tool_result_comes_back_in_a_single_user_turn()
{
var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var projectId = (await Projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var model = new ScriptedModelClient([
[
@@ -89,18 +82,21 @@ public class NovelAgentServiceTests : IDisposable
await BuildAgent(model).SendMessageAsync(projectId, new SendAgentMessageRequest("Add two characters."));
var secondRequest = model.Transcripts[1];
var resultTurn = secondRequest[^1];
var resultTurn = model.Transcripts[1][^1];
var listed = await Characters.ListAsync(projectId);
resultTurn.Role.Should().Be("user");
resultTurn.Content.OfType<AgentToolResultBlock>().Should().HaveCount(2);
(await _characters.ListAsync(projectId)).Should().HaveCount(2);
Assert.Multiple(() =>
{
Assert.That(resultTurn.Role, Is.EqualTo("user"));
Assert.That(resultTurn.Content.OfType<AgentToolResultBlock>().Count(), Is.EqualTo(2));
Assert.That(listed, Has.Count.EqualTo(2));
});
}
[Fact]
[Test]
public async Task A_failing_tool_is_reported_back_rather_than_thrown()
{
var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var projectId = (await Projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var model = new ScriptedModelClient([
[ToolUse("t1", "update_character", new { character_id = Guid.NewGuid().ToString(), name = "Ines" })],
@@ -111,16 +107,19 @@ public class NovelAgentServiceTests : IDisposable
projectId, new SendAgentMessageRequest("Rename her."));
var errorResult = model.Transcripts[1][^1].Content.OfType<AgentToolResultBlock>().Single();
errorResult.IsError.Should().BeTrue();
errorResult.Content.Should().Contain("was not found");
turn.Message.Content.Should().Contain("does not exist yet");
Assert.Multiple(() =>
{
Assert.That(errorResult.IsError, Is.True);
Assert.That(errorResult.Content, Does.Contain("was not found"));
Assert.That(turn.Message.Content, Does.Contain("does not exist yet"));
});
}
[Fact]
[Test]
public async Task Unknown_tools_are_reported_without_breaking_the_loop()
{
var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var projectId = (await Projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var model = new ScriptedModelClient([
[ToolUse("t1", "summon_muse", new { })],
@@ -130,14 +129,18 @@ public class NovelAgentServiceTests : IDisposable
await BuildAgent(model).SendMessageAsync(projectId, new SendAgentMessageRequest("Summon the muse."));
var result = model.Transcripts[1][^1].Content.OfType<AgentToolResultBlock>().Single();
result.IsError.Should().BeTrue();
result.Content.Should().Contain("No such tool");
Assert.Multiple(() =>
{
Assert.That(result.IsError, Is.True);
Assert.That(result.Content, Does.Contain("No such tool"));
});
}
[Fact]
[Test]
public async Task The_loop_stops_at_the_iteration_ceiling()
{
var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var projectId = (await Projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
// A model that only ever asks for more tools would otherwise loop forever.
var model = new ScriptedModelClient(
@@ -147,14 +150,17 @@ public class NovelAgentServiceTests : IDisposable
var turn = await BuildAgent(model).SendMessageAsync(
projectId, new SendAgentMessageRequest("Keep going forever."));
model.Transcripts.Should().HaveCount(4);
turn.Message.Content.Should().Contain("tool-call limit");
Assert.Multiple(() =>
{
Assert.That(model.Transcripts, Has.Count.EqualTo(4));
Assert.That(turn.Message.Content, Does.Contain("tool-call limit"));
});
}
[Fact]
[Test]
public async Task Follow_up_messages_continue_the_same_conversation()
{
var projectId = (await _projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var projectId = (await Projects.CreateAsync(new CreateProjectRequest("The Salt Road"))).Id;
var model = new ScriptedModelClient([
[new AgentTextBlock("First answer.")],
[new AgentTextBlock("Second answer.")]
@@ -165,35 +171,41 @@ public class NovelAgentServiceTests : IDisposable
var second = await agent.SendMessageAsync(
projectId, new SendAgentMessageRequest("Question two.", first.ConversationId));
second.ConversationId.Should().Be(first.ConversationId);
// The second request replays the earlier turns so the model has the history.
model.Transcripts[1].Should().HaveCount(3);
model.Transcripts[1].Select(m => m.Role).Should().Equal("user", "assistant", "user");
var conversation = await agent.GetConversationAsync(first.ConversationId);
conversation.Messages.Should().HaveCount(4);
Assert.Multiple(() =>
{
Assert.That(second.ConversationId, Is.EqualTo(first.ConversationId));
// The second request replays the earlier turns so the model has the history.
Assert.That(model.Transcripts[1], Has.Count.EqualTo(3));
Assert.That(
model.Transcripts[1].Select(m => m.Role),
Is.EqualTo(new[] { "user", "assistant", "user" }));
Assert.That(conversation.Messages, Has.Count.EqualTo(4));
});
}
[Fact]
[Test]
public void Every_tool_declares_an_object_schema_and_a_description()
{
_toolset.Definitions.Should().NotBeEmpty();
Assert.That(_toolset.Definitions, Is.Not.Empty);
foreach (var tool in _toolset.Definitions)
Assert.Multiple(() =>
{
tool.Description.Should().NotBeNullOrWhiteSpace();
tool.InputSchema.GetProperty("type").GetString().Should().Be("object");
tool.InputSchema.TryGetProperty("properties", out _).Should().BeTrue();
}
foreach (var tool in _toolset.Definitions)
{
Assert.That(string.IsNullOrWhiteSpace(tool.Description), Is.False, tool.Name);
Assert.That(tool.InputSchema.GetProperty("type").GetString(), Is.EqualTo("object"), tool.Name);
Assert.That(tool.InputSchema.TryGetProperty("properties", out _), Is.True, tool.Name);
}
_toolset.Definitions.Select(t => t.Name).Should().OnlyHaveUniqueItems();
Assert.That(_toolset.Definitions.Select(t => t.Name), Is.Unique);
});
}
private static AgentToolUseBlock ToolUse(string id, string name, object input) =>
new(id, name, JsonSerializer.SerializeToElement(input));
public void Dispose() => _db.Dispose();
}
/// <summary>