Add zip upload and mapped-path picker for outline import

Sandbox source paths under a configured Imports:RootPath, browse it
from the web dialog, upload a zip that extracts into staging, and
import a single markdown file (agent infers chapter vs character).
This commit is contained in:
James Wampler
2026-08-20 14:21:44 -07:00
parent e3d410da0b
commit 661f2917ea
22 changed files with 890 additions and 28 deletions
+1
View File
@@ -103,5 +103,6 @@ Several real bugs here — SQLite refusing ORDER BY DateTimeOffset, agent's mode
- Anthropic model id lives in `appsettings.json` under `Agent:Model`. Don't hardcode. - Anthropic model id lives in `appsettings.json` under `Agent:Model`. Don't hardcode.
- API key comes from `ANTHROPIC_API_KEY` or `Agent:ApiKey` — never commit one. App must stay fully usable without key; only agent endpoints require it. - API key comes from `ANTHROPIC_API_KEY` or `Agent:ApiKey` — never commit one. App must stay fully usable without key; only agent endpoints require it.
- EF migrations: `dotnet ef migrations add <Name> -p src/Novelly.Api -o Data/Migrations`. API migrates on boot. - EF migrations: `dotnet ef migrations add <Name> -p src/Novelly.Api -o Data/Migrations`. API migrates on boot.
- Outline import root lives in `appsettings.json` under `Imports:RootPath` (`Imports__RootPath` env var). When set, it's the only folder the browse/upload import endpoints and the source picker can reach; unset, those endpoints are disabled and the dialog falls back to a typed path with no sandbox. Created at boot if missing.
- `git push` runs `scripts/ci/prepush.sh` through Husky: build, test, then web build. Run `npm install` - `git push` runs `scripts/ci/prepush.sh` through Husky: build, test, then web build. Run `npm install`
once at repo root to install hook. once at repo root to install hook.
+1
View File
@@ -15,6 +15,7 @@ services:
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
Agent__Model: claude-sonnet-5 Agent__Model: claude-sonnet-5
Agent__Effort: high Agent__Effort: high
Imports__RootPath: /data/imports
volumes: volumes:
- /mnt/storage/apps/novelly/data:/data - /mnt/storage/apps/novelly/data:/data
networks: networks:
@@ -94,8 +94,11 @@ public static class NovellyServiceRegistration
services.Configure<AgentOptions>(configuration.GetSection(AgentOptions.SectionName)); services.Configure<AgentOptions>(configuration.GetSection(AgentOptions.SectionName));
services.AddScoped<IAgentModelClient, AnthropicAgentModelClient>(); services.AddScoped<IAgentModelClient, AnthropicAgentModelClient>();
services.Configure<ImportOptions>(configuration.GetSection(ImportOptions.SectionName));
services.AddSingleton(Channel.CreateUnbounded<Guid>()); services.AddSingleton(Channel.CreateUnbounded<Guid>());
services.AddScoped<ImportService>(); services.AddScoped<ImportService>();
services.AddScoped<ImportBrowseService>();
services.AddScoped<ImportZipExtractor>();
services.AddScoped<ImportAgentToolset>(); services.AddScoped<ImportAgentToolset>();
services.AddScoped<ImportAgentService>(); services.AddScoped<ImportAgentService>();
services.AddHostedService<ImportJobRunner>(); services.AddHostedService<ImportJobRunner>();
+65 -1
View File
@@ -23,7 +23,9 @@ public class ImportAgentService(
toolset.Initialize(sourceRoot, existingNovelId); toolset.Initialize(sourceRoot, existingNovelId);
var startingLedger = toolset.ReadLedgerOrNull(); var startingLedger = toolset.ReadLedgerOrNull();
var systemPrompt = BuildSystemPrompt(sourceRoot); var systemPrompt = ImportPaths.IsSingleFileSource(sourceRoot)
? BuildSingleFileSystemPrompt(sourceRoot)
: BuildSystemPrompt(sourceRoot);
var transcript = new List<AgentChatMessage> var transcript = new List<AgentChatMessage>
{ {
@@ -105,6 +107,9 @@ public class ImportAgentService(
private static string BuildSystemPrompt(string sourceRoot) => SystemPromptTemplate.Replace("{{SOURCE_ROOT}}", sourceRoot); private static string BuildSystemPrompt(string sourceRoot) => SystemPromptTemplate.Replace("{{SOURCE_ROOT}}", sourceRoot);
private static string BuildSingleFileSystemPrompt(string sourceRoot) =>
SingleFileSystemPromptTemplate.Replace("{{SOURCE_ROOT}}", sourceRoot);
private const string SystemPromptTemplate = """ private const string SystemPromptTemplate = """
You import a novel outline that already exists as markdown files on disk into this You import a novel outline that already exists as markdown files on disk into this
app's novel data. You are running unattended nobody will read your replies or app's novel data. You are running unattended nobody will read your replies or
@@ -199,4 +204,63 @@ public class ImportAgentService(
- If a tool call fails, stop that item and move on rather than retrying blindly - If a tool call fails, stop that item and move on rather than retrying blindly
the ledger stays at the last successful write either way. the ledger stays at the last successful write either way.
"""; """;
private const string SingleFileSystemPromptTemplate = """
You import a single outline file that already exists as markdown on disk into this
app's novel data. You are running unattended nobody will read your replies or
answer questions mid-run, so make the judgment calls yourself and record anything
genuinely ambiguous rather than stalling on it.
Your tools give you exactly two things: read-only access to the file under the
import source folder, and application tools that create the novel's chapters,
characters, beats and arcs. You cannot write or edit anything on disk except the
resume ledger, and you cannot read anything outside the source folder.
## Source file
The source root `{{SOURCE_ROOT}}` holds exactly one markdown file. Call
list_source_files to find its name, then read_source_file to read it. Decide what
kind of document it is before doing anything else:
- If it reads like a chapter outline (`# Chapter NN`, one or more summary
paragraphs, a beat table `| Beat | Character | What | Why |`) treat it as a single
chapter.
- If it reads like a character dossier (`# Name`, an italic tagline,
`## Appearance`, `## Background`, `## Motivation`) treat it as a single character.
`**Thread:**` (chapter files only) may name one character, several, or a character
plus a qualifier only auto-create an undossiered name from it when it names
exactly one clear proper name.
## The ledger
Before writing anything, call read_ledger. If it returns `{{}}`, this is a fresh
run. Call write_ledger with the full, updated ledger after every successful write.
## Passes
1. **Novel** skip if "novel" is in completedPasses or a novel id was already
supplied. Otherwise create one from whatever title/author information the file
gives, or a sensible placeholder title drawn from the file name if none is
present. Record novelId, mark "novel" done.
2. **The document** skip if already recorded. If it is a chapter: auto-create a
character stub (name only) for any single, unqualified name in the Thread or a
beat's Character column that isn't in the ledger yet, then create_chapter with
title, number (1 unless the file states otherwise), summary, and tags, then
create_beat for each table row with resolved character_ids. If it is a
character: create_character with occupation from the tagline and
appearance/backstory/want from Appearance/Background/Motivation; if it has a
`## Events` section, also update_character(importance: "Main") and add_arc_stage
for each bullet. Mark "characters", "chapters", and "arcs" all done once you've
handled the one document this run only ever has one item to place.
## Constraints
- Never invent plot content or character detail, and never guess which of several
candidate names an ambiguous reference means.
- Never write to disk except via write_ledger.
- Never call a create tool for something the ledger already records.
- If a tool call fails, stop and record what you have the ledger stays at the
last successful write either way.
""";
} }
@@ -0,0 +1,81 @@
using Microsoft.Extensions.Options;
namespace Novelly.Api.Imports;
public class ImportBrowseService(IOptions<ImportOptions> options, ILogger<ImportBrowseService> logger)
{
private readonly ImportOptions _options = options.Value;
public string? RootPath => _options.RootPath;
public ImportBrowseResponse List(string? relativePath)
{
var root = RequireRoot();
logger.LogInformation("Browsing import root at {RelativePath}", relativePath ?? "");
var target = string.IsNullOrWhiteSpace(relativePath) ? root : ImportPaths.ResolveWithin(root, relativePath);
if (!Directory.Exists(target))
throw new ArgumentException($"'{relativePath}' does not exist or is not a directory.", nameof(relativePath));
var normalizedRelative = Path.GetRelativePath(root, target).Replace(Path.DirectorySeparatorChar, '/');
if (normalizedRelative == ".")
{
normalizedRelative = "";
}
var parent = normalizedRelative == "" ? null : Path.GetRelativePath(root, Path.GetFullPath(Path.Combine(target, ".."))).Replace(Path.DirectorySeparatorChar, '/');
if (parent == ".")
{
parent = "";
}
var entries = Directory.EnumerateFileSystemEntries(target)
.Select(BuildEntry)
.Where(e => e is not null)
.Select(e => e!)
.OrderByDescending(e => e.IsDirectory)
.ThenBy(e => e.Name, StringComparer.Ordinal)
.ToArray();
return new ImportBrowseResponse(normalizedRelative, parent, entries);
ImportBrowseEntry? BuildEntry(string path)
{
var name = Path.GetFileName(path);
if (name.StartsWith('.'))
{
return null;
}
var isDirectory = Directory.Exists(path);
if (!isDirectory && !name.EndsWith(".md", StringComparison.OrdinalIgnoreCase))
{
return null;
}
var entryRelative = Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/');
var markdownCount = isDirectory ? ImportPaths.CountChapterFiles(path) : 0;
var looksImportable = isDirectory
? File.Exists(Path.Combine(path, "outline.md")) || markdownCount > 0
: true;
return new ImportBrowseEntry(name, entryRelative, path, isDirectory, markdownCount, looksImportable);
}
}
private string RequireRoot()
{
if (string.IsNullOrWhiteSpace(_options.RootPath))
throw new InvalidOperationException("No import root is configured (Imports:RootPath).");
var full = Path.GetFullPath(_options.RootPath);
Directory.CreateDirectory(full);
return full;
}
}
public record ImportBrowseEntry(string Name, string RelativePath, string SourceRoot, bool IsDirectory, int MarkdownFileCount, bool LooksImportable);
public record ImportBrowseResponse(string RelativePath, string? ParentRelativePath, IReadOnlyList<ImportBrowseEntry> Entries);
@@ -57,6 +57,8 @@ public class StartImportRequestValidator : IModelValidator<StartImportRequest>
} }
} }
public record ImportUploadResponse(string SourceRoot, string RelativePath, int MarkdownFileCount);
public static class ImportMapping public static class ImportMapping
{ {
public static ImportJobResponse ToResponse(this ImportJob job) => new( public static ImportJobResponse ToResponse(this ImportJob job) => new(
@@ -28,6 +28,24 @@ public static class ImportEndpoints
(await service.GetStatusAsync(id, ct))?.ToResponse().ToApiResult()) (await service.GetStatusAsync(id, ct))?.ToResponse().ToApiResult())
.WithSummary("Poll an import job's progress."); .WithSummary("Poll an import job's progress.");
imports.MapGet("/browse", (string? path, ImportBrowseService browse) =>
Results.Ok(browse.List(path)))
.WithSummary("List entries under the configured import root, for the source picker.");
imports.MapPost("/upload", (IFormFile file, ImportService service) =>
{
if (!file.FileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
throw new ArgumentException("Only .zip files can be uploaded.", nameof(file));
if (file.Length == 0)
throw new ArgumentException("The uploaded file is empty.", nameof(file));
using var stream = file.OpenReadStream();
return Results.Ok(service.UploadZip(stream, file.FileName));
})
.DisableAntiforgery()
.WithSummary("Upload a zip of an outline folder and stage it under the configured import root.");
return app; return app;
} }
} }
+8
View File
@@ -0,0 +1,8 @@
namespace Novelly.Api.Imports;
public class ImportOptions
{
public const string SectionName = "Imports";
public string? RootPath { get; set; }
}
+36 -3
View File
@@ -13,6 +13,7 @@ public record ImportLedger(
internal static class ImportPaths internal static class ImportPaths
{ {
private const string LedgerFileName = ".novelly-import.json"; private const string LedgerFileName = ".novelly-import.json";
public const string StagingFolderName = ".novelly-staging";
private static readonly JsonSerializerOptions LedgerOptions = new() private static readonly JsonSerializerOptions LedgerOptions = new()
{ {
@@ -20,7 +21,7 @@ internal static class ImportPaths
WriteIndented = true WriteIndented = true
}; };
public static string ResolveRoot(string sourceRoot) public static string ResolveRoot(string sourceRoot, string? importRoot = null)
{ {
if (string.IsNullOrWhiteSpace(sourceRoot)) if (string.IsNullOrWhiteSpace(sourceRoot))
throw new ArgumentException("'Source Root' must not be empty.", nameof(sourceRoot)); throw new ArgumentException("'Source Root' must not be empty.", nameof(sourceRoot));
@@ -38,25 +39,57 @@ internal static class ImportPaths
if (!Directory.Exists(full)) if (!Directory.Exists(full))
throw new ArgumentException($"'{full}' does not exist or is not a directory.", nameof(sourceRoot)); throw new ArgumentException($"'{full}' does not exist or is not a directory.", nameof(sourceRoot));
EnsureWithinImportRoot(importRoot, full, sourceRoot);
return full; return full;
} }
public static void EnsureWithinImportRoot(string? importRoot, string candidate, string originalInput)
{
if (importRoot is not null && !IsWithin(importRoot, candidate))
throw new ArgumentException($"'{originalInput}' is outside the configured import root.", nameof(originalInput));
}
public static bool IsSingleFileSource(string root)
{
if (Directory.EnumerateDirectories(root).Any())
{
return false;
}
return Directory.EnumerateFiles(root, "*.md", SearchOption.TopDirectoryOnly).Count() == 1;
}
public static string ResolveWithin(string root, string relativePath) public static string ResolveWithin(string root, string relativePath)
{ {
if (string.IsNullOrWhiteSpace(relativePath)) if (string.IsNullOrWhiteSpace(relativePath))
throw new ArgumentException("Path must not be empty."); throw new ArgumentException("Path must not be empty.");
var combined = Path.GetFullPath(Path.Combine(root, relativePath)); var combined = Path.GetFullPath(Path.Combine(root, relativePath));
var relativeToRoot = Path.GetRelativePath(root, combined);
if (relativeToRoot.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relativeToRoot)) if (!IsWithin(root, combined))
throw new ArgumentException($"'{relativePath}' escapes the import source folder."); throw new ArgumentException($"'{relativePath}' escapes the import source folder.");
return combined; return combined;
} }
private static bool IsWithin(string root, string candidate)
{
var relativeToRoot = Path.GetRelativePath(root, candidate);
return relativeToRoot == "." || !relativeToRoot.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relativeToRoot);
}
public static string LedgerPath(string root) => Path.Combine(root, LedgerFileName); public static string LedgerPath(string root) => Path.Combine(root, LedgerFileName);
public static string StagingRoot(string importRoot) => Path.Combine(importRoot, StagingFolderName);
public static string SanitizeForFolderName(string value)
{
var sanitized = new string(value.Select(c => char.IsLetterOrDigit(c) || c is '-' or '_' ? c : '-').ToArray());
sanitized = sanitized.Trim('-', '_');
return string.IsNullOrEmpty(sanitized) ? "import" : sanitized[..Math.Min(sanitized.Length, 60)];
}
public static ImportLedger? ReadLedger(string root) public static ImportLedger? ReadLedger(string root)
{ {
var path = LedgerPath(root); var path = LedgerPath(root);
+61 -2
View File
@@ -1,5 +1,6 @@
using System.Threading.Channels; using System.Threading.Channels;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using Novelly.Api.Common; using Novelly.Api.Common;
using Novelly.Api.Common.Validation; using Novelly.Api.Common.Validation;
using Novelly.Api.Data; using Novelly.Api.Data;
@@ -13,10 +14,34 @@ public class ImportService(
NovelService novels, NovelService novels,
Channel<Guid> queue, Channel<Guid> queue,
INovelUserContext userContext, INovelUserContext userContext,
IOptions<ImportOptions> importOptions,
ImportZipExtractor zipExtractor,
ILogger<ImportService> logger, ILogger<ImportService> logger,
IModelValidator<InspectImportRequest> inspectValidator, IModelValidator<InspectImportRequest> inspectValidator,
IModelValidator<StartImportRequest> startValidator) IModelValidator<StartImportRequest> startValidator)
{ {
private readonly string? _importRoot = importOptions.Value.RootPath is { } root ? Path.GetFullPath(root) : null;
public ImportUploadResponse UploadZip(Stream zipStream, string fileName)
{
if (_importRoot is null)
throw new InvalidOperationException("No import root is configured (Imports:RootPath).");
logger.LogInformation("Uploading import zip {FileName}", fileName);
Directory.CreateDirectory(_importRoot);
var stagingDir = Path.Combine(
ImportPaths.StagingRoot(_importRoot),
$"zip-{ImportPaths.SanitizeForFolderName(Path.GetFileNameWithoutExtension(fileName))}-{Guid.NewGuid():N}");
zipExtractor.Extract(zipStream, stagingDir);
var markdownCount = Directory.EnumerateFiles(stagingDir, "*.md", SearchOption.AllDirectories).Count();
var relativePath = Path.GetRelativePath(_importRoot, stagingDir).Replace(Path.DirectorySeparatorChar, '/');
return new ImportUploadResponse(stagingDir, relativePath, markdownCount);
}
public Task<ImportInspectionResponse> InspectAsync(InspectImportRequest request, CancellationToken ct = default) public Task<ImportInspectionResponse> InspectAsync(InspectImportRequest request, CancellationToken ct = default)
{ {
Guard.Null(request, nameof(request)); Guard.Null(request, nameof(request));
@@ -24,7 +49,7 @@ public class ImportService(
logger.LogInformation("Inspecting import source {SourceRoot}", request.SourceRoot); logger.LogInformation("Inspecting import source {SourceRoot}", request.SourceRoot);
var root = ImportPaths.ResolveRoot(request.SourceRoot); var root = ResolveSourceRoot(request.SourceRoot);
var ledger = ImportPaths.ReadLedger(root); var ledger = ImportPaths.ReadLedger(root);
var total = ImportPaths.CountChapterFiles(root); var total = ImportPaths.CountChapterFiles(root);
@@ -48,7 +73,7 @@ public class ImportService(
logger.LogInformation( logger.LogInformation(
"Starting import for {SourceRoot}, forceRestart {ForceRestart}", request.SourceRoot, request.ForceRestart); "Starting import for {SourceRoot}, forceRestart {ForceRestart}", request.SourceRoot, request.ForceRestart);
var root = ImportPaths.ResolveRoot(request.SourceRoot); var root = ResolveSourceRoot(request.SourceRoot);
if (request.ForceRestart) if (request.ForceRestart)
{ {
@@ -88,6 +113,40 @@ public class ImportService(
return job; return job;
} }
private string ResolveSourceRoot(string sourceRoot)
{
string full;
try
{
full = Path.GetFullPath(sourceRoot);
}
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
{
throw new ArgumentException($"'{sourceRoot}' is not a valid path.", nameof(sourceRoot), ex);
}
if (!File.Exists(full))
{
return ImportPaths.ResolveRoot(sourceRoot, _importRoot);
}
if (!full.EndsWith(".md", StringComparison.OrdinalIgnoreCase))
throw new ArgumentException($"'{sourceRoot}' is not a markdown file or a directory.", nameof(sourceRoot));
ImportPaths.EnsureWithinImportRoot(_importRoot, full, sourceRoot);
var stagingParent = _importRoot is not null
? ImportPaths.StagingRoot(_importRoot)
: Path.Combine(Path.GetTempPath(), "novelly-import-staging");
var stagingDir = Path.Combine(
stagingParent, $"file-{ImportPaths.SanitizeForFolderName(Path.GetFileNameWithoutExtension(full))}");
Directory.CreateDirectory(stagingDir);
File.Copy(full, Path.Combine(stagingDir, Path.GetFileName(full)), overwrite: true);
return stagingDir;
}
public async Task<ImportJob?> GetStatusAsync(Guid id, CancellationToken ct = default) public async Task<ImportJob?> GetStatusAsync(Guid id, CancellationToken ct = default)
{ {
Guard.Default(id, nameof(id)); Guard.Default(id, nameof(id));
@@ -0,0 +1,89 @@
using System.IO.Compression;
namespace Novelly.Api.Imports;
public class ImportZipExtractor(ILogger<ImportZipExtractor> logger)
{
private const int MaxEntryCount = 2000;
private const long MaxEntryUncompressedBytes = 10 * 1024 * 1024;
private const long MaxTotalUncompressedBytes = 100 * 1024 * 1024;
private static readonly string[] AllowedFileNames = [".novelly-import.json"];
public void Extract(Stream zipStream, string stagingDir)
{
Directory.CreateDirectory(stagingDir);
try
{
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read);
var entries = archive.Entries
.Where(e => !string.IsNullOrEmpty(e.Name))
.Where(e => !e.FullName.StartsWith("__MACOSX/", StringComparison.OrdinalIgnoreCase))
.Where(e => AllowedFileNames.Contains(e.Name) || !e.Name.StartsWith('.'))
.ToArray();
if (entries.Length == 0)
throw new ArgumentException("The zip file is empty.");
if (entries.Length > MaxEntryCount)
throw new ArgumentException($"The zip file has too many entries (max {MaxEntryCount}).");
var stripPrefix = FindCommonTopLevelDirectory(entries);
var totalBytes = 0L;
foreach (var entry in entries)
{
var relativePath = stripPrefix is null
? entry.FullName
: entry.FullName[(stripPrefix.Length + 1)..];
if (relativePath.Length == 0)
{
continue;
}
if (!relativePath.EndsWith(".md", StringComparison.OrdinalIgnoreCase) && !AllowedFileNames.Contains(entry.Name))
throw new ArgumentException($"'{entry.FullName}' is not a markdown file. Only .md files (and .novelly-import.json) are allowed.");
if (entry.Length > MaxEntryUncompressedBytes)
throw new ArgumentException($"'{entry.FullName}' is too large (max {MaxEntryUncompressedBytes / (1024 * 1024)} MB per file).");
totalBytes += entry.Length;
if (totalBytes > MaxTotalUncompressedBytes)
throw new ArgumentException($"The zip file is too large uncompressed (max {MaxTotalUncompressedBytes / (1024 * 1024)} MB).");
var destination = ImportPaths.ResolveWithin(stagingDir, relativePath);
Directory.CreateDirectory(Path.GetDirectoryName(destination)!);
using var entryStream = entry.Open();
using var fileStream = File.Create(destination);
entryStream.CopyTo(fileStream);
}
logger.LogInformation("Extracted import zip with {EntryCount} entries into staging folder", entries.Length);
}
catch
{
if (Directory.Exists(stagingDir))
{
Directory.Delete(stagingDir, recursive: true);
}
throw;
}
}
private static string? FindCommonTopLevelDirectory(IReadOnlyCollection<ZipArchiveEntry> entries)
{
var topLevelSegments = entries
.Select(e => e.FullName.Split('/', '\\')[0])
.Distinct()
.ToArray();
return topLevelSegments.Length == 1 && entries.All(e => e.FullName.Contains('/') || e.FullName.Contains('\\'))
? topLevelSegments[0]
: null;
}
}
+6
View File
@@ -68,6 +68,12 @@ using (var scope = app.Services.CreateScope())
await ServiceUser.EnsureSeededAsync(db, builder.Configuration[ServiceApiKeyAuthenticationHandler.ConfigurationKey], app.Logger); await ServiceUser.EnsureSeededAsync(db, builder.Configuration[ServiceApiKeyAuthenticationHandler.ConfigurationKey], app.Logger);
await ActivityBackfill.RunAsync(db, app.Logger); await ActivityBackfill.RunAsync(db, app.Logger);
var importRoot = builder.Configuration.GetSection(ImportOptions.SectionName)[nameof(ImportOptions.RootPath)];
if (!string.IsNullOrWhiteSpace(importRoot))
{
Directory.CreateDirectory(importRoot);
}
} }
app.UseSerilogRequestLogging(); app.UseSerilogRequestLogging();
+3
View File
@@ -35,5 +35,8 @@
}, },
"UiSettings": { "UiSettings": {
"ShowPronouns": false "ShowPronouns": false
},
"Imports": {
"RootPath": null
} }
} }
+1
View File
@@ -7,6 +7,7 @@ server {
} }
location /api/ { location /api/ {
client_max_body_size 50m;
proxy_pass http://api:8080/api/; proxy_pass http://api:8080/api/;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
+3 -1
View File
@@ -11,11 +11,12 @@ export class ApiError extends Error {
} }
async function request<T>(path: string, init?: RequestInit): Promise<T> { async function request<T>(path: string, init?: RequestInit): Promise<T> {
const isFormData = init?.body instanceof FormData
const response = await fetch(`${BASE}${path}`, { const response = await fetch(`${BASE}${path}`, {
...init, ...init,
credentials: 'include', credentials: 'include',
headers: { headers: {
'Content-Type': 'application/json', ...(isFormData ? {} : { 'Content-Type': 'application/json' }),
...init?.headers, ...init?.headers,
}, },
}) })
@@ -37,6 +38,7 @@ export const api = {
get: <T>(path: string) => request<T>(path), get: <T>(path: string) => request<T>(path),
post: <T>(path: string, body?: unknown) => post: <T>(path: string, body?: unknown) =>
request<T>(path, { method: 'POST', body: JSON.stringify(body ?? {}) }), request<T>(path, { method: 'POST', body: JSON.stringify(body ?? {}) }),
postForm: <T>(path: string, body: FormData) => request<T>(path, { method: 'POST', body }),
patch: <T>(path: string, body: unknown) => patch: <T>(path: string, body: unknown) =>
request<T>(path, { method: 'PATCH', body: JSON.stringify(body) }), request<T>(path, { method: 'PATCH', body: JSON.stringify(body) }),
put: <T>(path: string, body: unknown) => put: <T>(path: string, body: unknown) =>
+22
View File
@@ -12,9 +12,11 @@ import type {
ConversationSummary, ConversationSummary,
Beat, Beat,
Genre, Genre,
ImportBrowse,
ImportInspection, ImportInspection,
ImportJob, ImportJob,
ImportJobStatus, ImportJobStatus,
ImportUpload,
OpenQuestion, OpenQuestion,
LocationReferences, LocationReferences,
LocationSummary, LocationSummary,
@@ -47,6 +49,7 @@ export const keys = {
conversations: (novelId: string) => ['novels', novelId, 'conversations'] as const, conversations: (novelId: string) => ['novels', novelId, 'conversations'] as const,
conversation: (id: string) => ['conversations', id] as const, conversation: (id: string) => ['conversations', id] as const,
importJob: (id: string) => ['imports', id] as const, importJob: (id: string) => ['imports', id] as const,
importBrowse: (path: string) => ['imports', 'browse', path] as const,
novelActivity: (novelId: string) => ['novels', novelId, 'activity'] as const, novelActivity: (novelId: string) => ['novels', novelId, 'activity'] as const,
myActivity: ['activity'] as const, myActivity: ['activity'] as const,
} }
@@ -635,6 +638,25 @@ export function useStartImport() {
}) })
} }
export function useImportBrowse(path: string, enabled = true) {
return useQuery({
queryKey: keys.importBrowse(path),
queryFn: () => api.get<ImportBrowse>(`/api/imports/browse?path=${encodeURIComponent(path)}`),
enabled,
retry: false,
})
}
export function useUploadImportZip() {
return useMutation({
mutationFn: (file: File) => {
const form = new FormData()
form.append('file', file)
return api.postForm<ImportUpload>('/api/imports/upload', form)
},
})
}
const terminalImportStatuses: ImportJobStatus[] = ['Completed', 'Failed', 'Paused'] const terminalImportStatuses: ImportJobStatus[] = ['Completed', 'Failed', 'Paused']
export function useImportJob(jobId: string | undefined) { export function useImportJob(jobId: string | undefined) {
+21
View File
@@ -329,6 +329,27 @@ export interface ImportInspection {
completedPasses: string[] completedPasses: string[]
} }
export interface ImportBrowseEntry {
name: string
relativePath: string
sourceRoot: string
isDirectory: boolean
markdownFileCount: number
looksImportable: boolean
}
export interface ImportBrowse {
relativePath: string
parentRelativePath: string | null
entries: ImportBrowseEntry[]
}
export interface ImportUpload {
sourceRoot: string
relativePath: string
markdownFileCount: number
}
export interface ActivityDay { export interface ActivityDay {
date: string date: string
words: number words: number
+88 -21
View File
@@ -1,8 +1,11 @@
import { useEffect, useState, type FormEvent } from 'react' import { useEffect, useRef, useState, type FormEvent } from 'react'
import { useQueryClient } from '@tanstack/react-query' import { useQueryClient } from '@tanstack/react-query'
import { useImportJob, useInspectImport, useStartImport } from '../api/hooks' import { useImportJob, useInspectImport, useStartImport, useUploadImportZip } from '../api/hooks'
import type { ImportInspection, ImportJob } from '../api/types' import type { ImportInspection, ImportJob } from '../api/types'
import { ErrorNote, Modal, Spinner } from './ui' import { ErrorNote, Modal, Spinner } from './ui'
import { ImportSourcePicker } from './ImportSourcePicker'
type SourceMode = 'browse' | 'upload' | 'path'
export function ImportDialog({ export function ImportDialog({
onClose, onClose,
@@ -11,13 +14,16 @@ export function ImportDialog({
onClose: () => void onClose: () => void
onImported?: (novelId: string) => void onImported?: (novelId: string) => void
}) { }) {
const [sourceMode, setSourceMode] = useState<SourceMode>('browse')
const [sourceRoot, setSourceRoot] = useState('') const [sourceRoot, setSourceRoot] = useState('')
const [inspection, setInspection] = useState<ImportInspection | null>(null) const [inspection, setInspection] = useState<ImportInspection | null>(null)
const [jobId, setJobId] = useState<string>() const [jobId, setJobId] = useState<string>()
const [confirmingRestart, setConfirmingRestart] = useState(false) const [confirmingRestart, setConfirmingRestart] = useState(false)
const fileInputRef = useRef<HTMLInputElement>(null)
const inspect = useInspectImport() const inspect = useInspectImport()
const start = useStartImport() const start = useStartImport()
const upload = useUploadImportZip()
const job = useImportJob(jobId) const job = useImportJob(jobId)
const qc = useQueryClient() const qc = useQueryClient()
@@ -33,6 +39,15 @@ export function ImportDialog({
inspect.mutate(sourceRoot.trim(), { onSuccess: setInspection }) inspect.mutate(sourceRoot.trim(), { onSuccess: setInspection })
} }
const chooseSource = (root: string) => {
setSourceRoot(root)
inspect.mutate(root, { onSuccess: setInspection })
}
const uploadZip = (file: File) => {
upload.mutate(file, { onSuccess: (result) => chooseSource(result.sourceRoot) })
}
const beginImport = (forceRestart = false) => { const beginImport = (forceRestart = false) => {
start.mutate({ sourceRoot: sourceRoot.trim(), forceRestart }, { onSuccess: (created) => setJobId(created.id) }) start.mutate({ sourceRoot: sourceRoot.trim(), forceRestart }, { onSuccess: (created) => setJobId(created.id) })
} }
@@ -55,27 +70,79 @@ export function ImportDialog({
) )
} }
const busy = inspect.isPending || start.isPending || upload.isPending
return ( return (
<Modal title="Import outline" onClose={onClose}> <Modal title="Import outline" onClose={onClose}>
<form onSubmit={check} className="grid gap-3"> <form onSubmit={check} className="grid gap-3">
<label className="block"> {!inspection && (
<span className="label">Source folder</span> <div id="import-source-mode" className="flex gap-1">
<input {(['browse', 'upload', 'path'] as const).map((mode) => (
className="input" <button
autoFocus key={mode}
value={sourceRoot} type="button"
onChange={(e) => { className="btn"
setSourceRoot(e.target.value) style={sourceMode === mode ? { color: 'var(--accent)' } : undefined}
setInspection(null) disabled={busy}
}} onClick={() => setSourceMode(mode)}
placeholder="/home/you/Documents/Novels/my-outline" >
disabled={inspect.isPending || start.isPending} {mode === 'browse' ? 'Browse' : mode === 'upload' ? 'Upload zip' : 'Enter path'}
/> </button>
</label> ))}
<p className="text-sm muted"> </div>
Absolute path to the folder holding outline.md, its chapter files and character )}
dossiers.
</p> {!inspection && sourceMode === 'browse' && (
<ImportSourcePicker onSelect={chooseSource} disabled={busy} />
)}
{!inspection && sourceMode === 'upload' && (
<div className="grid gap-2">
<input
id="import-zip-input"
ref={fileInputRef}
type="file"
accept=".zip"
className="input"
disabled={busy}
onChange={(e) => {
const file = e.target.files?.[0]
if (file) uploadZip(file)
}}
/>
<p className="text-sm muted">
A zip of the folder holding outline.md, its chapter files and character
dossiers.
</p>
{upload.isPending && <Spinner label="Uploading" />}
{upload.error && <ErrorNote error={upload.error} />}
</div>
)}
{!inspection && sourceMode === 'path' && (
<>
<label className="block">
<span className="label">Source folder or file</span>
<input
className="input"
autoFocus
value={sourceRoot}
onChange={(e) => {
setSourceRoot(e.target.value)
setInspection(null)
}}
placeholder="/home/you/Documents/Novels/my-outline"
disabled={busy}
/>
</label>
<p className="text-sm muted">
Absolute path to the folder holding outline.md and its chapter/character
files, or to a single markdown file.
</p>
</>
)}
{inspection && <p className="text-sm muted">{sourceRoot}</p>}
{inspect.error && <ErrorNote error={inspect.error} />} {inspect.error && <ErrorNote error={inspect.error} />}
{start.error && <ErrorNote error={start.error} />} {start.error && <ErrorNote error={start.error} />}
@@ -96,7 +163,7 @@ export function ImportDialog({
<button type="button" className="btn" onClick={onClose}> <button type="button" className="btn" onClick={onClose}>
Cancel Cancel
</button> </button>
{!inspection && ( {!inspection && sourceMode === 'path' && (
<button <button
type="submit" type="submit"
className="btn btn-primary" className="btn btn-primary"
@@ -0,0 +1,91 @@
import { useState } from 'react'
import { useImportBrowse } from '../api/hooks'
import { ApiError } from '../api/client'
import { Spinner } from './ui'
export function ImportSourcePicker({
onSelect,
disabled,
}: {
onSelect: (sourceRoot: string) => void
disabled?: boolean
}) {
const [path, setPath] = useState('')
const browse = useImportBrowse(path)
if (browse.isError && browse.error instanceof ApiError && browse.error.status === 400) {
return (
<p id="import-picker-unavailable" className="text-sm muted">
No import folder is configured on this server.
</p>
)
}
if (browse.isLoading) {
return <Spinner label="Loading" />
}
if (!browse.data) {
return null
}
const segments = browse.data.relativePath ? browse.data.relativePath.split('/') : []
return (
<div id="import-source-picker" className="grid gap-2">
<nav className="flex flex-wrap items-center gap-1 text-sm">
<button type="button" className="btn" disabled={disabled} onClick={() => setPath('')}>
Import folder
</button>
{segments.map((segment, i) => (
<span key={i} className="flex items-center gap-1">
<span className="muted">/</span>
<button
type="button"
className="btn"
disabled={disabled}
onClick={() => setPath(segments.slice(0, i + 1).join('/'))}
>
{segment}
</button>
</span>
))}
</nav>
<ul
id="import-source-picker-entries"
className="grid gap-1 rounded-md p-1"
style={{ background: 'var(--surface-sunken)', maxHeight: '16rem', overflowY: 'auto' }}
>
{browse.data.entries.length === 0 && <li className="px-2 py-1 text-sm muted">Empty folder.</li>}
{browse.data.entries.map((entry) => (
<li key={entry.relativePath} className="flex items-center justify-between gap-2 px-2 py-1">
<button
type="button"
className="flex-1 text-left text-sm"
disabled={disabled}
onClick={() => (entry.isDirectory ? setPath(entry.relativePath) : onSelect(entry.sourceRoot))}
>
{entry.isDirectory ? '📁' : '📄'} {entry.name}
{entry.isDirectory && entry.looksImportable && (
<span className="ml-2 text-xs" style={{ color: 'var(--accent)' }}>
outline found
</span>
)}
</button>
{entry.isDirectory && (
<button
type="button"
className="btn"
disabled={disabled || !entry.looksImportable}
onClick={() => onSelect(entry.sourceRoot)}
>
Select
</button>
)}
</li>
))}
</ul>
</div>
)
}
@@ -0,0 +1,100 @@
using Microsoft.Extensions.Options;
using Novelly.Api.Imports;
namespace Novelly.Api.Tests;
[TestFixture]
public class ImportBrowseServiceTests
{
private string _root = null!;
private ImportBrowseService _browse = null!;
[SetUp]
public void SetUp()
{
_root = Directory.CreateTempSubdirectory("novelly-browse-test-").FullName;
_browse = new ImportBrowseService(
Options.Create(new ImportOptions { RootPath = _root }),
new CapturingLogger<ImportBrowseService>());
}
[TearDown]
public void TearDown()
{
if (Directory.Exists(_root))
{
Directory.Delete(_root, recursive: true);
}
}
[Test]
public void Listing_the_root_returns_directories_before_files_ordered_by_name()
{
File.WriteAllText(Path.Combine(_root, "notes.md"), "# Notes");
Directory.CreateDirectory(Path.Combine(_root, "zeta"));
Directory.CreateDirectory(Path.Combine(_root, "alpha"));
var listing = _browse.List(null);
Assert.That(listing.Entries.Select(e => e.Name), Is.EqualTo(new[] { "alpha", "zeta", "notes.md" }));
}
[Test]
public void A_folder_with_an_outline_file_is_flagged_importable()
{
var folder = Path.Combine(_root, "my-novel");
Directory.CreateDirectory(folder);
File.WriteAllText(Path.Combine(folder, "outline.md"), "# My Novel");
var listing = _browse.List(null);
Assert.That(listing.Entries.Single(e => e.Name == "my-novel").LooksImportable, Is.True);
}
[Test]
public void An_empty_folder_is_not_flagged_importable()
{
Directory.CreateDirectory(Path.Combine(_root, "empty"));
var listing = _browse.List(null);
Assert.That(listing.Entries.Single(e => e.Name == "empty").LooksImportable, Is.False);
}
[Test]
public void The_staging_folder_is_hidden_from_listings()
{
Directory.CreateDirectory(Path.Combine(_root, ".novelly-staging"));
Directory.CreateDirectory(Path.Combine(_root, "visible"));
var listing = _browse.List(null);
Assert.That(listing.Entries.Select(e => e.Name), Is.EqualTo(new[] { "visible" }));
}
[Test]
public void Escaping_the_import_root_is_rejected()
{
Assert.That(() => _browse.List(".."), Throws.TypeOf<ArgumentException>());
}
[Test]
public void Browsing_without_a_configured_root_fails()
{
var unconfigured = new ImportBrowseService(
Options.Create(new ImportOptions()),
new CapturingLogger<ImportBrowseService>());
Assert.That(() => unconfigured.List(null), Throws.TypeOf<InvalidOperationException>());
}
[Test]
public void Listing_a_subfolder_reports_its_parent()
{
Directory.CreateDirectory(Path.Combine(_root, "child"));
var listing = _browse.List("child");
Assert.That(listing.ParentRelativePath, Is.EqualTo(""));
}
}
@@ -1,4 +1,5 @@
using System.Threading.Channels; using System.Threading.Channels;
using Microsoft.Extensions.Options;
using Novelly.Api.Imports; using Novelly.Api.Imports;
using Novelly.Api.Novels; using Novelly.Api.Novels;
@@ -20,6 +21,8 @@ public class ImportServiceTests : ServiceTestFixture
Novels, Novels,
_queue, _queue,
UserContext, UserContext,
Options.Create(new ImportOptions()),
new ImportZipExtractor(new CapturingLogger<ImportZipExtractor>()),
new CapturingLogger<ImportService>(), new CapturingLogger<ImportService>(),
new InspectImportRequestValidator(), new InspectImportRequestValidator(),
new StartImportRequestValidator()); new StartImportRequestValidator());
@@ -32,6 +35,12 @@ public class ImportServiceTests : ServiceTestFixture
{ {
Directory.Delete(_root, recursive: true); Directory.Delete(_root, recursive: true);
} }
var staging = Path.Combine(Path.GetTempPath(), "novelly-import-staging");
if (Directory.Exists(staging))
{
Directory.Delete(staging, recursive: true);
}
} }
[Test] [Test]
@@ -131,6 +140,71 @@ public class ImportServiceTests : ServiceTestFixture
}); });
} }
[Test]
public async Task Starting_an_import_outside_a_configured_import_root_is_rejected()
{
var configuredRoot = Directory.CreateTempSubdirectory("novelly-import-root-").FullName;
try
{
var restricted = new ImportService(
Db.Context,
Novels,
_queue,
UserContext,
Options.Create(new ImportOptions { RootPath = configuredRoot }),
new ImportZipExtractor(new CapturingLogger<ImportZipExtractor>()),
new CapturingLogger<ImportService>(),
new InspectImportRequestValidator(),
new StartImportRequestValidator());
Assert.That(
async () => await restricted.StartOrResumeAsync(new StartImportRequest(_root)),
Throws.TypeOf<ArgumentException>());
}
finally
{
Directory.Delete(configuredRoot, recursive: true);
}
}
[Test]
public async Task Starting_an_import_of_a_single_markdown_file_stages_it_into_a_folder()
{
var file = Path.Combine(_root, "01-chapter.md");
File.WriteAllText(file, "# Chapter 1");
var job = await _imports.StartOrResumeAsync(new StartImportRequest(file));
Assert.Multiple(() =>
{
Assert.That(Directory.Exists(job.SourceRoot), Is.True);
Assert.That(File.Exists(Path.Combine(job.SourceRoot, "01-chapter.md")), Is.True);
});
}
[Test]
public async Task Starting_an_import_of_the_same_single_file_twice_dedupes_to_the_same_staged_job()
{
var file = Path.Combine(_root, "01-chapter.md");
File.WriteAllText(file, "# Chapter 1");
var first = await _imports.StartOrResumeAsync(new StartImportRequest(file));
var second = await _imports.StartOrResumeAsync(new StartImportRequest(file));
Assert.That(second.Id, Is.EqualTo(first.Id));
}
[Test]
public async Task Starting_an_import_of_a_non_markdown_file_is_rejected()
{
var file = Path.Combine(_root, "notes.txt");
File.WriteAllText(file, "not markdown");
Assert.That(
async () => await _imports.StartOrResumeAsync(new StartImportRequest(file)),
Throws.TypeOf<ArgumentException>());
}
private void WriteChapterFiles(int count) private void WriteChapterFiles(int count)
{ {
var outlines = Path.Combine(_root, "outlines"); var outlines = Path.Combine(_root, "outlines");
@@ -0,0 +1,116 @@
using System.IO.Compression;
using Novelly.Api.Imports;
namespace Novelly.Api.Tests;
[TestFixture]
public class ImportZipExtractorTests
{
private string _stagingParent = null!;
private ImportZipExtractor _extractor = null!;
[SetUp]
public void SetUp()
{
_stagingParent = Directory.CreateTempSubdirectory("novelly-zip-test-").FullName;
_extractor = new ImportZipExtractor(new CapturingLogger<ImportZipExtractor>());
}
[TearDown]
public void TearDown()
{
if (Directory.Exists(_stagingParent))
{
Directory.Delete(_stagingParent, recursive: true);
}
}
[Test]
public void A_zip_of_the_expected_folder_layout_extracts_intact()
{
using var zip = BuildZip(("outline.md", "# My Novel"), ("outlines/01-chapter.md", "# Chapter 1"));
var staging = StagingDir();
_extractor.Extract(zip, staging);
Assert.Multiple(() =>
{
Assert.That(File.Exists(Path.Combine(staging, "outline.md")), Is.True);
Assert.That(File.Exists(Path.Combine(staging, "outlines", "01-chapter.md")), Is.True);
});
}
[Test]
public void A_single_wrapper_directory_is_flattened_away()
{
using var zip = BuildZip(("my-novel/outline.md", "# My Novel"), ("my-novel/outlines/01-chapter.md", "# Chapter 1"));
var staging = StagingDir();
_extractor.Extract(zip, staging);
Assert.Multiple(() =>
{
Assert.That(File.Exists(Path.Combine(staging, "outline.md")), Is.True);
Assert.That(Directory.Exists(Path.Combine(staging, "my-novel")), Is.False);
});
}
[Test]
public void A_zip_slip_entry_is_rejected_and_staging_is_cleaned_up()
{
using var zip = BuildZip(("outline.md", "# ok"), ("../evil.md", "pwned"));
var staging = StagingDir();
Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf<ArgumentException>());
Assert.That(Directory.Exists(staging), Is.False);
}
[Test]
public void A_disallowed_file_extension_is_rejected()
{
using var zip = BuildZip(("outline.md", "# ok"), ("script.sh", "echo hi"));
var staging = StagingDir();
Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf<ArgumentException>());
Assert.That(Directory.Exists(staging), Is.False);
}
[Test]
public void The_ledger_file_is_allowed_through()
{
using var zip = BuildZip(("outline.md", "# ok"), (".novelly-import.json", "{}"));
var staging = StagingDir();
_extractor.Extract(zip, staging);
Assert.That(File.Exists(Path.Combine(staging, ".novelly-import.json")), Is.True);
}
[Test]
public void An_empty_zip_is_rejected()
{
using var zip = BuildZip();
var staging = StagingDir();
Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf<ArgumentException>());
}
private string StagingDir() => Path.Combine(_stagingParent, $"staging-{Guid.NewGuid():N}");
private static MemoryStream BuildZip(params (string Path, string Content)[] entries)
{
var stream = new MemoryStream();
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true))
{
foreach (var (path, content) in entries)
{
var entry = archive.CreateEntry(path);
using var writer = new StreamWriter(entry.Open());
writer.Write(content);
}
}
stream.Position = 0;
return stream;
}
}