From 661f2917eaafaeff220739e4c77b4ddd794b6896 Mon Sep 17 00:00:00 2001 From: James Wampler Date: Thu, 20 Aug 2026 11:38:35 -0700 Subject: [PATCH] Add zip upload and mapped-path picker for outline import Sandbox source paths under a configured Imports:RootPath, browse it from the web dialog, upload a zip that extracts into staging, and import a single markdown file (agent infers chapter vs character). --- CLAUDE.md | 1 + deploy/qa/docker-compose.qa.yml | 1 + .../Common/NovellyServiceRegistration.cs | 3 + src/Novelly.Api/Imports/ImportAgentService.cs | 66 +++++++++- .../Imports/ImportBrowseService.cs | 81 ++++++++++++ src/Novelly.Api/Imports/ImportContracts.cs | 2 + src/Novelly.Api/Imports/ImportEndpoints.cs | 18 +++ src/Novelly.Api/Imports/ImportOptions.cs | 8 ++ src/Novelly.Api/Imports/ImportPaths.cs | 39 +++++- src/Novelly.Api/Imports/ImportService.cs | 63 +++++++++- src/Novelly.Api/Imports/ImportZipExtractor.cs | 89 ++++++++++++++ src/Novelly.Api/Program.cs | 6 + src/Novelly.Api/appsettings.json | 3 + src/Novelly.Web/nginx.conf | 1 + src/Novelly.Web/src/api/client.ts | 4 +- src/Novelly.Web/src/api/hooks.ts | 22 ++++ src/Novelly.Web/src/api/types.ts | 21 ++++ .../src/components/ImportDialog.tsx | 109 ++++++++++++---- .../src/components/ImportSourcePicker.tsx | 91 ++++++++++++++ .../ImportBrowseServiceTests.cs | 100 +++++++++++++++ tests/Novelly.Api.Tests/ImportServiceTests.cs | 74 +++++++++++ .../ImportZipExtractorTests.cs | 116 ++++++++++++++++++ 22 files changed, 890 insertions(+), 28 deletions(-) create mode 100644 src/Novelly.Api/Imports/ImportBrowseService.cs create mode 100644 src/Novelly.Api/Imports/ImportOptions.cs create mode 100644 src/Novelly.Api/Imports/ImportZipExtractor.cs create mode 100644 src/Novelly.Web/src/components/ImportSourcePicker.tsx create mode 100644 tests/Novelly.Api.Tests/ImportBrowseServiceTests.cs create mode 100644 tests/Novelly.Api.Tests/ImportZipExtractorTests.cs diff --git a/CLAUDE.md b/CLAUDE.md index 63fadc0..871fe82 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,5 +103,6 @@ Several real bugs here — SQLite refusing ORDER BY DateTimeOffset, agent's mode - Anthropic model id lives in `appsettings.json` under `Agent:Model`. Don't hardcode. - API key comes from `ANTHROPIC_API_KEY` or `Agent:ApiKey` — never commit one. App must stay fully usable without key; only agent endpoints require it. - EF migrations: `dotnet ef migrations add -p src/Novelly.Api -o Data/Migrations`. API migrates on boot. +- Outline import root lives in `appsettings.json` under `Imports:RootPath` (`Imports__RootPath` env var). When set, it's the only folder the browse/upload import endpoints and the source picker can reach; unset, those endpoints are disabled and the dialog falls back to a typed path with no sandbox. Created at boot if missing. - `git push` runs `scripts/ci/prepush.sh` through Husky: build, test, then web build. Run `npm install` once at repo root to install hook. diff --git a/deploy/qa/docker-compose.qa.yml b/deploy/qa/docker-compose.qa.yml index 9dbef44..ede144a 100644 --- a/deploy/qa/docker-compose.qa.yml +++ b/deploy/qa/docker-compose.qa.yml @@ -15,6 +15,7 @@ services: ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} Agent__Model: claude-sonnet-5 Agent__Effort: high + Imports__RootPath: /data/imports volumes: - /mnt/storage/apps/novelly/data:/data networks: diff --git a/src/Novelly.Api/Common/NovellyServiceRegistration.cs b/src/Novelly.Api/Common/NovellyServiceRegistration.cs index 63a3f7f..2c08a11 100644 --- a/src/Novelly.Api/Common/NovellyServiceRegistration.cs +++ b/src/Novelly.Api/Common/NovellyServiceRegistration.cs @@ -94,8 +94,11 @@ public static class NovellyServiceRegistration services.Configure(configuration.GetSection(AgentOptions.SectionName)); services.AddScoped(); + services.Configure(configuration.GetSection(ImportOptions.SectionName)); services.AddSingleton(Channel.CreateUnbounded()); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddHostedService(); diff --git a/src/Novelly.Api/Imports/ImportAgentService.cs b/src/Novelly.Api/Imports/ImportAgentService.cs index 5ad5b4b..b815556 100644 --- a/src/Novelly.Api/Imports/ImportAgentService.cs +++ b/src/Novelly.Api/Imports/ImportAgentService.cs @@ -23,7 +23,9 @@ public class ImportAgentService( toolset.Initialize(sourceRoot, existingNovelId); var startingLedger = toolset.ReadLedgerOrNull(); - var systemPrompt = BuildSystemPrompt(sourceRoot); + var systemPrompt = ImportPaths.IsSingleFileSource(sourceRoot) + ? BuildSingleFileSystemPrompt(sourceRoot) + : BuildSystemPrompt(sourceRoot); var transcript = new List { @@ -105,6 +107,9 @@ public class ImportAgentService( private static string BuildSystemPrompt(string sourceRoot) => SystemPromptTemplate.Replace("{{SOURCE_ROOT}}", sourceRoot); + private static string BuildSingleFileSystemPrompt(string sourceRoot) => + SingleFileSystemPromptTemplate.Replace("{{SOURCE_ROOT}}", sourceRoot); + private const string SystemPromptTemplate = """ You import a novel outline that already exists as markdown files on disk into this app's novel data. You are running unattended — nobody will read your replies or @@ -199,4 +204,63 @@ public class ImportAgentService( - If a tool call fails, stop that item and move on rather than retrying blindly — the ledger stays at the last successful write either way. """; + + private const string SingleFileSystemPromptTemplate = """ + You import a single outline file that already exists as markdown on disk into this + app's novel data. You are running unattended — nobody will read your replies or + answer questions mid-run, so make the judgment calls yourself and record anything + genuinely ambiguous rather than stalling on it. + + Your tools give you exactly two things: read-only access to the file under the + import source folder, and application tools that create the novel's chapters, + characters, beats and arcs. You cannot write or edit anything on disk except the + resume ledger, and you cannot read anything outside the source folder. + + ## Source file + + The source root `{{SOURCE_ROOT}}` holds exactly one markdown file. Call + list_source_files to find its name, then read_source_file to read it. Decide what + kind of document it is before doing anything else: + + - If it reads like a chapter outline (`# Chapter NN`, one or more summary + paragraphs, a beat table `| Beat | Character | What | Why |`) treat it as a single + chapter. + - If it reads like a character dossier (`# Name`, an italic tagline, + `## Appearance`, `## Background`, `## Motivation`) treat it as a single character. + + `**Thread:**` (chapter files only) may name one character, several, or a character + plus a qualifier — only auto-create an undossiered name from it when it names + exactly one clear proper name. + + ## The ledger + + Before writing anything, call read_ledger. If it returns `{{}}`, this is a fresh + run. Call write_ledger with the full, updated ledger after every successful write. + + ## Passes + + 1. **Novel** — skip if "novel" is in completedPasses or a novel id was already + supplied. Otherwise create one from whatever title/author information the file + gives, or a sensible placeholder title drawn from the file name if none is + present. Record novelId, mark "novel" done. + 2. **The document** — skip if already recorded. If it is a chapter: auto-create a + character stub (name only) for any single, unqualified name in the Thread or a + beat's Character column that isn't in the ledger yet, then create_chapter with + title, number (1 unless the file states otherwise), summary, and tags, then + create_beat for each table row with resolved character_ids. If it is a + character: create_character with occupation from the tagline and + appearance/backstory/want from Appearance/Background/Motivation; if it has a + `## Events` section, also update_character(importance: "Main") and add_arc_stage + for each bullet. Mark "characters", "chapters", and "arcs" all done once you've + handled the one document — this run only ever has one item to place. + + ## Constraints + + - Never invent plot content or character detail, and never guess which of several + candidate names an ambiguous reference means. + - Never write to disk except via write_ledger. + - Never call a create tool for something the ledger already records. + - If a tool call fails, stop and record what you have — the ledger stays at the + last successful write either way. + """; } diff --git a/src/Novelly.Api/Imports/ImportBrowseService.cs b/src/Novelly.Api/Imports/ImportBrowseService.cs new file mode 100644 index 0000000..9f0403b --- /dev/null +++ b/src/Novelly.Api/Imports/ImportBrowseService.cs @@ -0,0 +1,81 @@ +using Microsoft.Extensions.Options; + +namespace Novelly.Api.Imports; + +public class ImportBrowseService(IOptions options, ILogger logger) +{ + private readonly ImportOptions _options = options.Value; + + public string? RootPath => _options.RootPath; + + public ImportBrowseResponse List(string? relativePath) + { + var root = RequireRoot(); + + logger.LogInformation("Browsing import root at {RelativePath}", relativePath ?? ""); + + var target = string.IsNullOrWhiteSpace(relativePath) ? root : ImportPaths.ResolveWithin(root, relativePath); + + if (!Directory.Exists(target)) + throw new ArgumentException($"'{relativePath}' does not exist or is not a directory.", nameof(relativePath)); + + var normalizedRelative = Path.GetRelativePath(root, target).Replace(Path.DirectorySeparatorChar, '/'); + if (normalizedRelative == ".") + { + normalizedRelative = ""; + } + + var parent = normalizedRelative == "" ? null : Path.GetRelativePath(root, Path.GetFullPath(Path.Combine(target, ".."))).Replace(Path.DirectorySeparatorChar, '/'); + if (parent == ".") + { + parent = ""; + } + + var entries = Directory.EnumerateFileSystemEntries(target) + .Select(BuildEntry) + .Where(e => e is not null) + .Select(e => e!) + .OrderByDescending(e => e.IsDirectory) + .ThenBy(e => e.Name, StringComparer.Ordinal) + .ToArray(); + + return new ImportBrowseResponse(normalizedRelative, parent, entries); + + ImportBrowseEntry? BuildEntry(string path) + { + var name = Path.GetFileName(path); + if (name.StartsWith('.')) + { + return null; + } + + var isDirectory = Directory.Exists(path); + if (!isDirectory && !name.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var entryRelative = Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/'); + var markdownCount = isDirectory ? ImportPaths.CountChapterFiles(path) : 0; + var looksImportable = isDirectory + ? File.Exists(Path.Combine(path, "outline.md")) || markdownCount > 0 + : true; + + return new ImportBrowseEntry(name, entryRelative, path, isDirectory, markdownCount, looksImportable); + } + } + + private string RequireRoot() + { + if (string.IsNullOrWhiteSpace(_options.RootPath)) + throw new InvalidOperationException("No import root is configured (Imports:RootPath)."); + + var full = Path.GetFullPath(_options.RootPath); + Directory.CreateDirectory(full); + return full; + } +} + +public record ImportBrowseEntry(string Name, string RelativePath, string SourceRoot, bool IsDirectory, int MarkdownFileCount, bool LooksImportable); + +public record ImportBrowseResponse(string RelativePath, string? ParentRelativePath, IReadOnlyList Entries); diff --git a/src/Novelly.Api/Imports/ImportContracts.cs b/src/Novelly.Api/Imports/ImportContracts.cs index 622b805..3bb1194 100644 --- a/src/Novelly.Api/Imports/ImportContracts.cs +++ b/src/Novelly.Api/Imports/ImportContracts.cs @@ -57,6 +57,8 @@ public class StartImportRequestValidator : IModelValidator } } +public record ImportUploadResponse(string SourceRoot, string RelativePath, int MarkdownFileCount); + public static class ImportMapping { public static ImportJobResponse ToResponse(this ImportJob job) => new( diff --git a/src/Novelly.Api/Imports/ImportEndpoints.cs b/src/Novelly.Api/Imports/ImportEndpoints.cs index bdc7386..c32746e 100644 --- a/src/Novelly.Api/Imports/ImportEndpoints.cs +++ b/src/Novelly.Api/Imports/ImportEndpoints.cs @@ -28,6 +28,24 @@ public static class ImportEndpoints (await service.GetStatusAsync(id, ct))?.ToResponse().ToApiResult()) .WithSummary("Poll an import job's progress."); + imports.MapGet("/browse", (string? path, ImportBrowseService browse) => + Results.Ok(browse.List(path))) + .WithSummary("List entries under the configured import root, for the source picker."); + + imports.MapPost("/upload", (IFormFile file, ImportService service) => + { + if (!file.FileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("Only .zip files can be uploaded.", nameof(file)); + + if (file.Length == 0) + throw new ArgumentException("The uploaded file is empty.", nameof(file)); + + using var stream = file.OpenReadStream(); + return Results.Ok(service.UploadZip(stream, file.FileName)); + }) + .DisableAntiforgery() + .WithSummary("Upload a zip of an outline folder and stage it under the configured import root."); + return app; } } diff --git a/src/Novelly.Api/Imports/ImportOptions.cs b/src/Novelly.Api/Imports/ImportOptions.cs new file mode 100644 index 0000000..bfde34c --- /dev/null +++ b/src/Novelly.Api/Imports/ImportOptions.cs @@ -0,0 +1,8 @@ +namespace Novelly.Api.Imports; + +public class ImportOptions +{ + public const string SectionName = "Imports"; + + public string? RootPath { get; set; } +} diff --git a/src/Novelly.Api/Imports/ImportPaths.cs b/src/Novelly.Api/Imports/ImportPaths.cs index 7aa7a01..849ed15 100644 --- a/src/Novelly.Api/Imports/ImportPaths.cs +++ b/src/Novelly.Api/Imports/ImportPaths.cs @@ -13,6 +13,7 @@ public record ImportLedger( internal static class ImportPaths { private const string LedgerFileName = ".novelly-import.json"; + public const string StagingFolderName = ".novelly-staging"; private static readonly JsonSerializerOptions LedgerOptions = new() { @@ -20,7 +21,7 @@ internal static class ImportPaths WriteIndented = true }; - public static string ResolveRoot(string sourceRoot) + public static string ResolveRoot(string sourceRoot, string? importRoot = null) { if (string.IsNullOrWhiteSpace(sourceRoot)) throw new ArgumentException("'Source Root' must not be empty.", nameof(sourceRoot)); @@ -38,25 +39,57 @@ internal static class ImportPaths if (!Directory.Exists(full)) throw new ArgumentException($"'{full}' does not exist or is not a directory.", nameof(sourceRoot)); + EnsureWithinImportRoot(importRoot, full, sourceRoot); + return full; } + public static void EnsureWithinImportRoot(string? importRoot, string candidate, string originalInput) + { + if (importRoot is not null && !IsWithin(importRoot, candidate)) + throw new ArgumentException($"'{originalInput}' is outside the configured import root.", nameof(originalInput)); + } + + public static bool IsSingleFileSource(string root) + { + if (Directory.EnumerateDirectories(root).Any()) + { + return false; + } + + return Directory.EnumerateFiles(root, "*.md", SearchOption.TopDirectoryOnly).Count() == 1; + } + public static string ResolveWithin(string root, string relativePath) { if (string.IsNullOrWhiteSpace(relativePath)) throw new ArgumentException("Path must not be empty."); var combined = Path.GetFullPath(Path.Combine(root, relativePath)); - var relativeToRoot = Path.GetRelativePath(root, combined); - if (relativeToRoot.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relativeToRoot)) + if (!IsWithin(root, combined)) throw new ArgumentException($"'{relativePath}' escapes the import source folder."); return combined; } + private static bool IsWithin(string root, string candidate) + { + var relativeToRoot = Path.GetRelativePath(root, candidate); + return relativeToRoot == "." || !relativeToRoot.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relativeToRoot); + } + public static string LedgerPath(string root) => Path.Combine(root, LedgerFileName); + public static string StagingRoot(string importRoot) => Path.Combine(importRoot, StagingFolderName); + + public static string SanitizeForFolderName(string value) + { + var sanitized = new string(value.Select(c => char.IsLetterOrDigit(c) || c is '-' or '_' ? c : '-').ToArray()); + sanitized = sanitized.Trim('-', '_'); + return string.IsNullOrEmpty(sanitized) ? "import" : sanitized[..Math.Min(sanitized.Length, 60)]; + } + public static ImportLedger? ReadLedger(string root) { var path = LedgerPath(root); diff --git a/src/Novelly.Api/Imports/ImportService.cs b/src/Novelly.Api/Imports/ImportService.cs index 7122700..e71089a 100644 --- a/src/Novelly.Api/Imports/ImportService.cs +++ b/src/Novelly.Api/Imports/ImportService.cs @@ -1,5 +1,6 @@ using System.Threading.Channels; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; using Novelly.Api.Common; using Novelly.Api.Common.Validation; using Novelly.Api.Data; @@ -13,10 +14,34 @@ public class ImportService( NovelService novels, Channel queue, INovelUserContext userContext, + IOptions importOptions, + ImportZipExtractor zipExtractor, ILogger logger, IModelValidator inspectValidator, IModelValidator startValidator) { + private readonly string? _importRoot = importOptions.Value.RootPath is { } root ? Path.GetFullPath(root) : null; + + public ImportUploadResponse UploadZip(Stream zipStream, string fileName) + { + if (_importRoot is null) + throw new InvalidOperationException("No import root is configured (Imports:RootPath)."); + + logger.LogInformation("Uploading import zip {FileName}", fileName); + + Directory.CreateDirectory(_importRoot); + var stagingDir = Path.Combine( + ImportPaths.StagingRoot(_importRoot), + $"zip-{ImportPaths.SanitizeForFolderName(Path.GetFileNameWithoutExtension(fileName))}-{Guid.NewGuid():N}"); + + zipExtractor.Extract(zipStream, stagingDir); + + var markdownCount = Directory.EnumerateFiles(stagingDir, "*.md", SearchOption.AllDirectories).Count(); + var relativePath = Path.GetRelativePath(_importRoot, stagingDir).Replace(Path.DirectorySeparatorChar, '/'); + + return new ImportUploadResponse(stagingDir, relativePath, markdownCount); + } + public Task InspectAsync(InspectImportRequest request, CancellationToken ct = default) { Guard.Null(request, nameof(request)); @@ -24,7 +49,7 @@ public class ImportService( logger.LogInformation("Inspecting import source {SourceRoot}", request.SourceRoot); - var root = ImportPaths.ResolveRoot(request.SourceRoot); + var root = ResolveSourceRoot(request.SourceRoot); var ledger = ImportPaths.ReadLedger(root); var total = ImportPaths.CountChapterFiles(root); @@ -48,7 +73,7 @@ public class ImportService( logger.LogInformation( "Starting import for {SourceRoot}, forceRestart {ForceRestart}", request.SourceRoot, request.ForceRestart); - var root = ImportPaths.ResolveRoot(request.SourceRoot); + var root = ResolveSourceRoot(request.SourceRoot); if (request.ForceRestart) { @@ -88,6 +113,40 @@ public class ImportService( return job; } + private string ResolveSourceRoot(string sourceRoot) + { + string full; + try + { + full = Path.GetFullPath(sourceRoot); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + throw new ArgumentException($"'{sourceRoot}' is not a valid path.", nameof(sourceRoot), ex); + } + + if (!File.Exists(full)) + { + return ImportPaths.ResolveRoot(sourceRoot, _importRoot); + } + + if (!full.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException($"'{sourceRoot}' is not a markdown file or a directory.", nameof(sourceRoot)); + + ImportPaths.EnsureWithinImportRoot(_importRoot, full, sourceRoot); + + var stagingParent = _importRoot is not null + ? ImportPaths.StagingRoot(_importRoot) + : Path.Combine(Path.GetTempPath(), "novelly-import-staging"); + var stagingDir = Path.Combine( + stagingParent, $"file-{ImportPaths.SanitizeForFolderName(Path.GetFileNameWithoutExtension(full))}"); + + Directory.CreateDirectory(stagingDir); + File.Copy(full, Path.Combine(stagingDir, Path.GetFileName(full)), overwrite: true); + + return stagingDir; + } + public async Task GetStatusAsync(Guid id, CancellationToken ct = default) { Guard.Default(id, nameof(id)); diff --git a/src/Novelly.Api/Imports/ImportZipExtractor.cs b/src/Novelly.Api/Imports/ImportZipExtractor.cs new file mode 100644 index 0000000..8a824e0 --- /dev/null +++ b/src/Novelly.Api/Imports/ImportZipExtractor.cs @@ -0,0 +1,89 @@ +using System.IO.Compression; + +namespace Novelly.Api.Imports; + +public class ImportZipExtractor(ILogger logger) +{ + private const int MaxEntryCount = 2000; + private const long MaxEntryUncompressedBytes = 10 * 1024 * 1024; + private const long MaxTotalUncompressedBytes = 100 * 1024 * 1024; + + private static readonly string[] AllowedFileNames = [".novelly-import.json"]; + + public void Extract(Stream zipStream, string stagingDir) + { + Directory.CreateDirectory(stagingDir); + + try + { + using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read); + + var entries = archive.Entries + .Where(e => !string.IsNullOrEmpty(e.Name)) + .Where(e => !e.FullName.StartsWith("__MACOSX/", StringComparison.OrdinalIgnoreCase)) + .Where(e => AllowedFileNames.Contains(e.Name) || !e.Name.StartsWith('.')) + .ToArray(); + + if (entries.Length == 0) + throw new ArgumentException("The zip file is empty."); + + if (entries.Length > MaxEntryCount) + throw new ArgumentException($"The zip file has too many entries (max {MaxEntryCount})."); + + var stripPrefix = FindCommonTopLevelDirectory(entries); + var totalBytes = 0L; + + foreach (var entry in entries) + { + var relativePath = stripPrefix is null + ? entry.FullName + : entry.FullName[(stripPrefix.Length + 1)..]; + + if (relativePath.Length == 0) + { + continue; + } + + if (!relativePath.EndsWith(".md", StringComparison.OrdinalIgnoreCase) && !AllowedFileNames.Contains(entry.Name)) + throw new ArgumentException($"'{entry.FullName}' is not a markdown file. Only .md files (and .novelly-import.json) are allowed."); + + if (entry.Length > MaxEntryUncompressedBytes) + throw new ArgumentException($"'{entry.FullName}' is too large (max {MaxEntryUncompressedBytes / (1024 * 1024)} MB per file)."); + + totalBytes += entry.Length; + if (totalBytes > MaxTotalUncompressedBytes) + throw new ArgumentException($"The zip file is too large uncompressed (max {MaxTotalUncompressedBytes / (1024 * 1024)} MB)."); + + var destination = ImportPaths.ResolveWithin(stagingDir, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + + using var entryStream = entry.Open(); + using var fileStream = File.Create(destination); + entryStream.CopyTo(fileStream); + } + + logger.LogInformation("Extracted import zip with {EntryCount} entries into staging folder", entries.Length); + } + catch + { + if (Directory.Exists(stagingDir)) + { + Directory.Delete(stagingDir, recursive: true); + } + + throw; + } + } + + private static string? FindCommonTopLevelDirectory(IReadOnlyCollection entries) + { + var topLevelSegments = entries + .Select(e => e.FullName.Split('/', '\\')[0]) + .Distinct() + .ToArray(); + + return topLevelSegments.Length == 1 && entries.All(e => e.FullName.Contains('/') || e.FullName.Contains('\\')) + ? topLevelSegments[0] + : null; + } +} diff --git a/src/Novelly.Api/Program.cs b/src/Novelly.Api/Program.cs index dfbdd82..3c97abc 100644 --- a/src/Novelly.Api/Program.cs +++ b/src/Novelly.Api/Program.cs @@ -68,6 +68,12 @@ using (var scope = app.Services.CreateScope()) await ServiceUser.EnsureSeededAsync(db, builder.Configuration[ServiceApiKeyAuthenticationHandler.ConfigurationKey], app.Logger); await ActivityBackfill.RunAsync(db, app.Logger); + + var importRoot = builder.Configuration.GetSection(ImportOptions.SectionName)[nameof(ImportOptions.RootPath)]; + if (!string.IsNullOrWhiteSpace(importRoot)) + { + Directory.CreateDirectory(importRoot); + } } app.UseSerilogRequestLogging(); diff --git a/src/Novelly.Api/appsettings.json b/src/Novelly.Api/appsettings.json index 9d09b10..7eee26a 100644 --- a/src/Novelly.Api/appsettings.json +++ b/src/Novelly.Api/appsettings.json @@ -35,5 +35,8 @@ }, "UiSettings": { "ShowPronouns": false + }, + "Imports": { + "RootPath": null } } diff --git a/src/Novelly.Web/nginx.conf b/src/Novelly.Web/nginx.conf index e0554ad..57478f5 100644 --- a/src/Novelly.Web/nginx.conf +++ b/src/Novelly.Web/nginx.conf @@ -7,6 +7,7 @@ server { } location /api/ { + client_max_body_size 50m; proxy_pass http://api:8080/api/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/src/Novelly.Web/src/api/client.ts b/src/Novelly.Web/src/api/client.ts index a489926..ec025bc 100644 --- a/src/Novelly.Web/src/api/client.ts +++ b/src/Novelly.Web/src/api/client.ts @@ -11,11 +11,12 @@ export class ApiError extends Error { } async function request(path: string, init?: RequestInit): Promise { + const isFormData = init?.body instanceof FormData const response = await fetch(`${BASE}${path}`, { ...init, credentials: 'include', headers: { - 'Content-Type': 'application/json', + ...(isFormData ? {} : { 'Content-Type': 'application/json' }), ...init?.headers, }, }) @@ -37,6 +38,7 @@ export const api = { get: (path: string) => request(path), post: (path: string, body?: unknown) => request(path, { method: 'POST', body: JSON.stringify(body ?? {}) }), + postForm: (path: string, body: FormData) => request(path, { method: 'POST', body }), patch: (path: string, body: unknown) => request(path, { method: 'PATCH', body: JSON.stringify(body) }), put: (path: string, body: unknown) => diff --git a/src/Novelly.Web/src/api/hooks.ts b/src/Novelly.Web/src/api/hooks.ts index 04cc71e..400ca38 100644 --- a/src/Novelly.Web/src/api/hooks.ts +++ b/src/Novelly.Web/src/api/hooks.ts @@ -12,9 +12,11 @@ import type { ConversationSummary, Beat, Genre, + ImportBrowse, ImportInspection, ImportJob, ImportJobStatus, + ImportUpload, OpenQuestion, LocationReferences, LocationSummary, @@ -47,6 +49,7 @@ export const keys = { conversations: (novelId: string) => ['novels', novelId, 'conversations'] as const, conversation: (id: string) => ['conversations', id] as const, importJob: (id: string) => ['imports', id] as const, + importBrowse: (path: string) => ['imports', 'browse', path] as const, novelActivity: (novelId: string) => ['novels', novelId, 'activity'] as const, myActivity: ['activity'] as const, } @@ -635,6 +638,25 @@ export function useStartImport() { }) } +export function useImportBrowse(path: string, enabled = true) { + return useQuery({ + queryKey: keys.importBrowse(path), + queryFn: () => api.get(`/api/imports/browse?path=${encodeURIComponent(path)}`), + enabled, + retry: false, + }) +} + +export function useUploadImportZip() { + return useMutation({ + mutationFn: (file: File) => { + const form = new FormData() + form.append('file', file) + return api.postForm('/api/imports/upload', form) + }, + }) +} + const terminalImportStatuses: ImportJobStatus[] = ['Completed', 'Failed', 'Paused'] export function useImportJob(jobId: string | undefined) { diff --git a/src/Novelly.Web/src/api/types.ts b/src/Novelly.Web/src/api/types.ts index 5741f25..acdbfb4 100644 --- a/src/Novelly.Web/src/api/types.ts +++ b/src/Novelly.Web/src/api/types.ts @@ -329,6 +329,27 @@ export interface ImportInspection { completedPasses: string[] } +export interface ImportBrowseEntry { + name: string + relativePath: string + sourceRoot: string + isDirectory: boolean + markdownFileCount: number + looksImportable: boolean +} + +export interface ImportBrowse { + relativePath: string + parentRelativePath: string | null + entries: ImportBrowseEntry[] +} + +export interface ImportUpload { + sourceRoot: string + relativePath: string + markdownFileCount: number +} + export interface ActivityDay { date: string words: number diff --git a/src/Novelly.Web/src/components/ImportDialog.tsx b/src/Novelly.Web/src/components/ImportDialog.tsx index 7b5cd1a..7671ad7 100644 --- a/src/Novelly.Web/src/components/ImportDialog.tsx +++ b/src/Novelly.Web/src/components/ImportDialog.tsx @@ -1,8 +1,11 @@ -import { useEffect, useState, type FormEvent } from 'react' +import { useEffect, useRef, useState, type FormEvent } from 'react' import { useQueryClient } from '@tanstack/react-query' -import { useImportJob, useInspectImport, useStartImport } from '../api/hooks' +import { useImportJob, useInspectImport, useStartImport, useUploadImportZip } from '../api/hooks' import type { ImportInspection, ImportJob } from '../api/types' import { ErrorNote, Modal, Spinner } from './ui' +import { ImportSourcePicker } from './ImportSourcePicker' + +type SourceMode = 'browse' | 'upload' | 'path' export function ImportDialog({ onClose, @@ -11,13 +14,16 @@ export function ImportDialog({ onClose: () => void onImported?: (novelId: string) => void }) { + const [sourceMode, setSourceMode] = useState('browse') const [sourceRoot, setSourceRoot] = useState('') const [inspection, setInspection] = useState(null) const [jobId, setJobId] = useState() const [confirmingRestart, setConfirmingRestart] = useState(false) + const fileInputRef = useRef(null) const inspect = useInspectImport() const start = useStartImport() + const upload = useUploadImportZip() const job = useImportJob(jobId) const qc = useQueryClient() @@ -33,6 +39,15 @@ export function ImportDialog({ inspect.mutate(sourceRoot.trim(), { onSuccess: setInspection }) } + const chooseSource = (root: string) => { + setSourceRoot(root) + inspect.mutate(root, { onSuccess: setInspection }) + } + + const uploadZip = (file: File) => { + upload.mutate(file, { onSuccess: (result) => chooseSource(result.sourceRoot) }) + } + const beginImport = (forceRestart = false) => { start.mutate({ sourceRoot: sourceRoot.trim(), forceRestart }, { onSuccess: (created) => setJobId(created.id) }) } @@ -55,27 +70,79 @@ export function ImportDialog({ ) } + const busy = inspect.isPending || start.isPending || upload.isPending + return (
- -

- Absolute path to the folder holding outline.md, its chapter files and character - dossiers. -

+ {!inspection && ( +
+ {(['browse', 'upload', 'path'] as const).map((mode) => ( + + ))} +
+ )} + + {!inspection && sourceMode === 'browse' && ( + + )} + + {!inspection && sourceMode === 'upload' && ( +
+ { + const file = e.target.files?.[0] + if (file) uploadZip(file) + }} + /> +

+ A zip of the folder holding outline.md, its chapter files and character + dossiers. +

+ {upload.isPending && } + {upload.error && } +
+ )} + + {!inspection && sourceMode === 'path' && ( + <> + +

+ Absolute path to the folder holding outline.md and its chapter/character + files, or to a single markdown file. +

+ + )} + + {inspection &&

{sourceRoot}

} {inspect.error && } {start.error && } @@ -96,7 +163,7 @@ export function ImportDialog({ - {!inspection && ( + {!inspection && sourceMode === 'path' && ( + {segments.map((segment, i) => ( + + / + + + ))} + + +
    + {browse.data.entries.length === 0 &&
  • Empty folder.
    • } + {browse.data.entries.map((entry) => ( +
  • + + {entry.isDirectory && ( + + )} +
    • + ))} +
+ + ) +} diff --git a/tests/Novelly.Api.Tests/ImportBrowseServiceTests.cs b/tests/Novelly.Api.Tests/ImportBrowseServiceTests.cs new file mode 100644 index 0000000..154d612 --- /dev/null +++ b/tests/Novelly.Api.Tests/ImportBrowseServiceTests.cs @@ -0,0 +1,100 @@ +using Microsoft.Extensions.Options; +using Novelly.Api.Imports; + +namespace Novelly.Api.Tests; + +[TestFixture] +public class ImportBrowseServiceTests +{ + private string _root = null!; + private ImportBrowseService _browse = null!; + + [SetUp] + public void SetUp() + { + _root = Directory.CreateTempSubdirectory("novelly-browse-test-").FullName; + _browse = new ImportBrowseService( + Options.Create(new ImportOptions { RootPath = _root }), + new CapturingLogger()); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + + [Test] + public void Listing_the_root_returns_directories_before_files_ordered_by_name() + { + File.WriteAllText(Path.Combine(_root, "notes.md"), "# Notes"); + Directory.CreateDirectory(Path.Combine(_root, "zeta")); + Directory.CreateDirectory(Path.Combine(_root, "alpha")); + + var listing = _browse.List(null); + + Assert.That(listing.Entries.Select(e => e.Name), Is.EqualTo(new[] { "alpha", "zeta", "notes.md" })); + } + + [Test] + public void A_folder_with_an_outline_file_is_flagged_importable() + { + var folder = Path.Combine(_root, "my-novel"); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "outline.md"), "# My Novel"); + + var listing = _browse.List(null); + + Assert.That(listing.Entries.Single(e => e.Name == "my-novel").LooksImportable, Is.True); + } + + [Test] + public void An_empty_folder_is_not_flagged_importable() + { + Directory.CreateDirectory(Path.Combine(_root, "empty")); + + var listing = _browse.List(null); + + Assert.That(listing.Entries.Single(e => e.Name == "empty").LooksImportable, Is.False); + } + + [Test] + public void The_staging_folder_is_hidden_from_listings() + { + Directory.CreateDirectory(Path.Combine(_root, ".novelly-staging")); + Directory.CreateDirectory(Path.Combine(_root, "visible")); + + var listing = _browse.List(null); + + Assert.That(listing.Entries.Select(e => e.Name), Is.EqualTo(new[] { "visible" })); + } + + [Test] + public void Escaping_the_import_root_is_rejected() + { + Assert.That(() => _browse.List(".."), Throws.TypeOf()); + } + + [Test] + public void Browsing_without_a_configured_root_fails() + { + var unconfigured = new ImportBrowseService( + Options.Create(new ImportOptions()), + new CapturingLogger()); + + Assert.That(() => unconfigured.List(null), Throws.TypeOf()); + } + + [Test] + public void Listing_a_subfolder_reports_its_parent() + { + Directory.CreateDirectory(Path.Combine(_root, "child")); + + var listing = _browse.List("child"); + + Assert.That(listing.ParentRelativePath, Is.EqualTo("")); + } +} diff --git a/tests/Novelly.Api.Tests/ImportServiceTests.cs b/tests/Novelly.Api.Tests/ImportServiceTests.cs index 154f8b3..af69ff3 100644 --- a/tests/Novelly.Api.Tests/ImportServiceTests.cs +++ b/tests/Novelly.Api.Tests/ImportServiceTests.cs @@ -1,4 +1,5 @@ using System.Threading.Channels; +using Microsoft.Extensions.Options; using Novelly.Api.Imports; using Novelly.Api.Novels; @@ -20,6 +21,8 @@ public class ImportServiceTests : ServiceTestFixture Novels, _queue, UserContext, + Options.Create(new ImportOptions()), + new ImportZipExtractor(new CapturingLogger()), new CapturingLogger(), new InspectImportRequestValidator(), new StartImportRequestValidator()); @@ -32,6 +35,12 @@ public class ImportServiceTests : ServiceTestFixture { Directory.Delete(_root, recursive: true); } + + var staging = Path.Combine(Path.GetTempPath(), "novelly-import-staging"); + if (Directory.Exists(staging)) + { + Directory.Delete(staging, recursive: true); + } } [Test] @@ -131,6 +140,71 @@ public class ImportServiceTests : ServiceTestFixture }); } + [Test] + public async Task Starting_an_import_outside_a_configured_import_root_is_rejected() + { + var configuredRoot = Directory.CreateTempSubdirectory("novelly-import-root-").FullName; + try + { + var restricted = new ImportService( + Db.Context, + Novels, + _queue, + UserContext, + Options.Create(new ImportOptions { RootPath = configuredRoot }), + new ImportZipExtractor(new CapturingLogger()), + new CapturingLogger(), + new InspectImportRequestValidator(), + new StartImportRequestValidator()); + + Assert.That( + async () => await restricted.StartOrResumeAsync(new StartImportRequest(_root)), + Throws.TypeOf()); + } + finally + { + Directory.Delete(configuredRoot, recursive: true); + } + } + + [Test] + public async Task Starting_an_import_of_a_single_markdown_file_stages_it_into_a_folder() + { + var file = Path.Combine(_root, "01-chapter.md"); + File.WriteAllText(file, "# Chapter 1"); + + var job = await _imports.StartOrResumeAsync(new StartImportRequest(file)); + + Assert.Multiple(() => + { + Assert.That(Directory.Exists(job.SourceRoot), Is.True); + Assert.That(File.Exists(Path.Combine(job.SourceRoot, "01-chapter.md")), Is.True); + }); + } + + [Test] + public async Task Starting_an_import_of_the_same_single_file_twice_dedupes_to_the_same_staged_job() + { + var file = Path.Combine(_root, "01-chapter.md"); + File.WriteAllText(file, "# Chapter 1"); + + var first = await _imports.StartOrResumeAsync(new StartImportRequest(file)); + var second = await _imports.StartOrResumeAsync(new StartImportRequest(file)); + + Assert.That(second.Id, Is.EqualTo(first.Id)); + } + + [Test] + public async Task Starting_an_import_of_a_non_markdown_file_is_rejected() + { + var file = Path.Combine(_root, "notes.txt"); + File.WriteAllText(file, "not markdown"); + + Assert.That( + async () => await _imports.StartOrResumeAsync(new StartImportRequest(file)), + Throws.TypeOf()); + } + private void WriteChapterFiles(int count) { var outlines = Path.Combine(_root, "outlines"); diff --git a/tests/Novelly.Api.Tests/ImportZipExtractorTests.cs b/tests/Novelly.Api.Tests/ImportZipExtractorTests.cs new file mode 100644 index 0000000..cdc5ef5 --- /dev/null +++ b/tests/Novelly.Api.Tests/ImportZipExtractorTests.cs @@ -0,0 +1,116 @@ +using System.IO.Compression; +using Novelly.Api.Imports; + +namespace Novelly.Api.Tests; + +[TestFixture] +public class ImportZipExtractorTests +{ + private string _stagingParent = null!; + private ImportZipExtractor _extractor = null!; + + [SetUp] + public void SetUp() + { + _stagingParent = Directory.CreateTempSubdirectory("novelly-zip-test-").FullName; + _extractor = new ImportZipExtractor(new CapturingLogger()); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_stagingParent)) + { + Directory.Delete(_stagingParent, recursive: true); + } + } + + [Test] + public void A_zip_of_the_expected_folder_layout_extracts_intact() + { + using var zip = BuildZip(("outline.md", "# My Novel"), ("outlines/01-chapter.md", "# Chapter 1")); + var staging = StagingDir(); + + _extractor.Extract(zip, staging); + + Assert.Multiple(() => + { + Assert.That(File.Exists(Path.Combine(staging, "outline.md")), Is.True); + Assert.That(File.Exists(Path.Combine(staging, "outlines", "01-chapter.md")), Is.True); + }); + } + + [Test] + public void A_single_wrapper_directory_is_flattened_away() + { + using var zip = BuildZip(("my-novel/outline.md", "# My Novel"), ("my-novel/outlines/01-chapter.md", "# Chapter 1")); + var staging = StagingDir(); + + _extractor.Extract(zip, staging); + + Assert.Multiple(() => + { + Assert.That(File.Exists(Path.Combine(staging, "outline.md")), Is.True); + Assert.That(Directory.Exists(Path.Combine(staging, "my-novel")), Is.False); + }); + } + + [Test] + public void A_zip_slip_entry_is_rejected_and_staging_is_cleaned_up() + { + using var zip = BuildZip(("outline.md", "# ok"), ("../evil.md", "pwned")); + var staging = StagingDir(); + + Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf()); + Assert.That(Directory.Exists(staging), Is.False); + } + + [Test] + public void A_disallowed_file_extension_is_rejected() + { + using var zip = BuildZip(("outline.md", "# ok"), ("script.sh", "echo hi")); + var staging = StagingDir(); + + Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf()); + Assert.That(Directory.Exists(staging), Is.False); + } + + [Test] + public void The_ledger_file_is_allowed_through() + { + using var zip = BuildZip(("outline.md", "# ok"), (".novelly-import.json", "{}")); + var staging = StagingDir(); + + _extractor.Extract(zip, staging); + + Assert.That(File.Exists(Path.Combine(staging, ".novelly-import.json")), Is.True); + } + + [Test] + public void An_empty_zip_is_rejected() + { + using var zip = BuildZip(); + var staging = StagingDir(); + + Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf()); + } + + private string StagingDir() => Path.Combine(_stagingParent, $"staging-{Guid.NewGuid():N}"); + + private static MemoryStream BuildZip(params (string Path, string Content)[] entries) + { + var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var (path, content) in entries) + { + var entry = archive.CreateEntry(path); + using var writer = new StreamWriter(entry.Open()); + writer.Write(content); + } + } + + stream.Position = 0; + return stream; + } +}