Add zip upload and mapped-path picker for outline import
Sandbox source paths under a configured Imports:RootPath, browse it from the web dialog, upload a zip that extracts into staging, and import a single markdown file (agent infers chapter vs character).
This commit is contained in:
James Wampler
@@ -103,5 +103,6 @@ Several real bugs here — SQLite refusing ORDER BY DateTimeOffset, agent's mode
|
||||
- Anthropic model id lives in `appsettings.json` under `Agent:Model`. Don't hardcode.
|
||||
- API key comes from `ANTHROPIC_API_KEY` or `Agent:ApiKey` — never commit one. App must stay fully usable without key; only agent endpoints require it.
|
||||
- EF migrations: `dotnet ef migrations add <Name> -p src/Novelly.Api -o Data/Migrations`. API migrates on boot.
|
||||
- Outline import root lives in `appsettings.json` under `Imports:RootPath` (`Imports__RootPath` env var). When set, it's the only folder the browse/upload import endpoints and the source picker can reach; unset, those endpoints are disabled and the dialog falls back to a typed path with no sandbox. Created at boot if missing.
|
||||
- `git push` runs `scripts/ci/prepush.sh` through Husky: build, test, then web build. Run `npm install`
|
||||
once at repo root to install hook.
|
||||
|
||||
@@ -15,6 +15,7 @@ services:
|
||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||
Agent__Model: claude-sonnet-5
|
||||
Agent__Effort: high
|
||||
Imports__RootPath: /data/imports
|
||||
volumes:
|
||||
- /mnt/storage/apps/novelly/data:/data
|
||||
networks:
|
||||
|
||||
@@ -94,8 +94,11 @@ public static class NovellyServiceRegistration
|
||||
services.Configure<AgentOptions>(configuration.GetSection(AgentOptions.SectionName));
|
||||
services.AddScoped<IAgentModelClient, AnthropicAgentModelClient>();
|
||||
|
||||
services.Configure<ImportOptions>(configuration.GetSection(ImportOptions.SectionName));
|
||||
services.AddSingleton(Channel.CreateUnbounded<Guid>());
|
||||
services.AddScoped<ImportService>();
|
||||
services.AddScoped<ImportBrowseService>();
|
||||
services.AddScoped<ImportZipExtractor>();
|
||||
services.AddScoped<ImportAgentToolset>();
|
||||
services.AddScoped<ImportAgentService>();
|
||||
services.AddHostedService<ImportJobRunner>();
|
||||
|
||||
@@ -23,7 +23,9 @@ public class ImportAgentService(
|
||||
toolset.Initialize(sourceRoot, existingNovelId);
|
||||
|
||||
var startingLedger = toolset.ReadLedgerOrNull();
|
||||
var systemPrompt = BuildSystemPrompt(sourceRoot);
|
||||
var systemPrompt = ImportPaths.IsSingleFileSource(sourceRoot)
|
||||
? BuildSingleFileSystemPrompt(sourceRoot)
|
||||
: BuildSystemPrompt(sourceRoot);
|
||||
|
||||
var transcript = new List<AgentChatMessage>
|
||||
{
|
||||
@@ -105,6 +107,9 @@ public class ImportAgentService(
|
||||
|
||||
private static string BuildSystemPrompt(string sourceRoot) => SystemPromptTemplate.Replace("{{SOURCE_ROOT}}", sourceRoot);
|
||||
|
||||
private static string BuildSingleFileSystemPrompt(string sourceRoot) =>
|
||||
SingleFileSystemPromptTemplate.Replace("{{SOURCE_ROOT}}", sourceRoot);
|
||||
|
||||
private const string SystemPromptTemplate = """
|
||||
You import a novel outline that already exists as markdown files on disk into this
|
||||
app's novel data. You are running unattended — nobody will read your replies or
|
||||
@@ -199,4 +204,63 @@ public class ImportAgentService(
|
||||
- If a tool call fails, stop that item and move on rather than retrying blindly —
|
||||
the ledger stays at the last successful write either way.
|
||||
""";
|
||||
|
||||
private const string SingleFileSystemPromptTemplate = """
|
||||
You import a single outline file that already exists as markdown on disk into this
|
||||
app's novel data. You are running unattended — nobody will read your replies or
|
||||
answer questions mid-run, so make the judgment calls yourself and record anything
|
||||
genuinely ambiguous rather than stalling on it.
|
||||
|
||||
Your tools give you exactly two things: read-only access to the file under the
|
||||
import source folder, and application tools that create the novel's chapters,
|
||||
characters, beats and arcs. You cannot write or edit anything on disk except the
|
||||
resume ledger, and you cannot read anything outside the source folder.
|
||||
|
||||
## Source file
|
||||
|
||||
The source root `{{SOURCE_ROOT}}` holds exactly one markdown file. Call
|
||||
list_source_files to find its name, then read_source_file to read it. Decide what
|
||||
kind of document it is before doing anything else:
|
||||
|
||||
- If it reads like a chapter outline (`# Chapter NN`, one or more summary
|
||||
paragraphs, a beat table `| Beat | Character | What | Why |`) treat it as a single
|
||||
chapter.
|
||||
- If it reads like a character dossier (`# Name`, an italic tagline,
|
||||
`## Appearance`, `## Background`, `## Motivation`) treat it as a single character.
|
||||
|
||||
`**Thread:**` (chapter files only) may name one character, several, or a character
|
||||
plus a qualifier — only auto-create an undossiered name from it when it names
|
||||
exactly one clear proper name.
|
||||
|
||||
## The ledger
|
||||
|
||||
Before writing anything, call read_ledger. If it returns `{{}}`, this is a fresh
|
||||
run. Call write_ledger with the full, updated ledger after every successful write.
|
||||
|
||||
## Passes
|
||||
|
||||
1. **Novel** — skip if "novel" is in completedPasses or a novel id was already
|
||||
supplied. Otherwise create one from whatever title/author information the file
|
||||
gives, or a sensible placeholder title drawn from the file name if none is
|
||||
present. Record novelId, mark "novel" done.
|
||||
2. **The document** — skip if already recorded. If it is a chapter: auto-create a
|
||||
character stub (name only) for any single, unqualified name in the Thread or a
|
||||
beat's Character column that isn't in the ledger yet, then create_chapter with
|
||||
title, number (1 unless the file states otherwise), summary, and tags, then
|
||||
create_beat for each table row with resolved character_ids. If it is a
|
||||
character: create_character with occupation from the tagline and
|
||||
appearance/backstory/want from Appearance/Background/Motivation; if it has a
|
||||
`## Events` section, also update_character(importance: "Main") and add_arc_stage
|
||||
for each bullet. Mark "characters", "chapters", and "arcs" all done once you've
|
||||
handled the one document — this run only ever has one item to place.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Never invent plot content or character detail, and never guess which of several
|
||||
candidate names an ambiguous reference means.
|
||||
- Never write to disk except via write_ledger.
|
||||
- Never call a create tool for something the ledger already records.
|
||||
- If a tool call fails, stop and record what you have — the ledger stays at the
|
||||
last successful write either way.
|
||||
""";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Novelly.Api.Imports;
|
||||
|
||||
public class ImportBrowseService(IOptions<ImportOptions> options, ILogger<ImportBrowseService> logger)
|
||||
{
|
||||
private readonly ImportOptions _options = options.Value;
|
||||
|
||||
public string? RootPath => _options.RootPath;
|
||||
|
||||
public ImportBrowseResponse List(string? relativePath)
|
||||
{
|
||||
var root = RequireRoot();
|
||||
|
||||
logger.LogInformation("Browsing import root at {RelativePath}", relativePath ?? "");
|
||||
|
||||
var target = string.IsNullOrWhiteSpace(relativePath) ? root : ImportPaths.ResolveWithin(root, relativePath);
|
||||
|
||||
if (!Directory.Exists(target))
|
||||
throw new ArgumentException($"'{relativePath}' does not exist or is not a directory.", nameof(relativePath));
|
||||
|
||||
var normalizedRelative = Path.GetRelativePath(root, target).Replace(Path.DirectorySeparatorChar, '/');
|
||||
if (normalizedRelative == ".")
|
||||
{
|
||||
normalizedRelative = "";
|
||||
}
|
||||
|
||||
var parent = normalizedRelative == "" ? null : Path.GetRelativePath(root, Path.GetFullPath(Path.Combine(target, ".."))).Replace(Path.DirectorySeparatorChar, '/');
|
||||
if (parent == ".")
|
||||
{
|
||||
parent = "";
|
||||
}
|
||||
|
||||
var entries = Directory.EnumerateFileSystemEntries(target)
|
||||
.Select(BuildEntry)
|
||||
.Where(e => e is not null)
|
||||
.Select(e => e!)
|
||||
.OrderByDescending(e => e.IsDirectory)
|
||||
.ThenBy(e => e.Name, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return new ImportBrowseResponse(normalizedRelative, parent, entries);
|
||||
|
||||
ImportBrowseEntry? BuildEntry(string path)
|
||||
{
|
||||
var name = Path.GetFileName(path);
|
||||
if (name.StartsWith('.'))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var isDirectory = Directory.Exists(path);
|
||||
if (!isDirectory && !name.EndsWith(".md", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var entryRelative = Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/');
|
||||
var markdownCount = isDirectory ? ImportPaths.CountChapterFiles(path) : 0;
|
||||
var looksImportable = isDirectory
|
||||
? File.Exists(Path.Combine(path, "outline.md")) || markdownCount > 0
|
||||
: true;
|
||||
|
||||
return new ImportBrowseEntry(name, entryRelative, path, isDirectory, markdownCount, looksImportable);
|
||||
}
|
||||
}
|
||||
|
||||
private string RequireRoot()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_options.RootPath))
|
||||
throw new InvalidOperationException("No import root is configured (Imports:RootPath).");
|
||||
|
||||
var full = Path.GetFullPath(_options.RootPath);
|
||||
Directory.CreateDirectory(full);
|
||||
return full;
|
||||
}
|
||||
}
|
||||
|
||||
public record ImportBrowseEntry(string Name, string RelativePath, string SourceRoot, bool IsDirectory, int MarkdownFileCount, bool LooksImportable);
|
||||
|
||||
public record ImportBrowseResponse(string RelativePath, string? ParentRelativePath, IReadOnlyList<ImportBrowseEntry> Entries);
|
||||
@@ -57,6 +57,8 @@ public class StartImportRequestValidator : IModelValidator<StartImportRequest>
|
||||
}
|
||||
}
|
||||
|
||||
public record ImportUploadResponse(string SourceRoot, string RelativePath, int MarkdownFileCount);
|
||||
|
||||
public static class ImportMapping
|
||||
{
|
||||
public static ImportJobResponse ToResponse(this ImportJob job) => new(
|
||||
|
||||
@@ -28,6 +28,24 @@ public static class ImportEndpoints
|
||||
(await service.GetStatusAsync(id, ct))?.ToResponse().ToApiResult())
|
||||
.WithSummary("Poll an import job's progress.");
|
||||
|
||||
imports.MapGet("/browse", (string? path, ImportBrowseService browse) =>
|
||||
Results.Ok(browse.List(path)))
|
||||
.WithSummary("List entries under the configured import root, for the source picker.");
|
||||
|
||||
imports.MapPost("/upload", (IFormFile file, ImportService service) =>
|
||||
{
|
||||
if (!file.FileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
|
||||
throw new ArgumentException("Only .zip files can be uploaded.", nameof(file));
|
||||
|
||||
if (file.Length == 0)
|
||||
throw new ArgumentException("The uploaded file is empty.", nameof(file));
|
||||
|
||||
using var stream = file.OpenReadStream();
|
||||
return Results.Ok(service.UploadZip(stream, file.FileName));
|
||||
})
|
||||
.DisableAntiforgery()
|
||||
.WithSummary("Upload a zip of an outline folder and stage it under the configured import root.");
|
||||
|
||||
return app;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace Novelly.Api.Imports;
|
||||
|
||||
public class ImportOptions
|
||||
{
|
||||
public const string SectionName = "Imports";
|
||||
|
||||
public string? RootPath { get; set; }
|
||||
}
|
||||
@@ -13,6 +13,7 @@ public record ImportLedger(
|
||||
internal static class ImportPaths
|
||||
{
|
||||
private const string LedgerFileName = ".novelly-import.json";
|
||||
public const string StagingFolderName = ".novelly-staging";
|
||||
|
||||
private static readonly JsonSerializerOptions LedgerOptions = new()
|
||||
{
|
||||
@@ -20,7 +21,7 @@ internal static class ImportPaths
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
public static string ResolveRoot(string sourceRoot)
|
||||
public static string ResolveRoot(string sourceRoot, string? importRoot = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sourceRoot))
|
||||
throw new ArgumentException("'Source Root' must not be empty.", nameof(sourceRoot));
|
||||
@@ -38,25 +39,57 @@ internal static class ImportPaths
|
||||
if (!Directory.Exists(full))
|
||||
throw new ArgumentException($"'{full}' does not exist or is not a directory.", nameof(sourceRoot));
|
||||
|
||||
EnsureWithinImportRoot(importRoot, full, sourceRoot);
|
||||
|
||||
return full;
|
||||
}
|
||||
|
||||
public static void EnsureWithinImportRoot(string? importRoot, string candidate, string originalInput)
|
||||
{
|
||||
if (importRoot is not null && !IsWithin(importRoot, candidate))
|
||||
throw new ArgumentException($"'{originalInput}' is outside the configured import root.", nameof(originalInput));
|
||||
}
|
||||
|
||||
public static bool IsSingleFileSource(string root)
|
||||
{
|
||||
if (Directory.EnumerateDirectories(root).Any())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return Directory.EnumerateFiles(root, "*.md", SearchOption.TopDirectoryOnly).Count() == 1;
|
||||
}
|
||||
|
||||
public static string ResolveWithin(string root, string relativePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
throw new ArgumentException("Path must not be empty.");
|
||||
|
||||
var combined = Path.GetFullPath(Path.Combine(root, relativePath));
|
||||
var relativeToRoot = Path.GetRelativePath(root, combined);
|
||||
|
||||
if (relativeToRoot.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relativeToRoot))
|
||||
if (!IsWithin(root, combined))
|
||||
throw new ArgumentException($"'{relativePath}' escapes the import source folder.");
|
||||
|
||||
return combined;
|
||||
}
|
||||
|
||||
private static bool IsWithin(string root, string candidate)
|
||||
{
|
||||
var relativeToRoot = Path.GetRelativePath(root, candidate);
|
||||
return relativeToRoot == "." || !relativeToRoot.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relativeToRoot);
|
||||
}
|
||||
|
||||
public static string LedgerPath(string root) => Path.Combine(root, LedgerFileName);
|
||||
|
||||
public static string StagingRoot(string importRoot) => Path.Combine(importRoot, StagingFolderName);
|
||||
|
||||
public static string SanitizeForFolderName(string value)
|
||||
{
|
||||
var sanitized = new string(value.Select(c => char.IsLetterOrDigit(c) || c is '-' or '_' ? c : '-').ToArray());
|
||||
sanitized = sanitized.Trim('-', '_');
|
||||
return string.IsNullOrEmpty(sanitized) ? "import" : sanitized[..Math.Min(sanitized.Length, 60)];
|
||||
}
|
||||
|
||||
public static ImportLedger? ReadLedger(string root)
|
||||
{
|
||||
var path = LedgerPath(root);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Novelly.Api.Common;
|
||||
using Novelly.Api.Common.Validation;
|
||||
using Novelly.Api.Data;
|
||||
@@ -13,10 +14,34 @@ public class ImportService(
|
||||
NovelService novels,
|
||||
Channel<Guid> queue,
|
||||
INovelUserContext userContext,
|
||||
IOptions<ImportOptions> importOptions,
|
||||
ImportZipExtractor zipExtractor,
|
||||
ILogger<ImportService> logger,
|
||||
IModelValidator<InspectImportRequest> inspectValidator,
|
||||
IModelValidator<StartImportRequest> startValidator)
|
||||
{
|
||||
private readonly string? _importRoot = importOptions.Value.RootPath is { } root ? Path.GetFullPath(root) : null;
|
||||
|
||||
public ImportUploadResponse UploadZip(Stream zipStream, string fileName)
|
||||
{
|
||||
if (_importRoot is null)
|
||||
throw new InvalidOperationException("No import root is configured (Imports:RootPath).");
|
||||
|
||||
logger.LogInformation("Uploading import zip {FileName}", fileName);
|
||||
|
||||
Directory.CreateDirectory(_importRoot);
|
||||
var stagingDir = Path.Combine(
|
||||
ImportPaths.StagingRoot(_importRoot),
|
||||
$"zip-{ImportPaths.SanitizeForFolderName(Path.GetFileNameWithoutExtension(fileName))}-{Guid.NewGuid():N}");
|
||||
|
||||
zipExtractor.Extract(zipStream, stagingDir);
|
||||
|
||||
var markdownCount = Directory.EnumerateFiles(stagingDir, "*.md", SearchOption.AllDirectories).Count();
|
||||
var relativePath = Path.GetRelativePath(_importRoot, stagingDir).Replace(Path.DirectorySeparatorChar, '/');
|
||||
|
||||
return new ImportUploadResponse(stagingDir, relativePath, markdownCount);
|
||||
}
|
||||
|
||||
public Task<ImportInspectionResponse> InspectAsync(InspectImportRequest request, CancellationToken ct = default)
|
||||
{
|
||||
Guard.Null(request, nameof(request));
|
||||
@@ -24,7 +49,7 @@ public class ImportService(
|
||||
|
||||
logger.LogInformation("Inspecting import source {SourceRoot}", request.SourceRoot);
|
||||
|
||||
var root = ImportPaths.ResolveRoot(request.SourceRoot);
|
||||
var root = ResolveSourceRoot(request.SourceRoot);
|
||||
var ledger = ImportPaths.ReadLedger(root);
|
||||
var total = ImportPaths.CountChapterFiles(root);
|
||||
|
||||
@@ -48,7 +73,7 @@ public class ImportService(
|
||||
logger.LogInformation(
|
||||
"Starting import for {SourceRoot}, forceRestart {ForceRestart}", request.SourceRoot, request.ForceRestart);
|
||||
|
||||
var root = ImportPaths.ResolveRoot(request.SourceRoot);
|
||||
var root = ResolveSourceRoot(request.SourceRoot);
|
||||
|
||||
if (request.ForceRestart)
|
||||
{
|
||||
@@ -88,6 +113,40 @@ public class ImportService(
|
||||
return job;
|
||||
}
|
||||
|
||||
private string ResolveSourceRoot(string sourceRoot)
|
||||
{
|
||||
string full;
|
||||
try
|
||||
{
|
||||
full = Path.GetFullPath(sourceRoot);
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
throw new ArgumentException($"'{sourceRoot}' is not a valid path.", nameof(sourceRoot), ex);
|
||||
}
|
||||
|
||||
if (!File.Exists(full))
|
||||
{
|
||||
return ImportPaths.ResolveRoot(sourceRoot, _importRoot);
|
||||
}
|
||||
|
||||
if (!full.EndsWith(".md", StringComparison.OrdinalIgnoreCase))
|
||||
throw new ArgumentException($"'{sourceRoot}' is not a markdown file or a directory.", nameof(sourceRoot));
|
||||
|
||||
ImportPaths.EnsureWithinImportRoot(_importRoot, full, sourceRoot);
|
||||
|
||||
var stagingParent = _importRoot is not null
|
||||
? ImportPaths.StagingRoot(_importRoot)
|
||||
: Path.Combine(Path.GetTempPath(), "novelly-import-staging");
|
||||
var stagingDir = Path.Combine(
|
||||
stagingParent, $"file-{ImportPaths.SanitizeForFolderName(Path.GetFileNameWithoutExtension(full))}");
|
||||
|
||||
Directory.CreateDirectory(stagingDir);
|
||||
File.Copy(full, Path.Combine(stagingDir, Path.GetFileName(full)), overwrite: true);
|
||||
|
||||
return stagingDir;
|
||||
}
|
||||
|
||||
public async Task<ImportJob?> GetStatusAsync(Guid id, CancellationToken ct = default)
|
||||
{
|
||||
Guard.Default(id, nameof(id));
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace Novelly.Api.Imports;
|
||||
|
||||
public class ImportZipExtractor(ILogger<ImportZipExtractor> logger)
|
||||
{
|
||||
private const int MaxEntryCount = 2000;
|
||||
private const long MaxEntryUncompressedBytes = 10 * 1024 * 1024;
|
||||
private const long MaxTotalUncompressedBytes = 100 * 1024 * 1024;
|
||||
|
||||
private static readonly string[] AllowedFileNames = [".novelly-import.json"];
|
||||
|
||||
public void Extract(Stream zipStream, string stagingDir)
|
||||
{
|
||||
Directory.CreateDirectory(stagingDir);
|
||||
|
||||
try
|
||||
{
|
||||
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read);
|
||||
|
||||
var entries = archive.Entries
|
||||
.Where(e => !string.IsNullOrEmpty(e.Name))
|
||||
.Where(e => !e.FullName.StartsWith("__MACOSX/", StringComparison.OrdinalIgnoreCase))
|
||||
.Where(e => AllowedFileNames.Contains(e.Name) || !e.Name.StartsWith('.'))
|
||||
.ToArray();
|
||||
|
||||
if (entries.Length == 0)
|
||||
throw new ArgumentException("The zip file is empty.");
|
||||
|
||||
if (entries.Length > MaxEntryCount)
|
||||
throw new ArgumentException($"The zip file has too many entries (max {MaxEntryCount}).");
|
||||
|
||||
var stripPrefix = FindCommonTopLevelDirectory(entries);
|
||||
var totalBytes = 0L;
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var relativePath = stripPrefix is null
|
||||
? entry.FullName
|
||||
: entry.FullName[(stripPrefix.Length + 1)..];
|
||||
|
||||
if (relativePath.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!relativePath.EndsWith(".md", StringComparison.OrdinalIgnoreCase) && !AllowedFileNames.Contains(entry.Name))
|
||||
throw new ArgumentException($"'{entry.FullName}' is not a markdown file. Only .md files (and .novelly-import.json) are allowed.");
|
||||
|
||||
if (entry.Length > MaxEntryUncompressedBytes)
|
||||
throw new ArgumentException($"'{entry.FullName}' is too large (max {MaxEntryUncompressedBytes / (1024 * 1024)} MB per file).");
|
||||
|
||||
totalBytes += entry.Length;
|
||||
if (totalBytes > MaxTotalUncompressedBytes)
|
||||
throw new ArgumentException($"The zip file is too large uncompressed (max {MaxTotalUncompressedBytes / (1024 * 1024)} MB).");
|
||||
|
||||
var destination = ImportPaths.ResolveWithin(stagingDir, relativePath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destination)!);
|
||||
|
||||
using var entryStream = entry.Open();
|
||||
using var fileStream = File.Create(destination);
|
||||
entryStream.CopyTo(fileStream);
|
||||
}
|
||||
|
||||
logger.LogInformation("Extracted import zip with {EntryCount} entries into staging folder", entries.Length);
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (Directory.Exists(stagingDir))
|
||||
{
|
||||
Directory.Delete(stagingDir, recursive: true);
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? FindCommonTopLevelDirectory(IReadOnlyCollection<ZipArchiveEntry> entries)
|
||||
{
|
||||
var topLevelSegments = entries
|
||||
.Select(e => e.FullName.Split('/', '\\')[0])
|
||||
.Distinct()
|
||||
.ToArray();
|
||||
|
||||
return topLevelSegments.Length == 1 && entries.All(e => e.FullName.Contains('/') || e.FullName.Contains('\\'))
|
||||
? topLevelSegments[0]
|
||||
: null;
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,12 @@ using (var scope = app.Services.CreateScope())
|
||||
|
||||
await ServiceUser.EnsureSeededAsync(db, builder.Configuration[ServiceApiKeyAuthenticationHandler.ConfigurationKey], app.Logger);
|
||||
await ActivityBackfill.RunAsync(db, app.Logger);
|
||||
|
||||
var importRoot = builder.Configuration.GetSection(ImportOptions.SectionName)[nameof(ImportOptions.RootPath)];
|
||||
if (!string.IsNullOrWhiteSpace(importRoot))
|
||||
{
|
||||
Directory.CreateDirectory(importRoot);
|
||||
}
|
||||
}
|
||||
|
||||
app.UseSerilogRequestLogging();
|
||||
|
||||
@@ -35,5 +35,8 @@
|
||||
},
|
||||
"UiSettings": {
|
||||
"ShowPronouns": false
|
||||
},
|
||||
"Imports": {
|
||||
"RootPath": null
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ server {
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
client_max_body_size 50m;
|
||||
proxy_pass http://api:8080/api/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
|
||||
@@ -11,11 +11,12 @@ export class ApiError extends Error {
|
||||
}
|
||||
|
||||
async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
const isFormData = init?.body instanceof FormData
|
||||
const response = await fetch(`${BASE}${path}`, {
|
||||
...init,
|
||||
credentials: 'include',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...(isFormData ? {} : { 'Content-Type': 'application/json' }),
|
||||
...init?.headers,
|
||||
},
|
||||
})
|
||||
@@ -37,6 +38,7 @@ export const api = {
|
||||
get: <T>(path: string) => request<T>(path),
|
||||
post: <T>(path: string, body?: unknown) =>
|
||||
request<T>(path, { method: 'POST', body: JSON.stringify(body ?? {}) }),
|
||||
postForm: <T>(path: string, body: FormData) => request<T>(path, { method: 'POST', body }),
|
||||
patch: <T>(path: string, body: unknown) =>
|
||||
request<T>(path, { method: 'PATCH', body: JSON.stringify(body) }),
|
||||
put: <T>(path: string, body: unknown) =>
|
||||
|
||||
@@ -12,9 +12,11 @@ import type {
|
||||
ConversationSummary,
|
||||
Beat,
|
||||
Genre,
|
||||
ImportBrowse,
|
||||
ImportInspection,
|
||||
ImportJob,
|
||||
ImportJobStatus,
|
||||
ImportUpload,
|
||||
OpenQuestion,
|
||||
LocationReferences,
|
||||
LocationSummary,
|
||||
@@ -47,6 +49,7 @@ export const keys = {
|
||||
conversations: (novelId: string) => ['novels', novelId, 'conversations'] as const,
|
||||
conversation: (id: string) => ['conversations', id] as const,
|
||||
importJob: (id: string) => ['imports', id] as const,
|
||||
importBrowse: (path: string) => ['imports', 'browse', path] as const,
|
||||
novelActivity: (novelId: string) => ['novels', novelId, 'activity'] as const,
|
||||
myActivity: ['activity'] as const,
|
||||
}
|
||||
@@ -635,6 +638,25 @@ export function useStartImport() {
|
||||
})
|
||||
}
|
||||
|
||||
export function useImportBrowse(path: string, enabled = true) {
|
||||
return useQuery({
|
||||
queryKey: keys.importBrowse(path),
|
||||
queryFn: () => api.get<ImportBrowse>(`/api/imports/browse?path=${encodeURIComponent(path)}`),
|
||||
enabled,
|
||||
retry: false,
|
||||
})
|
||||
}
|
||||
|
||||
export function useUploadImportZip() {
|
||||
return useMutation({
|
||||
mutationFn: (file: File) => {
|
||||
const form = new FormData()
|
||||
form.append('file', file)
|
||||
return api.postForm<ImportUpload>('/api/imports/upload', form)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const terminalImportStatuses: ImportJobStatus[] = ['Completed', 'Failed', 'Paused']
|
||||
|
||||
export function useImportJob(jobId: string | undefined) {
|
||||
|
||||
@@ -329,6 +329,27 @@ export interface ImportInspection {
|
||||
completedPasses: string[]
|
||||
}
|
||||
|
||||
export interface ImportBrowseEntry {
|
||||
name: string
|
||||
relativePath: string
|
||||
sourceRoot: string
|
||||
isDirectory: boolean
|
||||
markdownFileCount: number
|
||||
looksImportable: boolean
|
||||
}
|
||||
|
||||
export interface ImportBrowse {
|
||||
relativePath: string
|
||||
parentRelativePath: string | null
|
||||
entries: ImportBrowseEntry[]
|
||||
}
|
||||
|
||||
export interface ImportUpload {
|
||||
sourceRoot: string
|
||||
relativePath: string
|
||||
markdownFileCount: number
|
||||
}
|
||||
|
||||
export interface ActivityDay {
|
||||
date: string
|
||||
words: number
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { useEffect, useState, type FormEvent } from 'react'
|
||||
import { useEffect, useRef, useState, type FormEvent } from 'react'
|
||||
import { useQueryClient } from '@tanstack/react-query'
|
||||
import { useImportJob, useInspectImport, useStartImport } from '../api/hooks'
|
||||
import { useImportJob, useInspectImport, useStartImport, useUploadImportZip } from '../api/hooks'
|
||||
import type { ImportInspection, ImportJob } from '../api/types'
|
||||
import { ErrorNote, Modal, Spinner } from './ui'
|
||||
import { ImportSourcePicker } from './ImportSourcePicker'
|
||||
|
||||
type SourceMode = 'browse' | 'upload' | 'path'
|
||||
|
||||
export function ImportDialog({
|
||||
onClose,
|
||||
@@ -11,13 +14,16 @@ export function ImportDialog({
|
||||
onClose: () => void
|
||||
onImported?: (novelId: string) => void
|
||||
}) {
|
||||
const [sourceMode, setSourceMode] = useState<SourceMode>('browse')
|
||||
const [sourceRoot, setSourceRoot] = useState('')
|
||||
const [inspection, setInspection] = useState<ImportInspection | null>(null)
|
||||
const [jobId, setJobId] = useState<string>()
|
||||
const [confirmingRestart, setConfirmingRestart] = useState(false)
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
|
||||
const inspect = useInspectImport()
|
||||
const start = useStartImport()
|
||||
const upload = useUploadImportZip()
|
||||
const job = useImportJob(jobId)
|
||||
const qc = useQueryClient()
|
||||
|
||||
@@ -33,6 +39,15 @@ export function ImportDialog({
|
||||
inspect.mutate(sourceRoot.trim(), { onSuccess: setInspection })
|
||||
}
|
||||
|
||||
const chooseSource = (root: string) => {
|
||||
setSourceRoot(root)
|
||||
inspect.mutate(root, { onSuccess: setInspection })
|
||||
}
|
||||
|
||||
const uploadZip = (file: File) => {
|
||||
upload.mutate(file, { onSuccess: (result) => chooseSource(result.sourceRoot) })
|
||||
}
|
||||
|
||||
const beginImport = (forceRestart = false) => {
|
||||
start.mutate({ sourceRoot: sourceRoot.trim(), forceRestart }, { onSuccess: (created) => setJobId(created.id) })
|
||||
}
|
||||
@@ -55,11 +70,59 @@ export function ImportDialog({
|
||||
)
|
||||
}
|
||||
|
||||
const busy = inspect.isPending || start.isPending || upload.isPending
|
||||
|
||||
return (
|
||||
<Modal title="Import outline" onClose={onClose}>
|
||||
<form onSubmit={check} className="grid gap-3">
|
||||
{!inspection && (
|
||||
<div id="import-source-mode" className="flex gap-1">
|
||||
{(['browse', 'upload', 'path'] as const).map((mode) => (
|
||||
<button
|
||||
key={mode}
|
||||
type="button"
|
||||
className="btn"
|
||||
style={sourceMode === mode ? { color: 'var(--accent)' } : undefined}
|
||||
disabled={busy}
|
||||
onClick={() => setSourceMode(mode)}
|
||||
>
|
||||
{mode === 'browse' ? 'Browse' : mode === 'upload' ? 'Upload zip' : 'Enter path'}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!inspection && sourceMode === 'browse' && (
|
||||
<ImportSourcePicker onSelect={chooseSource} disabled={busy} />
|
||||
)}
|
||||
|
||||
{!inspection && sourceMode === 'upload' && (
|
||||
<div className="grid gap-2">
|
||||
<input
|
||||
id="import-zip-input"
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".zip"
|
||||
className="input"
|
||||
disabled={busy}
|
||||
onChange={(e) => {
|
||||
const file = e.target.files?.[0]
|
||||
if (file) uploadZip(file)
|
||||
}}
|
||||
/>
|
||||
<p className="text-sm muted">
|
||||
A zip of the folder holding outline.md, its chapter files and character
|
||||
dossiers.
|
||||
</p>
|
||||
{upload.isPending && <Spinner label="Uploading" />}
|
||||
{upload.error && <ErrorNote error={upload.error} />}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!inspection && sourceMode === 'path' && (
|
||||
<>
|
||||
<label className="block">
|
||||
<span className="label">Source folder</span>
|
||||
<span className="label">Source folder or file</span>
|
||||
<input
|
||||
className="input"
|
||||
autoFocus
|
||||
@@ -69,13 +132,17 @@ export function ImportDialog({
|
||||
setInspection(null)
|
||||
}}
|
||||
placeholder="/home/you/Documents/Novels/my-outline"
|
||||
disabled={inspect.isPending || start.isPending}
|
||||
disabled={busy}
|
||||
/>
|
||||
</label>
|
||||
<p className="text-sm muted">
|
||||
Absolute path to the folder holding outline.md, its chapter files and character
|
||||
dossiers.
|
||||
Absolute path to the folder holding outline.md and its chapter/character
|
||||
files, or to a single markdown file.
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
|
||||
{inspection && <p className="text-sm muted">{sourceRoot}</p>}
|
||||
|
||||
{inspect.error && <ErrorNote error={inspect.error} />}
|
||||
{start.error && <ErrorNote error={start.error} />}
|
||||
@@ -96,7 +163,7 @@ export function ImportDialog({
|
||||
<button type="button" className="btn" onClick={onClose}>
|
||||
Cancel
|
||||
</button>
|
||||
{!inspection && (
|
||||
{!inspection && sourceMode === 'path' && (
|
||||
<button
|
||||
type="submit"
|
||||
className="btn btn-primary"
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import { useState } from 'react'
|
||||
import { useImportBrowse } from '../api/hooks'
|
||||
import { ApiError } from '../api/client'
|
||||
import { Spinner } from './ui'
|
||||
|
||||
export function ImportSourcePicker({
|
||||
onSelect,
|
||||
disabled,
|
||||
}: {
|
||||
onSelect: (sourceRoot: string) => void
|
||||
disabled?: boolean
|
||||
}) {
|
||||
const [path, setPath] = useState('')
|
||||
const browse = useImportBrowse(path)
|
||||
|
||||
if (browse.isError && browse.error instanceof ApiError && browse.error.status === 400) {
|
||||
return (
|
||||
<p id="import-picker-unavailable" className="text-sm muted">
|
||||
No import folder is configured on this server.
|
||||
</p>
|
||||
)
|
||||
}
|
||||
|
||||
if (browse.isLoading) {
|
||||
return <Spinner label="Loading" />
|
||||
}
|
||||
|
||||
if (!browse.data) {
|
||||
return null
|
||||
}
|
||||
|
||||
const segments = browse.data.relativePath ? browse.data.relativePath.split('/') : []
|
||||
|
||||
return (
|
||||
<div id="import-source-picker" className="grid gap-2">
|
||||
<nav className="flex flex-wrap items-center gap-1 text-sm">
|
||||
<button type="button" className="btn" disabled={disabled} onClick={() => setPath('')}>
|
||||
Import folder
|
||||
</button>
|
||||
{segments.map((segment, i) => (
|
||||
<span key={i} className="flex items-center gap-1">
|
||||
<span className="muted">/</span>
|
||||
<button
|
||||
type="button"
|
||||
className="btn"
|
||||
disabled={disabled}
|
||||
onClick={() => setPath(segments.slice(0, i + 1).join('/'))}
|
||||
>
|
||||
{segment}
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</nav>
|
||||
|
||||
<ul
|
||||
id="import-source-picker-entries"
|
||||
className="grid gap-1 rounded-md p-1"
|
||||
style={{ background: 'var(--surface-sunken)', maxHeight: '16rem', overflowY: 'auto' }}
|
||||
>
|
||||
{browse.data.entries.length === 0 && <li className="px-2 py-1 text-sm muted">Empty folder.</li>}
|
||||
{browse.data.entries.map((entry) => (
|
||||
<li key={entry.relativePath} className="flex items-center justify-between gap-2 px-2 py-1">
|
||||
<button
|
||||
type="button"
|
||||
className="flex-1 text-left text-sm"
|
||||
disabled={disabled}
|
||||
onClick={() => (entry.isDirectory ? setPath(entry.relativePath) : onSelect(entry.sourceRoot))}
|
||||
>
|
||||
{entry.isDirectory ? '📁' : '📄'} {entry.name}
|
||||
{entry.isDirectory && entry.looksImportable && (
|
||||
<span className="ml-2 text-xs" style={{ color: 'var(--accent)' }}>
|
||||
outline found
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
{entry.isDirectory && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn"
|
||||
disabled={disabled || !entry.looksImportable}
|
||||
onClick={() => onSelect(entry.sourceRoot)}
|
||||
>
|
||||
Select
|
||||
</button>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using Novelly.Api.Imports;
|
||||
|
||||
namespace Novelly.Api.Tests;
|
||||
|
||||
[TestFixture]
|
||||
public class ImportBrowseServiceTests
|
||||
{
|
||||
private string _root = null!;
|
||||
private ImportBrowseService _browse = null!;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_root = Directory.CreateTempSubdirectory("novelly-browse-test-").FullName;
|
||||
_browse = new ImportBrowseService(
|
||||
Options.Create(new ImportOptions { RootPath = _root }),
|
||||
new CapturingLogger<ImportBrowseService>());
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Listing_the_root_returns_directories_before_files_ordered_by_name()
|
||||
{
|
||||
File.WriteAllText(Path.Combine(_root, "notes.md"), "# Notes");
|
||||
Directory.CreateDirectory(Path.Combine(_root, "zeta"));
|
||||
Directory.CreateDirectory(Path.Combine(_root, "alpha"));
|
||||
|
||||
var listing = _browse.List(null);
|
||||
|
||||
Assert.That(listing.Entries.Select(e => e.Name), Is.EqualTo(new[] { "alpha", "zeta", "notes.md" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void A_folder_with_an_outline_file_is_flagged_importable()
|
||||
{
|
||||
var folder = Path.Combine(_root, "my-novel");
|
||||
Directory.CreateDirectory(folder);
|
||||
File.WriteAllText(Path.Combine(folder, "outline.md"), "# My Novel");
|
||||
|
||||
var listing = _browse.List(null);
|
||||
|
||||
Assert.That(listing.Entries.Single(e => e.Name == "my-novel").LooksImportable, Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void An_empty_folder_is_not_flagged_importable()
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(_root, "empty"));
|
||||
|
||||
var listing = _browse.List(null);
|
||||
|
||||
Assert.That(listing.Entries.Single(e => e.Name == "empty").LooksImportable, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void The_staging_folder_is_hidden_from_listings()
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(_root, ".novelly-staging"));
|
||||
Directory.CreateDirectory(Path.Combine(_root, "visible"));
|
||||
|
||||
var listing = _browse.List(null);
|
||||
|
||||
Assert.That(listing.Entries.Select(e => e.Name), Is.EqualTo(new[] { "visible" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Escaping_the_import_root_is_rejected()
|
||||
{
|
||||
Assert.That(() => _browse.List(".."), Throws.TypeOf<ArgumentException>());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Browsing_without_a_configured_root_fails()
|
||||
{
|
||||
var unconfigured = new ImportBrowseService(
|
||||
Options.Create(new ImportOptions()),
|
||||
new CapturingLogger<ImportBrowseService>());
|
||||
|
||||
Assert.That(() => unconfigured.List(null), Throws.TypeOf<InvalidOperationException>());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Listing_a_subfolder_reports_its_parent()
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(_root, "child"));
|
||||
|
||||
var listing = _browse.List("child");
|
||||
|
||||
Assert.That(listing.ParentRelativePath, Is.EqualTo(""));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Novelly.Api.Imports;
|
||||
using Novelly.Api.Novels;
|
||||
|
||||
@@ -20,6 +21,8 @@ public class ImportServiceTests : ServiceTestFixture
|
||||
Novels,
|
||||
_queue,
|
||||
UserContext,
|
||||
Options.Create(new ImportOptions()),
|
||||
new ImportZipExtractor(new CapturingLogger<ImportZipExtractor>()),
|
||||
new CapturingLogger<ImportService>(),
|
||||
new InspectImportRequestValidator(),
|
||||
new StartImportRequestValidator());
|
||||
@@ -32,6 +35,12 @@ public class ImportServiceTests : ServiceTestFixture
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
|
||||
var staging = Path.Combine(Path.GetTempPath(), "novelly-import-staging");
|
||||
if (Directory.Exists(staging))
|
||||
{
|
||||
Directory.Delete(staging, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
@@ -131,6 +140,71 @@ public class ImportServiceTests : ServiceTestFixture
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Starting_an_import_outside_a_configured_import_root_is_rejected()
|
||||
{
|
||||
var configuredRoot = Directory.CreateTempSubdirectory("novelly-import-root-").FullName;
|
||||
try
|
||||
{
|
||||
var restricted = new ImportService(
|
||||
Db.Context,
|
||||
Novels,
|
||||
_queue,
|
||||
UserContext,
|
||||
Options.Create(new ImportOptions { RootPath = configuredRoot }),
|
||||
new ImportZipExtractor(new CapturingLogger<ImportZipExtractor>()),
|
||||
new CapturingLogger<ImportService>(),
|
||||
new InspectImportRequestValidator(),
|
||||
new StartImportRequestValidator());
|
||||
|
||||
Assert.That(
|
||||
async () => await restricted.StartOrResumeAsync(new StartImportRequest(_root)),
|
||||
Throws.TypeOf<ArgumentException>());
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(configuredRoot, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Starting_an_import_of_a_single_markdown_file_stages_it_into_a_folder()
|
||||
{
|
||||
var file = Path.Combine(_root, "01-chapter.md");
|
||||
File.WriteAllText(file, "# Chapter 1");
|
||||
|
||||
var job = await _imports.StartOrResumeAsync(new StartImportRequest(file));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(Directory.Exists(job.SourceRoot), Is.True);
|
||||
Assert.That(File.Exists(Path.Combine(job.SourceRoot, "01-chapter.md")), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Starting_an_import_of_the_same_single_file_twice_dedupes_to_the_same_staged_job()
|
||||
{
|
||||
var file = Path.Combine(_root, "01-chapter.md");
|
||||
File.WriteAllText(file, "# Chapter 1");
|
||||
|
||||
var first = await _imports.StartOrResumeAsync(new StartImportRequest(file));
|
||||
var second = await _imports.StartOrResumeAsync(new StartImportRequest(file));
|
||||
|
||||
Assert.That(second.Id, Is.EqualTo(first.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task Starting_an_import_of_a_non_markdown_file_is_rejected()
|
||||
{
|
||||
var file = Path.Combine(_root, "notes.txt");
|
||||
File.WriteAllText(file, "not markdown");
|
||||
|
||||
Assert.That(
|
||||
async () => await _imports.StartOrResumeAsync(new StartImportRequest(file)),
|
||||
Throws.TypeOf<ArgumentException>());
|
||||
}
|
||||
|
||||
private void WriteChapterFiles(int count)
|
||||
{
|
||||
var outlines = Path.Combine(_root, "outlines");
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
using System.IO.Compression;
|
||||
using Novelly.Api.Imports;
|
||||
|
||||
namespace Novelly.Api.Tests;
|
||||
|
||||
[TestFixture]
|
||||
public class ImportZipExtractorTests
|
||||
{
|
||||
private string _stagingParent = null!;
|
||||
private ImportZipExtractor _extractor = null!;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_stagingParent = Directory.CreateTempSubdirectory("novelly-zip-test-").FullName;
|
||||
_extractor = new ImportZipExtractor(new CapturingLogger<ImportZipExtractor>());
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
if (Directory.Exists(_stagingParent))
|
||||
{
|
||||
Directory.Delete(_stagingParent, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void A_zip_of_the_expected_folder_layout_extracts_intact()
|
||||
{
|
||||
using var zip = BuildZip(("outline.md", "# My Novel"), ("outlines/01-chapter.md", "# Chapter 1"));
|
||||
var staging = StagingDir();
|
||||
|
||||
_extractor.Extract(zip, staging);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(File.Exists(Path.Combine(staging, "outline.md")), Is.True);
|
||||
Assert.That(File.Exists(Path.Combine(staging, "outlines", "01-chapter.md")), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void A_single_wrapper_directory_is_flattened_away()
|
||||
{
|
||||
using var zip = BuildZip(("my-novel/outline.md", "# My Novel"), ("my-novel/outlines/01-chapter.md", "# Chapter 1"));
|
||||
var staging = StagingDir();
|
||||
|
||||
_extractor.Extract(zip, staging);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(File.Exists(Path.Combine(staging, "outline.md")), Is.True);
|
||||
Assert.That(Directory.Exists(Path.Combine(staging, "my-novel")), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void A_zip_slip_entry_is_rejected_and_staging_is_cleaned_up()
|
||||
{
|
||||
using var zip = BuildZip(("outline.md", "# ok"), ("../evil.md", "pwned"));
|
||||
var staging = StagingDir();
|
||||
|
||||
Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf<ArgumentException>());
|
||||
Assert.That(Directory.Exists(staging), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void A_disallowed_file_extension_is_rejected()
|
||||
{
|
||||
using var zip = BuildZip(("outline.md", "# ok"), ("script.sh", "echo hi"));
|
||||
var staging = StagingDir();
|
||||
|
||||
Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf<ArgumentException>());
|
||||
Assert.That(Directory.Exists(staging), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void The_ledger_file_is_allowed_through()
|
||||
{
|
||||
using var zip = BuildZip(("outline.md", "# ok"), (".novelly-import.json", "{}"));
|
||||
var staging = StagingDir();
|
||||
|
||||
_extractor.Extract(zip, staging);
|
||||
|
||||
Assert.That(File.Exists(Path.Combine(staging, ".novelly-import.json")), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void An_empty_zip_is_rejected()
|
||||
{
|
||||
using var zip = BuildZip();
|
||||
var staging = StagingDir();
|
||||
|
||||
Assert.That(() => _extractor.Extract(zip, staging), Throws.TypeOf<ArgumentException>());
|
||||
}
|
||||
|
||||
private string StagingDir() => Path.Combine(_stagingParent, $"staging-{Guid.NewGuid():N}");
|
||||
|
||||
private static MemoryStream BuildZip(params (string Path, string Content)[] entries)
|
||||
{
|
||||
var stream = new MemoryStream();
|
||||
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true))
|
||||
{
|
||||
foreach (var (path, content) in entries)
|
||||
{
|
||||
var entry = archive.CreateEntry(path);
|
||||
using var writer = new StreamWriter(entry.Open());
|
||||
writer.Write(content);
|
||||
}
|
||||
}
|
||||
|
||||
stream.Position = 0;
|
||||
return stream;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user