From e3f7990d3cf807924f69e9b68ecdf20df34a520f Mon Sep 17 00:00:00 2001 From: Sirus <5435333+Sirush@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:49:02 +0200 Subject: [PATCH] Webnovel download & sync system --- .../Controllers/AdminController.WebNovels.cs | 213 ++ Jiten.Api/Controllers/AdminController.cs | 7 + .../Dtos/Requests/AddWebNovelDeckRequest.cs | 19 + Jiten.Api/Jobs/FetchMetadataJob.cs | 65 +- Jiten.Api/Jobs/ParseJob.cs | 18 +- Jiten.Api/Jobs/WebNovelFetchJob.cs | 334 +++ Jiten.Api/Jobs/WebNovelImportJob.cs | 202 ++ Jiten.Api/Jobs/WebNovelQueues.cs | 18 + Jiten.Api/Jobs/WebNovelSyncSweepJob.cs | 138 + Jiten.Api/Program.cs | 43 + Jiten.Cli/CliOptions.cs | 8 + Jiten.Cli/Commands/WebNovelCommands.cs | 118 + Jiten.Cli/Program.cs | 7 + Jiten.Core/Data/WebNovel/WebNovelChapter.cs | 37 + Jiten.Core/Data/WebNovel/WebNovelProvider.cs | 9 + Jiten.Core/Data/WebNovel/WebNovelSource.cs | 70 + Jiten.Core/Extractors/EbookExtractor.cs | 28 +- Jiten.Core/Extractors/RubyHtmlHelper.cs | 36 + Jiten.Core/JitenDbContext.cs | 38 + .../20260711232624_WebNovelDecks.Designer.cs | 2268 +++++++++++++++++ .../20260711232624_WebNovelDecks.cs | 102 + .../Migrations/JitenDbContextModelSnapshot.cs | 116 + Jiten.Core/WebNovel/IWebNovelSource.cs | 33 + Jiten.Core/WebNovel/SubdeckChunker.cs | 227 ++ Jiten.Core/WebNovel/SyosetuSource.cs | 432 ++++ .../WebNovel/WebNovelMetadataExtensions.cs | 35 + Jiten.Core/WebNovel/WebNovelModels.cs | 68 + Jiten.Core/WebNovel/WebNovelSchedule.cs | 69 + Jiten.Core/WebNovel/WebNovelSourceResolver.cs | 22 + Jiten.Core/WebNovel/WebNovelUrlParser.cs | 61 + Jiten.Tests/RubyHtmlHelperTests.cs | 48 + Jiten.Tests/SubdeckChunkerTests.cs | 215 ++ Jiten.Tests/WebNovelScheduleTests.cs | 78 + .../app/components/dashboard/CoverTools.vue | 23 +- .../dashboard/WebNovelSyncPanel.vue | 174 ++ .../app/pages/dashboard/add-webnovel.vue | 227 ++ .../app/pages/dashboard/genre-mappings.vue | 8 + Jiten.Web/app/pages/dashboard/index.vue | 10 + Jiten.Web/app/pages/dashboard/media/[id].vue | 8 + .../app/pages/dashboard/tag-mappings.vue | 8 + Jiten.Web/app/utils/coverImage.ts | 31 + Jiten.Web/app/utils/mediaTypeMapper.ts | 2 +- 42 files changed, 5634 insertions(+), 39 deletions(-) create mode 100644 Jiten.Api/Controllers/AdminController.WebNovels.cs create mode 100644 Jiten.Api/Dtos/Requests/AddWebNovelDeckRequest.cs create mode 100644 Jiten.Api/Jobs/WebNovelFetchJob.cs create mode 100644 Jiten.Api/Jobs/WebNovelImportJob.cs create mode 100644 Jiten.Api/Jobs/WebNovelQueues.cs create mode 100644 Jiten.Api/Jobs/WebNovelSyncSweepJob.cs create mode 100644 Jiten.Cli/Commands/WebNovelCommands.cs create mode 100644 Jiten.Core/Data/WebNovel/WebNovelChapter.cs create mode 100644 Jiten.Core/Data/WebNovel/WebNovelProvider.cs create mode 100644 Jiten.Core/Data/WebNovel/WebNovelSource.cs create mode 100644 Jiten.Core/Extractors/RubyHtmlHelper.cs create mode 100644 Jiten.Core/Migrations/20260711232624_WebNovelDecks.Designer.cs create mode 100644 Jiten.Core/Migrations/20260711232624_WebNovelDecks.cs create mode 100644 Jiten.Core/WebNovel/IWebNovelSource.cs create mode 100644 Jiten.Core/WebNovel/SubdeckChunker.cs create mode 100644 Jiten.Core/WebNovel/SyosetuSource.cs create mode 100644 Jiten.Core/WebNovel/WebNovelMetadataExtensions.cs create mode 100644 Jiten.Core/WebNovel/WebNovelModels.cs create mode 100644 Jiten.Core/WebNovel/WebNovelSchedule.cs create mode 100644 Jiten.Core/WebNovel/WebNovelSourceResolver.cs create mode 100644 Jiten.Core/WebNovel/WebNovelUrlParser.cs create mode 100644 Jiten.Tests/RubyHtmlHelperTests.cs create mode 100644 Jiten.Tests/SubdeckChunkerTests.cs create mode 100644 Jiten.Tests/WebNovelScheduleTests.cs create mode 100644 Jiten.Web/app/components/dashboard/WebNovelSyncPanel.vue create mode 100644 Jiten.Web/app/pages/dashboard/add-webnovel.vue diff --git a/Jiten.Api/Controllers/AdminController.WebNovels.cs b/Jiten.Api/Controllers/AdminController.WebNovels.cs new file mode 100644 index 00000000..dccbbace --- /dev/null +++ b/Jiten.Api/Controllers/AdminController.WebNovels.cs @@ -0,0 +1,213 @@ +using Hangfire; +using Jiten.Api.Dtos.Requests; +using Jiten.Api.Jobs; +using Jiten.Core.Data; +using Jiten.Core.Data.WebNovel; +using Jiten.Core.WebNovel; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; + +namespace Jiten.Api.Controllers; + +public partial class AdminController +{ + /// + /// Work metadata straight from the provider, so the admin can review it (and generate a cover from the + /// title) before committing to an import. + /// + [HttpGet("webnovel-preview")] + public async Task PreviewWebNovel([FromQuery] string url, + [FromServices] IWebNovelSourceResolver sourceResolver) + { + if (!WebNovelUrlParser.TryParse(url, out var provider, out var sourceId)) + return BadRequest(new { Message = "Not a supported webnovel URL." }); + + if (!sourceResolver.IsSupported(provider)) + return BadRequest(new { Message = $"{provider} is not enabled yet." }); + + var existing = await dbContext.WebNovelSources + .FirstOrDefaultAsync(s => s.Provider == provider && s.SourceId == sourceId); + + if (existing != null) + return Conflict(new { Message = "This novel is already tracked.", DeckId = existing.DeckId }); + + try + { + var info = await sourceResolver.Resolve(provider).GetInfoAsync(sourceId); + + // A same-title webnovel deck would make InsertDeck skip the insert, so the import would fail + var titleConflict = await dbContext.Decks + .AnyAsync(d => d.OriginalTitle == info.Title && + d.MediaType == MediaType.WebNovel); + + return Ok(new + { + Provider = provider.ToString(), + info.SourceId, + info.Url, + info.Title, + TitleConflict = titleConflict, + info.Author, + info.Synopsis, + info.Genre, + info.Keywords, + info.EpisodeCount, + info.TotalCharacters, + info.IsCompleted, + info.IsOnHiatus, + info.IsOneShot, + info.IsR15, + FirstPublishedAt = info.FirstPublishedAt?.UtcDateTime, + LastUpdatedAt = info.LastUpdatedAt?.UtcDateTime, + EstimatedSubdecks = EstimateSubdecks(info.TotalCharacters) + }); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Webnovel preview failed for {Url}", url); + return StatusCode(StatusCodes.Status502BadGateway, + new { Message = "Could not read this novel from the source.", Details = ex.Message }); + } + } + + [HttpPost("add-webnovel-deck")] + [Consumes("multipart/form-data")] + public async Task AddWebNovelDeck([FromForm] AddWebNovelDeckRequest model, + [FromServices] IWebNovelSourceResolver sourceResolver) + { + if (!ModelState.IsValid) + return BadRequest(ModelState); + + if (!WebNovelUrlParser.TryParse(model.Url, out var provider, out var sourceId)) + return BadRequest(new { Message = "Not a supported webnovel URL." }); + + if (!sourceResolver.IsSupported(provider)) + return BadRequest(new { Message = $"{provider} is not enabled yet." }); + + if (await dbContext.WebNovelSources.AnyAsync(s => s.Provider == provider && s.SourceId == sourceId)) + return Conflict(new { Message = "This novel is already tracked." }); + + string? coverPath = null; + if (model.CoverImage is { Length: > 0 }) + { + var directory = Path.Join(config["StaticFilesPath"], "tmp", Guid.NewGuid().ToString()); + Directory.CreateDirectory(directory); + + coverPath = Path.Join(directory, "cover.jpg"); + await using var stream = new FileStream(coverPath, FileMode.Create); + await model.CoverImage.CopyToAsync(stream); + } + + // Fetching a long novel is 20+ minutes of polite requests, so it can never run in-request + var jobId = backgroundJobs.Enqueue( + job => job.Import(provider, sourceId, coverPath, model.ChunkCharBudget)); + + logger.LogInformation("Admin queued webnovel import for {Provider}/{SourceId} (job {JobId})", + provider, sourceId, jobId); + + return Accepted(new { Message = "Import queued.", JobId = jobId, Provider = provider.ToString(), SourceId = sourceId }); + } + + [HttpGet("webnovel/{deckId:int}")] + public async Task GetWebNovelSource(int deckId) + { + var tracked = await dbContext.WebNovelSources + .AsNoTracking() + .FirstOrDefaultAsync(s => s.DeckId == deckId); + + if (tracked == null) + return NotFound(); + + var subdecks = await dbContext.WebNovelChapters + .AsNoTracking() + .Where(c => c.DeckId == deckId) + .GroupBy(c => c.ChildDeckId) + .Select(g => new + { + ChildDeckId = g.Key, + StartEpisode = g.Min(c => c.EpisodeNumber), + EndEpisode = g.Max(c => c.EpisodeNumber), + EpisodeCount = g.Count(), + CharCount = g.Sum(c => c.CharCount) + }) + .OrderBy(g => g.StartEpisode) + .ToListAsync(); + + return Ok(new + { + tracked.DeckId, + Provider = tracked.Provider.ToString(), + tracked.SourceId, + Url = WebNovelUrlParser.BuildWorkUrl(tracked.Provider, tracked.SourceId), + tracked.LastEpisodeCount, + tracked.LastSourceUpdate, + tracked.LastSyncedAt, + tracked.NextCheckAt, + tracked.SyncEnabled, + tracked.CompletedAtSource, + tracked.OnHiatusAtSource, + tracked.ConsecutiveFailures, + tracked.LastError, + tracked.ChunkCharBudget, + tracked.PendingRevisionCount, + Subdecks = subdecks + }); + } + + [HttpPost("webnovel/{deckId:int}/sync")] + public async Task SyncWebNovelNow(int deckId) + { + if (!await dbContext.WebNovelSources.AnyAsync(s => s.DeckId == deckId)) + return NotFound(); + + var jobId = backgroundJobs.Enqueue(job => job.Sync(deckId)); + + return Accepted(new { Message = "Sync queued.", JobId = jobId }); + } + + /// + /// Re-fetches one subdeck's whole episode range, picking up revisions (改稿) to episodes already ingested. + /// + [HttpPost("webnovel/{deckId:int}/rebuild/{childDeckId:int}")] + public async Task RebuildWebNovelSubdeck(int deckId, int childDeckId) + { + if (!await dbContext.WebNovelChapters.AnyAsync(c => c.DeckId == deckId && c.ChildDeckId == childDeckId)) + return NotFound(); + + var jobId = backgroundJobs.Enqueue(job => job.RebuildSubdeck(deckId, childDeckId)); + + return Accepted(new { Message = "Rebuild queued.", JobId = jobId }); + } + + [HttpPost("webnovel/{deckId:int}/sync-enabled")] + public async Task SetWebNovelSyncEnabled(int deckId, [FromBody] SetWebNovelSyncEnabledRequest model) + { + var tracked = await dbContext.WebNovelSources.FirstOrDefaultAsync(s => s.DeckId == deckId); + if (tracked == null) + return NotFound(); + + tracked.SyncEnabled = model.Enabled; + + // Coming off a freeze shouldn't wait out a backoff window + if (model.Enabled) + { + tracked.NextCheckAt = DateTimeOffset.UtcNow; + tracked.ConsecutiveFailures = 0; + tracked.LastError = null; + } + + await dbContext.SaveChangesAsync(); + + return Ok(new { tracked.DeckId, tracked.SyncEnabled }); + } + + private static int EstimateSubdecks(long totalCharacters) => + totalCharacters <= 0 + ? 1 + : (int)Math.Max(1, Math.Ceiling(totalCharacters / (double)SubdeckChunker.DefaultCharBudget)); +} + +public class SetWebNovelSyncEnabledRequest +{ + public bool Enabled { get; set; } +} diff --git a/Jiten.Api/Controllers/AdminController.cs b/Jiten.Api/Controllers/AdminController.cs index 889e72d8..6575098f 100644 --- a/Jiten.Api/Controllers/AdminController.cs +++ b/Jiten.Api/Controllers/AdminController.cs @@ -1155,6 +1155,9 @@ public async Task FetchMetadata(int deckId) else backgroundJobs.Enqueue(job => job.FetchGoogleBooksMissingMetadata(deckId)); break; + case MediaType.WebNovel: + backgroundJobs.Enqueue(job => job.FetchSyosetuMissingMetadata(deckId)); + break; default: return NotFound("No fetch job for this media type."); } @@ -1204,6 +1207,10 @@ public async Task FetchAllMissingMetadata() } break; + case MediaType.WebNovel: + if (deck.Links.Any(l => l.LinkType == LinkType.Syosetsu)) + backgroundJobs.Enqueue(job => job.FetchSyosetuMissingMetadata(deck.DeckId)); + break; default: break; } diff --git a/Jiten.Api/Dtos/Requests/AddWebNovelDeckRequest.cs b/Jiten.Api/Dtos/Requests/AddWebNovelDeckRequest.cs new file mode 100644 index 00000000..1c5390d7 --- /dev/null +++ b/Jiten.Api/Dtos/Requests/AddWebNovelDeckRequest.cs @@ -0,0 +1,19 @@ +namespace Jiten.Api.Dtos.Requests; + +public class AddWebNovelDeckRequest +{ + /// + /// Novel URL (https://ncode.syosetu.com/n9669bk/) or a bare ncode + /// + public required string Url { get; set; } + + /// + /// Optional: uploaded or generated in the browser. Syosetu works have no cover art of their own. + /// + public IFormFile? CoverImage { get; set; } + + /// + /// Optional per-novel override of the subdeck character budget + /// + public int? ChunkCharBudget { get; set; } +} diff --git a/Jiten.Api/Jobs/FetchMetadataJob.cs b/Jiten.Api/Jobs/FetchMetadataJob.cs index 4f4176ca..c94ea252 100644 --- a/Jiten.Api/Jobs/FetchMetadataJob.cs +++ b/Jiten.Api/Jobs/FetchMetadataJob.cs @@ -2,11 +2,16 @@ using Jiten.Core; using Jiten.Core.Data; using Jiten.Core.Data.Providers; +using Jiten.Core.Data.WebNovel; +using Jiten.Core.WebNovel; using Microsoft.EntityFrameworkCore; namespace Jiten.Api.Jobs; -public class FetchMetadataJob(IDbContextFactory contextFactory, IConfiguration configuration) +public class FetchMetadataJob( + IDbContextFactory contextFactory, + IConfiguration configuration, + IWebNovelSourceResolver sourceResolver) { private const float ANILIST_DELAY = 2.2f; private const float GOOGLE_BOOKS_DELAY = 3f; @@ -366,6 +371,64 @@ public async Task FetchJikanMissingMetadata(int deckId) } } + /// + /// Refreshes a tracked webnovel from the source's metadata API + /// + [Queue(WebNovelQueues.SyosetuMetadata)] + public async Task FetchSyosetuMissingMetadata(int deckId) + { + await using var context = await contextFactory.CreateDbContextAsync(); + + var deck = await context.Decks + .Include(d => d.Links) + .Include(d => d.Titles) + .Include(d => d.DeckGenres) + .Include(d => d.DeckTags) + .FirstAsync(d => d.DeckId == deckId); + + var (provider, sourceId) = await ResolveWebNovelWork(context, deck); + + var info = await sourceResolver.Resolve(provider).GetInfoAsync(sourceId); + var metadata = info.ToMetadata(); + + if (deck.ReleaseDate == default && metadata.ReleaseDate != null) + deck.ReleaseDate = DateOnly.FromDateTime(metadata.ReleaseDate.Value); + + if (string.IsNullOrEmpty(deck.Description)) + deck.Description = metadata.Description?.Length > 2000 ? metadata.Description?[..2000] : metadata.Description; + + if (deck.Links.All(l => l.LinkType != LinkType.Syosetsu)) + deck.Links.Add(new Link { DeckId = deck.DeckId, LinkType = LinkType.Syosetsu, Url = info.Url }); + + await MetadataProviderHelper.ApplyGenreAndTagMappings(context, deck, metadata, LinkType.Syosetsu); + + await context.SaveChangesAsync(); + } + + /// + /// The ledger is the authority on which work a deck came from; the link is the fallback for a webnovel + /// deck that was added by hand rather than imported. + /// + private static async Task<(WebNovelProvider Provider, string SourceId)> ResolveWebNovelWork(JitenDbContext context, Deck deck) + { + var tracked = await context.WebNovelSources + .AsNoTracking() + .FirstOrDefaultAsync(s => s.DeckId == deck.DeckId); + + if (tracked != null) + return (tracked.Provider, tracked.SourceId); + + var link = deck.Links.FirstOrDefault(l => l.LinkType == LinkType.Syosetsu); + + if (link == null) + throw new Exception($"No Syosetsu link found for deck with ID {deck.DeckId}."); + + if (!WebNovelUrlParser.TryParse(link.Url, out var provider, out var sourceId)) + throw new Exception($"Syosetsu link '{link.Url}' on deck {deck.DeckId} is not a work URL."); + + return (provider, sourceId); + } + private static void MergeDictionaryEntries(Deck deck, List entries) { foreach (var entry in entries) diff --git a/Jiten.Api/Jobs/ParseJob.cs b/Jiten.Api/Jobs/ParseJob.cs index 8690a11b..833ce2bf 100644 --- a/Jiten.Api/Jobs/ParseJob.cs +++ b/Jiten.Api/Jobs/ParseJob.cs @@ -18,6 +18,15 @@ public class ParseJob( { [Queue("parse")] public async Task Parse(Metadata metadata, MediaType deckType, bool storeRawText = false) + { + await ParseAndGetDeckId(metadata, deckType, storeRawText); + } + + /// + /// Same as , but hands back the created deck. Callers that need to attach their own + /// rows to the new deck tree (webnovel ledger) run it inline instead of enqueuing it. + /// + public async Task ParseAndGetDeckId(Metadata metadata, MediaType deckType, bool storeRawText = false) { Deck deck = new(); string filePath = metadata.FilePath!; @@ -127,10 +136,13 @@ public async Task Parse(Metadata metadata, MediaType deckType, bool storeRawText await MetadataProviderHelper.ApplyGenreAndTagMappings(context, deck, metadata, firstLink.LinkType); } - var coverImage = await File.ReadAllBytesAsync(metadata.Image ?? throw new Exception("No cover image found.")); + // Sources with no cover art (webnovels) fall back to nocover.jpg, set above + var coverImage = !string.IsNullOrEmpty(metadata.Image) && File.Exists(metadata.Image) + ? await File.ReadAllBytesAsync(metadata.Image) + : []; // Insert the deck into the database - await JitenHelper.InsertDeck(contextFactory, deck, coverImage ?? [], false); + await JitenHelper.InsertDeck(contextFactory, deck, coverImage, false); // Process relations from metadata if (metadata.Relations.Count > 0) @@ -154,6 +166,8 @@ public async Task Parse(Metadata metadata, MediaType deckType, bool storeRawText // Notify IndexNow of the new (top-level) deck page so Bing can index it quickly. await indexNow.SubmitDeckAsync(deck.DeckId); + + return deck.DeckId; } /// diff --git a/Jiten.Api/Jobs/WebNovelFetchJob.cs b/Jiten.Api/Jobs/WebNovelFetchJob.cs new file mode 100644 index 00000000..1f5b84f2 --- /dev/null +++ b/Jiten.Api/Jobs/WebNovelFetchJob.cs @@ -0,0 +1,334 @@ +using Hangfire; +using Jiten.Core; +using Jiten.Core.Data; +using Jiten.Core.Data.WebNovel; +using Jiten.Core.WebNovel; +using Microsoft.EntityFrameworkCore; + +namespace Jiten.Api.Jobs; + +public class WebNovelFetchJob( + IDbContextFactory contextFactory, + IWebNovelSourceResolver sourceResolver, + IBackgroundJobClient backgroundJobs, + ILogger logger) +{ + /// + /// Appends any new episodes to the novel's subdecks. + /// + /// Subdeck rows and their raw text are updated by DeckId. The InsertDeck(update: true) path must + /// never be used here: JitenHelper.CollectChildDeckUpdates matches children by OriginalTitle and + /// orphan-deletes the ones it can't match, and the open subdeck's title changes on every append — that + /// would delete the subdeck and every user's progress on it. + /// + [Queue(WebNovelQueues.Syosetu)] + [AutomaticRetry(Attempts = 1)] + public async Task Sync(int parentDeckId) + { + await using var context = await contextFactory.CreateDbContextAsync(); + + var tracked = await context.WebNovelSources + .Include(s => s.Chapters) + .FirstOrDefaultAsync(s => s.DeckId == parentDeckId); + + if (tracked == null) + { + logger.LogWarning("WebNovelSync: deck {DeckId} is not a tracked webnovel", parentDeckId); + return; + } + + try + { + var source = sourceResolver.Resolve(tracked.Provider); + + var info = await source.GetInfoAsync(tracked.SourceId); + var toc = await source.GetTocAsync(tracked.SourceId); + + var ledger = tracked.Chapters.ToDictionary(c => c.EpisodeNumber); + + // Episode numbers at the source are positional: deleting one renumbers everything after it, + // so the ledger no longer lines up with the table of contents. Appending would attach the + // wrong text to the wrong episode — stop and surface it instead. + if (toc.Count < ledger.Count) + { + throw new InvalidOperationException( + $"The source lists {toc.Count} episodes but {ledger.Count} are ingested — episodes were " + + "deleted or renumbered at the source. Rebuild the affected subdecks or re-import the novel."); + } + + var newEpisodes = toc.Where(e => !ledger.ContainsKey(e.Number)).OrderBy(e => e.Number).ToList(); + + // Episodes we already hold whose text changed at the source (改稿). Applying these means + // rebuilding the subdeck that holds them, so they are surfaced for a manual refresh instead. + var revisedCount = toc.Count(e => ledger.TryGetValue(e.Number, out var known) && + e.UpdatedAt != null && known.SourceUpdatedAt != null && + e.UpdatedAt > known.SourceUpdatedAt); + + if (newEpisodes.Count == 0) + { + ApplySuccessState(tracked, info, revisedCount); + await context.SaveChangesAsync(); + logger.LogInformation("WebNovelSync: deck {DeckId} has no new episodes ({Revised} revised)", + parentDeckId, revisedCount); + return; + } + + logger.LogInformation("WebNovelSync: deck {DeckId} has {Count} new episodes", parentDeckId, newEpisodes.Count); + + var fetched = new List<(WebNovelEpisodeRef Reference, ChunkEpisode Chunk, string Text)>(); + foreach (var reference in newEpisodes) + { + var text = await source.GetEpisodeTextAsync(tracked.SourceId, reference); + fetched.Add((reference, + new ChunkEpisode(reference.Number, reference.Title, SubdeckChunker.CountCharacters(text)), + text)); + } + + var existingChunks = await BuildExistingChunksAsync(context, tracked); + var plans = SubdeckChunker.Plan(existingChunks, + fetched.Select(f => f.Chunk).ToList(), + tracked.ChunkCharBudget ?? SubdeckChunker.DefaultCharBudget); + + var textByEpisode = fetched.ToDictionary(f => f.Chunk.Number, f => f.Text); + var referenceByEpisode = fetched.ToDictionary(f => f.Chunk.Number, f => f.Reference); + + // One transaction around the whole apply: appended raw text must never be committed without the + // matching ledger rows and success state, or the subdeck's text and words silently diverge. + await using var transaction = await context.Database.BeginTransactionAsync(); + + var changedChildIds = await ApplyPlansAsync(context, tracked, plans, textByEpisode, referenceByEpisode); + + tracked.LastEpisodeCount = tracked.Chapters.Count; + ApplySuccessState(tracked, info, revisedCount); + await context.SaveChangesAsync(); + await transaction.CommitAsync(); + + // Reparses exactly the changed subdecks by DeckId, re-aggregates the parent and preserves progress + backgroundJobs.Enqueue(job => job.ParseNewSubdecks(parentDeckId, changedChildIds)); + + logger.LogInformation("WebNovelSync: deck {DeckId} appended {Episodes} episodes across {Subdecks} subdecks", + parentDeckId, newEpisodes.Count, changedChildIds.Count); + } + catch (Exception ex) + { + // This context's tracker may hold half-applied changes (or be the thing that threw), so the + // failure bookkeeping runs on its own context. + await RecordFailureAsync(parentDeckId, ex); + throw; + } + } + + private static void ApplySuccessState(WebNovelSource tracked, WebNovelInfo info, int revisedCount) + { + tracked.CompletedAtSource = info.IsCompleted; + tracked.OnHiatusAtSource = info.IsOnHiatus; + tracked.PendingRevisionCount = revisedCount; + tracked.LastSourceUpdate = info.LastUpdatedAt; + tracked.LastSyncedAt = DateTimeOffset.UtcNow; + tracked.NextCheckAt = WebNovelSchedule.NextCheck(info.IsCompleted); + tracked.ConsecutiveFailures = 0; + tracked.LastError = null; + } + + private async Task RecordFailureAsync(int parentDeckId, Exception ex) + { + try + { + await using var context = await contextFactory.CreateDbContextAsync(); + var tracked = await context.WebNovelSources.FirstOrDefaultAsync(s => s.DeckId == parentDeckId); + if (tracked == null) + return; + + tracked.ConsecutiveFailures++; + tracked.LastError = WebNovelImportJob.Truncate(ex.Message, 1000); + tracked.NextCheckAt = WebNovelSchedule.NextCheckAfterFailure(tracked.ConsecutiveFailures); + await context.SaveChangesAsync(); + + if (tracked.ConsecutiveFailures >= 3) + { + logger.LogError(ex, "WebNovelSync: deck {DeckId} has failed {Count} times in a row — the site's markup " + + "has probably changed, re-check the narou.rb site definitions", + parentDeckId, tracked.ConsecutiveFailures); + } + else + { + logger.LogWarning(ex, "WebNovelSync: deck {DeckId} failed", parentDeckId); + } + } + catch (Exception saveEx) + { + logger.LogError(saveEx, "WebNovelSync: could not record the failure for deck {DeckId}", parentDeckId); + } + } + + /// + /// Re-fetches every episode in one subdeck and replaces its text, picking up revisions (改稿) to episodes + /// that were already ingested. Manual, because a rebuild costs one request per episode in the range. + /// + [Queue(WebNovelQueues.Syosetu)] + [AutomaticRetry(Attempts = 1)] + public async Task RebuildSubdeck(int parentDeckId, int childDeckId) + { + await using var context = await contextFactory.CreateDbContextAsync(); + + var tracked = await context.WebNovelSources + .Include(s => s.Chapters) + .FirstOrDefaultAsync(s => s.DeckId == parentDeckId); + + if (tracked == null) + return; + + var chapters = tracked.Chapters.Where(c => c.ChildDeckId == childDeckId) + .OrderBy(c => c.EpisodeNumber) + .ToList(); + + if (chapters.Count == 0) + { + logger.LogWarning("WebNovelRebuild: subdeck {ChildDeckId} holds no episodes", childDeckId); + return; + } + + var source = sourceResolver.Resolve(tracked.Provider); + var toc = (await source.GetTocAsync(tracked.SourceId)).ToDictionary(e => e.Number); + + var texts = new List(); + foreach (var chapter in chapters) + { + if (!toc.TryGetValue(chapter.EpisodeNumber, out var reference)) + { + // The episode was deleted at the source; keep the range intact and skip it + logger.LogWarning("WebNovelRebuild: episode {Episode} no longer exists at the source", + chapter.EpisodeNumber); + continue; + } + + var text = await source.GetEpisodeTextAsync(tracked.SourceId, reference); + texts.Add(text); + + chapter.Title = WebNovelImportJob.Truncate(reference.Title, 500); + chapter.SourceUpdatedAt = reference.UpdatedAt; + chapter.CharCount = SubdeckChunker.CountCharacters(text); + } + + var rawText = await context.DeckRawTexts.FirstOrDefaultAsync(t => t.DeckId == childDeckId); + if (rawText == null) + { + rawText = new DeckRawText { DeckId = childDeckId }; + context.DeckRawTexts.Add(rawText); + } + + rawText.RawText = string.Join("\n\n", texts); + + // Recompute over the whole ledger: this rebuild refreshed its own chapters' SourceUpdatedAt, but + // other subdecks may still hold revised episodes awaiting their own rebuild. + tracked.PendingRevisionCount = tracked.Chapters.Count(c => toc.TryGetValue(c.EpisodeNumber, out var e) && + e.UpdatedAt != null && c.SourceUpdatedAt != null && + e.UpdatedAt > c.SourceUpdatedAt); + await context.SaveChangesAsync(); + + backgroundJobs.Enqueue(job => job.ParseNewSubdecks(parentDeckId, new List { childDeckId })); + + logger.LogInformation("WebNovelRebuild: rebuilt subdeck {ChildDeckId} from {Count} episodes", + childDeckId, texts.Count); + } + + private static async Task> BuildExistingChunksAsync(JitenDbContext context, WebNovelSource tracked) + { + if (tracked.Chapters.Count == 0) + return []; + + var orderByChild = await context.Decks + .Where(d => d.ParentDeckId == tracked.DeckId) + .Select(d => new { d.DeckId, d.DeckOrder }) + .ToDictionaryAsync(d => d.DeckId, d => d.DeckOrder); + + return tracked.Chapters + .GroupBy(c => c.ChildDeckId) + .Select(g => new ExistingChunk( + ChunkIndex: orderByChild.GetValueOrDefault(g.Key), + ChildDeckId: g.Key, + StartEpisode: g.Min(c => c.EpisodeNumber), + EndEpisode: g.Max(c => c.EpisodeNumber), + EpisodeCount: g.Count(), + CharCount: g.Sum(c => c.CharCount))) + .OrderBy(c => c.ChunkIndex) + .ToList(); + } + + /// + /// Extends the open subdeck and opens new ones, all keyed by DeckId so existing subdecks keep their identity. + /// + private async Task> ApplyPlansAsync(JitenDbContext context, + WebNovelSource tracked, + List plans, + Dictionary textByEpisode, + Dictionary referenceByEpisode) + { + var parent = await context.Decks.FirstAsync(d => d.DeckId == tracked.DeckId); + var changedChildIds = new List(); + + foreach (var plan in plans) + { + var appendedText = string.Join("\n\n", plan.EpisodesToAppend.Select(e => textByEpisode[e.Number])); + int childDeckId; + + if (plan.ChildDeckId is { } existingChildId) + { + var child = await context.Decks.FirstAsync(d => d.DeckId == existingChildId); + var rawText = await context.DeckRawTexts.FirstOrDefaultAsync(t => t.DeckId == existingChildId); + + if (rawText == null) + { + // Raw-text storage was off when this subdeck was created; there is nothing to append to + logger.LogError("WebNovelSync: subdeck {ChildDeckId} has no stored raw text, cannot append", + existingChildId); + continue; + } + + rawText.RawText = $"{rawText.RawText}\n\n{appendedText}"; + + // The open subdeck's episode range grew + child.OriginalTitle = plan.Title; + child.LastUpdate = DateTimeOffset.UtcNow; + + childDeckId = existingChildId; + } + else + { + var child = new Deck + { + ParentDeckId = tracked.DeckId, + DeckOrder = plan.ChunkIndex, + MediaType = parent.MediaType, + OriginalTitle = plan.Title, + DifficultyOverride = -1, + CreationDate = DateTimeOffset.UtcNow, + LastUpdate = DateTimeOffset.UtcNow, + RawText = new DeckRawText(appendedText) + }; + + context.Decks.Add(child); + await context.SaveChangesAsync(); + + childDeckId = child.DeckId; + } + + foreach (var episode in plan.EpisodesToAppend) + { + tracked.Chapters.Add(new WebNovelChapter + { + DeckId = tracked.DeckId, + EpisodeNumber = episode.Number, + ChildDeckId = childDeckId, + Title = WebNovelImportJob.Truncate(episode.Title, 500), + SourceUpdatedAt = referenceByEpisode[episode.Number].UpdatedAt, + CharCount = episode.CharCount + }); + } + + changedChildIds.Add(childDeckId); + } + + return changedChildIds; + } +} diff --git a/Jiten.Api/Jobs/WebNovelImportJob.cs b/Jiten.Api/Jobs/WebNovelImportJob.cs new file mode 100644 index 00000000..7602e3b0 --- /dev/null +++ b/Jiten.Api/Jobs/WebNovelImportJob.cs @@ -0,0 +1,202 @@ +using Hangfire; +using Jiten.Core; +using Jiten.Core.Data; +using Jiten.Core.Data.Providers; +using Jiten.Core.Data.WebNovel; +using Jiten.Core.WebNovel; +using Microsoft.EntityFrameworkCore; + +namespace Jiten.Api.Jobs; + +public class WebNovelImportJob( + IDbContextFactory contextFactory, + IWebNovelSourceResolver sourceResolver, + ParseJob parseJob, + IConfiguration config, + ILogger logger) +{ + /// + /// Fetches a whole novel and creates the parent deck plus its chapter-range subdecks. + /// + /// Runs on the per-site queue: a 1,000-episode novel is ~20 minutes of polite fetching, so it must + /// never run in-request, and it must serialise with syncs so the site never sees two requests at once. + /// + [Queue(WebNovelQueues.Syosetu)] + [AutomaticRetry(Attempts = 1)] + public async Task Import(WebNovelProvider provider, string sourceId, string? coverPath, int? chunkCharBudget) + { + await using var context = await contextFactory.CreateDbContextAsync(); + + if (await context.WebNovelSources.AnyAsync(s => s.Provider == provider && s.SourceId == sourceId)) + { + logger.LogWarning("WebNovelImport: {Provider}/{SourceId} is already tracked, skipping", provider, sourceId); + return; + } + + var source = sourceResolver.Resolve(provider); + + var info = await source.GetInfoAsync(sourceId); + + // InsertDeck dedupes new decks on (OriginalTitle, MediaType) and silently keeps the existing one, which + // would leave us with a ledger pointing at someone else's deck. Catch it before the long fetch. + if (await context.Decks.AnyAsync(d => d.OriginalTitle == info.Title && d.MediaType == MediaType.WebNovel)) + { + throw new InvalidOperationException( + $"A webnovel deck titled '{info.Title}' already exists. Rename or remove it before importing {sourceId}."); + } + + var toc = await source.GetTocAsync(sourceId); + + if (toc.Count == 0) + throw new InvalidOperationException($"{provider}/{sourceId} has no episodes."); + + logger.LogInformation("WebNovelImport: {Title} ({SourceId}) — {Episodes} episodes, ~{Chars} chars", + info.Title, sourceId, toc.Count, info.TotalCharacters); + + var budget = chunkCharBudget ?? SubdeckChunker.DefaultCharBudget; + + var workingDirectory = Path.Join(config["StaticFilesPath"], "tmp", $"webnovel-{sourceId}-{Guid.NewGuid()}"); + Directory.CreateDirectory(workingDirectory); + + try + { + var episodes = new List<(WebNovelEpisodeRef Reference, ChunkEpisode Chunk, string Text)>(); + + foreach (var reference in toc) + { + var text = await source.GetEpisodeTextAsync(sourceId, reference); + episodes.Add((reference, + new ChunkEpisode(reference.Number, reference.Title, SubdeckChunker.CountCharacters(text)), + text)); + } + + var plans = SubdeckChunker.Plan([], episodes.Select(e => e.Chunk).ToList(), budget); + var textByEpisode = episodes.ToDictionary(e => e.Chunk.Number, e => e.Text); + + var metadata = info.ToMetadata(); + + foreach (var plan in plans) + { + var chunkPath = Path.Join(workingDirectory, $"chunk-{plan.ChunkIndex:D4}.txt"); + await File.WriteAllTextAsync(chunkPath, JoinEpisodes(plan, textByEpisode)); + + metadata.Children.Add(new Metadata + { + FilePath = chunkPath, + OriginalTitle = plan.Title + }); + } + + if (!string.IsNullOrEmpty(coverPath) && File.Exists(coverPath)) + metadata.Image = coverPath; + + var parentDeckId = await parseJob.ParseAndGetDeckId(metadata, MediaType.WebNovel, storeRawText: true); + + await WriteLedgerAsync(parentDeckId, provider, sourceId, info, plans, episodes, chunkCharBudget); + + // Only on success: a Hangfire retry of a failed import still needs the uploaded cover + DeleteCoverDirectory(coverPath); + + logger.LogInformation("WebNovelImport: created deck {DeckId} with {Subdecks} subdecks for {Title}", + parentDeckId, plans.Count, info.Title); + } + finally + { + try + { + Directory.Delete(workingDirectory, recursive: true); + } + catch (Exception ex) + { + logger.LogWarning(ex, "WebNovelImport: could not clean up {Directory}", workingDirectory); + } + } + } + + private void DeleteCoverDirectory(string? coverPath) + { + if (string.IsNullOrEmpty(coverPath)) + return; + + try + { + var directory = Path.GetDirectoryName(coverPath); + if (!string.IsNullOrEmpty(directory) && Directory.Exists(directory)) + Directory.Delete(directory, recursive: true); + } + catch (Exception ex) + { + logger.LogWarning(ex, "WebNovelImport: could not clean up the cover at {CoverPath}", coverPath); + } + } + + private static string JoinEpisodes(ChunkPlan plan, Dictionary textByEpisode) => + string.Join("\n\n", plan.EpisodesToAppend.Select(e => textByEpisode[e.Number])); + + private async Task WriteLedgerAsync(int parentDeckId, WebNovelProvider provider, string sourceId, WebNovelInfo info, + List plans, + List<(WebNovelEpisodeRef Reference, ChunkEpisode Chunk, string Text)> episodes, + int? chunkCharBudget) + { + await using var context = await contextFactory.CreateDbContextAsync(); + + // Children were created in chunk order, so DeckOrder maps back to the chunk that produced them + var childIdsByOrder = await context.Decks + .Where(d => d.ParentDeckId == parentDeckId) + .OrderBy(d => d.DeckOrder) + .Select(d => new { d.DeckId, d.DeckOrder }) + .ToDictionaryAsync(d => d.DeckOrder, d => d.DeckId); + + // No subdecks means the deck wasn't actually created (a same-title deck already existed and the insert + // was skipped). Writing the ledger now would bind this novel to a deck it doesn't own. + if (childIdsByOrder.Count == 0) + { + throw new InvalidOperationException( + $"Deck {parentDeckId} has no subdecks after import — the insert was skipped, so {sourceId} was not tracked."); + } + + context.WebNovelSources.Add(new WebNovelSource + { + DeckId = parentDeckId, + Provider = provider, + SourceId = sourceId, + LastEpisodeCount = episodes.Count, + LastSourceUpdate = info.LastUpdatedAt, + LastSyncedAt = DateTimeOffset.UtcNow, + NextCheckAt = WebNovelSchedule.NextCheck(info.IsCompleted), + CompletedAtSource = info.IsCompleted, + OnHiatusAtSource = info.IsOnHiatus, + ChunkCharBudget = chunkCharBudget + }); + + var referenceByNumber = episodes.ToDictionary(e => e.Chunk.Number, e => e.Reference); + + foreach (var plan in plans) + { + if (!childIdsByOrder.TryGetValue(plan.ChunkIndex, out var childDeckId)) + { + logger.LogError("WebNovelImport: no subdeck was created for chunk {Chunk} of deck {DeckId}", + plan.ChunkIndex, parentDeckId); + continue; + } + + foreach (var episode in plan.EpisodesToAppend) + { + context.WebNovelChapters.Add(new WebNovelChapter + { + DeckId = parentDeckId, + EpisodeNumber = episode.Number, + ChildDeckId = childDeckId, + Title = Truncate(episode.Title, 500), + SourceUpdatedAt = referenceByNumber[episode.Number].UpdatedAt, + CharCount = episode.CharCount + }); + } + } + + await context.SaveChangesAsync(); + } + + internal static string Truncate(string value, int maxLength) => + value.Length <= maxLength ? value : value[..maxLength]; +} diff --git a/Jiten.Api/Jobs/WebNovelQueues.cs b/Jiten.Api/Jobs/WebNovelQueues.cs new file mode 100644 index 00000000..25d385c2 --- /dev/null +++ b/Jiten.Api/Jobs/WebNovelQueues.cs @@ -0,0 +1,18 @@ +namespace Jiten.Api.Jobs; + +/// +/// One single-worker queue per site. Imports and syncs for a site serialise on the same queue, so the +/// politeness rate holds however many novels are queued; different sites still run concurrently. +/// +public static class WebNovelQueues +{ + public const string Syosetu = "webnovel-syosetu"; + + /// + /// Metadata refreshes are a single API call, so they get their own queue rather than waiting out the + /// hours-long import sitting in front of them + /// + public const string SyosetuMetadata = "webnovel-syosetu-metadata"; + + public static readonly string[] All = [Syosetu, SyosetuMetadata]; +} diff --git a/Jiten.Api/Jobs/WebNovelSyncSweepJob.cs b/Jiten.Api/Jobs/WebNovelSyncSweepJob.cs new file mode 100644 index 00000000..324d3d4a --- /dev/null +++ b/Jiten.Api/Jobs/WebNovelSyncSweepJob.cs @@ -0,0 +1,138 @@ +using Hangfire; +using Jiten.Core; +using Jiten.Core.Data.WebNovel; +using Jiten.Core.WebNovel; +using Microsoft.EntityFrameworkCore; + +namespace Jiten.Api.Jobs; + +public class WebNovelSyncSweepJob( + IDbContextFactory contextFactory, + IWebNovelSourceResolver sourceResolver, + IBackgroundJobClient backgroundJobs, + ILogger logger) +{ + /// + /// Polls every tracked novel's update state and enqueues a fetch for the ones worth syncing. + /// + /// For Syosetu this is two API calls per 1,000 novels — the metadata API reports episode count and last + /// update, so no episode or table-of-contents page is touched until there is something to fetch. + /// + /// Polling is daily but syncs are batched (): a sync reparses + /// the subdeck and rewrites the parent's aggregated words however few episodes it ingests, so a + /// daily-updating serial is left to accumulate ~15 episodes (or two weeks) between syncs. + /// + [Queue("default")] + public async Task Sweep() + { + await using var context = await contextFactory.CreateDbContextAsync(); + + var now = DateTimeOffset.UtcNow; + var due = await context.WebNovelSources + .Where(s => s.SyncEnabled && s.NextCheckAt <= now) + .ToListAsync(); + + if (due.Count == 0) + { + logger.LogInformation("WebNovelSweep: nothing due"); + return; + } + + var dirtyCount = 0; + + foreach (var group in due.GroupBy(s => s.Provider)) + { + if (!sourceResolver.IsSupported(group.Key)) + { + logger.LogWarning("WebNovelSweep: provider {Provider} is not enabled, skipping {Count} novels", + group.Key, group.Count()); + continue; + } + + try + { + dirtyCount += await SweepProviderAsync(context, sourceResolver.Resolve(group.Key), group.ToList()); + } + catch (Exception ex) + { + logger.LogError(ex, "WebNovelSweep: polling {Provider} failed", group.Key); + } + } + + await context.SaveChangesAsync(); + + logger.LogInformation("WebNovelSweep: polled {Due} novels, {Dirty} due a sync", due.Count, dirtyCount); + } + + private async Task SweepProviderAsync(JitenDbContext context, IWebNovelSource source, List tracked) + { + var polled = source is IBatchPollableSource batch + ? await batch.BatchPollAsync(tracked.Select(s => s.SourceId)) + : await PollIndividuallyAsync(source, tracked); + + var dirty = 0; + + foreach (var novel in tracked) + { + if (!polled.TryGetValue(novel.SourceId, out var info)) + { + // The work was deleted or made private at the source + novel.ConsecutiveFailures++; + novel.LastError = "Not found at the source."; + novel.NextCheckAt = WebNovelSchedule.NextCheckAfterFailure(novel.ConsecutiveFailures); + logger.LogWarning("WebNovelSweep: {SourceId} was not returned by the source", novel.SourceId); + continue; + } + + novel.CompletedAtSource = info.IsCompleted; + novel.OnHiatusAtSource = info.IsOnHiatus; + + // Fewer episodes than ingested means deletions/renumbering at the source: every later episode + // shifted position, so a sync would append the wrong text. Alert instead of syncing. + if (info.EpisodeCount < novel.LastEpisodeCount) + { + novel.ConsecutiveFailures++; + novel.LastError = $"The source lists {info.EpisodeCount} episodes but {novel.LastEpisodeCount} are " + + "ingested — episodes were deleted or renumbered. Rebuild the affected subdecks or re-import."; + novel.NextCheckAt = WebNovelSchedule.NextCheckAfterFailure(novel.ConsecutiveFailures); + logger.LogWarning("WebNovelSweep: {SourceId} episode count dropped from {Known} to {Polled}", + novel.SourceId, novel.LastEpisodeCount, info.EpisodeCount); + continue; + } + + if (WebNovelSchedule.ShouldSync(novel, info)) + { + dirty++; + var deckId = novel.DeckId; + backgroundJobs.Enqueue(job => job.Sync(deckId)); + + // The fetch job owns the next check time from here + continue; + } + + novel.NextCheckAt = WebNovelSchedule.NextCheck(info.IsCompleted); + + // The novel polled healthy — a stale failure state (e.g. temporarily private) shouldn't linger + novel.ConsecutiveFailures = 0; + novel.LastError = null; + + // Only mark as synced when nothing is pending. A below-threshold backlog keeps LastSyncedAt at + // the last ingest, so the max-lag clock in ShouldSync keeps running instead of resetting daily. + if (!WebNovelSchedule.IsDirty(novel, info)) + novel.LastSyncedAt = DateTimeOffset.UtcNow; + } + + return dirty; + } + + private static async Task> PollIndividuallyAsync(IWebNovelSource source, + List tracked) + { + var polled = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var novel in tracked) + polled[novel.SourceId] = await source.GetInfoAsync(novel.SourceId); + + return polled; + } +} diff --git a/Jiten.Api/Program.cs b/Jiten.Api/Program.cs index fc7a4295..a1c68240 100644 --- a/Jiten.Api/Program.cs +++ b/Jiten.Api/Program.cs @@ -12,6 +12,7 @@ using Jiten.Api.Authentication; using Jiten.Core; using Jiten.Core.Data.Authentication; +using Jiten.Core.WebNovel; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.HttpOverrides; @@ -115,6 +116,22 @@ }); builder.Services.AddSingleton(); +builder.Services.AddHttpClient(SyosetuSource.HttpClientName, client => + { + client.DefaultRequestHeaders.Add("User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + client.Timeout = TimeSpan.FromSeconds(30); + }) + // The API's gzip=5 body is inflated by hand; this covers the HTML pages + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); + +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + // OpenTelemetry Configuration var otelConfig = builder.Configuration.GetSection("OpenTelemetry"); var enableOtlpExporter = otelConfig.GetValue("EnableOtlpExporter"); @@ -532,6 +549,9 @@ bool IsTrustedSsr(HttpContext ctx) // Hangfire jobs builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -593,6 +613,24 @@ bool IsTrustedSsr(HttpContext ctx) options.WorkerCount = 1; }); + +builder.Services.AddHangfireServer((options) => +{ + options.ServerName = "WebNovelSyosetuServer"; + options.Queues = [WebNovelQueues.Syosetu]; + options.WorkerCount = 1; + // A large novel can take hours + options.ShutdownTimeout = TimeSpan.FromHours(3); + options.StopTimeout = TimeSpan.FromHours(3); +}); + +builder.Services.AddHangfireServer((options) => +{ + options.ServerName = "WebNovelSyosetuMetadataServer"; + options.Queues = [WebNovelQueues.SyosetuMetadata]; + options.WorkerCount = 1; +}); + builder.Services.AddHangfireServer((options) => { options.ServerName = "CoverageServer"; @@ -668,6 +706,11 @@ bool IsTrustedSsr(HttpContext ctx) "embed-pending-decks", job => job.EmbedPending(), "*/30 * * * *"); + + recurringJobs.AddOrUpdate( + "webnovel-sync-sweep", + job => job.Sweep(), + Cron.Daily(5)); } app.UseResponseCompression(); diff --git a/Jiten.Cli/CliOptions.cs b/Jiten.Cli/CliOptions.cs index 37e9a156..afc637d7 100644 --- a/Jiten.Cli/CliOptions.cs +++ b/Jiten.Cli/CliOptions.cs @@ -317,4 +317,12 @@ public class CliOptions [Option(longName: "min-shared", Required = false, HelpText = "Override the gated minimum shared-distinctive-word count for --explain probing.")] public int? MinShared { get; set; } + + [Option(longName: "webnovel-test", Required = false, + HelpText = "Dry run a webnovel URL or ncode: metadata, table of contents, first episode and the subdeck split. Writes nothing.")] + public string? WebNovelTest { get; set; } + + [Option(longName: "chunk-chars", Required = false, + HelpText = "Subdeck character budget for --webnovel-test (default: 150000).")] + public int? ChunkChars { get; set; } } diff --git a/Jiten.Cli/Commands/WebNovelCommands.cs b/Jiten.Cli/Commands/WebNovelCommands.cs new file mode 100644 index 00000000..018898a9 --- /dev/null +++ b/Jiten.Cli/Commands/WebNovelCommands.cs @@ -0,0 +1,118 @@ +using Jiten.Core.WebNovel; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Jiten.Cli.Commands; + +public class WebNovelCommands +{ + /// + /// Dry run: fetches a novel's metadata, table of contents and first episode, and reports how it would be + /// split into subdecks. Writes nothing. Use it to check the site's markup after a failed sync. + /// + public async Task Test(string url, int? chunkCharBudget) + { + if (!WebNovelUrlParser.TryParse(url, out var provider, out var sourceId)) + { + Console.WriteLine($"'{url}' is not a supported webnovel URL or ncode."); + return; + } + + var services = new ServiceCollection(); + services.AddHttpClient(SyosetuSource.HttpClientName, client => + { + client.DefaultRequestHeaders.Add("User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + client.Timeout = TimeSpan.FromSeconds(30); + }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + var source = new SyosetuSource(httpClientFactory, NullLogger.Instance, provider); + + Console.WriteLine($"Provider: {provider} Source id: {sourceId}"); + Console.WriteLine("Fetching metadata..."); + + var info = await source.GetInfoAsync(sourceId); + + Console.WriteLine(); + Console.WriteLine($" Title : {info.Title}"); + Console.WriteLine($" Author : {info.Author}"); + Console.WriteLine($" Genre : {info.Genre}"); + Console.WriteLine($" Keywords : {string.Join(", ", info.Keywords)}"); + Console.WriteLine($" Episodes : {info.EpisodeCount}"); + Console.WriteLine($" Characters : {info.TotalCharacters:N0}"); + Console.WriteLine($" First up : {info.FirstPublishedAt:yyyy-MM-dd}"); + Console.WriteLine($" Last up : {info.LastUpdatedAt:yyyy-MM-dd HH:mm}"); + Console.WriteLine($" One-shot : {info.IsOneShot} Completed: {info.IsCompleted} Hiatus: {info.IsOnHiatus} R15: {info.IsR15}"); + Console.WriteLine($" Synopsis : {Preview(info.Synopsis, 120)}"); + + Console.WriteLine(); + Console.WriteLine("Fetching table of contents..."); + var toc = await source.GetTocAsync(sourceId); + Console.WriteLine($" {toc.Count} episodes listed (API reported {info.EpisodeCount})"); + + if (toc.Count == 0) + return; + + foreach (var episode in toc.Take(3)) + { + Console.WriteLine($" #{episode.Number,-4} {Preview(episode.Title, 40),-42} " + + $"updated {episode.UpdatedAt:yyyy-MM-dd HH:mm} section: {episode.SectionTitle}"); + } + + if (toc.Count > 3) + Console.WriteLine($" ... #{toc[^1].Number} {Preview(toc[^1].Title, 40)}"); + + Console.WriteLine(); + Console.WriteLine($"Fetching episode #{toc[0].Number}..."); + var text = await source.GetEpisodeTextAsync(sourceId, toc[0]); + + var characters = SubdeckChunker.CountCharacters(text); + var rubyCount = text.Count(c => c == '{'); + + Console.WriteLine($" Raw length : {text.Length:N0}"); + Console.WriteLine($" Counted chars : {characters:N0} (furigana + whitespace excluded)"); + Console.WriteLine($" Ruby annotations: {rubyCount}"); + Console.WriteLine(); + Console.WriteLine(" --- first 300 chars ---"); + Console.WriteLine(" " + Preview(text, 300).Replace("\n", "\n ")); + Console.WriteLine(" ---"); + + // Project the whole work using this episode's length as the per-episode estimate + var budget = chunkCharBudget ?? SubdeckChunker.DefaultCharBudget; + var averageChars = info.EpisodeCount > 0 && info.TotalCharacters > 0 + ? (int)(info.TotalCharacters / info.EpisodeCount) + : characters; + + var projected = SubdeckChunker.Plan([], + toc.Select(e => new ChunkEpisode(e.Number, e.Title, averageChars)).ToList(), + budget); + + Console.WriteLine(); + Console.WriteLine($"Subdeck projection (budget {budget:N0} chars, ~{averageChars:N0} chars/episode):"); + Console.WriteLine($" {projected.Count} subdecks"); + + foreach (var plan in projected.Take(5)) + Console.WriteLine($" [{plan.ChunkIndex}] {plan.Title} ({plan.EpisodesToAppend.Count} episodes)"); + + if (projected.Count > 5) + Console.WriteLine($" ... [{projected[^1].ChunkIndex}] {projected[^1].Title} ({projected[^1].EpisodesToAppend.Count} episodes)"); + } + + private static string Preview(string? value, int maxLength) + { + if (string.IsNullOrEmpty(value)) + return ""; + + var single = value.Length <= maxLength ? value : value[..maxLength] + "…"; + return single; + } +} diff --git a/Jiten.Cli/Program.cs b/Jiten.Cli/Program.cs index 5ad32629..99e64e9f 100644 --- a/Jiten.Cli/Program.cs +++ b/Jiten.Cli/Program.cs @@ -43,6 +43,13 @@ private static async Task ExecuteCommands(CliContext context, CliOptions options var metadataCommands = new MetadataCommands(); var benchmarkCommands = new BenchmarkCommands(context); var rubyExtractCommands = new RubyExtractCommands(context); + var webNovelCommands = new WebNovelCommands(); + + if (!string.IsNullOrEmpty(options.WebNovelTest)) + { + await webNovelCommands.Test(options.WebNovelTest, options.ChunkChars); + return; + } // Import commands if (options.Import) diff --git a/Jiten.Core/Data/WebNovel/WebNovelChapter.cs b/Jiten.Core/Data/WebNovel/WebNovelChapter.cs new file mode 100644 index 00000000..b3270397 --- /dev/null +++ b/Jiten.Core/Data/WebNovel/WebNovelChapter.cs @@ -0,0 +1,37 @@ +using System.ComponentModel.DataAnnotations; + +namespace Jiten.Core.Data.WebNovel; + +/// +/// Per-episode ledger. Subdeck ranges, revision detection and child identity all derive from this, +/// never from deck titles — subdeck titles change as the open subdeck grows. +/// +public class WebNovelChapter +{ + /// + /// Parent deck (the tracked novel) + /// + public int DeckId { get; set; } + + /// + /// 1-based episode index at the source + /// + public int EpisodeNumber { get; set; } + + /// + /// Subdeck this episode's text lives in + /// + public int ChildDeckId { get; set; } + + [MaxLength(500)] + public string Title { get; set; } = string.Empty; + + /// + /// Latest publish/revision (改稿) timestamp from the table of contents + /// + public DateTimeOffset? SourceUpdatedAt { get; set; } + + public int CharCount { get; set; } + + public WebNovelSource Source { get; set; } = null!; +} diff --git a/Jiten.Core/Data/WebNovel/WebNovelProvider.cs b/Jiten.Core/Data/WebNovel/WebNovelProvider.cs new file mode 100644 index 00000000..cf4cee71 --- /dev/null +++ b/Jiten.Core/Data/WebNovel/WebNovelProvider.cs @@ -0,0 +1,9 @@ +namespace Jiten.Core.Data.WebNovel; + +public enum WebNovelProvider +{ + Syosetu = 1, + SyosetuNovel18 = 2, + Kakuyomu = 3, + Hameln = 4 +} diff --git a/Jiten.Core/Data/WebNovel/WebNovelSource.cs b/Jiten.Core/Data/WebNovel/WebNovelSource.cs new file mode 100644 index 00000000..058324e2 --- /dev/null +++ b/Jiten.Core/Data/WebNovel/WebNovelSource.cs @@ -0,0 +1,70 @@ +using System.ComponentModel.DataAnnotations; + +namespace Jiten.Core.Data.WebNovel; + +/// +/// One tracked webnovel. Keyed on the parent deck; chapter-range subdecks hang off that deck. +/// +public class WebNovelSource +{ + /// + /// Parent deck holding the chapter-range subdecks + /// + public int DeckId { get; set; } + + public Deck Deck { get; set; } = null!; + + public WebNovelProvider Provider { get; set; } + + /// + /// Provider-specific work id (Syosetu ncode, lowercase) + /// + [MaxLength(64)] + public string SourceId { get; set; } = string.Empty; + + /// + /// Number of episodes ingested so far + /// + public int LastEpisodeCount { get; set; } + + /// + /// Latest episode timestamp reported by the source at the last sync (Narou general_lastup) + /// + public DateTimeOffset? LastSourceUpdate { get; set; } + + public DateTimeOffset? LastSyncedAt { get; set; } + + /// + /// Staggered next poll; also used to back off after failures + /// + public DateTimeOffset NextCheckAt { get; set; } + + public bool SyncEnabled { get; set; } = true; + + /// + /// Finished at the source (Narou end=0) — polled monthly instead of weekly + /// + public bool CompletedAtSource { get; set; } + + /// + /// Serialisation stopped at the source (Narou isstop) + /// + public bool OnHiatusAtSource { get; set; } + + public int ConsecutiveFailures { get; set; } + + [MaxLength(1000)] + public string? LastError { get; set; } + + /// + /// Per-novel override of the subdeck character budget; null uses the global default + /// + public int? ChunkCharBudget { get; set; } + + /// + /// Episodes revised at the source inside already-closed subdecks, awaiting a manual refresh + /// + public int PendingRevisionCount { get; set; } + + public List Chapters { get; set; } = new(); +} diff --git a/Jiten.Core/Extractors/EbookExtractor.cs b/Jiten.Core/Extractors/EbookExtractor.cs index 53bdddcc..79d40a85 100644 --- a/Jiten.Core/Extractors/EbookExtractor.cs +++ b/Jiten.Core/Extractors/EbookExtractor.cs @@ -150,33 +150,7 @@ private async Task ExtractTextFromChapter(string htmlContent) if (body == null) return ""; - foreach (var rubyElement in body.QuerySelectorAll("ruby").ToList()) - { - string baseText = ""; - var rbElements = rubyElement.QuerySelectorAll("rb"); - if (rbElements.Any()) - { - baseText = string.Concat(rbElements.Select(rb => rb.TextContent)); - } - else - { - baseText = string.Concat( - rubyElement.ChildNodes - .Where(cn => cn.NodeType == NodeType.Text || (cn is IElement el && - !el.TagName.Equals("RT", StringComparison.OrdinalIgnoreCase) && - !el.TagName.Equals("RP", StringComparison.OrdinalIgnoreCase))) - .Select(cn => cn.TextContent) - ); - } - - var rtText = string.Concat(rubyElement.QuerySelectorAll("rt").Select(rt => rt.TextContent)).Trim(); - var trimmedBase = baseText.Trim(); - var replacement = !string.IsNullOrEmpty(rtText) && trimmedBase.Length > 0 - ? $"{{{trimmedBase}'{rtText}}}" - : trimmedBase; - - rubyElement.Parent?.ReplaceChild(document.CreateTextNode(replacement), rubyElement); - } + RubyHtmlHelper.InlineRubyAnnotations(body, document); var textNodes = body.Descendants() .Where(n => n.ParentElement != null && diff --git a/Jiten.Core/Extractors/RubyHtmlHelper.cs b/Jiten.Core/Extractors/RubyHtmlHelper.cs new file mode 100644 index 00000000..2a2b150a --- /dev/null +++ b/Jiten.Core/Extractors/RubyHtmlHelper.cs @@ -0,0 +1,36 @@ +using AngleSharp.Dom; + +namespace Jiten.Core; + +public static class RubyHtmlHelper +{ + /// + /// Replaces every <ruby> element under with the parser's inline furigana + /// format ({base'reading}), so readings survive into DeckRawText and feed FuriganaHintExtractor. + /// Handles both <rb>-wrapped bases (epub) and bare text-node bases (Syosetu). + /// + public static void InlineRubyAnnotations(IElement root, IDocument document) + { + foreach (var rubyElement in root.QuerySelectorAll("ruby").ToList()) + { + var rbElements = rubyElement.QuerySelectorAll("rb"); + var baseText = rbElements.Any() + ? string.Concat(rbElements.Select(rb => rb.TextContent)) + : string.Concat(rubyElement.ChildNodes + .Where(cn => cn.NodeType == NodeType.Text || + (cn is IElement el && + !el.TagName.Equals("RT", StringComparison.OrdinalIgnoreCase) && + !el.TagName.Equals("RP", StringComparison.OrdinalIgnoreCase))) + .Select(cn => cn.TextContent)); + + var rtText = string.Concat(rubyElement.QuerySelectorAll("rt").Select(rt => rt.TextContent)).Trim(); + var trimmedBase = baseText.Trim(); + + var replacement = !string.IsNullOrEmpty(rtText) && trimmedBase.Length > 0 + ? $"{{{trimmedBase}'{rtText}}}" + : trimmedBase; + + rubyElement.Parent?.ReplaceChild(document.CreateTextNode(replacement), rubyElement); + } + } +} diff --git a/Jiten.Core/JitenDbContext.cs b/Jiten.Core/JitenDbContext.cs index d6f22614..b98882ab 100644 --- a/Jiten.Core/JitenDbContext.cs +++ b/Jiten.Core/JitenDbContext.cs @@ -1,5 +1,6 @@ using Jiten.Core.Data; using Jiten.Core.Data.JMDict; +using Jiten.Core.Data.WebNovel; using Microsoft.Extensions.Configuration; namespace Jiten.Core; @@ -44,6 +45,9 @@ public class JitenDbContext : DbContext public DbSet DeckRelationships { get; set; } + public DbSet WebNovelSources { get; set; } + public DbSet WebNovelChapters { get; set; } + public DbSet WordSets { get; set; } public DbSet WordSetMembers { get; set; } @@ -210,6 +214,40 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasForeignKey(drt => drt.DeckId); }); + modelBuilder.Entity(entity => + { + entity.ToTable("WebNovelSources", "jiten"); + entity.HasKey(s => s.DeckId); + entity.Property(s => s.DeckId).ValueGeneratedNever(); + + entity.HasOne(s => s.Deck) + .WithOne() + .HasForeignKey(s => s.DeckId) + .OnDelete(DeleteBehavior.Cascade); + + entity.HasIndex(s => new { s.Provider, s.SourceId }) + .IsUnique() + .HasDatabaseName("IX_WebNovelSources_Provider_SourceId"); + + // The sweeper scans for novels due a check + entity.HasIndex(s => new { s.SyncEnabled, s.NextCheckAt }) + .HasDatabaseName("IX_WebNovelSources_SyncEnabled_NextCheckAt"); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("WebNovelChapters", "jiten"); + entity.HasKey(c => new { c.DeckId, c.EpisodeNumber }); + + entity.HasOne(c => c.Source) + .WithMany(s => s.Chapters) + .HasForeignKey(c => c.DeckId) + .OnDelete(DeleteBehavior.Cascade); + + entity.HasIndex(c => c.ChildDeckId) + .HasDatabaseName("IX_WebNovelChapters_ChildDeckId"); + }); + modelBuilder.Entity(entity => { entity.ToTable("DeckStats", "jiten"); diff --git a/Jiten.Core/Migrations/20260711232624_WebNovelDecks.Designer.cs b/Jiten.Core/Migrations/20260711232624_WebNovelDecks.Designer.cs new file mode 100644 index 00000000..f84afb97 --- /dev/null +++ b/Jiten.Core/Migrations/20260711232624_WebNovelDecks.Designer.cs @@ -0,0 +1,2268 @@ +// +using System; +using System.Collections.Generic; +using Jiten.Core; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Jiten.Core.Migrations +{ + [DbContext(typeof(JitenDbContext))] + [Migration("20260711232624_WebNovelDecks")] + partial class WebNovelDecks + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("jiten") + .HasAnnotation("ProductVersion", "9.0.2") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "fuzzystrmatch"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Jiten.Core.Data.BlacklistedComparisonDeck", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)"); + + b.HasKey("Id"); + + b.HasIndex("DeckId"); + + b.HasIndex("UserId", "DeckId") + .IsUnique() + .HasDatabaseName("IX_BlacklistedComparisonDecks_UserDeck"); + + b.ToTable("BlacklistedComparisonDecks", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Deck", b => + { + b.Property("DeckId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("DeckId")); + + b.Property("CharacterCount") + .HasColumnType("integer"); + + b.Property("CoverName") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("DeckOrder") + .HasColumnType("integer"); + + b.Property("Description") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("DialoguePercentage") + .HasColumnType("real"); + + b.Property("Difficulty") + .HasColumnType("real"); + + b.Property("DifficultyOverride") + .HasColumnType("real"); + + b.Property("EnglishTitle") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ExternalRating") + .HasColumnType("smallint"); + + b.Property("HideAverageSentenceLength") + .HasColumnType("boolean"); + + b.Property("HideDialoguePercentage") + .HasColumnType("boolean"); + + b.Property("LastUpdate") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaType") + .HasColumnType("integer"); + + b.Property("OriginalFileName") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("OriginalTitle") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ParentDeckId") + .HasColumnType("integer"); + + b.Property("ReleaseDate") + .HasColumnType("date"); + + b.Property("RomajiTitle") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("SentenceCount") + .HasColumnType("integer"); + + b.Property("SpeechDuration") + .HasColumnType("bigint"); + + b.Property("SpeechMoraCount") + .HasColumnType("bigint"); + + b.Property("UniqueKanjiCount") + .HasColumnType("integer"); + + b.Property("UniqueKanjiUsedOnceCount") + .HasColumnType("integer"); + + b.Property("UniqueWordCount") + .HasColumnType("integer"); + + b.Property("UniqueWordUsedOnceCount") + .HasColumnType("integer"); + + b.Property("WordCount") + .HasColumnType("integer"); + + b.HasKey("DeckId"); + + b.HasIndex("CharacterCount") + .HasDatabaseName("IX_CharacterCount"); + + b.HasIndex("Difficulty") + .HasDatabaseName("IX_Difficulty"); + + b.HasIndex("EnglishTitle") + .HasDatabaseName("IX_EnglishTitle"); + + b.HasIndex("ExternalRating") + .HasDatabaseName("IX_ExternalRating"); + + b.HasIndex("MediaType") + .HasDatabaseName("IX_MediaType"); + + b.HasIndex("OriginalTitle") + .HasDatabaseName("IX_OriginalTitle"); + + b.HasIndex("ReleaseDate") + .HasDatabaseName("IX_ReleaseDate"); + + b.HasIndex("RomajiTitle") + .HasDatabaseName("IX_RomajiTitle"); + + b.HasIndex("UniqueKanjiCount") + .HasDatabaseName("IX_UniqueKanjiCount"); + + b.HasIndex("ParentDeckId", "MediaType") + .HasDatabaseName("IX_ParentDeckId_MediaType"); + + b.ToTable("Decks", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckDictionaryEntry", b => + { + b.Property("DeckDictionaryEntryId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("DeckDictionaryEntryId")); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("EntryType") + .HasColumnType("integer"); + + b.Property("Surface") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.HasKey("DeckDictionaryEntryId"); + + b.HasIndex("DeckId", "Surface") + .IsUnique() + .HasDatabaseName("IX_DeckDictionaryEntry_DeckId_Surface"); + + b.ToTable("DeckDictionaryEntries", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckDifficulty", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("DecilesJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Difficulty") + .HasPrecision(4, 2) + .HasColumnType("numeric(4,2)"); + + b.Property("DistinctVoterCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("EasierVoteCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("HarderVoteCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("LastUpdated") + .HasColumnType("timestamp with time zone"); + + b.Property("NEffective") + .ValueGeneratedOnAdd() + .HasPrecision(6, 2) + .HasColumnType("numeric(6,2)") + .HasDefaultValue(0m); + + b.Property("Peak") + .HasPrecision(4, 2) + .HasColumnType("numeric(4,2)"); + + b.Property("ProgressionJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("UserAdjustment") + .ValueGeneratedOnAdd() + .HasPrecision(4, 2) + .HasColumnType("numeric(4,2)") + .HasDefaultValue(0m); + + b.HasKey("DeckId"); + + b.ToTable("DeckDifficulty", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckEmbedding", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("Signature") + .HasColumnType("bytea"); + + b.Property("Vector") + .IsRequired() + .HasColumnType("bytea"); + + b.HasKey("DeckId"); + + b.ToTable("DeckEmbeddings", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckEmbeddingSpace", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("Components") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("Dimension") + .HasColumnType("integer"); + + b.Property("Idf") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("Mean") + .IsRequired() + .HasColumnType("bytea"); + + b.HasKey("Id"); + + b.ToTable("DeckEmbeddingSpaces", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckGenre", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("Genre") + .HasColumnType("integer"); + + b.HasKey("DeckId", "Genre"); + + b.HasIndex("DeckId") + .HasDatabaseName("IX_DeckGenres_DeckId"); + + b.HasIndex("Genre") + .HasDatabaseName("IX_DeckGenres_Genre"); + + b.ToTable("DeckGenres", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckRawText", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("RawText") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("DeckId"); + + b.HasIndex("DeckId") + .HasDatabaseName("IX_DeckRawText_DeckId"); + + b.ToTable("DeckRawTexts", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckRelationship", b => + { + b.Property("SourceDeckId") + .HasColumnType("integer"); + + b.Property("TargetDeckId") + .HasColumnType("integer"); + + b.Property("RelationshipType") + .HasColumnType("integer"); + + b.HasKey("SourceDeckId", "TargetDeckId", "RelationshipType"); + + b.HasIndex("SourceDeckId") + .HasDatabaseName("IX_DeckRelationships_SourceDeckId"); + + b.HasIndex("TargetDeckId") + .HasDatabaseName("IX_DeckRelationships_TargetDeckId"); + + b.ToTable("DeckRelationships", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckStats", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("ComputedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CoverageCurve") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("CoverageSamplesJson") + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.HasKey("DeckId"); + + b.HasIndex("ComputedAt") + .HasDatabaseName("IX_DeckStats_ComputedAt"); + + b.ToTable("DeckStats", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckTag", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("TagId") + .HasColumnType("integer"); + + b.Property("Percentage") + .HasColumnType("smallint"); + + b.HasKey("DeckId", "TagId"); + + b.HasIndex("Percentage") + .HasDatabaseName("IX_DeckTags_Percentage"); + + b.HasIndex("TagId") + .HasDatabaseName("IX_DeckTags_TagId"); + + b.ToTable("DeckTags", "jiten", t => + { + t.HasCheckConstraint("CK_DeckTags_Percentage", "\"Percentage\" >= 0 AND \"Percentage\" <= 100"); + }); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckTitle", b => + { + b.Property("DeckTitleId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("DeckTitleId")); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TitleNoSpaces") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("text") + .HasComputedColumnSql("REPLACE(\"Title\", ' ', '')", true); + + b.Property("TitleType") + .HasColumnType("integer"); + + b.HasKey("DeckTitleId"); + + b.HasIndex("Title") + .HasDatabaseName("IX_DeckTitles_Title"); + + b.HasIndex("DeckId", "TitleType") + .HasDatabaseName("IX_DeckTitles_DeckId_TitleType"); + + b.ToTable("DeckTitles", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckWord", b => + { + b.Property("DeckWordId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("DeckWordId")); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("Occurrences") + .HasColumnType("integer"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.Property("WordId") + .HasColumnType("integer"); + + b.HasKey("DeckWordId"); + + b.HasIndex("DeckId") + .HasDatabaseName("IX_DeckWords_DeckId_IncWordIdReadingIndexOcc"); + + b.HasIndex("WordId", "ReadingIndex") + .HasDatabaseName("IX_DeckWords_WordId_ReadingIndex_IncDeckIdOcc"); + + b.HasIndex("DeckId", "WordId", "ReadingIndex") + .IsUnique() + .HasDatabaseName("IX_DeckWords_DeckId_WordId_ReadingIndex"); + + b.ToTable("DeckWords", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyRankGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("MediaTypeGroup") + .HasColumnType("integer"); + + b.Property("SortIndex") + .HasColumnType("integer"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "MediaTypeGroup", "SortIndex") + .IsUnique() + .HasDatabaseName("IX_DifficultyRankGroups_UserGroupSort"); + + b.ToTable("DifficultyRankGroups", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyRankItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("GroupId") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)"); + + b.HasKey("Id"); + + b.HasIndex("DeckId"); + + b.HasIndex("GroupId") + .HasDatabaseName("IX_DifficultyRankItems_GroupId"); + + b.HasIndex("UserId", "DeckId") + .IsUnique() + .HasDatabaseName("IX_DifficultyRankItems_UserDeck"); + + b.ToTable("DifficultyRankItems", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyRating", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("Rating") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)"); + + b.HasKey("Id"); + + b.HasIndex("DeckId"); + + b.HasIndex("UserId", "DeckId") + .IsUnique() + .HasDatabaseName("IX_DifficultyRatings_UserDeck"); + + b.ToTable("DifficultyRatings", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyVote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeckHighId") + .HasColumnType("integer"); + + b.Property("DeckLowId") + .HasColumnType("integer"); + + b.Property("IsValid") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.Property("Outcome") + .HasColumnType("integer"); + + b.Property("Source") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)"); + + b.HasKey("Id"); + + b.HasIndex("DeckHighId") + .HasDatabaseName("IX_DifficultyVotes_DeckHighId"); + + b.HasIndex("DeckLowId") + .HasDatabaseName("IX_DifficultyVotes_DeckLowId"); + + b.HasIndex("UserId") + .HasDatabaseName("IX_DifficultyVotes_UserId"); + + b.HasIndex("UserId", "DeckLowId", "DeckHighId") + .IsUnique() + .HasDatabaseName("IX_DifficultyVotes_Unique") + .HasFilter("\"IsValid\" = true"); + + b.ToTable("DifficultyVotes", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExampleSentence", b => + { + b.Property("SentenceId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("SentenceId")); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("Difficulty") + .HasColumnType("real"); + + b.Property("Position") + .HasColumnType("integer"); + + b.Property("Text") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("SentenceId"); + + b.HasIndex("DeckId") + .HasDatabaseName("IX_ExampleSentence_DeckId"); + + b.HasIndex("DeckId", "Difficulty") + .HasDatabaseName("IX_ExampleSentence_DeckId_Difficulty"); + + b.ToTable("ExampleSentences", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExampleSentenceWord", b => + { + b.Property("ExampleSentenceId") + .HasColumnType("integer"); + + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("Position") + .HasColumnType("smallint"); + + b.Property("Length") + .HasColumnType("smallint"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.HasKey("ExampleSentenceId", "WordId", "Position"); + + b.HasIndex("WordId", "ReadingIndex") + .HasDatabaseName("IX_ExampleSentenceWord_WordIdReadingIndex"); + + b.ToTable("ExampleSentenceWords", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExternalGenreMapping", b => + { + b.Property("ExternalGenreMappingId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("ExternalGenreMappingId")); + + b.Property("ExternalGenreName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("JitenGenre") + .HasColumnType("integer"); + + b.Property("Provider") + .HasColumnType("integer"); + + b.HasKey("ExternalGenreMappingId"); + + b.HasIndex("Provider", "ExternalGenreName") + .HasDatabaseName("IX_ExternalGenreMapping_Provider_ExternalName"); + + b.HasIndex("Provider", "ExternalGenreName", "JitenGenre") + .IsUnique() + .HasDatabaseName("IX_ExternalGenreMapping_Provider_ExternalName_JitenGenre"); + + b.ToTable("ExternalGenreMappings", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExternalTagMapping", b => + { + b.Property("ExternalTagMappingId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("ExternalTagMappingId")); + + b.Property("ExternalTagName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Provider") + .HasColumnType("integer"); + + b.Property("TagId") + .HasColumnType("integer"); + + b.HasKey("ExternalTagMappingId"); + + b.HasIndex("TagId"); + + b.HasIndex("Provider", "ExternalTagName") + .HasDatabaseName("IX_ExternalTagMapping_Provider_ExternalName"); + + b.HasIndex("Provider", "ExternalTagName", "TagId") + .IsUnique() + .HasDatabaseName("IX_ExternalTagMapping_Provider_ExternalName_TagId"); + + b.ToTable("ExternalTagMappings", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictCrossReference", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("FromSenseIndex") + .HasColumnType("integer"); + + b.Property("FromWordId") + .HasColumnType("integer"); + + b.Property("RawText") + .IsRequired() + .HasColumnType("text"); + + b.Property("TargetDict") + .HasColumnType("int"); + + b.Property("TargetKanji") + .HasColumnType("text"); + + b.Property("TargetReading") + .HasColumnType("text"); + + b.Property("TargetSenseIndex") + .HasColumnType("smallint"); + + b.Property("TargetWordId") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("TargetWordId") + .HasDatabaseName("IX_CrossReferences_TargetWordId"); + + b.HasIndex("FromWordId", "Type") + .HasDatabaseName("IX_CrossReferences_FromWordId_Type"); + + b.ToTable("CrossReferences", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictDefinition", b => + { + b.Property("DefinitionId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("DefinitionId")); + + b.PrimitiveCollection>("Dial") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text[]") + .HasDefaultValueSql("'{}'"); + + b.PrimitiveCollection>("EnglishMeanings") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection>("Field") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text[]") + .HasDefaultValueSql("'{}'"); + + b.PrimitiveCollection>("GlossTypes") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text[]") + .HasDefaultValueSql("'{}'"); + + b.Property("IsActiveInLatestSource") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.PrimitiveCollection>("Misc") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text[]") + .HasDefaultValueSql("'{}'"); + + b.PrimitiveCollection>("PartsOfSpeech") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection>("Pos") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text[]") + .HasDefaultValueSql("'{}'"); + + b.PrimitiveCollection>("RestrictedToReadingIndices") + .HasColumnType("smallint[]"); + + b.Property("SenseIndex") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.PrimitiveCollection>("SenseInfo") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text[]") + .HasDefaultValueSql("'{}'"); + + b.Property("WordId") + .HasColumnType("integer"); + + b.HasKey("DefinitionId"); + + b.HasIndex("WordId", "SenseIndex") + .HasDatabaseName("IX_Definitions_WordId_SenseIndex"); + + b.ToTable("Definitions", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictLookup", b => + { + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("LookupKey") + .HasColumnType("text"); + + b.HasKey("WordId", "LookupKey"); + + b.ToTable("Lookups", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWord", b => + { + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("EntryInfoJson") + .HasColumnType("jsonb"); + + b.Property("LanguageSourcesJson") + .HasColumnType("jsonb"); + + b.Property("Origin") + .HasColumnType("int"); + + b.PrimitiveCollection>("PartsOfSpeech") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection>("PitchAccents") + .HasColumnType("int[]"); + + b.PrimitiveCollection>("Priorities") + .HasColumnType("text[]"); + + b.HasKey("WordId"); + + b.ToTable("Words", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWordComposition", b => + { + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.Property("Position") + .HasColumnType("smallint"); + + b.Property("ComponentReadingIndex") + .HasColumnType("smallint"); + + b.Property("ComponentSurface") + .IsRequired() + .HasColumnType("text"); + + b.Property("ComponentWordId") + .HasColumnType("integer"); + + b.HasKey("WordId", "ReadingIndex", "Position"); + + b.HasIndex("ComponentWordId") + .HasDatabaseName("IX_WordComposition_ComponentWordId"); + + b.HasIndex("WordId", "ReadingIndex") + .HasDatabaseName("IX_WordComposition_WordId_ReadingIndex"); + + b.ToTable("WordCompositions", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWordForm", b => + { + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.Property("FormType") + .HasColumnType("smallint"); + + b.PrimitiveCollection>("InfoTags") + .HasColumnType("text[]"); + + b.Property("IsActiveInLatestSource") + .HasColumnType("boolean"); + + b.Property("IsNoKanji") + .HasColumnType("boolean"); + + b.Property("IsObsolete") + .HasColumnType("boolean"); + + b.Property("IsSearchOnly") + .HasColumnType("boolean"); + + b.PrimitiveCollection>("Priorities") + .HasColumnType("text[]"); + + b.Property("RubyText") + .IsRequired() + .HasColumnType("text"); + + b.Property("Text") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("WordId", "ReadingIndex"); + + b.HasIndex("WordId") + .HasDatabaseName("IX_WordForms_WordId"); + + b.HasIndex("WordId", "FormType", "Text") + .IsUnique() + .HasDatabaseName("IX_WordForms_WordId_FormType_Text"); + + b.ToTable("WordForms", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWordFormFrequency", b => + { + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.Property("FrequencyPercentage") + .ValueGeneratedOnAdd() + .HasColumnType("double precision") + .HasDefaultValue(0.0); + + b.Property("FrequencyRank") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("ObservedFrequency") + .ValueGeneratedOnAdd() + .HasColumnType("double precision") + .HasDefaultValue(0.0); + + b.Property("UsedInMediaAmount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.HasKey("WordId", "ReadingIndex"); + + b.HasIndex("FrequencyRank") + .HasDatabaseName("IX_WordFormFrequencies_FrequencyRank"); + + b.ToTable("WordFormFrequencies", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWordFrequency", b => + { + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("FrequencyRank") + .HasColumnType("integer"); + + b.Property("ObservedFrequency") + .HasColumnType("double precision"); + + b.Property("UsedInMediaAmount") + .HasColumnType("integer"); + + b.HasKey("WordId"); + + b.ToTable("WordFrequencies", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.Kanji", b => + { + b.Property("Character") + .HasColumnType("text"); + + b.Property("FrequencyRank") + .HasColumnType("integer"); + + b.Property("Grade") + .HasColumnType("smallint"); + + b.Property("JlptLevel") + .HasColumnType("smallint"); + + b.PrimitiveCollection>("KunReadings") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection>("Meanings") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection>("OnReadings") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("StrokeCount") + .HasColumnType("smallint"); + + b.HasKey("Character"); + + b.HasIndex("FrequencyRank") + .HasDatabaseName("IX_Kanji_FrequencyRank"); + + b.HasIndex("JlptLevel") + .HasDatabaseName("IX_Kanji_JlptLevel"); + + b.HasIndex("StrokeCount") + .HasDatabaseName("IX_Kanji_StrokeCount"); + + b.ToTable("Kanji", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.KanjiReadingWord", b => + { + b.Property("KanjiCharacter") + .HasColumnType("text"); + + b.Property("Reading") + .HasColumnType("text"); + + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.HasKey("KanjiCharacter", "Reading", "WordId", "ReadingIndex"); + + b.HasIndex("WordId") + .HasDatabaseName("IX_KanjiReadingWords_WordId"); + + b.HasIndex("KanjiCharacter", "Reading") + .HasDatabaseName("IX_KanjiReadingWords_KanjiCharacter_Reading"); + + b.ToTable("KanjiReadingWords", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.WordKanji", b => + { + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.Property("KanjiCharacter") + .HasColumnType("text"); + + b.Property("Position") + .HasColumnType("smallint"); + + b.HasKey("WordId", "ReadingIndex", "KanjiCharacter", "Position"); + + b.HasIndex("KanjiCharacter") + .HasDatabaseName("IX_WordKanji_KanjiCharacter"); + + b.HasIndex("WordId", "ReadingIndex") + .HasDatabaseName("IX_WordKanji_WordId_ReadingIndex"); + + b.ToTable("WordKanji", "jmdict"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Link", b => + { + b.Property("LinkId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("LinkId")); + + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("LinkType") + .HasColumnType("integer"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("LinkId"); + + b.HasIndex("DeckId"); + + b.ToTable("Links", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequest", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdminNote") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("ExternalLinkType") + .HasColumnType("integer"); + + b.Property("ExternalUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("FulfilledDeckId") + .HasColumnType("integer"); + + b.Property("MediaType") + .HasColumnType("integer"); + + b.Property("RequesterId") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UpvoteCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.HasKey("Id"); + + b.HasIndex("FulfilledDeckId"); + + b.HasIndex("MediaType") + .HasDatabaseName("IX_MediaRequest_MediaType"); + + b.HasIndex("RequesterId") + .HasDatabaseName("IX_MediaRequest_RequesterId"); + + b.HasIndex("Title") + .HasDatabaseName("IX_MediaRequest_Title"); + + b.HasIndex("Status", "CreatedAt") + .IsDescending(false, true) + .HasDatabaseName("IX_MediaRequest_Status_CreatedAt"); + + b.HasIndex("Status", "UpvoteCount") + .IsDescending(false, true) + .HasDatabaseName("IX_MediaRequest_Status_UpvoteCount"); + + b.ToTable("MediaRequests", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestComment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaRequestId") + .HasColumnType("integer"); + + b.Property("ParentCommentId") + .HasColumnType("integer"); + + b.Property("Text") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("MediaRequestId") + .HasDatabaseName("IX_MediaRequestComment_MediaRequestId"); + + b.HasIndex("ParentCommentId") + .HasDatabaseName("IX_MediaRequestComment_ParentCommentId"); + + b.HasIndex("UserId") + .HasDatabaseName("IX_MediaRequestComment_UserId"); + + b.ToTable("MediaRequestComments", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestSubscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaRequestId") + .HasColumnType("integer"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("MediaRequestId", "UserId") + .IsUnique() + .HasDatabaseName("IX_MediaRequestSubscription_RequestId_UserId"); + + b.ToTable("MediaRequestSubscriptions", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestUpload", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdminNote") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("AdminReviewed") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FileDeleted") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("FileSize") + .HasColumnType("bigint"); + + b.Property("MediaRequestCommentId") + .HasColumnType("integer"); + + b.Property("MediaRequestId") + .HasColumnType("integer"); + + b.Property("OriginalFileCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("StoragePath") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Id"); + + b.HasIndex("MediaRequestCommentId") + .IsUnique() + .HasDatabaseName("IX_MediaRequestUpload_CommentId"); + + b.HasIndex("MediaRequestId") + .HasDatabaseName("IX_MediaRequestUpload_RequestId"); + + b.ToTable("MediaRequestUploads", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestUpvote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaRequestId") + .HasColumnType("integer"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("MediaRequestId", "UserId") + .IsUnique() + .HasDatabaseName("IX_MediaRequestUpvote_RequestId_UserId"); + + b.ToTable("MediaRequestUpvotes", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Notification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsRead") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false); + + b.Property("LinkUrl") + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.Property("Message") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("ReadAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedAt") + .HasDatabaseName("IX_Notification_CreatedAt"); + + b.HasIndex("UserId", "IsRead", "CreatedAt") + .IsDescending(false, false, true) + .HasDatabaseName("IX_Notification_UserId_IsRead_CreatedAt"); + + b.ToTable("Notifications", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.RequestActivityLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Detail") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("IpAddress") + .HasMaxLength(45) + .HasColumnType("character varying(45)"); + + b.Property("MediaRequestId") + .HasColumnType("integer"); + + b.Property("TargetUserId") + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("Action", "CreatedAt") + .HasDatabaseName("IX_RequestActivityLog_Action_CreatedAt"); + + b.HasIndex("MediaRequestId", "CreatedAt") + .HasDatabaseName("IX_RequestActivityLog_RequestId_CreatedAt"); + + b.HasIndex("UserId", "CreatedAt") + .HasDatabaseName("IX_RequestActivityLog_UserId_CreatedAt"); + + b.ToTable("RequestActivityLogs", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.SkippedComparison", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeckHighId") + .HasColumnType("integer"); + + b.Property("DeckLowId") + .HasColumnType("integer"); + + b.Property("Permanent") + .HasColumnType("boolean"); + + b.Property("UserId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "DeckLowId", "DeckHighId") + .IsUnique() + .HasDatabaseName("IX_SkippedComparisons_UserPair"); + + b.ToTable("SkippedComparisons", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Tag", b => + { + b.Property("TagId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("TagId")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.HasKey("TagId"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("IX_Tags_Name"); + + b.ToTable("Tags", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelChapter", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("EpisodeNumber") + .HasColumnType("integer"); + + b.Property("CharCount") + .HasColumnType("integer"); + + b.Property("ChildDeckId") + .HasColumnType("integer"); + + b.Property("SourceUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("DeckId", "EpisodeNumber"); + + b.HasIndex("ChildDeckId") + .HasDatabaseName("IX_WebNovelChapters_ChildDeckId"); + + b.ToTable("WebNovelChapters", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelSource", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("ChunkCharBudget") + .HasColumnType("integer"); + + b.Property("CompletedAtSource") + .HasColumnType("boolean"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer"); + + b.Property("LastEpisodeCount") + .HasColumnType("integer"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("LastSourceUpdate") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSyncedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("NextCheckAt") + .HasColumnType("timestamp with time zone"); + + b.Property("OnHiatusAtSource") + .HasColumnType("boolean"); + + b.Property("PendingRevisionCount") + .HasColumnType("integer"); + + b.Property("Provider") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SyncEnabled") + .HasColumnType("boolean"); + + b.HasKey("DeckId"); + + b.HasIndex("Provider", "SourceId") + .IsUnique() + .HasDatabaseName("IX_WebNovelSources_Provider_SourceId"); + + b.HasIndex("SyncEnabled", "NextCheckAt") + .HasDatabaseName("IX_WebNovelSources_SyncEnabled_NextCheckAt"); + + b.ToTable("WebNovelSources", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WordSet", b => + { + b.Property("SetId") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("SetId")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("WordCount") + .HasColumnType("integer"); + + b.HasKey("SetId"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("IX_WordSet_Slug"); + + b.ToTable("WordSets", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WordSetMember", b => + { + b.Property("SetId") + .HasColumnType("integer"); + + b.Property("WordId") + .HasColumnType("integer"); + + b.Property("ReadingIndex") + .HasColumnType("smallint"); + + b.Property("Position") + .HasColumnType("integer"); + + b.HasKey("SetId", "WordId", "ReadingIndex"); + + b.HasIndex("WordId", "ReadingIndex") + .HasDatabaseName("IX_WordSetMember_WordId_ReadingIndex"); + + b.ToTable("WordSetMembers", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.BlacklistedComparisonDeck", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany() + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Deck", b => + { + b.HasOne("Jiten.Core.Data.Deck", "ParentDeck") + .WithMany("Children") + .HasForeignKey("ParentDeckId") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("ParentDeck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckDictionaryEntry", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany("DictionaryEntries") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckDifficulty", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithOne("DeckDifficulty") + .HasForeignKey("Jiten.Core.Data.DeckDifficulty", "DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckEmbedding", b => + { + b.HasOne("Jiten.Core.Data.Deck", null) + .WithOne() + .HasForeignKey("Jiten.Core.Data.DeckEmbedding", "DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckGenre", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany("DeckGenres") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckRawText", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithOne("RawText") + .HasForeignKey("Jiten.Core.Data.DeckRawText", "DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckRelationship", b => + { + b.HasOne("Jiten.Core.Data.Deck", "SourceDeck") + .WithMany("RelationshipsAsSource") + .HasForeignKey("SourceDeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("FK_DeckRelationships_Decks_SourceDeckId"); + + b.HasOne("Jiten.Core.Data.Deck", "TargetDeck") + .WithMany("RelationshipsAsTarget") + .HasForeignKey("TargetDeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("FK_DeckRelationships_Decks_TargetDeckId"); + + b.Navigation("SourceDeck"); + + b.Navigation("TargetDeck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckStats", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithOne("DeckStats") + .HasForeignKey("Jiten.Core.Data.DeckStats", "DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckTag", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany("DeckTags") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.Tag", "Tag") + .WithMany("DeckTags") + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + + b.Navigation("Tag"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckTitle", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany("Titles") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DeckWord", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany("DeckWords") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyRankItem", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany() + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.DifficultyRankGroup", "Group") + .WithMany("Items") + .HasForeignKey("GroupId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyRating", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany() + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyVote", b => + { + b.HasOne("Jiten.Core.Data.Deck", "DeckHigh") + .WithMany() + .HasForeignKey("DeckHighId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.Deck", "DeckLow") + .WithMany() + .HasForeignKey("DeckLowId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DeckHigh"); + + b.Navigation("DeckLow"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExampleSentence", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany("ExampleSentences") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExampleSentenceWord", b => + { + b.HasOne("Jiten.Core.Data.ExampleSentence", "ExampleSentence") + .WithMany("Words") + .HasForeignKey("ExampleSentenceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", "Word") + .WithMany() + .HasForeignKey("WordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ExampleSentence"); + + b.Navigation("Word"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExternalTagMapping", b => + { + b.HasOne("Jiten.Core.Data.Tag", "Tag") + .WithMany() + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Tag"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictDefinition", b => + { + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", null) + .WithMany("Definitions") + .HasForeignKey("WordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictLookup", b => + { + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", null) + .WithMany("Lookups") + .HasForeignKey("WordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWordComposition", b => + { + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", "Component") + .WithMany() + .HasForeignKey("ComponentWordId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", "Word") + .WithMany() + .HasForeignKey("WordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Component"); + + b.Navigation("Word"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWordForm", b => + { + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", null) + .WithMany("Forms") + .HasForeignKey("WordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWordFrequency", b => + { + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", null) + .WithMany() + .HasForeignKey("WordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.WordKanji", b => + { + b.HasOne("Jiten.Core.Data.JMDict.Kanji", "Kanji") + .WithMany("WordKanjis") + .HasForeignKey("KanjiCharacter") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.JMDict.JmDictWord", "Word") + .WithMany() + .HasForeignKey("WordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Kanji"); + + b.Navigation("Word"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Link", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithMany("Links") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequest", b => + { + b.HasOne("Jiten.Core.Data.Deck", "FulfilledDeck") + .WithMany() + .HasForeignKey("FulfilledDeckId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("FulfilledDeck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestComment", b => + { + b.HasOne("Jiten.Core.Data.MediaRequest", "MediaRequest") + .WithMany("Comments") + .HasForeignKey("MediaRequestId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.MediaRequestComment", "ParentComment") + .WithMany("AdminComments") + .HasForeignKey("ParentCommentId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("MediaRequest"); + + b.Navigation("ParentComment"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestSubscription", b => + { + b.HasOne("Jiten.Core.Data.MediaRequest", "MediaRequest") + .WithMany("Subscriptions") + .HasForeignKey("MediaRequestId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MediaRequest"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestUpload", b => + { + b.HasOne("Jiten.Core.Data.MediaRequestComment", "Comment") + .WithOne("Upload") + .HasForeignKey("Jiten.Core.Data.MediaRequestUpload", "MediaRequestCommentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Jiten.Core.Data.MediaRequest", "MediaRequest") + .WithMany("Uploads") + .HasForeignKey("MediaRequestId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Comment"); + + b.Navigation("MediaRequest"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestUpvote", b => + { + b.HasOne("Jiten.Core.Data.MediaRequest", "MediaRequest") + .WithMany("Upvotes") + .HasForeignKey("MediaRequestId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MediaRequest"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelChapter", b => + { + b.HasOne("Jiten.Core.Data.WebNovel.WebNovelSource", "Source") + .WithMany("Chapters") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelSource", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithOne() + .HasForeignKey("Jiten.Core.Data.WebNovel.WebNovelSource", "DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WordSetMember", b => + { + b.HasOne("Jiten.Core.Data.WordSet", "Set") + .WithMany("Members") + .HasForeignKey("SetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Set"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Deck", b => + { + b.Navigation("Children"); + + b.Navigation("DeckDifficulty"); + + b.Navigation("DeckGenres"); + + b.Navigation("DeckStats"); + + b.Navigation("DeckTags"); + + b.Navigation("DeckWords"); + + b.Navigation("DictionaryEntries"); + + b.Navigation("ExampleSentences"); + + b.Navigation("Links"); + + b.Navigation("RawText"); + + b.Navigation("RelationshipsAsSource"); + + b.Navigation("RelationshipsAsTarget"); + + b.Navigation("Titles"); + }); + + modelBuilder.Entity("Jiten.Core.Data.DifficultyRankGroup", b => + { + b.Navigation("Items"); + }); + + modelBuilder.Entity("Jiten.Core.Data.ExampleSentence", b => + { + b.Navigation("Words"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.JmDictWord", b => + { + b.Navigation("Definitions"); + + b.Navigation("Forms"); + + b.Navigation("Lookups"); + }); + + modelBuilder.Entity("Jiten.Core.Data.JMDict.Kanji", b => + { + b.Navigation("WordKanjis"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequest", b => + { + b.Navigation("Comments"); + + b.Navigation("Subscriptions"); + + b.Navigation("Uploads"); + + b.Navigation("Upvotes"); + }); + + modelBuilder.Entity("Jiten.Core.Data.MediaRequestComment", b => + { + b.Navigation("AdminComments"); + + b.Navigation("Upload"); + }); + + modelBuilder.Entity("Jiten.Core.Data.Tag", b => + { + b.Navigation("DeckTags"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelSource", b => + { + b.Navigation("Chapters"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WordSet", b => + { + b.Navigation("Members"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Jiten.Core/Migrations/20260711232624_WebNovelDecks.cs b/Jiten.Core/Migrations/20260711232624_WebNovelDecks.cs new file mode 100644 index 00000000..a4f8a8c4 --- /dev/null +++ b/Jiten.Core/Migrations/20260711232624_WebNovelDecks.cs @@ -0,0 +1,102 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Jiten.Core.Migrations +{ + /// + public partial class WebNovelDecks : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "WebNovelSources", + schema: "jiten", + columns: table => new + { + DeckId = table.Column(type: "integer", nullable: false), + Provider = table.Column(type: "integer", nullable: false), + SourceId = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + LastEpisodeCount = table.Column(type: "integer", nullable: false), + LastSourceUpdate = table.Column(type: "timestamp with time zone", nullable: true), + LastSyncedAt = table.Column(type: "timestamp with time zone", nullable: true), + NextCheckAt = table.Column(type: "timestamp with time zone", nullable: false), + SyncEnabled = table.Column(type: "boolean", nullable: false), + CompletedAtSource = table.Column(type: "boolean", nullable: false), + OnHiatusAtSource = table.Column(type: "boolean", nullable: false), + ConsecutiveFailures = table.Column(type: "integer", nullable: false), + LastError = table.Column(type: "character varying(1000)", maxLength: 1000, nullable: true), + ChunkCharBudget = table.Column(type: "integer", nullable: true), + PendingRevisionCount = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_WebNovelSources", x => x.DeckId); + table.ForeignKey( + name: "FK_WebNovelSources_Decks_DeckId", + column: x => x.DeckId, + principalSchema: "jiten", + principalTable: "Decks", + principalColumn: "DeckId", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "WebNovelChapters", + schema: "jiten", + columns: table => new + { + DeckId = table.Column(type: "integer", nullable: false), + EpisodeNumber = table.Column(type: "integer", nullable: false), + ChildDeckId = table.Column(type: "integer", nullable: false), + Title = table.Column(type: "character varying(500)", maxLength: 500, nullable: false), + SourceUpdatedAt = table.Column(type: "timestamp with time zone", nullable: true), + CharCount = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_WebNovelChapters", x => new { x.DeckId, x.EpisodeNumber }); + table.ForeignKey( + name: "FK_WebNovelChapters_WebNovelSources_DeckId", + column: x => x.DeckId, + principalSchema: "jiten", + principalTable: "WebNovelSources", + principalColumn: "DeckId", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_WebNovelChapters_ChildDeckId", + schema: "jiten", + table: "WebNovelChapters", + column: "ChildDeckId"); + + migrationBuilder.CreateIndex( + name: "IX_WebNovelSources_Provider_SourceId", + schema: "jiten", + table: "WebNovelSources", + columns: new[] { "Provider", "SourceId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_WebNovelSources_SyncEnabled_NextCheckAt", + schema: "jiten", + table: "WebNovelSources", + columns: new[] { "SyncEnabled", "NextCheckAt" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "WebNovelChapters", + schema: "jiten"); + + migrationBuilder.DropTable( + name: "WebNovelSources", + schema: "jiten"); + } + } +} diff --git a/Jiten.Core/Migrations/JitenDbContextModelSnapshot.cs b/Jiten.Core/Migrations/JitenDbContextModelSnapshot.cs index 1497882c..9713a624 100644 --- a/Jiten.Core/Migrations/JitenDbContextModelSnapshot.cs +++ b/Jiten.Core/Migrations/JitenDbContextModelSnapshot.cs @@ -1603,6 +1603,95 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("Tags", "jiten"); }); + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelChapter", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("EpisodeNumber") + .HasColumnType("integer"); + + b.Property("CharCount") + .HasColumnType("integer"); + + b.Property("ChildDeckId") + .HasColumnType("integer"); + + b.Property("SourceUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("DeckId", "EpisodeNumber"); + + b.HasIndex("ChildDeckId") + .HasDatabaseName("IX_WebNovelChapters_ChildDeckId"); + + b.ToTable("WebNovelChapters", "jiten"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelSource", b => + { + b.Property("DeckId") + .HasColumnType("integer"); + + b.Property("ChunkCharBudget") + .HasColumnType("integer"); + + b.Property("CompletedAtSource") + .HasColumnType("boolean"); + + b.Property("ConsecutiveFailures") + .HasColumnType("integer"); + + b.Property("LastEpisodeCount") + .HasColumnType("integer"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("LastSourceUpdate") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSyncedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("NextCheckAt") + .HasColumnType("timestamp with time zone"); + + b.Property("OnHiatusAtSource") + .HasColumnType("boolean"); + + b.Property("PendingRevisionCount") + .HasColumnType("integer"); + + b.Property("Provider") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SyncEnabled") + .HasColumnType("boolean"); + + b.HasKey("DeckId"); + + b.HasIndex("Provider", "SourceId") + .IsUnique() + .HasDatabaseName("IX_WebNovelSources_Provider_SourceId"); + + b.HasIndex("SyncEnabled", "NextCheckAt") + .HasDatabaseName("IX_WebNovelSources_SyncEnabled_NextCheckAt"); + + b.ToTable("WebNovelSources", "jiten"); + }); + modelBuilder.Entity("Jiten.Core.Data.WordSet", b => { b.Property("SetId") @@ -2052,6 +2141,28 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("MediaRequest"); }); + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelChapter", b => + { + b.HasOne("Jiten.Core.Data.WebNovel.WebNovelSource", "Source") + .WithMany("Chapters") + .HasForeignKey("DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelSource", b => + { + b.HasOne("Jiten.Core.Data.Deck", "Deck") + .WithOne() + .HasForeignKey("Jiten.Core.Data.WebNovel.WebNovelSource", "DeckId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Deck"); + }); + modelBuilder.Entity("Jiten.Core.Data.WordSetMember", b => { b.HasOne("Jiten.Core.Data.WordSet", "Set") @@ -2139,6 +2250,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("DeckTags"); }); + modelBuilder.Entity("Jiten.Core.Data.WebNovel.WebNovelSource", b => + { + b.Navigation("Chapters"); + }); + modelBuilder.Entity("Jiten.Core.Data.WordSet", b => { b.Navigation("Members"); diff --git a/Jiten.Core/WebNovel/IWebNovelSource.cs b/Jiten.Core/WebNovel/IWebNovelSource.cs new file mode 100644 index 00000000..e6e1531e --- /dev/null +++ b/Jiten.Core/WebNovel/IWebNovelSource.cs @@ -0,0 +1,33 @@ +using Jiten.Core.Data.WebNovel; + +namespace Jiten.Core.WebNovel; + +/// +/// A webnovel site. Implementations are responsible for their own politeness throttling — callers may +/// fetch in a tight loop, so the rate limit must hold inside the source, not at the call site. +/// +public interface IWebNovelSource +{ + WebNovelProvider Provider { get; } + + Task GetInfoAsync(string sourceId, CancellationToken ct = default); + + /// + /// Full table of contents, ordered by episode number. One-shots return a single entry. + /// + Task> GetTocAsync(string sourceId, CancellationToken ct = default); + + /// + /// Episode body, ruby inlined as {base'reading}. + /// + Task GetEpisodeTextAsync(string sourceId, WebNovelEpisodeRef episode, CancellationToken ct = default); +} + +/// +/// A source whose API can report the update state of many works at once. The sweeper uses this to poll +/// every tracked novel in a couple of requests instead of one table-of-contents fetch each. +/// +public interface IBatchPollableSource +{ + Task> BatchPollAsync(IEnumerable sourceIds, CancellationToken ct = default); +} diff --git a/Jiten.Core/WebNovel/SubdeckChunker.cs b/Jiten.Core/WebNovel/SubdeckChunker.cs new file mode 100644 index 00000000..7dc91492 --- /dev/null +++ b/Jiten.Core/WebNovel/SubdeckChunker.cs @@ -0,0 +1,227 @@ +using System.Text.RegularExpressions; + +namespace Jiten.Core.WebNovel; + +/// +/// An episode about to be placed into a subdeck. +/// +public record ChunkEpisode(int Number, string Title, int CharCount); + +/// +/// A subdeck that already exists. Only the last one can still grow. +/// +public record ExistingChunk(int ChunkIndex, int ChildDeckId, int StartEpisode, int EndEpisode, int EpisodeCount, int CharCount); + +/// +/// One subdeck to create or extend. is null when the subdeck doesn't exist yet. +/// +public record ChunkPlan +{ + public int ChunkIndex { get; init; } + public int? ChildDeckId { get; init; } + public List EpisodesToAppend { get; init; } = new(); + + /// + /// Episode range of the subdeck once this plan is applied, counting episodes already in it + /// + public int StartEpisode { get; init; } + public int EndEpisode { get; init; } + + public bool IsNew => ChildDeckId == null; + + public string Title => SubdeckChunker.BuildTitle(StartEpisode, EndEpisode); +} + +public static partial class SubdeckChunker +{ + /// + /// Roughly a light-novel volume, matching the size of existing per-volume novel decks + /// + public const int DefaultCharBudget = 150_000; + + /// + /// Backstop for works with micro-chapters, which would otherwise never reach the budget + /// + public const int MaxEpisodesPerChunk = 150; + + /// + /// Subdecks prefer to end on a round episode number — 第1話〜第60話 rather than 第1話〜第57話 — so the + /// split lands where a reader would expect it. Steps are tried largest first, and one is only used when + /// the subdeck holds at least twice its size, which keeps the rounding a small nudge rather than a + /// wholesale redraw of the boundary. Works with very long chapters fall through and split at the budget. + /// + private static readonly int[] BoundarySteps = [10, 5]; + + /// + /// Assigns episodes to subdecks by character budget, splitting on a round episode number where possible. + /// + /// Only the last subdeck is ever appended to; once it reaches the budget it closes and a new one opens. + /// Closed subdecks are immutable ranges, so no episode ever moves between subdecks and user progress + /// (SRS, coverage, known-word marks) stays stable. + /// + public static List Plan(IReadOnlyList existing, + IReadOnlyList newEpisodes, + int charBudget = DefaultCharBudget, + int maxEpisodesPerChunk = MaxEpisodesPerChunk) + { + if (newEpisodes.Count == 0) + return []; + + var plans = new List(); + var lastChunk = existing.OrderBy(c => c.ChunkIndex).LastOrDefault(); + var nextIndex = (lastChunk?.ChunkIndex ?? 0) + 1; + var episodes = newEpisodes.OrderBy(e => e.Number).ToList(); + + // The open subdeck is simply the last one — unless it is already full, in which case we open a new one + var open = lastChunk != null && !IsFull(lastChunk.CharCount, lastChunk.EpisodeCount, charBudget, maxEpisodesPerChunk) + ? new OpenChunk(lastChunk.ChunkIndex, lastChunk.ChildDeckId, lastChunk.StartEpisode, + lastChunk.EndEpisode, lastChunk.EpisodeCount, lastChunk.CharCount) + : null; + + var index = 0; + while (index < episodes.Count) + { + open ??= new OpenChunk(nextIndex, null, episodes[index].Number, sealedEnd: null, 0, 0); + + open.Append(episodes[index]); + index++; + + if (!IsFull(open.CharCount, open.EpisodeCount, charBudget, maxEpisodesPerChunk)) + continue; + + open.TargetEnd ??= PreferredEnd(open); + + // The round boundary is still ahead of us: keep the subdeck open and grow into it + if (open.LastEpisode < open.TargetEnd) + continue; + + // Rounding down hands the overshooting episodes back to the next subdeck + index -= open.TruncateAfter(open.TargetEnd.Value); + + if (open.Appended.Count > 0) + plans.Add(open.ToPlan()); + + if (open.ChildDeckId == null && open.Appended.Count > 0) + nextIndex++; + + open = null; + } + + // A partially filled subdeck stays open for the next sync + if (open is { Appended.Count: > 0 }) + plans.Add(open.ToPlan()); + + return plans; + } + + /// + /// The episode to split on, given the budget ran out at . Rounds to the + /// nearest round number, but never before episodes the subdeck already holds: those are parsed and + /// published, so the boundary can only ever move forward over them. + /// + private static int PreferredEnd(OpenChunk open) + { + foreach (var step in BoundarySteps) + { + if (open.EpisodeCount < step * 2) + continue; + + var rounded = (int)Math.Round(open.LastEpisode / (double)step, MidpointRounding.AwayFromZero) * step; + + // Rounding down would cut episodes the subdeck already published; take the next boundary up instead, + // leaving the subdeck open until the source publishes enough episodes to reach it + if (rounded < open.MinimumEnd) + rounded = (open.MinimumEnd + step - 1) / step * step; + + return rounded; + } + + return open.LastEpisode; + } + + private sealed class OpenChunk(int chunkIndex, int? childDeckId, int startEpisode, int? sealedEnd, int episodeCount, + int charCount) + { + public int? ChildDeckId { get; } = childDeckId; + public int EpisodeCount { get; private set; } = episodeCount; + public int CharCount { get; private set; } = charCount; + public List Appended { get; } = []; + + /// + /// Where this subdeck will close, once decided. Fixed on the first overflow so that growing into the + /// boundary can't keep moving it. + /// + public int? TargetEnd { get; set; } + + public int LastEpisode => Appended.Count > 0 ? Appended[^1].Number : sealedEnd ?? startEpisode; + + /// + /// The earliest episode this subdeck can end on: it must keep everything already written to it, and a + /// brand-new subdeck must still hold its first episode. + /// + public int MinimumEnd => sealedEnd ?? startEpisode; + + public void Append(ChunkEpisode episode) + { + Appended.Add(episode); + EpisodeCount++; + CharCount += episode.CharCount; + } + + /// + /// Drops the episodes past the boundary so the next subdeck can take them, and reports how many were + /// handed back. Only ever touches episodes appended in this pass — keeps the + /// boundary at or beyond what is already on disk. + /// + public int TruncateAfter(int endEpisode) + { + var removed = 0; + + while (Appended.Count > 0 && Appended[^1].Number > endEpisode) + { + CharCount -= Appended[^1].CharCount; + EpisodeCount--; + Appended.RemoveAt(Appended.Count - 1); + removed++; + } + + return removed; + } + + public ChunkPlan ToPlan() => new() + { + ChunkIndex = chunkIndex, + ChildDeckId = ChildDeckId, + EpisodesToAppend = Appended, + StartEpisode = startEpisode, + EndEpisode = Appended[^1].Number + }; + } + + private static bool IsFull(int charCount, int episodeCount, int charBudget, int maxEpisodesPerChunk) => + charCount >= charBudget || episodeCount >= maxEpisodesPerChunk; + + /// + /// Subdeck title, e.g. 第1話〜第60話. The open subdeck is renamed as it grows. + /// + public static string BuildTitle(int startEpisode, int endEpisode) => + startEpisode == endEpisode + ? $"第{startEpisode}話" + : $"第{startEpisode}話〜第{endEpisode}話"; + + /// + /// Length of the text as a reader sees it: inline furigana ({漢字'かんじ}) contributes only its base, + /// and whitespace doesn't count — mirroring how Deck.CharacterCount is measured. + /// + public static int CountCharacters(string annotatedText) + { + if (string.IsNullOrEmpty(annotatedText)) + return 0; + + var stripped = FuriganaAnnotation().Replace(annotatedText, "$1"); + return stripped.Count(c => !char.IsWhiteSpace(c)); + } + + [GeneratedRegex(@"\{([^'{}]+)'([^}]+)\}")] + private static partial Regex FuriganaAnnotation(); +} diff --git a/Jiten.Core/WebNovel/SyosetuSource.cs b/Jiten.Core/WebNovel/SyosetuSource.cs new file mode 100644 index 00000000..2917dbde --- /dev/null +++ b/Jiten.Core/WebNovel/SyosetuSource.cs @@ -0,0 +1,432 @@ +using System.Globalization; +using System.IO.Compression; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; +using Jiten.Core.Data.WebNovel; +using Microsoft.Extensions.Logging; + +namespace Jiten.Core.WebNovel; + +/// +/// 小説家になろう. Metadata comes from the official API; episode text is scraped. +/// Politeness matches narou.rb's shipped defaults (0.7s between requests, a longer pause every 10). +/// +public partial class SyosetuSource : IWebNovelSource, IBatchPollableSource +{ + public const string HttpClientName = "webnovel-syosetu"; + + /// + /// narou.rb download.interval + /// + private static readonly TimeSpan RequestInterval = TimeSpan.FromMilliseconds(700); + + /// + /// narou.rb download.wait-steps: an extra pause every N requests, Syosetu only + /// + private const int WaitStepEvery = 10; + private static readonly TimeSpan WaitStepPause = TimeSpan.FromSeconds(5); + + private const int MaxRetries = 4; + + /// + /// The API caps a batch query at 500 ncodes + /// + public const int BatchPollChunkSize = 500; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + private readonly SemaphoreSlim _throttle = new(1, 1); + private DateTimeOffset _lastRequest = DateTimeOffset.MinValue; + private int _requestCount; + + public WebNovelProvider Provider { get; } + + private bool IsNovel18 => Provider == WebNovelProvider.SyosetuNovel18; + private string Host => IsNovel18 ? "novel18.syosetu.com" : "ncode.syosetu.com"; + private string ApiUrl => IsNovel18 + ? "https://api.syosetu.com/novel18api/api/" + : "https://api.syosetu.com/novelapi/api/"; + + public SyosetuSource(IHttpClientFactory httpClientFactory, ILogger logger, + WebNovelProvider provider = WebNovelProvider.Syosetu) + { + _httpClientFactory = httpClientFactory; + _logger = logger; + Provider = provider; + } + + public async Task GetInfoAsync(string sourceId, CancellationToken ct = default) + { + var results = await QueryApiAsync([sourceId], ct); + + if (!results.TryGetValue(sourceId, out var info)) + throw new InvalidOperationException($"Syosetu API returned no work for ncode {sourceId}."); + + return info; + } + + /// + /// Polls every tracked novel's update state in ceil(N/500) API calls. Update signal is + /// general_lastup + general_all_no, so this never touches an episode page. + /// + public async Task> BatchPollAsync(IEnumerable sourceIds, + CancellationToken ct = default) + { + var all = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var chunk in sourceIds.Distinct(StringComparer.OrdinalIgnoreCase).Chunk(BatchPollChunkSize)) + { + foreach (var (ncode, info) in await QueryApiAsync(chunk, ct)) + all[ncode] = info; + } + + return all; + } + + private async Task> QueryApiAsync(IReadOnlyCollection ncodes, + CancellationToken ct) + { + // gzip=5 and of= keep the response small; docs ask for both on bulk queries + var url = $"{ApiUrl}?out=json&gzip=5&lim={ncodes.Count}" + + "&of=t-n-w-s-g-k-gf-gl-nt-e-ga-l-ir-ist" + + $"&ncode={string.Join('-', ncodes)}"; + + var json = await FetchGzipJsonAsync(url, ct); + + using var doc = JsonDocument.Parse(json); + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + + // First array element is {"allcount":N}; the rest are works + foreach (var element in doc.RootElement.EnumerateArray().Skip(1)) + { + var info = ParseInfo(element); + if (!string.IsNullOrEmpty(info.SourceId)) + results[info.SourceId] = info; + } + + return results; + } + + private WebNovelInfo ParseInfo(JsonElement e) + { + var ncode = (GetString(e, "ncode") ?? string.Empty).ToLowerInvariant(); + + var info = new WebNovelInfo + { + Provider = Provider, + SourceId = ncode, + Url = WebNovelUrlParser.BuildWorkUrl(Provider, ncode), + Title = GetString(e, "title") ?? string.Empty, + Author = GetString(e, "writer"), + Synopsis = GetString(e, "story")?.Trim(), + Genre = GenreLabel(GetInt(e, "genre")), + FirstPublishedAt = ParseJst(GetString(e, "general_firstup")), + LastUpdatedAt = ParseJst(GetString(e, "general_lastup")), + EpisodeCount = GetInt(e, "general_all_no"), + TotalCharacters = GetInt(e, "length"), + // noveltype 2 = 短編: a single page with no table of contents + IsOneShot = GetInt(e, "noveltype") == 2, + // end is 0 for both completed serials and one-shots, 1 while serialising + IsCompleted = GetInt(e, "end") == 0, + IsOnHiatus = GetInt(e, "isstop") == 1, + IsR15 = GetInt(e, "isr15") == 1, + IsAdultOnly = IsNovel18 + }; + + var keywords = GetString(e, "keyword"); + if (!string.IsNullOrWhiteSpace(keywords)) + { + info.Keywords = keywords + .Split([' ', '　'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Distinct() + .ToList(); + } + + // A one-shot always has exactly one page (the API usually reports 1, but normalise defensively) + if (info.IsOneShot) + info.EpisodeCount = 1; + + return info; + } + + public async Task> GetTocAsync(string sourceId, CancellationToken ct = default) + { + var info = await GetInfoAsync(sourceId, ct); + + if (info.IsOneShot) + { + return + [ + new WebNovelEpisodeRef + { + Number = 1, + Title = info.Title, + UpdatedAt = info.LastUpdatedAt, + IsOneShot = true + } + ]; + } + + var episodes = new List(); + var parser = new HtmlParser(); + var page = 1; + + while (true) + { + var url = page == 1 + ? $"https://{Host}/{sourceId}/" + : $"https://{Host}/{sourceId}/?p={page}"; + + var html = await FetchStringAsync(url, ct); + using var document = await parser.ParseDocumentAsync(html, ct); + + var before = episodes.Count; + string? sectionTitle = null; + + // Chapter headings and episode rows are siblings, so walk them in document order + foreach (var node in document.QuerySelectorAll(".p-eplist__chapter-title, .p-eplist__sublist")) + { + if (node.ClassList.Contains("p-eplist__chapter-title")) + { + sectionTitle = node.TextContent.Trim(); + continue; + } + + var link = node.QuerySelector("a.p-eplist__subtitle"); + var href = link?.GetAttribute("href"); + if (link == null || string.IsNullOrEmpty(href)) + continue; + + var number = EpisodeNumberFromHref(href); + if (number == null) + continue; + + episodes.Add(new WebNovelEpisodeRef + { + Number = number.Value, + Title = link.TextContent.Trim(), + UpdatedAt = ParseTocUpdate(node.QuerySelector(".p-eplist__update")), + SectionTitle = sectionTitle + }); + } + + if (episodes.Count == before) + break; + + if (document.QuerySelector("a.c-pager__item--next") == null) + break; + + page++; + } + + return episodes.OrderBy(e => e.Number).ToList(); + } + + public async Task GetEpisodeTextAsync(string sourceId, WebNovelEpisodeRef episode, + CancellationToken ct = default) + { + // A one-shot's body lives on the work page itself; /{ncode}/1/ is a 404 for it + var url = episode.IsOneShot + ? $"https://{Host}/{sourceId}/" + : $"https://{Host}/{sourceId}/{episode.Number}/"; + + var html = await FetchStringAsync(url, ct); + + var parser = new HtmlParser(); + using var document = await parser.ParseDocumentAsync(html, ct); + + // preface/afterword carry the same js-novel-text class as the body, distinguished by a modifier + var blocks = document.QuerySelectorAll("div.js-novel-text").ToList(); + var body = blocks.FirstOrDefault(b => !b.ClassList.Contains("p-novel__text--preface") && + !b.ClassList.Contains("p-novel__text--afterword")); + + if (body == null) + throw new InvalidOperationException($"No novel text found at {url} (markup may have changed)."); + + return ExtractBlockText(body, document); + } + + private static string ExtractBlockText(IElement block, IDocument document) + { + RubyHtmlHelper.InlineRubyAnnotations(block, document); + + var sb = new StringBuilder(); + foreach (var paragraph in block.QuerySelectorAll("p")) + sb.AppendLine(paragraph.TextContent.Trim()); + + // Blank paragraphs are


; collapse the runs they leave behind + return BlankLineRuns().Replace(sb.ToString(), "\n\n").Trim(); + } + + private static int? EpisodeNumberFromHref(string href) + { + var match = EpisodeHrefPattern().Match(href); + return match.Success && int.TryParse(match.Groups[1].Value, out var n) ? n : null; + } + + /// + /// The update cell holds the publish date plus, when the episode was revised, a 改稿 span whose + /// title attribute carries the revision date. The later of the two is what we track. + /// + private static DateTimeOffset? ParseTocUpdate(IElement? cell) + { + if (cell == null) + return null; + + var revisionSpan = cell.QuerySelector("span[title]")?.GetAttribute("title"); + if (!string.IsNullOrEmpty(revisionSpan)) + { + var revised = ParseJst(revisionSpan.Replace("改稿", string.Empty).Trim()); + if (revised != null) + return revised; + } + + var text = cell.ChildNodes + .Where(n => n.NodeType == NodeType.Text) + .Select(n => n.TextContent.Trim()) + .FirstOrDefault(t => !string.IsNullOrEmpty(t)); + + return ParseJst(text); + } + + /// + /// Syosetu renders every timestamp in JST with no zone marker. Returned as UTC: these land in + /// timestamptz columns, and Npgsql only writes DateTimeOffsets whose offset is zero. + /// + private static DateTimeOffset? ParseJst(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return null; + + string[] formats = ["yyyy-MM-dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyy/MM/dd HH:mm:ss"]; + + if (DateTime.TryParseExact(value.Trim(), formats, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var parsed)) + { + return new DateTimeOffset(parsed, TimeSpan.FromHours(9)).ToUniversalTime(); + } + + return null; + } + + private async Task FetchStringAsync(string url, CancellationToken ct) + { + using var response = await SendAsync(url, ct); + return await response.Content.ReadAsStringAsync(ct); + } + + /// + /// The API's gzip=5 returns a gzip body regardless of Accept-Encoding, so it is inflated by hand. + /// + private async Task FetchGzipJsonAsync(string url, CancellationToken ct) + { + using var response = await SendAsync(url, ct); + + await using var stream = await response.Content.ReadAsStreamAsync(ct); + await using var gzip = new GZipStream(stream, CompressionMode.Decompress); + using var reader = new StreamReader(gzip, Encoding.UTF8); + + return await reader.ReadToEndAsync(ct); + } + + private async Task SendAsync(string url, CancellationToken ct) + { + for (var attempt = 0; ; attempt++) + { + await ThrottleAsync(ct); + + var client = _httpClientFactory.CreateClient(HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, url); + + if (IsNovel18) + request.Headers.Add("Cookie", "over18=yes"); + + var response = await client.SendAsync(request, ct); + + // 503 is Syosetu's "Too many access" — back off rather than hammer + if ((response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable || + (int)response.StatusCode == 429) && attempt < MaxRetries) + { + response.Dispose(); + + var backoff = TimeSpan.FromSeconds(Math.Pow(2, attempt + 1)); + _logger.LogWarning("Syosetu throttled us on {Url}, backing off {Backoff}s (attempt {Attempt})", + url, backoff.TotalSeconds, attempt + 1); + await Task.Delay(backoff, ct); + continue; + } + + if (!response.IsSuccessStatusCode) + { + var status = response.StatusCode; + response.Dispose(); + throw new HttpRequestException($"Request to {url} failed with status {(int)status} ({status}).", + inner: null, statusCode: status); + } + + return response; + } + } + + private async Task ThrottleAsync(CancellationToken ct) + { + await _throttle.WaitAsync(ct); + try + { + var sinceLast = DateTimeOffset.UtcNow - _lastRequest; + if (sinceLast < RequestInterval) + await Task.Delay(RequestInterval - sinceLast, ct); + + if (++_requestCount % WaitStepEvery == 0) + await Task.Delay(WaitStepPause, ct); + + _lastRequest = DateTimeOffset.UtcNow; + } + finally + { + _throttle.Release(); + } + } + + private static string? GetString(JsonElement e, string name) => + e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() : null; + + private static int GetInt(JsonElement e, string name) => + e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number ? v.GetInt32() : 0; + + private static string? GenreLabel(int code) => code switch + { + 101 => "異世界〔恋愛〕", + 102 => "現実世界〔恋愛〕", + 201 => "ハイファンタジー〔ファンタジー〕", + 202 => "ローファンタジー〔ファンタジー〕", + 301 => "純文学〔文芸〕", + 302 => "ヒューマンドラマ〔文芸〕", + 303 => "歴史〔文芸〕", + 304 => "推理〔文芸〕", + 305 => "ホラー〔文芸〕", + 306 => "アクション〔文芸〕", + 307 => "コメディー〔文芸〕", + 401 => "VRゲーム〔SF〕", + 402 => "宇宙〔SF〕", + 403 => "空想科学〔SF〕", + 404 => "パニック〔SF〕", + 9901 => "童話〔その他〕", + 9902 => "詩〔その他〕", + 9903 => "エッセイ〔その他〕", + 9904 => "リプレイ〔その他〕", + 9999 => "その他〔その他〕", + 9801 => "ノンジャンル〔ノンジャンル〕", + _ => null + }; + + [GeneratedRegex(@"/(\d+)/?$")] + private static partial Regex EpisodeHrefPattern(); + + [GeneratedRegex(@"\n{3,}")] + private static partial Regex BlankLineRuns(); +} diff --git a/Jiten.Core/WebNovel/WebNovelMetadataExtensions.cs b/Jiten.Core/WebNovel/WebNovelMetadataExtensions.cs new file mode 100644 index 00000000..9503e07f --- /dev/null +++ b/Jiten.Core/WebNovel/WebNovelMetadataExtensions.cs @@ -0,0 +1,35 @@ +using Jiten.Core.Data; +using Jiten.Core.Data.Providers; + +namespace Jiten.Core.WebNovel; + +public static class WebNovelMetadataExtensions +{ + /// + /// Provider info as deck metadata. Shared by the import (which creates the deck) and the metadata + /// refresh (which re-applies genre and tag mappings), so both see the same genres, keywords and links. + /// + public static Metadata ToMetadata(this WebNovelInfo info) + { + var metadata = new Metadata + { + OriginalTitle = info.Title, + Description = info.Synopsis, + ReleaseDate = info.FirstPublishedAt?.UtcDateTime, + IsAdultOnly = info.IsAdultOnly, + Links = [new Link { LinkType = LinkType.Syosetsu, Url = info.Url }], + Tags = info.Keywords.Select(k => new MetadataTag { Name = k, Percentage = 100 }).ToList() + }; + + if (!string.IsNullOrEmpty(info.Genre)) + { + metadata.Genres.Add(info.Genre); + + // Also offered to the tag mappings: a Narou genre carries information no Jiten genre holds + // (異世界〔恋愛〕 is Romance *and* Isekai), and the two mapping tables are separate lookups. + metadata.Tags.Add(new MetadataTag { Name = info.Genre, Percentage = 100 }); + } + + return metadata; + } +} diff --git a/Jiten.Core/WebNovel/WebNovelModels.cs b/Jiten.Core/WebNovel/WebNovelModels.cs new file mode 100644 index 00000000..3196c734 --- /dev/null +++ b/Jiten.Core/WebNovel/WebNovelModels.cs @@ -0,0 +1,68 @@ +using Jiten.Core.Data.WebNovel; + +namespace Jiten.Core.WebNovel; + +/// +/// Work-level metadata, from the provider's API where one exists. +/// +public class WebNovelInfo +{ + public WebNovelProvider Provider { get; set; } + public string SourceId { get; set; } = string.Empty; + public string Url { get; set; } = string.Empty; + + public string Title { get; set; } = string.Empty; + public string? Author { get; set; } + public string? Synopsis { get; set; } + + /// + /// Provider genre label (e.g. ハイファンタジー〔ファンタジー〕), mapped via ExternalGenreMappings + /// + public string? Genre { get; set; } + + public List Keywords { get; set; } = new(); + + public DateTimeOffset? FirstPublishedAt { get; set; } + public DateTimeOffset? LastUpdatedAt { get; set; } + + public int EpisodeCount { get; set; } + public long TotalCharacters { get; set; } + + /// + /// 短編 — the work is a single page and has no table of contents + /// + public bool IsOneShot { get; set; } + + public bool IsCompleted { get; set; } + public bool IsOnHiatus { get; set; } + public bool IsAdultOnly { get; set; } + public bool IsR15 { get; set; } +} + +/// +/// One entry in the table of contents. +/// +public class WebNovelEpisodeRef +{ + /// + /// 1-based index at the source + /// + public int Number { get; set; } + + public string Title { get; set; } = string.Empty; + + /// + /// Latest publish or revision (改稿) timestamp shown in the table of contents + /// + public DateTimeOffset? UpdatedAt { get; set; } + + /// + /// Enclosing 章 heading, when the work uses them + /// + public string? SectionTitle { get; set; } + + /// + /// The episode is a 短編's single page: its body lives on the work page itself, /{n}/ does not exist + /// + public bool IsOneShot { get; set; } +} diff --git a/Jiten.Core/WebNovel/WebNovelSchedule.cs b/Jiten.Core/WebNovel/WebNovelSchedule.cs new file mode 100644 index 00000000..0b19261b --- /dev/null +++ b/Jiten.Core/WebNovel/WebNovelSchedule.cs @@ -0,0 +1,69 @@ +using Jiten.Core.Data.WebNovel; + +namespace Jiten.Core.WebNovel; + +public static class WebNovelSchedule +{ + /// + /// Slightly under a day so a daily sweep always picks an active novel up + /// + private static readonly TimeSpan ActiveInterval = TimeSpan.FromHours(20); + + /// + /// Finished works still get the occasional epilogue or 番外編 + /// + private static readonly TimeSpan CompletedInterval = TimeSpan.FromDays(30); + + private static readonly TimeSpan MaxBackoff = TimeSpan.FromDays(16); + + public static DateTimeOffset NextCheck(bool completedAtSource) => + DateTimeOffset.UtcNow + (completedAtSource ? CompletedInterval : ActiveInterval); + + /// + /// Backs off exponentially after failures. Repeated failures usually mean the site's markup changed, + /// so there is no point retrying hard. + /// + public static DateTimeOffset NextCheckAfterFailure(int consecutiveFailures) + { + var backoff = TimeSpan.FromDays(Math.Pow(2, Math.Clamp(consecutiveFailures, 1, 4))); + return DateTimeOffset.UtcNow + (backoff > MaxBackoff ? MaxBackoff : backoff); + } + + /// + /// A sync is only worth its cost once this many episodes have accumulated: each one reparses the open + /// subdeck and rewrites the parent's aggregated word data, whether it ingests 1 episode or 20. + /// ~15 median episodes is a quarter of a subdeck — enough to visibly move the deck's statistics. + /// + public const int MinEpisodesForSync = 15; + + /// + /// Slow novels still land within this window, however few episodes accumulated. Matches the Narou API + /// docs' cache-expiry guidance. + /// + public static readonly TimeSpan MaxSyncLag = TimeSpan.FromDays(14); + + /// + /// A novel has pending changes when the source reports episodes we haven't ingested, or a newer + /// timestamp than the one we last saw (a revision to an existing episode). + /// + public static bool IsDirty(WebNovelSource tracked, WebNovelInfo polled) => + polled.EpisodeCount > tracked.LastEpisodeCount || + (polled.LastUpdatedAt != null && + (tracked.LastSourceUpdate == null || polled.LastUpdatedAt > tracked.LastSourceUpdate)); + + /// + /// Whether pending changes are worth a sync yet: enough episodes accumulated, or anything at all has + /// been pending for . The sweep must leave LastSyncedAt untouched while + /// a below-threshold backlog exists, so the lag clock measures time since the last ingest. + /// + public static bool ShouldSync(WebNovelSource tracked, WebNovelInfo polled) + { + if (!IsDirty(tracked, polled)) + return false; + + if (polled.EpisodeCount - tracked.LastEpisodeCount >= MinEpisodesForSync) + return true; + + return tracked.LastSyncedAt == null || DateTimeOffset.UtcNow - tracked.LastSyncedAt >= MaxSyncLag; + } +} diff --git a/Jiten.Core/WebNovel/WebNovelSourceResolver.cs b/Jiten.Core/WebNovel/WebNovelSourceResolver.cs new file mode 100644 index 00000000..91d74db5 --- /dev/null +++ b/Jiten.Core/WebNovel/WebNovelSourceResolver.cs @@ -0,0 +1,22 @@ +using Jiten.Core.Data.WebNovel; + +namespace Jiten.Core.WebNovel; + +public interface IWebNovelSourceResolver +{ + IWebNovelSource Resolve(WebNovelProvider provider); + bool IsSupported(WebNovelProvider provider); +} + +public class WebNovelSourceResolver(IEnumerable sources) : IWebNovelSourceResolver +{ + private readonly Dictionary _sources = + sources.ToDictionary(s => s.Provider); + + public IWebNovelSource Resolve(WebNovelProvider provider) => + _sources.TryGetValue(provider, out var source) + ? source + : throw new NotSupportedException($"Webnovel provider {provider} is not enabled."); + + public bool IsSupported(WebNovelProvider provider) => _sources.ContainsKey(provider); +} diff --git a/Jiten.Core/WebNovel/WebNovelUrlParser.cs b/Jiten.Core/WebNovel/WebNovelUrlParser.cs new file mode 100644 index 00000000..95f3f91e --- /dev/null +++ b/Jiten.Core/WebNovel/WebNovelUrlParser.cs @@ -0,0 +1,61 @@ +using System.Text.RegularExpressions; +using Jiten.Core.Data.WebNovel; + +namespace Jiten.Core.WebNovel; + +public static partial class WebNovelUrlParser +{ + [GeneratedRegex(@"^[nN]\d{4}[a-zA-Z]{1,2}$")] + private static partial Regex NcodePattern(); + + /// + /// Accepts a novel URL or a bare ncode. Returns false when the input isn't a supported source. + /// + public static bool TryParse(string input, out WebNovelProvider provider, out string sourceId) + { + provider = default; + sourceId = string.Empty; + + if (string.IsNullOrWhiteSpace(input)) + return false; + + input = input.Trim(); + + // Bare ncode (n9669bk) + if (NcodePattern().IsMatch(input)) + { + provider = WebNovelProvider.Syosetu; + sourceId = input.ToLowerInvariant(); + return true; + } + + if (!Uri.TryCreate(input, UriKind.Absolute, out var uri)) + return false; + + var host = uri.Host.ToLowerInvariant(); + var firstSegment = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + + if (string.IsNullOrEmpty(firstSegment) || !NcodePattern().IsMatch(firstSegment)) + return false; + + provider = host switch + { + "ncode.syosetu.com" => WebNovelProvider.Syosetu, + "novel18.syosetu.com" => WebNovelProvider.SyosetuNovel18, + _ => default + }; + + if (provider == default) + return false; + + sourceId = firstSegment.ToLowerInvariant(); + return true; + } + + public static string BuildWorkUrl(WebNovelProvider provider, string sourceId) => provider switch + { + WebNovelProvider.Syosetu => $"https://ncode.syosetu.com/{sourceId}/", + WebNovelProvider.SyosetuNovel18 => $"https://novel18.syosetu.com/{sourceId}/", + _ => throw new NotSupportedException($"No URL template for provider {provider}") + }; +} diff --git a/Jiten.Tests/RubyHtmlHelperTests.cs b/Jiten.Tests/RubyHtmlHelperTests.cs new file mode 100644 index 00000000..215278da --- /dev/null +++ b/Jiten.Tests/RubyHtmlHelperTests.cs @@ -0,0 +1,48 @@ +using AngleSharp.Html.Parser; +using FluentAssertions; +using Jiten.Core; +using Xunit; + +namespace Jiten.Tests; + +public class RubyHtmlHelperTests +{ + private static string Inline(string html) + { + var document = new HtmlParser().ParseDocument($"{html}"); + RubyHtmlHelper.InlineRubyAnnotations(document.Body!, document); + return document.Body!.TextContent; + } + + [Fact] + public void SyosetuRuby_BecomesInlineFuriganaAnnotation() + { + // Syosetu wraps the base in a bare text node — there is no + var html = "

俺の愛機（パソコン）にバットを

"; + + Inline(html).Should().Be("俺の{愛機'パソコン}にバットを"); + } + + [Fact] + public void EpubRuby_WithRbElement_BecomesInlineFuriganaAnnotation() + { + var html = "

漢字(かんじ)

"; + + Inline(html).Should().Be("{漢字'かんじ}"); + } + + [Fact] + public void RubyWithoutReading_KeepsOnlyTheBase() + { + Inline("

漢字

").Should().Be("漢字"); + } + + [Fact] + public void MultipleRuby_AreAllConverted() + { + var html = "

古代長耳族(ハイエルフ)と" + + "長耳族(エルフ)

"; + + Inline(html).Should().Be("{古代長耳族'ハイエルフ}と{長耳族'エルフ}"); + } +} diff --git a/Jiten.Tests/SubdeckChunkerTests.cs b/Jiten.Tests/SubdeckChunkerTests.cs new file mode 100644 index 00000000..7259a2e6 --- /dev/null +++ b/Jiten.Tests/SubdeckChunkerTests.cs @@ -0,0 +1,215 @@ +using FluentAssertions; +using Jiten.Core.WebNovel; +using Xunit; + +namespace Jiten.Tests; + +public class SubdeckChunkerTests +{ + private static List Episodes(int from, int count, int charsEach) => + Enumerable.Range(from, count) + .Select(n => new ChunkEpisode(n, $"第{n}話", charsEach)) + .ToList(); + + [Fact] + public void FirstImport_MedianEpisodes_SplitsAtBudget() + { + // 2,500 chars is the median Narou episode: a 150k budget holds 60 of them + var plans = SubdeckChunker.Plan([], Episodes(1, 130, 2_500)); + + plans.Should().HaveCount(3); + plans[0].StartEpisode.Should().Be(1); + plans[0].EndEpisode.Should().Be(60); + plans[1].StartEpisode.Should().Be(61); + plans[1].EndEpisode.Should().Be(120); + + // The trailing partial subdeck stays open for the next sync + plans[2].StartEpisode.Should().Be(121); + plans[2].EndEpisode.Should().Be(130); + plans.Should().OnlyContain(p => p.IsNew); + plans.Select(p => p.ChunkIndex).Should().Equal(1, 2, 3); + } + + [Fact] + public void MicroChapters_CapAtMaxEpisodes() + { + // 300-char chapters would need 500 episodes to reach the budget; the episode cap closes first + var plans = SubdeckChunker.Plan([], Episodes(1, 200, 300)); + + plans[0].EpisodesToAppend.Should().HaveCount(SubdeckChunker.MaxEpisodesPerChunk); + plans[0].EndEpisode.Should().Be(150); + plans[1].StartEpisode.Should().Be(151); + } + + [Fact] + public void SingleEpisodeLargerThanBudget_GetsItsOwnSubdeck() + { + var plans = SubdeckChunker.Plan([], [new ChunkEpisode(1, "長編", 400_000), new ChunkEpisode(2, "続き", 1_000)]); + + plans.Should().HaveCount(2); + plans[0].EpisodesToAppend.Should().ContainSingle(); + plans[0].Title.Should().Be("第1話"); + plans[1].StartEpisode.Should().Be(2); + } + + [Fact] + public void Sync_ExtendsOpenSubdeck_KeepingItsDeckId() + { + // Open subdeck: episodes 121-130, well under budget + var existing = new List { new(1, ChildDeckId: 42, StartEpisode: 121, EndEpisode: 130, EpisodeCount: 10, CharCount: 25_000) }; + + var plans = SubdeckChunker.Plan(existing, Episodes(131, 2, 2_500)); + + plans.Should().ContainSingle(); + plans[0].IsNew.Should().BeFalse(); + plans[0].ChildDeckId.Should().Be(42); + plans[0].ChunkIndex.Should().Be(1); + + // Only the new episodes are appended; the range covers everything the subdeck now holds + plans[0].EpisodesToAppend.Should().HaveCount(2); + plans[0].StartEpisode.Should().Be(121); + plans[0].EndEpisode.Should().Be(132); + plans[0].Title.Should().Be("第121話〜第132話"); + } + + [Fact] + public void Sync_ClosesOpenSubdeck_AndOpensNext() + { + // 148k of a 150k budget: one more episode closes it + var existing = new List { new(2, ChildDeckId: 42, StartEpisode: 61, EndEpisode: 119, EpisodeCount: 59, CharCount: 148_000) }; + + var plans = SubdeckChunker.Plan(existing, Episodes(120, 3, 2_500)); + + plans.Should().HaveCount(2); + + plans[0].ChildDeckId.Should().Be(42); + plans[0].EpisodesToAppend.Should().ContainSingle(); + plans[0].EndEpisode.Should().Be(120); + + plans[1].IsNew.Should().BeTrue(); + plans[1].ChunkIndex.Should().Be(3); + plans[1].StartEpisode.Should().Be(121); + plans[1].EndEpisode.Should().Be(122); + } + + [Fact] + public void Sync_WhenLastSubdeckIsFull_OpensNewOne() + { + var existing = new List { new(1, ChildDeckId: 42, StartEpisode: 1, EndEpisode: 60, EpisodeCount: 60, CharCount: 150_000) }; + + var plans = SubdeckChunker.Plan(existing, Episodes(61, 1, 2_500)); + + plans.Should().ContainSingle(); + plans[0].IsNew.Should().BeTrue(); + plans[0].ChunkIndex.Should().Be(2); + plans[0].ChildDeckId.Should().BeNull(); + plans[0].Title.Should().Be("第61話"); + } + + [Fact] + public void NoNewEpisodes_PlansNothing() + { + var existing = new List { new(1, 42, 1, 60, 60, 150_000) }; + + SubdeckChunker.Plan(existing, []).Should().BeEmpty(); + } + + [Fact] + public void ExactBudgetBoundary_ClosesSubdeck() + { + var plans = SubdeckChunker.Plan([], Episodes(1, 2, 75_000)); + + plans.Should().ContainSingle(); + plans[0].EndEpisode.Should().Be(2); + + // Exactly at budget counts as full, so the next episode opens a new subdeck + var next = SubdeckChunker.Plan([new ExistingChunk(1, 42, 1, 2, 2, 150_000)], Episodes(3, 1, 1_000)); + next[0].IsNew.Should().BeTrue(); + } + + [Fact] + public void BudgetRunsOutJustPastABoundary_RoundsDown() + { + // 2,840-char episodes fill the budget on episode 53; the split belongs at 50 + var plans = SubdeckChunker.Plan([], Episodes(1, 130, 2_840)); + + plans.Select(p => (p.StartEpisode, p.EndEpisode)).Should().Equal((1, 50), (51, 100), (101, 130)); + plans[0].EpisodesToAppend.Should().HaveCount(50); + } + + [Fact] + public void BudgetRunsOutJustBeforeABoundary_GrowsIntoIt() + { + // 2,640-char episodes fill the budget on episode 57; the subdeck keeps taking episodes up to 60 + var plans = SubdeckChunker.Plan([], Episodes(1, 130, 2_640)); + + plans.Select(p => (p.StartEpisode, p.EndEpisode)).Should().Equal((1, 60), (61, 120), (121, 130)); + plans[0].Title.Should().Be("第1話〜第60話"); + } + + [Fact] + public void OpenSubdeckAlreadyPastTheBoundary_CanOnlyMoveForward() + { + // Episodes 1-63 are already parsed into subdeck 42, so the split can't be pulled back to 60 + var existing = new List { new(1, ChildDeckId: 42, StartEpisode: 1, EndEpisode: 63, EpisodeCount: 63, CharCount: 148_000) }; + + var plans = SubdeckChunker.Plan(existing, Episodes(64, 12, 2_500)); + + plans[0].ChildDeckId.Should().Be(42); + plans[0].EndEpisode.Should().Be(70); + plans[0].EpisodesToAppend.Should().HaveCount(7); + + plans[1].IsNew.Should().BeTrue(); + plans[1].StartEpisode.Should().Be(71); + } + + [Fact] + public void OpenSubdeckSittingOnABoundary_IsLeftAloneAndTheNextOneStartsFresh() + { + // Subdeck 42 holds 1-60 and is just short of the budget. One more episode would overflow it, and 60 is + // already where it wants to end — so it takes nothing and closes as it stands. + var existing = new List { new(1, ChildDeckId: 42, StartEpisode: 1, EndEpisode: 60, EpisodeCount: 60, CharCount: 148_000) }; + + var plans = SubdeckChunker.Plan(existing, Episodes(61, 65, 2_500)); + + plans.Should().OnlyContain(p => p.IsNew); + plans.Select(p => (p.StartEpisode, p.EndEpisode)).Should().Equal((61, 120), (121, 125)); + plans[0].ChunkIndex.Should().Be(2); + } + + [Fact] + public void MidLengthChapters_FallBackToTheSmallerBoundary() + { + // 11k-char episodes only fit 14 to a subdeck — too few to round to tens, so it rounds to fives + var plans = SubdeckChunker.Plan([], Episodes(1, 30, 11_000)); + + plans.Select(p => (p.StartEpisode, p.EndEpisode)).Should().Equal((1, 15), (16, 30)); + } + + [Fact] + public void VeryLongChapters_SplitAtTheBudgetWithNoRounding() + { + // 8 episodes to a subdeck: rounding to tens or fives would redraw the boundary, not nudge it + var plans = SubdeckChunker.Plan([], Episodes(1, 20, 20_000)); + + plans.Select(p => (p.StartEpisode, p.EndEpisode)).Should().Equal((1, 8), (9, 16), (17, 20)); + } + + [Fact] + public void CountCharacters_IgnoresFuriganaReadingsAndWhitespace() + { + // The reading in {愛機'パソコン} is an annotation, not text the reader counts + SubdeckChunker.CountCharacters("俺の{愛機'パソコン}だ。").Should().Be(6); + SubdeckChunker.CountCharacters("あ い\nう").Should().Be(3); + SubdeckChunker.CountCharacters("").Should().Be(0); + } + + [Fact] + public void CustomBudget_IsHonoured() + { + var plans = SubdeckChunker.Plan([], Episodes(1, 40, 2_500), charBudget: 50_000); + + plans[0].EpisodesToAppend.Should().HaveCount(20); + plans.Should().HaveCount(2); + } +} diff --git a/Jiten.Tests/WebNovelScheduleTests.cs b/Jiten.Tests/WebNovelScheduleTests.cs new file mode 100644 index 00000000..21593135 --- /dev/null +++ b/Jiten.Tests/WebNovelScheduleTests.cs @@ -0,0 +1,78 @@ +using FluentAssertions; +using Jiten.Core.Data.WebNovel; +using Jiten.Core.WebNovel; +using Xunit; + +namespace Jiten.Tests; + +public class WebNovelScheduleTests +{ + private static WebNovelSource Tracked(int episodes, DateTimeOffset? lastSynced, DateTimeOffset? lastSourceUpdate = null) => new() + { + LastEpisodeCount = episodes, + LastSyncedAt = lastSynced, + LastSourceUpdate = lastSourceUpdate + }; + + private static WebNovelInfo Polled(int episodes, DateTimeOffset? lastUpdated = null) => new() + { + EpisodeCount = episodes, + LastUpdatedAt = lastUpdated + }; + + [Fact] + public void CleanNovel_NeverSyncs() + { + var tracked = Tracked(100, lastSynced: DateTimeOffset.UtcNow.AddDays(-30)); + + WebNovelSchedule.ShouldSync(tracked, Polled(100)).Should().BeFalse(); + } + + [Fact] + public void SmallBacklog_WaitsForMoreEpisodes() + { + // 5 new episodes, synced recently: not worth a reparse yet + var tracked = Tracked(100, lastSynced: DateTimeOffset.UtcNow.AddDays(-3)); + + WebNovelSchedule.ShouldSync(tracked, Polled(105)).Should().BeFalse(); + } + + [Fact] + public void EpisodeThreshold_TriggersSync() + { + var tracked = Tracked(100, lastSynced: DateTimeOffset.UtcNow.AddDays(-1)); + + WebNovelSchedule.ShouldSync(tracked, Polled(100 + WebNovelSchedule.MinEpisodesForSync)).Should().BeTrue(); + } + + [Fact] + public void MaxLag_FlushesSmallBacklog() + { + // A single pending episode still lands once it has waited out the lag window + var tracked = Tracked(100, lastSynced: DateTimeOffset.UtcNow - WebNovelSchedule.MaxSyncLag); + + WebNovelSchedule.ShouldSync(tracked, Polled(101)).Should().BeTrue(); + } + + [Fact] + public void NeverSynced_DirtyNovel_SyncsImmediately() + { + var tracked = Tracked(100, lastSynced: null); + + WebNovelSchedule.ShouldSync(tracked, Polled(101)).Should().BeTrue(); + } + + [Fact] + public void RevisionOnlyChange_WaitsForMaxLag() + { + // Newer lastup but no new episodes (改稿): dirty, but only worth a sync at the lag window + var seen = DateTimeOffset.UtcNow.AddDays(-10); + var revised = DateTimeOffset.UtcNow.AddDays(-1); + + var recent = Tracked(100, lastSynced: DateTimeOffset.UtcNow.AddDays(-2), lastSourceUpdate: seen); + WebNovelSchedule.ShouldSync(recent, Polled(100, revised)).Should().BeFalse(); + + var stale = Tracked(100, lastSynced: DateTimeOffset.UtcNow - WebNovelSchedule.MaxSyncLag, lastSourceUpdate: seen); + WebNovelSchedule.ShouldSync(stale, Polled(100, revised)).Should().BeTrue(); + } +} diff --git a/Jiten.Web/app/components/dashboard/CoverTools.vue b/Jiten.Web/app/components/dashboard/CoverTools.vue index aa94b151..347f4ee3 100644 --- a/Jiten.Web/app/components/dashboard/CoverTools.vue +++ b/Jiten.Web/app/components/dashboard/CoverTools.vue @@ -17,6 +17,7 @@ canvasToFile, shuffleOptions, defaultCoverOptions, + fallbackPalette, type CoverOptions, type CoverPalette, type BackgroundStyle, @@ -51,11 +52,12 @@ let currentCanvas: HTMLCanvasElement | null = null; let rafId: number | null = null; - const styleOptions: { label: string; value: BackgroundStyle }[] = [ + + const styleOptions = computed<{ label: string; value: BackgroundStyle }[]>(() => [ { label: 'Gradient', value: 'gradient' }, { label: 'Solid', value: 'solid' }, - { label: 'Blurred art', value: 'blurred' }, - ]; + ...(props.source ? [{ label: 'Blurred art', value: 'blurred' as const }] : []), + ]); const orientationOptions: { label: string; value: Orientation }[] = [ { label: 'Horizontal', value: 'horizontal' }, { label: 'Vertical (tategaki)', value: 'vertical' }, @@ -128,16 +130,21 @@ } async function openGenerator() { - if (!props.source) return; generatorVisible.value = true; loadingSource.value = true; previewUrl.value = null; sourceImage = null; currentCanvas = null; try { - const blob = await resolveBlob(props.source); - sourceImage = await loadImage(blob); - palette.value = extractPalette(sourceImage); + + if (props.source) { + const blob = await resolveBlob(props.source); + sourceImage = await loadImage(blob); + palette.value = extractPalette(sourceImage); + } else { + palette.value = fallbackPalette(props.title); + } + await ensureTitleFont(); options.value = defaultCoverOptions(palette.value, props.title, props.subtitle ?? ''); swatchTarget.value = 'bg1'; @@ -198,7 +205,7 @@ Rotate right - diff --git a/Jiten.Web/app/components/dashboard/WebNovelSyncPanel.vue b/Jiten.Web/app/components/dashboard/WebNovelSyncPanel.vue new file mode 100644 index 00000000..b36f13fa --- /dev/null +++ b/Jiten.Web/app/components/dashboard/WebNovelSyncPanel.vue @@ -0,0 +1,174 @@ + + + diff --git a/Jiten.Web/app/pages/dashboard/add-webnovel.vue b/Jiten.Web/app/pages/dashboard/add-webnovel.vue new file mode 100644 index 00000000..2e4a8d0e --- /dev/null +++ b/Jiten.Web/app/pages/dashboard/add-webnovel.vue @@ -0,0 +1,227 @@ + + + diff --git a/Jiten.Web/app/pages/dashboard/genre-mappings.vue b/Jiten.Web/app/pages/dashboard/genre-mappings.vue index fb460705..a906e61d 100644 --- a/Jiten.Web/app/pages/dashboard/genre-mappings.vue +++ b/Jiten.Web/app/pages/dashboard/genre-mappings.vue @@ -225,6 +225,10 @@ onMounted(() => { optionValue="value" placeholder="Filter by Genre" showClear + filter + filterPlaceholder="Search genre..." + autoFilterFocus + resetFilterOnHide class="w-full" /> { optionLabel="label" optionValue="value" placeholder="Select Jiten Genre" + filter + filterPlaceholder="Search genre..." + autoFilterFocus + resetFilterOnHide class="w-full" /> diff --git a/Jiten.Web/app/pages/dashboard/index.vue b/Jiten.Web/app/pages/dashboard/index.vue index c185442b..3a7acb8d 100644 --- a/Jiten.Web/app/pages/dashboard/index.vue +++ b/Jiten.Web/app/pages/dashboard/index.vue @@ -56,6 +56,16 @@ + + + + +