Webnovel download & sync system

Jiten.Api/Controllers/AdminController.WebNovels.cs

@@ -0,0 +1,213 @@
+using Hangfire;
+using Jiten.Api.Dtos.Requests;
+using Jiten.Api.Jobs;
+using Jiten.Core.Data;
+using Jiten.Core.Data.WebNovel;
+using Jiten.Core.WebNovel;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+
+namespace Jiten.Api.Controllers;
+
+public partial class AdminController
+{
+    /// <summary>
+    /// Work metadata straight from the provider, so the admin can review it (and generate a cover from the
+    /// title) before committing to an import.
+    /// </summary>
+    [HttpGet("webnovel-preview")]
+    public async Task<IActionResult> PreviewWebNovel([FromQuery] string url,
+                                                     [FromServices] IWebNovelSourceResolver sourceResolver)
+    {
+        if (!WebNovelUrlParser.TryParse(url, out var provider, out var sourceId))
+            return BadRequest(new { Message = "Not a supported webnovel URL." });
+
+        if (!sourceResolver.IsSupported(provider))
+            return BadRequest(new { Message = $"{provider} is not enabled yet." });
+
+        var existing = await dbContext.WebNovelSources
+                                      .FirstOrDefaultAsync(s => s.Provider == provider && s.SourceId == sourceId);
+
+        if (existing != null)
+            return Conflict(new { Message = "This novel is already tracked.", DeckId = existing.DeckId });
+
+        try
+        {
+            var info = await sourceResolver.Resolve(provider).GetInfoAsync(sourceId);
+
+            // A same-title webnovel deck would make InsertDeck skip the insert, so the import would fail
+            var titleConflict = await dbContext.Decks
+                                               .AnyAsync(d => d.OriginalTitle == info.Title &&
+                                                              d.MediaType == MediaType.WebNovel);
+
+            return Ok(new
+            {
+                Provider = provider.ToString(),
+                info.SourceId,
+                info.Url,
+                info.Title,
+                TitleConflict = titleConflict,
+                info.Author,
+                info.Synopsis,
+                info.Genre,
+                info.Keywords,
+                info.EpisodeCount,
+                info.TotalCharacters,
+                info.IsCompleted,
+                info.IsOnHiatus,
+                info.IsOneShot,
+                info.IsR15,
+                FirstPublishedAt = info.FirstPublishedAt?.UtcDateTime,
+                LastUpdatedAt = info.LastUpdatedAt?.UtcDateTime,
+                EstimatedSubdecks = EstimateSubdecks(info.TotalCharacters)
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogWarning(ex, "Webnovel preview failed for {Url}", url);
+            return StatusCode(StatusCodes.Status502BadGateway,
+                              new { Message = "Could not read this novel from the source.", Details = ex.Message });
+        }
+    }
+
+    [HttpPost("add-webnovel-deck")]
+    [Consumes("multipart/form-data")]
+    public async Task<IActionResult> AddWebNovelDeck([FromForm] AddWebNovelDeckRequest model,
+                                                     [FromServices] IWebNovelSourceResolver sourceResolver)
+    {
+        if (!ModelState.IsValid)
+            return BadRequest(ModelState);
+
+        if (!WebNovelUrlParser.TryParse(model.Url, out var provider, out var sourceId))
+            return BadRequest(new { Message = "Not a supported webnovel URL." });
+
+        if (!sourceResolver.IsSupported(provider))
+            return BadRequest(new { Message = $"{provider} is not enabled yet." });
+
+        if (await dbContext.WebNovelSources.AnyAsync(s => s.Provider == provider && s.SourceId == sourceId))
+            return Conflict(new { Message = "This novel is already tracked." });
+
+        string? coverPath = null;
+        if (model.CoverImage is { Length: > 0 })
+        {
+            var directory = Path.Join(config["StaticFilesPath"], "tmp", Guid.NewGuid().ToString());
+            Directory.CreateDirectory(directory);
+
+            coverPath = Path.Join(directory, "cover.jpg");
+            await using var stream = new FileStream(coverPath, FileMode.Create);
+            await model.CoverImage.CopyToAsync(stream);
+        }
+
+        // Fetching a long novel is 20+ minutes of polite requests, so it can never run in-request
+        var jobId = backgroundJobs.Enqueue<WebNovelImportJob>(
+            job => job.Import(provider, sourceId, coverPath, model.ChunkCharBudget));
+
+        logger.LogInformation("Admin queued webnovel import for {Provider}/{SourceId} (job {JobId})",
+                              provider, sourceId, jobId);
+
+        return Accepted(new { Message = "Import queued.", JobId = jobId, Provider = provider.ToString(), SourceId = sourceId });
+    }
+
+    [HttpGet("webnovel/{deckId:int}")]
+    public async Task<IActionResult> GetWebNovelSource(int deckId)
+    {
+        var tracked = await dbContext.WebNovelSources
+                                     .AsNoTracking()
+                                     .FirstOrDefaultAsync(s => s.DeckId == deckId);
+
+        if (tracked == null)
+            return NotFound();
+
+        var subdecks = await dbContext.WebNovelChapters
+                                      .AsNoTracking()
+                                      .Where(c => c.DeckId == deckId)
+                                      .GroupBy(c => c.ChildDeckId)
+                                      .Select(g => new
+                                      {
+                                          ChildDeckId = g.Key,
+                                          StartEpisode = g.Min(c => c.EpisodeNumber),
+                                          EndEpisode = g.Max(c => c.EpisodeNumber),
+                                          EpisodeCount = g.Count(),
+                                          CharCount = g.Sum(c => c.CharCount)
+                                      })
+                                      .OrderBy(g => g.StartEpisode)
+                                      .ToListAsync();
+
+        return Ok(new
+        {
+            tracked.DeckId,
+            Provider = tracked.Provider.ToString(),
+            tracked.SourceId,
+            Url = WebNovelUrlParser.BuildWorkUrl(tracked.Provider, tracked.SourceId),
+            tracked.LastEpisodeCount,
+            tracked.LastSourceUpdate,
+            tracked.LastSyncedAt,
+            tracked.NextCheckAt,
+            tracked.SyncEnabled,
+            tracked.CompletedAtSource,
+            tracked.OnHiatusAtSource,
+            tracked.ConsecutiveFailures,
+            tracked.LastError,
+            tracked.ChunkCharBudget,
+            tracked.PendingRevisionCount,
+            Subdecks = subdecks
+        });
+    }
+
+    [HttpPost("webnovel/{deckId:int}/sync")]
+    public async Task<IActionResult> SyncWebNovelNow(int deckId)
+    {
+        if (!await dbContext.WebNovelSources.AnyAsync(s => s.DeckId == deckId))
+            return NotFound();
+
+        var jobId = backgroundJobs.Enqueue<WebNovelFetchJob>(job => job.Sync(deckId));
+
+        return Accepted(new { Message = "Sync queued.", JobId = jobId });
+    }
+
+    /// <summary>
+    /// Re-fetches one subdeck's whole episode range, picking up revisions (改稿) to episodes already ingested.
+    /// </summary>
+    [HttpPost("webnovel/{deckId:int}/rebuild/{childDeckId:int}")]
+    public async Task<IActionResult> RebuildWebNovelSubdeck(int deckId, int childDeckId)
+    {
+        if (!await dbContext.WebNovelChapters.AnyAsync(c => c.DeckId == deckId && c.ChildDeckId == childDeckId))
+            return NotFound();
+
+        var jobId = backgroundJobs.Enqueue<WebNovelFetchJob>(job => job.RebuildSubdeck(deckId, childDeckId));
+
+        return Accepted(new { Message = "Rebuild queued.", JobId = jobId });
+    }
+
+    [HttpPost("webnovel/{deckId:int}/sync-enabled")]
+    public async Task<IActionResult> SetWebNovelSyncEnabled(int deckId, [FromBody] SetWebNovelSyncEnabledRequest model)
+    {
+        var tracked = await dbContext.WebNovelSources.FirstOrDefaultAsync(s => s.DeckId == deckId);
+        if (tracked == null)
+            return NotFound();
+
+        tracked.SyncEnabled = model.Enabled;
+
+        // Coming off a freeze shouldn't wait out a backoff window
+        if (model.Enabled)
+        {
+            tracked.NextCheckAt = DateTimeOffset.UtcNow;
+            tracked.ConsecutiveFailures = 0;
+            tracked.LastError = null;
+        }
+
+        await dbContext.SaveChangesAsync();
+
+        return Ok(new { tracked.DeckId, tracked.SyncEnabled });
+    }
+
+    private static int EstimateSubdecks(long totalCharacters) =>
+        totalCharacters <= 0
+            ? 1
+            : (int)Math.Max(1, Math.Ceiling(totalCharacters / (double)SubdeckChunker.DefaultCharBudget));
+}
+
+public class SetWebNovelSyncEnabledRequest
+{
+    public bool Enabled { get; set; }
+}

Jiten.Api/Controllers/AdminController.cs

@@ -1155,6 +1155,9 @@ public async Task<IActionResult> FetchMetadata(int deckId)
                 else
                     backgroundJobs.Enqueue<FetchMetadataJob>(job => job.FetchGoogleBooksMissingMetadata(deckId));
                 break;
+            case MediaType.WebNovel:
+                backgroundJobs.Enqueue<FetchMetadataJob>(job => job.FetchSyosetuMissingMetadata(deckId));
+                break;
             default:
                 return NotFound("No fetch job for this media type.");
         }
@@ -1204,6 +1207,10 @@ public async Task<IActionResult> FetchAllMissingMetadata()
                     }
 
                     break;
+                case MediaType.WebNovel:
+                    if (deck.Links.Any(l => l.LinkType == LinkType.Syosetsu))
+                        backgroundJobs.Enqueue<FetchMetadataJob>(job => job.FetchSyosetuMissingMetadata(deck.DeckId));
+                    break;
                 default:
                     break;
             }

Jiten.Api/Dtos/Requests/AddWebNovelDeckRequest.cs

@@ -0,0 +1,19 @@
+namespace Jiten.Api.Dtos.Requests;
+
+public class AddWebNovelDeckRequest
+{
+    /// <summary>
+    /// Novel URL (https://ncode.syosetu.com/n9669bk/) or a bare ncode
+    /// </summary>
+    public required string Url { get; set; }
+
+    /// <summary>
+    /// Optional: uploaded or generated in the browser. Syosetu works have no cover art of their own.
+    /// </summary>
+    public IFormFile? CoverImage { get; set; }
+
+    /// <summary>
+    /// Optional per-novel override of the subdeck character budget
+    /// </summary>
+    public int? ChunkCharBudget { get; set; }
+}

Jiten.Api/Jobs/FetchMetadataJob.cs

@@ -2,11 +2,16 @@
 using Jiten.Core;
 using Jiten.Core.Data;
 using Jiten.Core.Data.Providers;
+using Jiten.Core.Data.WebNovel;
+using Jiten.Core.WebNovel;
 using Microsoft.EntityFrameworkCore;
 
 namespace Jiten.Api.Jobs;
 
-public class FetchMetadataJob(IDbContextFactory<JitenDbContext> contextFactory, IConfiguration configuration)
+public class FetchMetadataJob(
+    IDbContextFactory<JitenDbContext> contextFactory,
+    IConfiguration configuration,
+    IWebNovelSourceResolver sourceResolver)
 {
     private const float ANILIST_DELAY = 2.2f;
     private const float GOOGLE_BOOKS_DELAY = 3f;
@@ -366,6 +371,64 @@ public async Task FetchJikanMissingMetadata(int deckId)
         }
     }
 
+    /// <summary>
+    /// Refreshes a tracked webnovel from the source's metadata API
+    /// </summary>
+    [Queue(WebNovelQueues.SyosetuMetadata)]
+    public async Task FetchSyosetuMissingMetadata(int deckId)
+    {
+        await using var context = await contextFactory.CreateDbContextAsync();
+
+        var deck = await context.Decks
+                                .Include(d => d.Links)
+                                .Include(d => d.Titles)
+                                .Include(d => d.DeckGenres)
+                                .Include(d => d.DeckTags)
+                                .FirstAsync(d => d.DeckId == deckId);
+
+        var (provider, sourceId) = await ResolveWebNovelWork(context, deck);
+
+        var info = await sourceResolver.Resolve(provider).GetInfoAsync(sourceId);
+        var metadata = info.ToMetadata();
+
+        if (deck.ReleaseDate == default && metadata.ReleaseDate != null)
+            deck.ReleaseDate = DateOnly.FromDateTime(metadata.ReleaseDate.Value);
+
+        if (string.IsNullOrEmpty(deck.Description))
+            deck.Description = metadata.Description?.Length > 2000 ? metadata.Description?[..2000] : metadata.Description;
+
+        if (deck.Links.All(l => l.LinkType != LinkType.Syosetsu))
+            deck.Links.Add(new Link { DeckId = deck.DeckId, LinkType = LinkType.Syosetsu, Url = info.Url });
+
+        await MetadataProviderHelper.ApplyGenreAndTagMappings(context, deck, metadata, LinkType.Syosetsu);
+
+        await context.SaveChangesAsync();
+    }
+
+    /// <summary>
+    /// The ledger is the authority on which work a deck came from; the link is the fallback for a webnovel
+    /// deck that was added by hand rather than imported.
+    /// </summary>
+    private static async Task<(WebNovelProvider Provider, string SourceId)> ResolveWebNovelWork(JitenDbContext context, Deck deck)
+    {
+        var tracked = await context.WebNovelSources
+                                   .AsNoTracking()
+                                   .FirstOrDefaultAsync(s => s.DeckId == deck.DeckId);
+
+        if (tracked != null)
+            return (tracked.Provider, tracked.SourceId);
+
+        var link = deck.Links.FirstOrDefault(l => l.LinkType == LinkType.Syosetsu);
+
+        if (link == null)
+            throw new Exception($"No Syosetsu link found for deck with ID {deck.DeckId}.");
+
+        if (!WebNovelUrlParser.TryParse(link.Url, out var provider, out var sourceId))
+            throw new Exception($"Syosetsu link '{link.Url}' on deck {deck.DeckId} is not a work URL.");
+
+        return (provider, sourceId);
+    }
+
     private static void MergeDictionaryEntries(Deck deck, List<DeckDictionaryEntry> entries)
     {
         foreach (var entry in entries)

Jiten.Api/Jobs/ParseJob.cs

@@ -18,6 +18,15 @@ public class ParseJob(
 {
     [Queue("parse")]
     public async Task Parse(Metadata metadata, MediaType deckType, bool storeRawText = false)
+    {
+        await ParseAndGetDeckId(metadata, deckType, storeRawText);
+    }
+
+    /// <summary>
+    /// Same as <see cref="Parse"/>, but hands back the created deck. Callers that need to attach their own
+    /// rows to the new deck tree (webnovel ledger) run it inline instead of enqueuing it.
+    /// </summary>
+    public async Task<int> ParseAndGetDeckId(Metadata metadata, MediaType deckType, bool storeRawText = false)
     {
         Deck deck = new();
         string filePath = metadata.FilePath!;
@@ -127,10 +136,13 @@ public async Task Parse(Metadata metadata, MediaType deckType, bool storeRawText
             await MetadataProviderHelper.ApplyGenreAndTagMappings(context, deck, metadata, firstLink.LinkType);
         }
 
-        var coverImage = await File.ReadAllBytesAsync(metadata.Image ?? throw new Exception("No cover image found."));
+        // Sources with no cover art (webnovels) fall back to nocover.jpg, set above
+        var coverImage = !string.IsNullOrEmpty(metadata.Image) && File.Exists(metadata.Image)
+            ? await File.ReadAllBytesAsync(metadata.Image)
+            : [];
 
         // Insert the deck into the database
-        await JitenHelper.InsertDeck(contextFactory, deck, coverImage ?? [], false);
+        await JitenHelper.InsertDeck(contextFactory, deck, coverImage, false);
 
         // Process relations from metadata
         if (metadata.Relations.Count > 0)
@@ -154,6 +166,8 @@ public async Task Parse(Metadata metadata, MediaType deckType, bool storeRawText
 
         // Notify IndexNow of the new (top-level) deck page so Bing can index it quickly.
         await indexNow.SubmitDeckAsync(deck.DeckId);
+
+        return deck.DeckId;
     }
 
     /// <summary>