feat: complete TODO parity with microposts and local rewriter

.rubocop_todo.yml

@@ -1,20 +1,12 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2026-05-13 07:38:03 UTC using RuboCop version 1.86.1.
+# on 2026-05-13 08:31:40 UTC using RuboCop version 1.86.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 1
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: EnforcedStyle, IndentationWidth.
-# SupportedStyles: with_first_argument, with_fixed_indentation
-Layout/ArgumentAlignment:
-  Exclude:
-    - 'lib/archaeo/warc_support.rb'
-
-# Offense count: 10
+# Offense count: 9
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https
@@ -25,22 +17,24 @@ Layout/LineLength:
     - 'lib/archaeo/page.rb'
     - 'lib/archaeo/path_sanitizer.rb'
     - 'lib/archaeo/url_rewriter.rb'
-    - 'lib/archaeo/warc_support.rb'
     - 'spec/archaeo/asset_extractor_spec.rb'
 
-# Offense count: 1
+# Offense count: 2
 # This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: AllowInHeredoc.
-Layout/TrailingWhitespace:
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: aligned, indented, indented_relative_to_receiver
+Layout/MultilineMethodCallIndentation:
   Exclude:
-    - 'lib/archaeo/warc_support.rb'
+    - 'lib/archaeo/page.rb'
 
-# Offense count: 1
+# Offense count: 3
 Lint/UselessConstantScoping:
   Exclude:
     - 'lib/archaeo/bulk_downloader.rb'
+    - 'lib/archaeo/local_rewriter.rb'
+    - 'lib/archaeo/page.rb'
 
-# Offense count: 26
+# Offense count: 28
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
 Metrics/AbcSize:
   Exclude:
@@ -53,6 +47,7 @@ Metrics/AbcSize:
     - 'lib/archaeo/configuration.rb'
     - 'lib/archaeo/content_tracker.rb'
     - 'lib/archaeo/coverage_report.rb'
+    - 'lib/archaeo/local_rewriter.rb'
     - 'lib/archaeo/parallel_cdx.rb'
     - 'lib/archaeo/pattern_filter.rb'
     - 'lib/archaeo/progress_report.rb'
@@ -73,7 +68,7 @@ Metrics/CyclomaticComplexity:
     - 'lib/archaeo/pattern_filter.rb'
     - 'lib/archaeo/url_rewriter.rb'
 
-# Offense count: 45
+# Offense count: 48
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 32
@@ -96,11 +91,12 @@ Metrics/PerceivedComplexity:
     - 'lib/archaeo/pattern_filter.rb'
     - 'lib/archaeo/url_rewriter.rb'
 
-# Offense count: 2
+# Offense count: 6
 # Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
 # AllowedNames: as, at, by, cc, db, id, if, in, io, ip, of, on, os, pp, to
 Naming/MethodParameterName:
   Exclude:
+    - 'lib/archaeo/page.rb'
     - 'spec/archaeo/coverage_report_spec.rb'
     - 'spec/archaeo/download_scheduler_spec.rb'
 
@@ -118,7 +114,15 @@ Performance/StringBytesize:
   Exclude:
     - 'lib/archaeo/encoding_detector.rb'
 
-# Offense count: 112
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: SkipBlocks, EnforcedStyle, OnlyStaticConstants.
+# SupportedStyles: described_class, explicit
+RSpec/DescribedClass:
+  Exclude:
+    - 'spec/archaeo/page_microposts_spec.rb'
+
+# Offense count: 131
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 36
@@ -153,7 +157,7 @@ RSpec/MultipleDescribes:
     - 'spec/archaeo/path_sanitizer_spec.rb'
     - 'spec/archaeo/warc_support_spec.rb'
 
-# Offense count: 133
+# Offense count: 145
 RSpec/MultipleExpectations:
   Max: 11
 
@@ -175,8 +179,11 @@ RSpec/StubbedMock:
   Exclude:
     - 'spec/archaeo/parallel_cdx_spec.rb'
 
-# Offense count: 1
+# Offense count: 3
 # This cop supports safe autocorrection (--autocorrect).
-Style/ExplicitBlockArgument:
+# Configuration parameters: EnforcedStyleForMultiline.
+# SupportedStylesForMultiline: comma, consistent_comma, diff_comma, no_comma
+Style/TrailingCommaInArguments:
   Exclude:
-    - 'lib/archaeo/warc_support.rb'
+    - 'lib/archaeo/cli.rb'
+    - 'lib/archaeo/page.rb'

lib/archaeo.rb

@@ -69,4 +69,6 @@ def initialize(message, status_code:, url:, page:)
   autoload :ContentChangeReport, "archaeo/content_tracker"
   autoload :ArchiveSearch, "archaeo/archive_search"
   autoload :SearchResult, "archaeo/archive_search"
+  autoload :LocalRewriter, "archaeo/local_rewriter"
+  autoload :LocalRewriteSummary, "archaeo/local_rewriter"
 end

lib/archaeo/cli.rb

@@ -194,6 +194,32 @@ def rewrite(url, timestamp)
       end
     end
 
+    desc "rewrite-local INPUT_DIR",
+         "Rewrite previously downloaded files to use local paths"
+    option :output, desc: "Output directory (default: rewrite in-place)",
+                    required: false
+    option :prefix, desc: "Local path prefix", default: "local"
+    option :rewrite_js, type: :boolean, default: false,
+                        desc: "Rewrite URLs in JavaScript strings"
+    option :rewrite_absolute, type: :boolean, default: false,
+                              desc: "Rewrite all absolute archive URLs"
+    def rewrite_local(input_dir)
+      handle_errors do
+        output_dir = options[:output] || input_dir
+        local_rewriter = LocalRewriter.new(
+          prefix: options[:prefix],
+          rewrite_js: options[:rewrite_js],
+          rewrite_absolute: options[:rewrite_absolute],
+        )
+        summary = local_rewriter.rewrite_directory(input_dir, output_dir)
+        color = build_color
+        warn color.success(
+          "Rewrote #{summary.rewritten}/#{summary.total} files " \
+          "in #{summary.elapsed.round(1)}s",
+        )
+      end
+    end
+
     desc "diff URL TIMESTAMP_A TIMESTAMP_B",
          "Compare assets of two archived snapshots"
     option :format, desc: "Output format (table, json)", default: "table"

lib/archaeo/local_rewriter.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Archaeo
+  LocalRewriteSummary = Struct.new(
+    :total, :rewritten, :skipped, :elapsed,
+    keyword_init: true
+  )
+
+  # Rewrites previously downloaded files by converting archive URLs
+  # to local paths. Operates on files already on disk without fetching.
+  class LocalRewriter
+    def initialize(prefix: "local", rewrite_js: false,
+                   rewrite_absolute: false)
+      @prefix = prefix
+      @rewrite_js = rewrite_js
+      @rewrite_absolute = rewrite_absolute
+    end
+
+    def rewrite_directory(input_dir, output_dir)
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      files = gather_files(input_dir)
+      rewritten = 0
+      skipped = 0
+
+      files.each do |path|
+        rel = path.sub(%r{\A#{Regexp.escape(input_dir)}/?}, "")
+        out_path = File.join(output_dir, rel)
+
+        result = rewrite_file(path, out_path)
+        result ? rewritten += 1 : skipped += 1
+      end
+
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+      LocalRewriteSummary.new(
+        total: files.size, rewritten: rewritten,
+        skipped: skipped, elapsed: elapsed
+      )
+    end
+
+    def rewrite_file(input_path, output_path)
+      content = File.read(input_path)
+      return nil unless rewrite_candidate?(content)
+
+      FileUtils.mkdir_p(File.dirname(output_path))
+      rewriter = build_rewriter
+      rewritten = apply_rewriting(rewriter, content, input_path)
+      File.write(output_path, rewritten)
+      true
+    end
+
+    private
+
+    def gather_files(dir)
+      Dir.glob(File.join(dir, "**", "*"))
+        .select { |f| File.file?(f) && text_file?(f) }
+    end
+
+    def text_file?(path)
+      ext = File.extname(path).downcase
+      TEXT_EXTENSIONS.include?(ext)
+    end
+
+    TEXT_EXTENSIONS = %w[
+      .html .htm .xhtml .css .js .json .xml .txt
+      .svg .md .yaml .yml .rss .atom
+    ].freeze
+
+    def rewrite_candidate?(content)
+      content.include?("web.archive.org")
+    end
+
+    def build_rewriter
+      UrlRewriter.new(
+        "https://web.archive.org", @prefix,
+        rewrite_js: @rewrite_js,
+        rewrite_absolute: @rewrite_absolute
+      )
+    end
+
+    def apply_rewriting(rewriter, content, path)
+      ext = File.extname(path).downcase
+      case ext
+      when ".html", ".htm", ".xhtml"
+        rewriter.rewrite_html(content)
+      when ".css"
+        rewriter.rewrite_css(content)
+      when ".js"
+        rewriter.rewrite_js(content)
+      else
+        rewrite_mixed(rewriter, content)
+      end
+    end
+
+    def rewrite_mixed(rewriter, content)
+      if content.include?("<") && content.include?(">")
+        rewriter.rewrite_html(content)
+      elsif content.include?("url(")
+        rewriter.rewrite_css(content)
+      else
+        rewriter.rewrite_js(content)
+      end
+    end
+  end
+end

lib/archaeo/page.rb

@@ -139,6 +139,16 @@ def scripts
       end
     end
 
+    def microposts
+      return [] unless html?
+
+      @microposts ||= begin
+        doc = Nokogiri::HTML(@raw_content)
+        containers = find_article_containers(doc)
+        containers.filter_map { |el| extract_micropost(el) }
+      end
+    end
+
     def to_h
       {
         content_type: @content_type,
@@ -255,5 +265,58 @@ def extract_form_fields(form)
       end
       inputs.reject { |f| f[:name].empty? }
     end
+
+    ARTICLE_SELECTORS = %w[
+      article [role=article] .post .entry .blog-post
+      .hentry .post-content .entry-content .article-content
+      .story .story-body .news-article
+    ].freeze
+
+    def find_article_containers(doc)
+      found = ARTICLE_SELECTORS
+        .filter_map { |sel| doc.css(sel) }
+        .flat_map(&:to_a)
+      found.any? ? found.uniq : [doc.at_css("body") || doc]
+    end
+
+    def extract_micropost(element)
+      title = extract_micropost_title(element)
+      body = extract_micropost_body(element)
+      return nil if body.nil? || body.strip.empty?
+
+      { title: title, body: body.strip,
+        date: extract_micropost_date(element),
+        author: extract_micropost_author(element) }
+    end
+
+    def extract_micropost_title(el)
+      heading = el.at_css("h1, h2, h3, [class*=title], [class*=heading]")
+      heading&.text&.strip
+    end
+
+    def extract_micropost_body(el)
+      paragraphs = el.css("p").map(&:text).join("\n")
+      return nil if paragraphs.strip.empty?
+
+      paragraphs
+    end
+
+    def extract_micropost_date(el)
+      time = el.at_css("time[datetime]")
+      return time["datetime"] if time
+
+      date_el = el.at_css(
+        "[class*=date], [class*=time], [class*=published], " \
+        "[property='datePublished']",
+      )
+      date_el&.text&.strip
+    end
+
+    def extract_micropost_author(el)
+      author_el = el.at_css(
+        "[class*=author], [rel=author], [property='author']",
+      )
+      author_el&.text&.strip
+    end
   end
 end