diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 0133d43..8ff8667 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,20 +1,12 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2026-05-13 07:38:03 UTC using RuboCop version 1.86.1. +# on 2026-05-13 08:31:40 UTC using RuboCop version 1.86.1. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 1 -# This cop supports safe autocorrection (--autocorrect). -# Configuration parameters: EnforcedStyle, IndentationWidth. -# SupportedStyles: with_first_argument, with_fixed_indentation -Layout/ArgumentAlignment: - Exclude: - - 'lib/archaeo/warc_support.rb' - -# Offense count: 10 +# Offense count: 9 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings. # URISchemes: http, https @@ -25,22 +17,24 @@ Layout/LineLength: - 'lib/archaeo/page.rb' - 'lib/archaeo/path_sanitizer.rb' - 'lib/archaeo/url_rewriter.rb' - - 'lib/archaeo/warc_support.rb' - 'spec/archaeo/asset_extractor_spec.rb' -# Offense count: 1 +# Offense count: 2 # This cop supports safe autocorrection (--autocorrect). -# Configuration parameters: AllowInHeredoc. -Layout/TrailingWhitespace: +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: aligned, indented, indented_relative_to_receiver +Layout/MultilineMethodCallIndentation: Exclude: - - 'lib/archaeo/warc_support.rb' + - 'lib/archaeo/page.rb' -# Offense count: 1 +# Offense count: 3 Lint/UselessConstantScoping: Exclude: - 'lib/archaeo/bulk_downloader.rb' + - 'lib/archaeo/local_rewriter.rb' + - 'lib/archaeo/page.rb' -# Offense count: 26 +# Offense count: 28 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max. Metrics/AbcSize: Exclude: @@ -53,6 +47,7 @@ Metrics/AbcSize: - 'lib/archaeo/configuration.rb' - 'lib/archaeo/content_tracker.rb' - 'lib/archaeo/coverage_report.rb' + - 'lib/archaeo/local_rewriter.rb' - 'lib/archaeo/parallel_cdx.rb' - 'lib/archaeo/pattern_filter.rb' - 'lib/archaeo/progress_report.rb' @@ -73,7 +68,7 @@ Metrics/CyclomaticComplexity: - 'lib/archaeo/pattern_filter.rb' - 'lib/archaeo/url_rewriter.rb' -# Offense count: 45 +# Offense count: 48 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. Metrics/MethodLength: Max: 32 @@ -96,11 +91,12 @@ Metrics/PerceivedComplexity: - 'lib/archaeo/pattern_filter.rb' - 'lib/archaeo/url_rewriter.rb' -# Offense count: 2 +# Offense count: 6 # Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames. # AllowedNames: as, at, by, cc, db, id, if, in, io, ip, of, on, os, pp, to Naming/MethodParameterName: Exclude: + - 'lib/archaeo/page.rb' - 'spec/archaeo/coverage_report_spec.rb' - 'spec/archaeo/download_scheduler_spec.rb' @@ -118,7 +114,15 @@ Performance/StringBytesize: Exclude: - 'lib/archaeo/encoding_detector.rb' -# Offense count: 112 +# Offense count: 1 +# This cop supports unsafe autocorrection (--autocorrect-all). +# Configuration parameters: SkipBlocks, EnforcedStyle, OnlyStaticConstants. +# SupportedStyles: described_class, explicit +RSpec/DescribedClass: + Exclude: + - 'spec/archaeo/page_microposts_spec.rb' + +# Offense count: 131 # Configuration parameters: CountAsOne. RSpec/ExampleLength: Max: 36 @@ -153,7 +157,7 @@ RSpec/MultipleDescribes: - 'spec/archaeo/path_sanitizer_spec.rb' - 'spec/archaeo/warc_support_spec.rb' -# Offense count: 133 +# Offense count: 145 RSpec/MultipleExpectations: Max: 11 @@ -175,8 +179,11 @@ RSpec/StubbedMock: Exclude: - 'spec/archaeo/parallel_cdx_spec.rb' -# Offense count: 1 +# Offense count: 3 # This cop supports safe autocorrection (--autocorrect). -Style/ExplicitBlockArgument: +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, diff_comma, no_comma +Style/TrailingCommaInArguments: Exclude: - - 'lib/archaeo/warc_support.rb' + - 'lib/archaeo/cli.rb' + - 'lib/archaeo/page.rb' diff --git a/lib/archaeo.rb b/lib/archaeo.rb index 9a45976..d7abd2c 100644 --- a/lib/archaeo.rb +++ b/lib/archaeo.rb @@ -69,4 +69,6 @@ def initialize(message, status_code:, url:, page:) autoload :ContentChangeReport, "archaeo/content_tracker" autoload :ArchiveSearch, "archaeo/archive_search" autoload :SearchResult, "archaeo/archive_search" + autoload :LocalRewriter, "archaeo/local_rewriter" + autoload :LocalRewriteSummary, "archaeo/local_rewriter" end diff --git a/lib/archaeo/cli.rb b/lib/archaeo/cli.rb index f586f3a..77c8939 100644 --- a/lib/archaeo/cli.rb +++ b/lib/archaeo/cli.rb @@ -194,6 +194,32 @@ def rewrite(url, timestamp) end end + desc "rewrite-local INPUT_DIR", + "Rewrite previously downloaded files to use local paths" + option :output, desc: "Output directory (default: rewrite in-place)", + required: false + option :prefix, desc: "Local path prefix", default: "local" + option :rewrite_js, type: :boolean, default: false, + desc: "Rewrite URLs in JavaScript strings" + option :rewrite_absolute, type: :boolean, default: false, + desc: "Rewrite all absolute archive URLs" + def rewrite_local(input_dir) + handle_errors do + output_dir = options[:output] || input_dir + local_rewriter = LocalRewriter.new( + prefix: options[:prefix], + rewrite_js: options[:rewrite_js], + rewrite_absolute: options[:rewrite_absolute], + ) + summary = local_rewriter.rewrite_directory(input_dir, output_dir) + color = build_color + warn color.success( + "Rewrote #{summary.rewritten}/#{summary.total} files " \ + "in #{summary.elapsed.round(1)}s", + ) + end + end + desc "diff URL TIMESTAMP_A TIMESTAMP_B", "Compare assets of two archived snapshots" option :format, desc: "Output format (table, json)", default: "table" diff --git a/lib/archaeo/local_rewriter.rb b/lib/archaeo/local_rewriter.rb new file mode 100644 index 0000000..f315225 --- /dev/null +++ b/lib/archaeo/local_rewriter.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require "fileutils" + +module Archaeo + LocalRewriteSummary = Struct.new( + :total, :rewritten, :skipped, :elapsed, + keyword_init: true + ) + + # Rewrites previously downloaded files by converting archive URLs + # to local paths. Operates on files already on disk without fetching. + class LocalRewriter + def initialize(prefix: "local", rewrite_js: false, + rewrite_absolute: false) + @prefix = prefix + @rewrite_js = rewrite_js + @rewrite_absolute = rewrite_absolute + end + + def rewrite_directory(input_dir, output_dir) + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + files = gather_files(input_dir) + rewritten = 0 + skipped = 0 + + files.each do |path| + rel = path.sub(%r{\A#{Regexp.escape(input_dir)}/?}, "") + out_path = File.join(output_dir, rel) + + result = rewrite_file(path, out_path) + result ? rewritten += 1 : skipped += 1 + end + + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time + LocalRewriteSummary.new( + total: files.size, rewritten: rewritten, + skipped: skipped, elapsed: elapsed + ) + end + + def rewrite_file(input_path, output_path) + content = File.read(input_path) + return nil unless rewrite_candidate?(content) + + FileUtils.mkdir_p(File.dirname(output_path)) + rewriter = build_rewriter + rewritten = apply_rewriting(rewriter, content, input_path) + File.write(output_path, rewritten) + true + end + + private + + def gather_files(dir) + Dir.glob(File.join(dir, "**", "*")) + .select { |f| File.file?(f) && text_file?(f) } + end + + def text_file?(path) + ext = File.extname(path).downcase + TEXT_EXTENSIONS.include?(ext) + end + + TEXT_EXTENSIONS = %w[ + .html .htm .xhtml .css .js .json .xml .txt + .svg .md .yaml .yml .rss .atom + ].freeze + + def rewrite_candidate?(content) + content.include?("web.archive.org") + end + + def build_rewriter + UrlRewriter.new( + "https://web.archive.org", @prefix, + rewrite_js: @rewrite_js, + rewrite_absolute: @rewrite_absolute + ) + end + + def apply_rewriting(rewriter, content, path) + ext = File.extname(path).downcase + case ext + when ".html", ".htm", ".xhtml" + rewriter.rewrite_html(content) + when ".css" + rewriter.rewrite_css(content) + when ".js" + rewriter.rewrite_js(content) + else + rewrite_mixed(rewriter, content) + end + end + + def rewrite_mixed(rewriter, content) + if content.include?("<") && content.include?(">") + rewriter.rewrite_html(content) + elsif content.include?("url(") + rewriter.rewrite_css(content) + else + rewriter.rewrite_js(content) + end + end + end +end diff --git a/lib/archaeo/page.rb b/lib/archaeo/page.rb index 81d53fd..15e4322 100644 --- a/lib/archaeo/page.rb +++ b/lib/archaeo/page.rb @@ -139,6 +139,16 @@ def scripts end end + def microposts + return [] unless html? + + @microposts ||= begin + doc = Nokogiri::HTML(@raw_content) + containers = find_article_containers(doc) + containers.filter_map { |el| extract_micropost(el) } + end + end + def to_h { content_type: @content_type, @@ -255,5 +265,58 @@ def extract_form_fields(form) end inputs.reject { |f| f[:name].empty? } end + + ARTICLE_SELECTORS = %w[ + article [role=article] .post .entry .blog-post + .hentry .post-content .entry-content .article-content + .story .story-body .news-article + ].freeze + + def find_article_containers(doc) + found = ARTICLE_SELECTORS + .filter_map { |sel| doc.css(sel) } + .flat_map(&:to_a) + found.any? ? found.uniq : [doc.at_css("body") || doc] + end + + def extract_micropost(element) + title = extract_micropost_title(element) + body = extract_micropost_body(element) + return nil if body.nil? || body.strip.empty? + + { title: title, body: body.strip, + date: extract_micropost_date(element), + author: extract_micropost_author(element) } + end + + def extract_micropost_title(el) + heading = el.at_css("h1, h2, h3, [class*=title], [class*=heading]") + heading&.text&.strip + end + + def extract_micropost_body(el) + paragraphs = el.css("p").map(&:text).join("\n") + return nil if paragraphs.strip.empty? + + paragraphs + end + + def extract_micropost_date(el) + time = el.at_css("time[datetime]") + return time["datetime"] if time + + date_el = el.at_css( + "[class*=date], [class*=time], [class*=published], " \ + "[property='datePublished']", + ) + date_el&.text&.strip + end + + def extract_micropost_author(el) + author_el = el.at_css( + "[class*=author], [rel=author], [property='author']", + ) + author_el&.text&.strip + end end end diff --git a/spec/archaeo/local_rewriter_spec.rb b/spec/archaeo/local_rewriter_spec.rb new file mode 100644 index 0000000..ecbbed8 --- /dev/null +++ b/spec/archaeo/local_rewriter_spec.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require "spec_helper" +require "tmpdir" +require "fileutils" + +RSpec.describe Archaeo::LocalRewriter do + let(:tmpdir) { Dir.mktmpdir } + let(:output_dir) { File.join(tmpdir, "output") } + + after { FileUtils.rm_rf(tmpdir) } + + def write_input_file(name, content) + path = File.join(tmpdir, "input", name) + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, content) + path + end + + it "rewrites HTML files with archive URLs" do + write_input_file("page.html", <<~HTML) + + + + HTML + + rewriter = described_class.new(prefix: "local") + summary = rewriter.rewrite_directory( + File.join(tmpdir, "input"), output_dir + ) + + expect(summary.rewritten).to eq(1) + expect(summary.total).to eq(1) + output = File.read(File.join(output_dir, "page.html")) + expect(output).not_to include("web.archive.org") + expect(output).to include("local") + end + + it "rewrites CSS files with archive URLs" do + write_input_file("style.css", <<~CSS) + .bg { background: url('https://web.archive.org/web/20220615/https://example.com/bg.png'); } + CSS + + rewriter = described_class.new(prefix: "assets") + summary = rewriter.rewrite_directory( + File.join(tmpdir, "input"), output_dir + ) + + expect(summary.rewritten).to eq(1) + output = File.read(File.join(output_dir, "style.css")) + expect(output).not_to include("web.archive.org") + expect(output).to include("assets") + end + + it "rewrites JS files when rewrite_js is enabled" do + write_input_file("app.js", <<~JS) + var url = 'https://web.archive.org/web/20220615/https://example.com/api'; + JS + + rewriter = described_class.new(prefix: "local", rewrite_js: true) + summary = rewriter.rewrite_directory( + File.join(tmpdir, "input"), output_dir + ) + + expect(summary.rewritten).to eq(1) + output = File.read(File.join(output_dir, "app.js")) + expect(output).not_to include("web.archive.org") + end + + it "skips files without archive URLs" do + write_input_file("clean.html", "Hello") + + rewriter = described_class.new(prefix: "local") + summary = rewriter.rewrite_directory( + File.join(tmpdir, "input"), output_dir + ) + + expect(summary.skipped).to eq(1) + expect(summary.rewritten).to eq(0) + end + + it "skips binary files" do + binary_path = File.join(tmpdir, "input", "image.png") + FileUtils.mkdir_p(File.dirname(binary_path)) + File.binwrite(binary_path, "\x89PNG\r\n\x1a\n") + + rewriter = described_class.new(prefix: "local") + summary = rewriter.rewrite_directory( + File.join(tmpdir, "input"), output_dir + ) + + expect(summary.total).to eq(0) + end + + it "handles nested directory structures" do + write_input_file("sub/deep/page.html", <<~HTML) + + Link + + HTML + + rewriter = described_class.new(prefix: "local") + summary = rewriter.rewrite_directory( + File.join(tmpdir, "input"), output_dir + ) + + expect(summary.rewritten).to eq(1) + output = File.read(File.join(output_dir, "sub/deep/page.html")) + expect(output).not_to include("web.archive.org") + end + + it "rewrites in-place when no output directory specified" do + input_dir = File.join(tmpdir, "input") + write_input_file("page.html", <<~HTML) + + + + HTML + + rewriter = described_class.new(prefix: "local") + summary = rewriter.rewrite_directory(input_dir, input_dir) + + expect(summary.rewritten).to eq(1) + output = File.read(File.join(input_dir, "page.html")) + expect(output).not_to include("web.archive.org") + end + + it "reports elapsed time" do + write_input_file("page.html", <<~HTML) + + + + HTML + + rewriter = described_class.new(prefix: "local") + summary = rewriter.rewrite_directory( + File.join(tmpdir, "input"), output_dir + ) + + expect(summary.elapsed).to be >= 0 + end + + describe "#rewrite_file" do + it "returns nil for files without archive URLs" do + input = File.join(tmpdir, "clean.html") + output = File.join(tmpdir, "out.html") + File.write(input, "Hello") + + rewriter = described_class.new(prefix: "local") + expect(rewriter.rewrite_file(input, output)).to be_nil + end + + it "returns true for successfully rewritten files" do + input = File.join(tmpdir, "page.html") + output = File.join(tmpdir, "out.html") + File.write(input, <<~HTML) + + + + HTML + + rewriter = described_class.new(prefix: "local") + expect(rewriter.rewrite_file(input, output)).to be true + expect(File.read(output)).to include("local") + end + end +end diff --git a/spec/archaeo/page_microposts_spec.rb b/spec/archaeo/page_microposts_spec.rb new file mode 100644 index 0000000..ec46cbb --- /dev/null +++ b/spec/archaeo/page_microposts_spec.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Archaeo::Page, "#microposts" do + def build_page(html) + Archaeo::Page.new( + content: html, + content_type: "text/html", + status_code: 200, + archive_url: "https://web.archive.org/web/20220615000000/https://example.com/", + original_url: "https://example.com/", + timestamp: "20220615000000", + ) + end + + it "extracts microposts from article elements" do + page = build_page(<<~HTML) + +
+

First Post

+ +

This is the first article content.

+

It has multiple paragraphs.

+
+ + HTML + + posts = page.microposts + expect(posts.size).to eq(1) + expect(posts[0][:title]).to eq("First Post") + expect(posts[0][:body]).to include("first article content") + expect(posts[0][:date]).to eq("2022-06-01") + end + + it "extracts author when present" do + page = build_page(<<~HTML) + +
+

Post Title

+ Jane Doe +

Article body text here.

+
+ + HTML + + posts = page.microposts + expect(posts[0][:author]).to eq("Jane Doe") + end + + it "extracts multiple microposts" do + page = build_page(<<~HTML) + +
+

Post One

+

Content one.

+
+
+

Post Two

+

Content two.

+
+ + HTML + + posts = page.microposts + expect(posts.size).to eq(2) + expect(posts[0][:title]).to eq("Post One") + expect(posts[1][:title]).to eq("Post Two") + end + + it "extracts from role=article" do + page = build_page(<<~HTML) + +
+

ARIA Article

+

Body text.

+
+ + HTML + + posts = page.microposts + expect(posts.size).to eq(1) + expect(posts[0][:title]).to eq("ARIA Article") + end + + it "extracts from common blog post classes" do + page = build_page(<<~HTML) + +
+

Blog Post

+

Blog content here.

+
+ + HTML + + posts = page.microposts + expect(posts.size).to eq(1) + expect(posts[0][:title]).to eq("Blog Post") + end + + it "falls back to body when no article containers found" do + page = build_page(<<~HTML) + +

Page Title

+

Some body text content here.

+

Another paragraph with text.

+ + HTML + + posts = page.microposts + expect(posts.size).to eq(1) + expect(posts[0][:body]).to include("body text content") + end + + it "returns empty array for non-HTML pages" do + page = described_class.new( + content: '{"key": "value"}', + content_type: "application/json", + status_code: 200, + archive_url: "https://web.archive.org/web/20220615/https://example.com/api", + original_url: "https://example.com/api", + timestamp: "20220615000000", + ) + + expect(page.microposts).to eq([]) + end + + it "skips containers with no paragraph text" do + page = build_page(<<~HTML) + +
+ just an image +
+ + HTML + + expect(page.microposts).to eq([]) + end + + it "extracts date from time element with datetime attribute" do + page = build_page(<<~HTML) + +
+

Dated Post

+ +

Content here.

+
+ + HTML + + posts = page.microposts + expect(posts[0][:date]).to eq("2022-06-15T10:30:00Z") + end + + it "extracts date from class-based date elements" do + page = build_page(<<~HTML) + +
+

Post

+ 2022-06-15 +

Content here.

+
+ + HTML + + posts = page.microposts + expect(posts[0][:date]).to eq("2022-06-15") + end +end