Blog Post
+Blog content here.
+diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 0133d43..8ff8667 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,20 +1,12 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2026-05-13 07:38:03 UTC using RuboCop version 1.86.1. +# on 2026-05-13 08:31:40 UTC using RuboCop version 1.86.1. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 1 -# This cop supports safe autocorrection (--autocorrect). -# Configuration parameters: EnforcedStyle, IndentationWidth. -# SupportedStyles: with_first_argument, with_fixed_indentation -Layout/ArgumentAlignment: - Exclude: - - 'lib/archaeo/warc_support.rb' - -# Offense count: 10 +# Offense count: 9 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings. # URISchemes: http, https @@ -25,22 +17,24 @@ Layout/LineLength: - 'lib/archaeo/page.rb' - 'lib/archaeo/path_sanitizer.rb' - 'lib/archaeo/url_rewriter.rb' - - 'lib/archaeo/warc_support.rb' - 'spec/archaeo/asset_extractor_spec.rb' -# Offense count: 1 +# Offense count: 2 # This cop supports safe autocorrection (--autocorrect). -# Configuration parameters: AllowInHeredoc. -Layout/TrailingWhitespace: +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: aligned, indented, indented_relative_to_receiver +Layout/MultilineMethodCallIndentation: Exclude: - - 'lib/archaeo/warc_support.rb' + - 'lib/archaeo/page.rb' -# Offense count: 1 +# Offense count: 3 Lint/UselessConstantScoping: Exclude: - 'lib/archaeo/bulk_downloader.rb' + - 'lib/archaeo/local_rewriter.rb' + - 'lib/archaeo/page.rb' -# Offense count: 26 +# Offense count: 28 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max. Metrics/AbcSize: Exclude: @@ -53,6 +47,7 @@ Metrics/AbcSize: - 'lib/archaeo/configuration.rb' - 'lib/archaeo/content_tracker.rb' - 'lib/archaeo/coverage_report.rb' + - 'lib/archaeo/local_rewriter.rb' - 'lib/archaeo/parallel_cdx.rb' - 'lib/archaeo/pattern_filter.rb' - 'lib/archaeo/progress_report.rb' @@ -73,7 +68,7 @@ Metrics/CyclomaticComplexity: - 'lib/archaeo/pattern_filter.rb' - 'lib/archaeo/url_rewriter.rb' -# Offense count: 45 +# Offense count: 48 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. Metrics/MethodLength: Max: 32 @@ -96,11 +91,12 @@ Metrics/PerceivedComplexity: - 'lib/archaeo/pattern_filter.rb' - 'lib/archaeo/url_rewriter.rb' -# Offense count: 2 +# Offense count: 6 # Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames. # AllowedNames: as, at, by, cc, db, id, if, in, io, ip, of, on, os, pp, to Naming/MethodParameterName: Exclude: + - 'lib/archaeo/page.rb' - 'spec/archaeo/coverage_report_spec.rb' - 'spec/archaeo/download_scheduler_spec.rb' @@ -118,7 +114,15 @@ Performance/StringBytesize: Exclude: - 'lib/archaeo/encoding_detector.rb' -# Offense count: 112 +# Offense count: 1 +# This cop supports unsafe autocorrection (--autocorrect-all). +# Configuration parameters: SkipBlocks, EnforcedStyle, OnlyStaticConstants. +# SupportedStyles: described_class, explicit +RSpec/DescribedClass: + Exclude: + - 'spec/archaeo/page_microposts_spec.rb' + +# Offense count: 131 # Configuration parameters: CountAsOne. RSpec/ExampleLength: Max: 36 @@ -153,7 +157,7 @@ RSpec/MultipleDescribes: - 'spec/archaeo/path_sanitizer_spec.rb' - 'spec/archaeo/warc_support_spec.rb' -# Offense count: 133 +# Offense count: 145 RSpec/MultipleExpectations: Max: 11 @@ -175,8 +179,11 @@ RSpec/StubbedMock: Exclude: - 'spec/archaeo/parallel_cdx_spec.rb' -# Offense count: 1 +# Offense count: 3 # This cop supports safe autocorrection (--autocorrect). -Style/ExplicitBlockArgument: +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, diff_comma, no_comma +Style/TrailingCommaInArguments: Exclude: - - 'lib/archaeo/warc_support.rb' + - 'lib/archaeo/cli.rb' + - 'lib/archaeo/page.rb' diff --git a/lib/archaeo.rb b/lib/archaeo.rb index 9a45976..d7abd2c 100644 --- a/lib/archaeo.rb +++ b/lib/archaeo.rb @@ -69,4 +69,6 @@ def initialize(message, status_code:, url:, page:) autoload :ContentChangeReport, "archaeo/content_tracker" autoload :ArchiveSearch, "archaeo/archive_search" autoload :SearchResult, "archaeo/archive_search" + autoload :LocalRewriter, "archaeo/local_rewriter" + autoload :LocalRewriteSummary, "archaeo/local_rewriter" end diff --git a/lib/archaeo/cli.rb b/lib/archaeo/cli.rb index f586f3a..77c8939 100644 --- a/lib/archaeo/cli.rb +++ b/lib/archaeo/cli.rb @@ -194,6 +194,32 @@ def rewrite(url, timestamp) end end + desc "rewrite-local INPUT_DIR", + "Rewrite previously downloaded files to use local paths" + option :output, desc: "Output directory (default: rewrite in-place)", + required: false + option :prefix, desc: "Local path prefix", default: "local" + option :rewrite_js, type: :boolean, default: false, + desc: "Rewrite URLs in JavaScript strings" + option :rewrite_absolute, type: :boolean, default: false, + desc: "Rewrite all absolute archive URLs" + def rewrite_local(input_dir) + handle_errors do + output_dir = options[:output] || input_dir + local_rewriter = LocalRewriter.new( + prefix: options[:prefix], + rewrite_js: options[:rewrite_js], + rewrite_absolute: options[:rewrite_absolute], + ) + summary = local_rewriter.rewrite_directory(input_dir, output_dir) + color = build_color + warn color.success( + "Rewrote #{summary.rewritten}/#{summary.total} files " \ + "in #{summary.elapsed.round(1)}s", + ) + end + end + desc "diff URL TIMESTAMP_A TIMESTAMP_B", "Compare assets of two archived snapshots" option :format, desc: "Output format (table, json)", default: "table" diff --git a/lib/archaeo/local_rewriter.rb b/lib/archaeo/local_rewriter.rb new file mode 100644 index 0000000..f315225 --- /dev/null +++ b/lib/archaeo/local_rewriter.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require "fileutils" + +module Archaeo + LocalRewriteSummary = Struct.new( + :total, :rewritten, :skipped, :elapsed, + keyword_init: true + ) + + # Rewrites previously downloaded files by converting archive URLs + # to local paths. Operates on files already on disk without fetching. + class LocalRewriter + def initialize(prefix: "local", rewrite_js: false, + rewrite_absolute: false) + @prefix = prefix + @rewrite_js = rewrite_js + @rewrite_absolute = rewrite_absolute + end + + def rewrite_directory(input_dir, output_dir) + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + files = gather_files(input_dir) + rewritten = 0 + skipped = 0 + + files.each do |path| + rel = path.sub(%r{\A#{Regexp.escape(input_dir)}/?}, "") + out_path = File.join(output_dir, rel) + + result = rewrite_file(path, out_path) + result ? rewritten += 1 : skipped += 1 + end + + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time + LocalRewriteSummary.new( + total: files.size, rewritten: rewritten, + skipped: skipped, elapsed: elapsed + ) + end + + def rewrite_file(input_path, output_path) + content = File.read(input_path) + return nil unless rewrite_candidate?(content) + + FileUtils.mkdir_p(File.dirname(output_path)) + rewriter = build_rewriter + rewritten = apply_rewriting(rewriter, content, input_path) + File.write(output_path, rewritten) + true + end + + private + + def gather_files(dir) + Dir.glob(File.join(dir, "**", "*")) + .select { |f| File.file?(f) && text_file?(f) } + end + + def text_file?(path) + ext = File.extname(path).downcase + TEXT_EXTENSIONS.include?(ext) + end + + TEXT_EXTENSIONS = %w[ + .html .htm .xhtml .css .js .json .xml .txt + .svg .md .yaml .yml .rss .atom + ].freeze + + def rewrite_candidate?(content) + content.include?("web.archive.org") + end + + def build_rewriter + UrlRewriter.new( + "https://web.archive.org", @prefix, + rewrite_js: @rewrite_js, + rewrite_absolute: @rewrite_absolute + ) + end + + def apply_rewriting(rewriter, content, path) + ext = File.extname(path).downcase + case ext + when ".html", ".htm", ".xhtml" + rewriter.rewrite_html(content) + when ".css" + rewriter.rewrite_css(content) + when ".js" + rewriter.rewrite_js(content) + else + rewrite_mixed(rewriter, content) + end + end + + def rewrite_mixed(rewriter, content) + if content.include?("<") && content.include?(">") + rewriter.rewrite_html(content) + elsif content.include?("url(") + rewriter.rewrite_css(content) + else + rewriter.rewrite_js(content) + end + end + end +end diff --git a/lib/archaeo/page.rb b/lib/archaeo/page.rb index 81d53fd..15e4322 100644 --- a/lib/archaeo/page.rb +++ b/lib/archaeo/page.rb @@ -139,6 +139,16 @@ def scripts end end + def microposts + return [] unless html? + + @microposts ||= begin + doc = Nokogiri::HTML(@raw_content) + containers = find_article_containers(doc) + containers.filter_map { |el| extract_micropost(el) } + end + end + def to_h { content_type: @content_type, @@ -255,5 +265,58 @@ def extract_form_fields(form) end inputs.reject { |f| f[:name].empty? } end + + ARTICLE_SELECTORS = %w[ + article [role=article] .post .entry .blog-post + .hentry .post-content .entry-content .article-content + .story .story-body .news-article + ].freeze + + def find_article_containers(doc) + found = ARTICLE_SELECTORS + .filter_map { |sel| doc.css(sel) } + .flat_map(&:to_a) + found.any? ? found.uniq : [doc.at_css("body") || doc] + end + + def extract_micropost(element) + title = extract_micropost_title(element) + body = extract_micropost_body(element) + return nil if body.nil? || body.strip.empty? + + { title: title, body: body.strip, + date: extract_micropost_date(element), + author: extract_micropost_author(element) } + end + + def extract_micropost_title(el) + heading = el.at_css("h1, h2, h3, [class*=title], [class*=heading]") + heading&.text&.strip + end + + def extract_micropost_body(el) + paragraphs = el.css("p").map(&:text).join("\n") + return nil if paragraphs.strip.empty? + + paragraphs + end + + def extract_micropost_date(el) + time = el.at_css("time[datetime]") + return time["datetime"] if time + + date_el = el.at_css( + "[class*=date], [class*=time], [class*=published], " \ + "[property='datePublished']", + ) + date_el&.text&.strip + end + + def extract_micropost_author(el) + author_el = el.at_css( + "[class*=author], [rel=author], [property='author']", + ) + author_el&.text&.strip + end end end diff --git a/spec/archaeo/local_rewriter_spec.rb b/spec/archaeo/local_rewriter_spec.rb new file mode 100644 index 0000000..ecbbed8 --- /dev/null +++ b/spec/archaeo/local_rewriter_spec.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require "spec_helper" +require "tmpdir" +require "fileutils" + +RSpec.describe Archaeo::LocalRewriter do + let(:tmpdir) { Dir.mktmpdir } + let(:output_dir) { File.join(tmpdir, "output") } + + after { FileUtils.rm_rf(tmpdir) } + + def write_input_file(name, content) + path = File.join(tmpdir, "input", name) + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, content) + path + end + + it "rewrites HTML files with archive URLs" do + write_input_file("page.html", <<~HTML) +
+
+
+ HTML
+
+ rewriter = described_class.new(prefix: "local")
+ summary = rewriter.rewrite_directory(
+ File.join(tmpdir, "input"), output_dir
+ )
+
+ expect(summary.rewritten).to eq(1)
+ expect(summary.total).to eq(1)
+ output = File.read(File.join(output_dir, "page.html"))
+ expect(output).not_to include("web.archive.org")
+ expect(output).to include("local")
+ end
+
+ it "rewrites CSS files with archive URLs" do
+ write_input_file("style.css", <<~CSS)
+ .bg { background: url('https://web.archive.org/web/20220615/https://example.com/bg.png'); }
+ CSS
+
+ rewriter = described_class.new(prefix: "assets")
+ summary = rewriter.rewrite_directory(
+ File.join(tmpdir, "input"), output_dir
+ )
+
+ expect(summary.rewritten).to eq(1)
+ output = File.read(File.join(output_dir, "style.css"))
+ expect(output).not_to include("web.archive.org")
+ expect(output).to include("assets")
+ end
+
+ it "rewrites JS files when rewrite_js is enabled" do
+ write_input_file("app.js", <<~JS)
+ var url = 'https://web.archive.org/web/20220615/https://example.com/api';
+ JS
+
+ rewriter = described_class.new(prefix: "local", rewrite_js: true)
+ summary = rewriter.rewrite_directory(
+ File.join(tmpdir, "input"), output_dir
+ )
+
+ expect(summary.rewritten).to eq(1)
+ output = File.read(File.join(output_dir, "app.js"))
+ expect(output).not_to include("web.archive.org")
+ end
+
+ it "skips files without archive URLs" do
+ write_input_file("clean.html", "Hello")
+
+ rewriter = described_class.new(prefix: "local")
+ summary = rewriter.rewrite_directory(
+ File.join(tmpdir, "input"), output_dir
+ )
+
+ expect(summary.skipped).to eq(1)
+ expect(summary.rewritten).to eq(0)
+ end
+
+ it "skips binary files" do
+ binary_path = File.join(tmpdir, "input", "image.png")
+ FileUtils.mkdir_p(File.dirname(binary_path))
+ File.binwrite(binary_path, "\x89PNG\r\n\x1a\n")
+
+ rewriter = described_class.new(prefix: "local")
+ summary = rewriter.rewrite_directory(
+ File.join(tmpdir, "input"), output_dir
+ )
+
+ expect(summary.total).to eq(0)
+ end
+
+ it "handles nested directory structures" do
+ write_input_file("sub/deep/page.html", <<~HTML)
+
+ Link
+
+ HTML
+
+ rewriter = described_class.new(prefix: "local")
+ summary = rewriter.rewrite_directory(
+ File.join(tmpdir, "input"), output_dir
+ )
+
+ expect(summary.rewritten).to eq(1)
+ output = File.read(File.join(output_dir, "sub/deep/page.html"))
+ expect(output).not_to include("web.archive.org")
+ end
+
+ it "rewrites in-place when no output directory specified" do
+ input_dir = File.join(tmpdir, "input")
+ write_input_file("page.html", <<~HTML)
+
+
+
+ HTML
+
+ rewriter = described_class.new(prefix: "local")
+ summary = rewriter.rewrite_directory(input_dir, input_dir)
+
+ expect(summary.rewritten).to eq(1)
+ output = File.read(File.join(input_dir, "page.html"))
+ expect(output).not_to include("web.archive.org")
+ end
+
+ it "reports elapsed time" do
+ write_input_file("page.html", <<~HTML)
+
+
+
+ HTML
+
+ rewriter = described_class.new(prefix: "local")
+ summary = rewriter.rewrite_directory(
+ File.join(tmpdir, "input"), output_dir
+ )
+
+ expect(summary.elapsed).to be >= 0
+ end
+
+ describe "#rewrite_file" do
+ it "returns nil for files without archive URLs" do
+ input = File.join(tmpdir, "clean.html")
+ output = File.join(tmpdir, "out.html")
+ File.write(input, "Hello")
+
+ rewriter = described_class.new(prefix: "local")
+ expect(rewriter.rewrite_file(input, output)).to be_nil
+ end
+
+ it "returns true for successfully rewritten files" do
+ input = File.join(tmpdir, "page.html")
+ output = File.join(tmpdir, "out.html")
+ File.write(input, <<~HTML)
+
+
+
+ HTML
+
+ rewriter = described_class.new(prefix: "local")
+ expect(rewriter.rewrite_file(input, output)).to be true
+ expect(File.read(output)).to include("local")
+ end
+ end
+end
diff --git a/spec/archaeo/page_microposts_spec.rb b/spec/archaeo/page_microposts_spec.rb
new file mode 100644
index 0000000..ec46cbb
--- /dev/null
+++ b/spec/archaeo/page_microposts_spec.rb
@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe Archaeo::Page, "#microposts" do
+ def build_page(html)
+ Archaeo::Page.new(
+ content: html,
+ content_type: "text/html",
+ status_code: 200,
+ archive_url: "https://web.archive.org/web/20220615000000/https://example.com/",
+ original_url: "https://example.com/",
+ timestamp: "20220615000000",
+ )
+ end
+
+ it "extracts microposts from article elements" do
+ page = build_page(<<~HTML)
+
+ This is the first article content.
+It has multiple paragraphs.
+Article body text here.
+Content one.
+Content two.
+Body text.
+Blog content here.
+Some body text content here.
+Another paragraph with text.
+ + HTML + + posts = page.microposts + expect(posts.size).to eq(1) + expect(posts[0][:body]).to include("body text content") + end + + it "returns empty array for non-HTML pages" do + page = described_class.new( + content: '{"key": "value"}', + content_type: "application/json", + status_code: 200, + archive_url: "https://web.archive.org/web/20220615/https://example.com/api", + original_url: "https://example.com/api", + timestamp: "20220615000000", + ) + + expect(page.microposts).to eq([]) + end + + it "skips containers with no paragraph text" do + page = build_page(<<~HTML) + +
+ Content here.
+Content here.
+