diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 8ff8667..34fda49 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2026-05-13 08:31:40 UTC using RuboCop version 1.86.1. +# on 2026-05-13 13:11:16 UTC using RuboCop version 1.86.1. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -19,14 +19,6 @@ Layout/LineLength: - 'lib/archaeo/url_rewriter.rb' - 'spec/archaeo/asset_extractor_spec.rb' -# Offense count: 2 -# This cop supports safe autocorrection (--autocorrect). -# Configuration parameters: EnforcedStyle, IndentationWidth. -# SupportedStyles: aligned, indented, indented_relative_to_receiver -Layout/MultilineMethodCallIndentation: - Exclude: - - 'lib/archaeo/page.rb' - # Offense count: 3 Lint/UselessConstantScoping: Exclude: @@ -46,7 +38,7 @@ Metrics/AbcSize: - 'lib/archaeo/cli.rb' - 'lib/archaeo/configuration.rb' - 'lib/archaeo/content_tracker.rb' - - 'lib/archaeo/coverage_report.rb' + - 'lib/archaeo/coverage_analyzer.rb' - 'lib/archaeo/local_rewriter.rb' - 'lib/archaeo/parallel_cdx.rb' - 'lib/archaeo/pattern_filter.rb' @@ -114,14 +106,6 @@ Performance/StringBytesize: Exclude: - 'lib/archaeo/encoding_detector.rb' -# Offense count: 1 -# This cop supports unsafe autocorrection (--autocorrect-all). -# Configuration parameters: SkipBlocks, EnforcedStyle, OnlyStaticConstants. -# SupportedStyles: described_class, explicit -RSpec/DescribedClass: - Exclude: - - 'spec/archaeo/page_microposts_spec.rb' - # Offense count: 131 # Configuration parameters: CountAsOne. RSpec/ExampleLength: @@ -178,12 +162,3 @@ RSpec/RepeatedExampleGroupDescription: RSpec/StubbedMock: Exclude: - 'spec/archaeo/parallel_cdx_spec.rb' - -# Offense count: 3 -# This cop supports safe autocorrection (--autocorrect). -# Configuration parameters: EnforcedStyleForMultiline. -# SupportedStylesForMultiline: comma, consistent_comma, diff_comma, no_comma -Style/TrailingCommaInArguments: - Exclude: - - 'lib/archaeo/cli.rb' - - 'lib/archaeo/page.rb' diff --git a/lib/archaeo.rb b/lib/archaeo.rb index d7abd2c..85f7f9e 100644 --- a/lib/archaeo.rb +++ b/lib/archaeo.rb @@ -71,4 +71,7 @@ def initialize(message, status_code:, url:, page:) autoload :SearchResult, "archaeo/archive_search" autoload :LocalRewriter, "archaeo/local_rewriter" autoload :LocalRewriteSummary, "archaeo/local_rewriter" + autoload :CoverageAnalyzer, "archaeo/coverage_analyzer" + autoload :HealthReport, "archaeo/health_report" + autoload :HealthDetail, "archaeo/health_report" end diff --git a/lib/archaeo/archive_health_check.rb b/lib/archaeo/archive_health_check.rb index da808c2..c111f82 100644 --- a/lib/archaeo/archive_health_check.rb +++ b/lib/archaeo/archive_health_check.rb @@ -1,20 +1,6 @@ # frozen_string_literal: true module Archaeo - # Verifies that archived snapshots are still accessible. - # - # Checks each snapshot by performing HEAD requests to the - # archive URL and reporting accessibility status. - HealthReport = Struct.new( - :total, :accessible, :missing, :errors, :details, - keyword_init: true - ) - - HealthDetail = Struct.new( - :snapshot, :status, :error, - keyword_init: true - ) - class ArchiveHealthCheck def initialize(client: HttpClient.new, cdx_api: nil) @client = client diff --git a/lib/archaeo/bulk_downloader.rb b/lib/archaeo/bulk_downloader.rb index 92cd6f2..f214003 100644 --- a/lib/archaeo/bulk_downloader.rb +++ b/lib/archaeo/bulk_downloader.rb @@ -245,9 +245,6 @@ def fetch_and_save(snapshot) page = fetch_page(snapshot) validate_page_status(page, snapshot) write_page_file(page, snapshot) - rescue StandardError - FileUtils.rm_f(tmp_path) if defined?(tmp_path) - raise end def fetch_page(snapshot) @@ -272,6 +269,9 @@ def write_page_file(page, snapshot) File.binwrite(tmp_path, page.content) File.rename(tmp_path, filename) page.content + rescue StandardError + FileUtils.rm_f(tmp_path) if defined?(tmp_path) + raise end EXTENSION_MAP = { diff --git a/lib/archaeo/coverage_analyzer.rb b/lib/archaeo/coverage_analyzer.rb new file mode 100644 index 0000000..1060689 --- /dev/null +++ b/lib/archaeo/coverage_analyzer.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Archaeo + # Builds a CoverageReport from CDX snapshot data. + class CoverageAnalyzer + def initialize(cdx_api: nil) + @cdx_api = cdx_api + end + + def analyze(url, from: nil, to: nil) + cdx = @cdx_api || CdxApi.new + snapshots = cdx.snapshots(url, from: from, to: to).to_a + + unique_urls = snapshots.map(&:original_url).uniq + status_dist = compute_status_distribution(snapshots) + gaps = compute_temporal_gaps(snapshots) + + CoverageReport.new( + url: url, + total_urls: unique_urls.size, + archived_urls: snapshots.count(&:success?), + status_distribution: status_dist, + temporal_gaps: gaps, + ) + end + + private + + def compute_status_distribution(snapshots) + snapshots.each_with_object(Hash.new(0)) do |snap, counts| + counts[snap.status_code] += 1 + end + end + + def compute_temporal_gaps(snapshots) + return [] if snapshots.size < 2 + + sorted = snapshots.sort_by(&:timestamp) + gaps = [] + sorted.each_cons(2) do |a, b| + diff_days = (b.timestamp.to_time - a.timestamp.to_time) / 86400 + next unless diff_days > 30 + + gaps << { from: a.timestamp.to_s, to: b.timestamp.to_s, + gap_days: diff_days.round } + end + gaps + end + end +end diff --git a/lib/archaeo/coverage_report.rb b/lib/archaeo/coverage_report.rb index 2107ffa..769f5a1 100644 --- a/lib/archaeo/coverage_report.rb +++ b/lib/archaeo/coverage_report.rb @@ -51,51 +51,4 @@ def as_json(*) to_h end end - - # Builds a CoverageReport from CDX snapshot data. - class CoverageAnalyzer - def initialize(cdx_api: nil) - @cdx_api = cdx_api - end - - def analyze(url, from: nil, to: nil) - cdx = @cdx_api || CdxApi.new - snapshots = cdx.snapshots(url, from: from, to: to).to_a - - unique_urls = snapshots.map(&:original_url).uniq - status_dist = compute_status_distribution(snapshots) - gaps = compute_temporal_gaps(snapshots) - - CoverageReport.new( - url: url, - total_urls: unique_urls.size, - archived_urls: snapshots.count(&:success?), - status_distribution: status_dist, - temporal_gaps: gaps, - ) - end - - private - - def compute_status_distribution(snapshots) - snapshots.each_with_object(Hash.new(0)) do |snap, counts| - counts[snap.status_code] += 1 - end - end - - def compute_temporal_gaps(snapshots) - return [] if snapshots.size < 2 - - sorted = snapshots.sort_by(&:timestamp) - gaps = [] - sorted.each_cons(2) do |a, b| - diff_days = (b.timestamp.to_time - a.timestamp.to_time) / 86400 - next unless diff_days > 30 - - gaps << { from: a.timestamp.to_s, to: b.timestamp.to_s, - gap_days: diff_days.round } - end - gaps - end - end end diff --git a/lib/archaeo/download_state.rb b/lib/archaeo/download_state.rb index a0f4b6f..be82c81 100644 --- a/lib/archaeo/download_state.rb +++ b/lib/archaeo/download_state.rb @@ -73,18 +73,15 @@ def file_exists?(timestamp, base_dir: @output_dir) def stale_entries(base_dir: @output_dir) @mutex.synchronize do entries.reject do |e| - find_file(base_dir, - e["ts"]) && File.exist?(find_file(base_dir, e["ts"])) + path = find_file(base_dir, e["ts"]) + path && File.exist?(path) end end end def cleanup_stale(base_dir: @output_dir) @mutex.synchronize do - stale = entries.reject do |e| - path = find_file(base_dir, e["ts"]) - path && File.exist?(path) - end + stale = stale_entries(base_dir: base_dir) @entries = entries - stale @entries_key = nil save diff --git a/lib/archaeo/health_report.rb b/lib/archaeo/health_report.rb new file mode 100644 index 0000000..6fd1235 --- /dev/null +++ b/lib/archaeo/health_report.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Archaeo + HealthReport = Struct.new( + :total, :accessible, :missing, :errors, :details, + keyword_init: true + ) + + HealthDetail = Struct.new( + :snapshot, :status, :error, + keyword_init: true + ) +end diff --git a/lib/archaeo/page.rb b/lib/archaeo/page.rb index 15e4322..b776659 100644 --- a/lib/archaeo/page.rb +++ b/lib/archaeo/page.rb @@ -59,21 +59,15 @@ def binary? end def title - @title ||= begin - doc = Nokogiri::HTML(@raw_content) - doc.at_css("title")&.text&.strip - rescue StandardError - nil - end + @title ||= html_doc.at_css("title")&.text&.strip end def links return [] unless html? @links ||= begin - doc = Nokogiri::HTML(@raw_content) base = @archive_url || @original_url - doc.css("a[href]").map do |anchor| + html_doc.css("a[href]").map do |anchor| href = resolve_page_url(anchor["href"], base) { href: href, text: anchor.text.strip, external: href && !href.include?(original_domain) } @@ -85,9 +79,8 @@ def meta_tags return {} unless html? @meta_tags ||= begin - doc = Nokogiri::HTML(@raw_content) - result = extract_meta_entries(doc) - canonical = doc.at_css('link[rel="canonical"]') + result = extract_meta_entries(html_doc) + canonical = html_doc.at_css('link[rel="canonical"]') result["canonical"] = canonical["href"].to_s if canonical result end @@ -96,46 +89,34 @@ def meta_tags def headings return [] unless html? - @headings ||= begin - doc = Nokogiri::HTML(@raw_content) - doc.css("h1, h2, h3, h4, h5, h6").map do |el| - { level: el.name[1].to_i, text: el.text.strip } - end + @headings ||= html_doc.css("h1, h2, h3, h4, h5, h6").map do |el| + { level: el.name[1].to_i, text: el.text.strip } end end def images return [] unless html? - @images ||= begin - doc = Nokogiri::HTML(@raw_content) - doc.css("img[src]").map do |el| - { src: el["src"], alt: el["alt"].to_s, - width: el["width"]&.to_i, height: el["height"]&.to_i } - end + @images ||= html_doc.css("img[src]").map do |el| + { src: el["src"], alt: el["alt"].to_s, + width: el["width"]&.to_i, height: el["height"]&.to_i } end end def forms return [] unless html? - @forms ||= begin - doc = Nokogiri::HTML(@raw_content) - doc.css("form").map do |form| - { action: form["action"].to_s, method: (form["method"] || "GET").upcase, - fields: extract_form_fields(form) } - end + @forms ||= html_doc.css("form").map do |form| + { action: form["action"].to_s, method: (form["method"] || "GET").upcase, + fields: extract_form_fields(form) } end end def scripts return [] unless html? - @scripts ||= begin - doc = Nokogiri::HTML(@raw_content) - doc.css("script").map do |el| - { src: el["src"].to_s, type: el["type"].to_s } - end + @scripts ||= html_doc.css("script").map do |el| + { src: el["src"].to_s, type: el["type"].to_s } end end @@ -143,8 +124,7 @@ def microposts return [] unless html? @microposts ||= begin - doc = Nokogiri::HTML(@raw_content) - containers = find_article_containers(doc) + containers = find_article_containers(html_doc) containers.filter_map { |el| extract_micropost(el) } end end @@ -162,15 +142,7 @@ def to_h end def as_json(*) - { - content_type: @content_type, - status_code: @status_code, - archive_url: @archive_url, - original_url: @original_url, - timestamp: @timestamp.to_s, - size: size, - encoding: encoding.to_s, - } + to_h.transform_values { |v| v.is_a?(Timestamp) ? v.to_s : v } end def inspect @@ -179,6 +151,10 @@ def inspect private + def html_doc + @html_doc ||= Nokogiri::HTML(@raw_content) + end + def detect_encoding charset = extract_charset(@content_type) return Encoding.find(charset) if charset @@ -199,11 +175,10 @@ def extract_charset(content_type) end def detect_html_charset - doc = Nokogiri::HTML(@raw_content) - node = doc.at_css("meta[charset]") + node = html_doc.at_css("meta[charset]") return node["charset"] if node - content = doc.at_css('meta[http-equiv="Content-Type"]')&.[]("content") + content = html_doc.at_css('meta[http-equiv="Content-Type"]')&.[]("content") return nil unless content match = content.match(/charset=([^\s;]+)/i) @@ -250,8 +225,8 @@ def extract_meta_entries(doc) def resolve_page_url(href, base) return href unless href - return href if href.start_with?("http", "//", "data:", "#", - "javascript:") + return href if href.start_with?("http:", "https:", "//", "data:", + "#", "javascript:") return nil unless base URI.join(base, href).to_s diff --git a/lib/archaeo/snapshot_diff.rb b/lib/archaeo/snapshot_diff.rb index f4e6f00..a4a0a38 100644 --- a/lib/archaeo/snapshot_diff.rb +++ b/lib/archaeo/snapshot_diff.rb @@ -104,7 +104,6 @@ def extract_assets(page) end def count_elements(page) - require "nokogiri" doc = Nokogiri::HTML(page.content) counts = Hash.new(0) doc.css("*").each { |el| counts[el.name] += 1 }