diff --git a/app/services/azure_reconciliation/directory_lister.rb b/app/services/azure_reconciliation/directory_lister.rb new file mode 100644 index 00000000..01f4b489 --- /dev/null +++ b/app/services/azure_reconciliation/directory_lister.rb @@ -0,0 +1,80 @@ +# Lists every file in one Azure File Share directory. +# +# The azure_file_shares gem's `list` returns only the first page (up to 5,000 +# entries) and drops the continuation marker, so it undercounts large folders. +# This follows NextMarker until the listing is complete, reusing the gem's +# request signing. +module AzureReconciliation + class DirectoryLister + class Error < StandardError; end + + PAGE_SIZE = 5000 + + def initialize(share:, files: AzureFileShares.client.files, connection: nil) + @share = share + @files = files + @connection = connection + end + + # Returns { file_name => byte_size }, or nil if the directory does not exist. + def files_in(directory) + result = {} + marker = nil + + loop do + body = fetch_page(directory, marker) + return nil if body.nil? + + xml = Nokogiri::XML(body) + xml.xpath("//Entries/File").each do |file| + result[file.at_xpath("Name").text] = file.at_xpath("Properties/Content-Length")&.text.to_i + end + + marker = xml.at_xpath("//NextMarker")&.text.presence + break if marker.nil? + end + + result + end + + private + + attr_reader :share, :files + + def fetch_page(directory, marker) + path = "/#{share}/#{encode_path(directory)}" + query = { restype: "directory", comp: "list", maxresults: PAGE_SIZE.to_s } + query[:marker] = marker if marker + + headers = { + "x-ms-date" => Time.now.utc.httpdate, + "x-ms-version" => files.client.api_version, + } + headers["Authorization"] = files.send(:calculate_authorization_header, :get, path, headers, query) + + response = connection.get("#{files.file_base_url}#{path}?#{URI.encode_www_form(query)}", nil, headers) + + return nil if response.status == 404 + return response.body if response.success? + + raise Error, "Listing #{directory} failed with #{response.status} #{error_detail(response.body)}" + end + + # Azure's AuthenticationErrorDetail holds the string it signed, which is + # what to compare against when a signature is rejected. + def error_detail(body) + text = body.to_s.dup.force_encoding(Encoding::UTF_8).scrub.delete_prefix("") + xml = Nokogiri::XML(text) + parts = %w[Code Message AuthenticationErrorDetail].filter_map { |tag| xml.at_xpath("//Error/#{tag}")&.text&.strip } + parts.any? ? parts.join("\n") : text[0, 500] + end + + def encode_path(directory) + directory.split("/").reject(&:blank?).map { |segment| ERB::Util.url_encode(segment) }.join("/") + end + + def connection + @connection ||= files.send(:create_file_connection) + end + end +end diff --git a/app/services/azure_reconciliation/report.rb b/app/services/azure_reconciliation/report.rb new file mode 100644 index 00000000..487f7179 --- /dev/null +++ b/app/services/azure_reconciliation/report.rb @@ -0,0 +1,215 @@ +# Compares topics, their documents, ActiveStorage and the Azure File Share, +# and returns one Row per discrepancy. Read only: it changes nothing. +# +# Expected Azure locations mirror FileManager: regular files go to the Pi and +# v2 Content folders, videos to v2 VideoContent, and archived topics' files to +# the matching _Archive folder. Shadow-copy topics hold documents waiting to be +# deleted from Azure, so their files should be gone. +module AzureReconciliation + class Report + Row = Struct.new( + :category, :language, :topic_id, :title, :document_id, :directory, + :file_name, :expected_size, :azure_size, :note, + keyword_init: true, + ) + + def initialize(lister:, languages: Language.all, check_blobs: true) + @lister = lister + @languages = languages + @check_blobs = check_blobs + end + + def rows + @rows ||= languages.flat_map { |language| LanguageCheck.new(language, lister, check_blobs).rows } + end + + def to_csv + CSV.generate do |csv| + csv << Row.members + rows.each { |row| csv << row.to_a } + end + end + + # { [language, category] => count } + def summary + rows.group_by { |row| [ row.language, row.category ] }.transform_values(&:size).sort.to_h + end + + private + + attr_reader :lister, :languages, :check_blobs + + class LanguageCheck + def initialize(language, lister, check_blobs) + @language = language + @lister = lister + @check_blobs = check_blobs + @rows = [] + @claimed = Set.new + @live_names = {} + end + + def rows + load_azure + topics = Topic.unscoped.where(language:).includes(:provider, documents_attachments: :blob).to_a + live, shadows = topics.partition { |topic| !topic.shadow_copy? } + + live.each { |topic| check_live_topic(topic) } + shadows.each { |topic| check_shadow_topic(topic) } + report_orphans(live) + report_duplicates(live) + + @rows + end + + private + + attr_reader :language, :lister, :check_blobs, :azure, :claimed, :live_names + + def prefix + language.file_storage_prefix + end + + def pi = "#{prefix}CMES-Pi/assets/Content" + def v2 = "#{prefix}CMES-v2/assets/Content" + def video = "#{prefix}CMES-v2/assets/VideoContent" + def pi_archive = "#{prefix}CMES-Pi_Archive" + def v2_archive = "#{prefix}CMES-v2_Archive" + + def directories + [ pi, v2, video, pi_archive, v2_archive ] + end + + # [active folder, archive folder] for each copy of a document. + def folder_pairs(document) + return [ [ video, v2_archive ] ] if document.content_type == "video/mp4" + + [ [ pi, pi_archive ], [ v2, v2_archive ] ] + end + + def load_azure + @azure = directories.index_with do |directory| + files = lister.files_in(directory) + add(category: "missing_directory", directory:) if files.nil? + files || {} + end + end + + def check_live_topic(topic) + if topic.documents_attachments.empty? + add(category: "topic_without_documents", topic:, note: topic.state) + return + end + + topic.documents_attachments.each do |document| + name = file_name(topic, document) or next + live_names[name] ||= topic + + if check_blobs && !document.blob.service.exist?(document.blob.key) + add(category: "missing_in_storage", topic:, document:, file_name: name, expected_size: document.byte_size) + end + + if document.byte_size.zero? + add(category: "empty_document", topic:, document:, file_name: name, expected_size: 0) + next + end + + folder_pairs(document).each do |active, archive| + expected, other = topic.archived? ? [ archive, active ] : [ active, archive ] + check_location(topic, document, name, expected, other) + end + end + end + + def check_location(topic, document, name, expected, other) + attrs = { topic:, document:, file_name: name, directory: expected, expected_size: document.byte_size } + + if azure[expected].key?(name) + claimed << [ expected, name ] + size = azure[expected][name] + add(category: "size_mismatch", azure_size: size, **attrs) if size != document.byte_size + elsif azure[other].key?(name) + claimed << [ other, name ] + add(category: "wrong_folder", azure_size: azure[other][name], note: "found in #{other}", **attrs) + else + add(category: "missing_in_azure", **attrs) + end + end + + # A shadow copy's file sharing a name with a current document is that + # document's file, so only unclaimed files count as failed deletes. + def check_shadow_topic(shadow) + shadow.documents_attachments.each do |document| + name = file_name(shadow, document) or next + + folder_pairs(document).flatten.each do |directory| + next unless azure[directory].key?(name) + next if claimed.include?([ directory, name ]) + + claimed << [ directory, name ] + add( + category: "failed_delete", topic_id: shadow.document_prefix.presence&.to_i, title: shadow.title, + document:, directory:, file_name: name, azure_size: azure[directory][name], + note: "shadow topic #{shadow.id}", + ) + end + end + end + + def report_orphans(live) + topics_by_prefix = live.index_by { |topic| topic.doc_prefix.to_s } + + azure.each do |directory, files| + files.each do |name, size| + next if claimed.include?([ directory, name ]) + + attrs = { directory:, file_name: name, azure_size: size } + if (topic = live_names[name]) + add(category: "orphan_stale_copy", topic:, **attrs) + elsif (topic = topic_for_name(name, topics_by_prefix)) + add(category: "orphan_unmatched_name", topic:, **attrs) + else + add(category: "orphan_no_topic", **attrs) + end + end + end + end + + # File names start with "_"; a prefix may itself contain underscores. + def topic_for_name(name, topics_by_prefix) + name.to_enum(:scan, "_").each do + topic = topics_by_prefix[name[0...Regexp.last_match.begin(0)]] + return topic if topic + end + nil + end + + def report_duplicates(live) + live + .group_by { |topic| [ topic.provider_id, topic.title.to_s.squish.downcase, topic.published_at.to_date ] } + .each_value do |group| + next if group.size < 2 + + ids = group.map(&:id).sort + group.each do |topic| + add(category: "duplicate_topic", topic:, note: "#{topic.state}; copies: #{ids.join(' ')}") + end + end + end + + def file_name(topic, document) + topic.custom_file_name(document) + rescue StandardError => e + add(category: "broken_topic", topic:, document:, note: "#{e.class}: #{e.message}") + nil + end + + def add(topic: nil, document: nil, **attrs) + attrs[:topic_id] ||= topic&.id + attrs[:title] ||= topic&.title + attrs[:document_id] ||= document&.id + @rows << Row.new(language: language.name, **attrs) + end + end + end +end diff --git a/lib/tasks/reconcile.rake b/lib/tasks/reconcile.rake new file mode 100644 index 00000000..478fa3ac --- /dev/null +++ b/lib/tasks/reconcile.rake @@ -0,0 +1,34 @@ +namespace :reconcile do + desc <<~DESC.squish + Read-only report of discrepancies between topics, their documents, ActiveStorage and the Azure File Share. + ENV: LANGUAGE= (default all), SKIP_BLOB_CHECK=1 (skip checking each file exists in storage), + DEST= (default tmp/reconciliation-.csv) or DEST=- to print the CSV to stdout. + DESC + task azure: :environment do + share = ENV.fetch("AZURE_STORAGE_SHARE_NAME") + languages = if ENV["LANGUAGE"].present? + Language.where(id: ENV["LANGUAGE"]).or(Language.where(name: ENV["LANGUAGE"])).to_a.presence or abort "LANGUAGE not found" + else + Language.order(:name).to_a + end + dest = ENV["DEST"].presence || Rails.root.join("tmp", "reconciliation-#{Time.current.strftime('%Y%m%d-%H%M%S')}.csv").to_s + + warn "Reconciling share #{share} for #{languages.map(&:name).join(', ')}" + report = AzureReconciliation::Report.new( + lister: AzureReconciliation::DirectoryLister.new(share:), + languages:, + check_blobs: ENV["SKIP_BLOB_CHECK"].blank?, + ) + + if dest == "-" + $stdout.write(report.to_csv) + else + File.write(dest, report.to_csv) + warn "Wrote #{report.rows.size} rows to #{dest}" + end + + warn "\nSummary (language, category, count):" + report.summary.each { |(language, category), count| warn format(" %-20s %-25s %6d", language, category, count) } + warn " No discrepancies found." if report.summary.empty? + end +end diff --git a/spec/services/azure_reconciliation/directory_lister_spec.rb b/spec/services/azure_reconciliation/directory_lister_spec.rb new file mode 100644 index 00000000..a6617295 --- /dev/null +++ b/spec/services/azure_reconciliation/directory_lister_spec.rb @@ -0,0 +1,80 @@ +require "rails_helper" + +RSpec.describe AzureReconciliation::DirectoryLister do + subject(:lister) { described_class.new(share: "contentshare", files:, connection:) } + + let(:files) do + double( + "AzureFileShares::Operations::FileOperations", + file_base_url: "https://acct.file.core.windows.net", + client: double(api_version: "2025-05-05"), + ) + end + let(:stubs) { Faraday::Adapter::Test::Stubs.new } + let(:connection) { Faraday.new { |f| f.adapter :test, stubs } } + + before do + allow(files).to receive(:calculate_authorization_header).and_return("SharedKey acct:sig") + end + + def page(names, next_marker: nil) + entries = names.map do |name, size| + "#{name}#{size}" + end.join + <<~XML + + #{entries}sub + #{next_marker} + XML + end + + it "follows the next marker until every page is read" do + stubs.get("/contentshare/CMES-Pi/assets/Content") do |env| + if env.params["marker"] == "page2" + [ 200, {}, page({ "2_b.pdf" => 20 }) ] + else + [ 200, {}, page({ "1_a.pdf" => 10 }, next_marker: "page2") ] + end + end + + expect(lister.files_in("CMES-Pi/assets/Content")).to eq( + "1_a.pdf" => 10, + "2_b.pdf" => 20, + ) + end + + it "signs the marker as part of the query" do + stubs.get("/contentshare/Content") { [ 200, {}, page({}) ] } + + lister.files_in("Content") + + expect(files).to have_received(:calculate_authorization_header) + .with(:get, "/contentshare/Content", hash_including("x-ms-version" => "2025-05-05"), hash_including(comp: "list", restype: "directory")) + end + + it "returns nil when the directory does not exist" do + stubs.get("/contentshare/ES_CMES-Pi_Archive") { [ 404, {}, "" ] } + + expect(lister.files_in("ES_CMES-Pi_Archive")).to be_nil + end + + it "raises on other errors" do + stubs.get("/contentshare/Content") { [ 500, {}, "boom" ] } + + expect { lister.files_in("Content") }.to raise_error(AzureReconciliation::DirectoryLister::Error, /500/) + end + + it "includes Azure's authentication detail when the signature is rejected" do + body = <<~XML + AuthenticationFailed + Server failed to authenticate the request.#{' padding' * 60} + Server used following string to sign: 'GET\\n\\n...' + XML + stubs.get("/contentshare/Content") { [ 403, {}, "\uFEFF#{body}".b ] } + + expect { lister.files_in("Content") }.to raise_error( + AzureReconciliation::DirectoryLister::Error, + /403 AuthenticationFailed.*Server used following string to sign/m, + ) + end +end diff --git a/spec/services/azure_reconciliation/report_spec.rb b/spec/services/azure_reconciliation/report_spec.rb new file mode 100644 index 00000000..a0e9e6fe --- /dev/null +++ b/spec/services/azure_reconciliation/report_spec.rb @@ -0,0 +1,214 @@ +require "rails_helper" + +RSpec.describe AzureReconciliation::Report do + subject(:report) { described_class.new(lister:, languages: [ language ], check_blobs:) } + + let(:language) { create(:language, name: "English") } + let(:provider) { create(:provider, file_name_prefix: "emdoc") } + let(:check_blobs) { false } + let(:azure) { Hash.new { |h, k| h[k] = {} } } + let(:lister) { FakeLister.new(azure) } + + let(:pi) { "CMES-Pi/assets/Content" } + let(:v2) { "CMES-v2/assets/Content" } + let(:video) { "CMES-v2/assets/VideoContent" } + let(:pi_archive) { "CMES-Pi_Archive" } + let(:v2_archive) { "CMES-v2_Archive" } + + FakeLister = Struct.new(:azure) do + def files_in(directory) + azure.key?(directory) ? azure[directory] : nil + end + end + + def create_topic(state: :active, title: "Sepsis", published_at: Time.zone.local(2026, 5, 2), **attrs) + create(:topic, language:, provider:, title:, state:, published_at:, **attrs) + end + + def attach(topic, content: "pdf-bytes", name: "sepsis.pdf", content_type: "application/pdf") + topic.documents.attach( + io: StringIO.new(content), + filename: "#{Topic::INTERNAL_FILENAME_PREFIX}_#{name}", + content_type:, + ) + topic.documents.last + end + + def place(topic, document, *directories, size: document.byte_size) + directories.each { |dir| azure[dir][topic.custom_file_name(document)] = size } + end + + def categories + report.rows.map(&:category) + end + + before do + [ pi, v2, video, pi_archive, v2_archive ].each { |dir| azure[dir] } + end + + it "reports nothing when every document is where it should be" do + topic = create_topic + place(topic, attach(topic), pi, v2) + + expect(report.rows).to be_empty + end + + it "reports active topics with no documents" do + topic = create_topic + + expect(report.rows).to contain_exactly(have_attributes(category: "topic_without_documents", topic_id: topic.id)) + end + + it "reports a document missing from one Azure folder" do + topic = create_topic + document = attach(topic) + place(topic, document, pi) + + expect(report.rows).to contain_exactly( + have_attributes(category: "missing_in_azure", topic_id: topic.id, document_id: document.id, directory: v2), + ) + end + + it "reports a size mismatch" do + topic = create_topic + document = attach(topic) + place(topic, document, pi, v2) + place(topic, document, v2, size: 3) + + expect(report.rows).to contain_exactly( + have_attributes(category: "size_mismatch", directory: v2, expected_size: document.byte_size, azure_size: 3), + ) + end + + it "reports 0 byte documents" do + topic = create_topic + attach(topic, content: "") + + expect(categories).to include("empty_document") + end + + it "expects videos only in the video folder" do + topic = create_topic + document = attach(topic, name: "talk.mp4", content_type: "video/mp4") + place(topic, document, video) + + expect(report.rows).to be_empty + end + + it "expects archived topics' files in the archive folders" do + topic = create_topic(state: :archived) + place(topic, attach(topic), pi_archive, v2_archive) + + expect(report.rows).to be_empty + end + + it "reports a file left in the active folder after archiving as wrong_folder" do + topic = create_topic(state: :archived) + document = attach(topic) + place(topic, document, pi, v2_archive) + + expect(report.rows).to contain_exactly( + have_attributes(category: "wrong_folder", directory: pi_archive, note: "found in #{pi}"), + ) + end + + it "reports a stale copy when the file is in both the active and archive folders" do + topic = create_topic(state: :archived) + place(topic, attach(topic), pi_archive, v2_archive, pi) + + expect(report.rows).to contain_exactly(have_attributes(category: "orphan_stale_copy", directory: pi, topic_id: topic.id)) + end + + describe "pending deletes" do + let(:topic) { create_topic } + let(:shadow) { create_topic(shadow_copy: true, document_prefix: topic.id) } + + it "reports a deleted document still in Azure as failed_delete" do + place(topic, attach(topic, name: "new.pdf"), pi, v2) + place(shadow, attach(shadow, name: "old.pdf"), pi) + + expect(report.rows).to contain_exactly(have_attributes(category: "failed_delete", directory: pi, topic_id: topic.id)) + end + + it "does not report a delete when a current document has the same file name" do + place(topic, attach(topic), pi, v2) + attach(shadow) + + expect(report.rows).to be_empty + end + end + + describe "files in Azure that match no document" do + it "reports a file for an existing topic under another name" do + topic = create_topic + place(topic, attach(topic), pi, v2) + azure[pi]["#{topic.id}_emdoc_2025_1_sepsis.pdf"] = 10 + + expect(report.rows).to contain_exactly( + have_attributes(category: "orphan_unmatched_name", topic_id: topic.id, file_name: "#{topic.id}_emdoc_2025_1_sepsis.pdf"), + ) + end + + it "reports a file whose topic no longer exists" do + azure[v2]["999999_emdoc_2026_5_gone.pdf"] = 10 + + expect(report.rows).to contain_exactly(have_attributes(category: "orphan_no_topic", topic_id: nil, azure_size: 10)) + end + end + + it "reports duplicate topics" do + first = create_topic(title: "Journal Feed Weekly Wrap Up May 2") + second = create_topic(title: "journal feed weekly wrap up may 2 ") + [ first, second ].each { |topic| place(topic, attach(topic, name: "wrap.pdf"), pi, v2) } + + expect(report.rows.select { _1.category == "duplicate_topic" }.map(&:topic_id)).to contain_exactly(first.id, second.id) + end + + describe "#to_csv and #summary" do + before do + create_topic(title: "Empty one") + create_topic(title: "Empty two") + end + + it "writes one CSV line per row with a header" do + csv = CSV.parse(report.to_csv, headers: true) + + expect(csv.headers).to eq(AzureReconciliation::Report::Row.members.map(&:to_s)) + expect(csv.map { _1["title"] }).to contain_exactly("Empty one", "Empty two") + end + + it "counts rows by language and category" do + expect(report.summary).to eq({ [ "English", "topic_without_documents" ] => 2 }) + end + end + + it "reports an Azure folder that does not exist" do + azure.delete(video) + + expect(report.rows).to contain_exactly(have_attributes(category: "missing_directory", directory: video)) + end + + it "uses the language prefix for folder names" do + spanish = create(:language, name: "Spanish") + topic = create(:topic, language: spanish, provider:) + place(topic, attach(topic), "SP_#{pi}", "SP_#{v2}") + %w[SP_CMES-v2/assets/VideoContent SP_CMES-Pi_Archive SP_CMES-v2_Archive].each { |dir| azure[dir] } + + rows = described_class.new(lister:, languages: [ spanish ], check_blobs: false).rows + + expect(rows).to be_empty + end + + context "when checking blobs in storage" do + let(:check_blobs) { true } + + it "reports a document whose file is missing from storage" do + topic = create_topic + document = attach(topic) + place(topic, document, pi, v2) + document.blob.service.delete(document.blob.key) + + expect(report.rows).to contain_exactly(have_attributes(category: "missing_in_storage", document_id: document.id)) + end + end +end