imp: adds docker registry repair packages

* docker-registry-repair: for repairing specific images
* docker-registry-tail: for tailing the registry service to
  verify/repair images as they are pushed to the registry server
* Ruby script parameterization added for nix module usage

Author: johnalotoski

overlay.nix

@@ -42,6 +42,8 @@ in
       cue = prev.callPackage ./pkgs/cue.nix {};
       devShell = final.callPackage ./pkgs/dev-shell.nix {};
       docker-distribution = prev.callPackage ./pkgs/docker-distribution.nix {};
+      docker-registry-repair = prev.callPackage ./pkgs/docker-registry/default.nix {name = "docker-registry-repair";};
+      docker-registry-tail = prev.callPackage ./pkgs/docker-registry/default.nix {name = "docker-registry-tail";};
 
       # Pin docker and containerd to avoid unexpected cluster wide docker daemon restarts
       # during metal deploy resulting in OCI jobs being killed or behaving unexpectedly

pkgs/docker-registry/default.nix

@@ -0,0 +1,29 @@
+{
+  name,
+  coreutils,
+  lib,
+  ruby,
+  stdenv,
+  writeShellApplication,
+}: let
+  inherit name;
+  wrapperApp = writeShellApplication {
+    inherit name;
+    runtimeInputs = [coreutils ruby];
+    text = ''exec ruby "$(dirname "$(readlink -f "$0")")/.${name}-wrapped" "$@"'';
+  };
+in
+  stdenv.mkDerivation rec {
+    inherit name;
+    src = ./${name}.rb;
+
+    dontUnpack = true;
+    dontPatch = true;
+    dontConfigure = true;
+    dontBuild = true;
+
+    installPhase = ''
+      install -Dm555 "$src" "$out/bin/.${name}-wrapped"
+      cp ${wrapperApp}/bin/${name} $out/bin
+    '';
+  }

pkgs/docker-registry/docker-registry-repair.rb

@@ -0,0 +1,115 @@
+require 'digest/sha2'
+require 'json'
+require 'fileutils'
+require 'optparse'
+
+OPTIONS = {
+  delete_tag: false,
+  dry_run: false,
+  registry: '/var/lib/docker-registry/docker/registry/v2',
+  repo: nil,
+  tag: nil,
+}
+
+op = OptionParser.new do |parser|
+  parser.banner = 'Usage: docker-registry-repair [options]'
+
+  parser.on '-d', '--dry-run [FLAG]', TrueClass, 'avoid deleting anything' do |v|
+    OPTIONS[:dry_run] = v.nil? ? true : v
+  end
+
+  parser.on '-r', '--repo REPO', 'repository part of the image name, like `cardano-public-documentation`' do |v|
+    OPTIONS[:repo] = v
+  end
+
+  parser.on '-t', '--tag TAG', 'tag of the image, the part after the `:`' do |v|
+    OPTIONS[:tag] = v
+  end
+
+  parser.on '--delete-tag [FLAG]', TrueClass, 'also delete all tag references' do |v|
+    OPTIONS[:delete_tag] = v.nil? ? true : v
+  end
+
+  parser.on '--registry-path PATH', "the registry path, defaults to: #{OPTIONS[:registry]}" do |v|
+    OPTIONS[:registry] = v
+  end
+end
+
+op.parse!
+
+def dry_run?; OPTIONS[:dry_run] end
+def delete_tag?; OPTIONS[:delete_tag] end
+def repo; OPTIONS[:repo] end
+def tag; OPTIONS[:tag] end
+def registry; OPTIONS[:registry] end
+def prefix(hash) hash[/sha256:(..)/, 1] end
+def suffix(hash) hash[/sha256:(.*)/, 1] end
+def blob(hash) "#{registry}/blobs/sha256/#{prefix(hash)}/#{suffix(hash)}/data" end
+
+def rm(path)
+  puts "removing #{path}"
+  FileUtils.rm_rf(path) unless dry_run?
+end
+
+def remove_layers(hash)
+  `redis-cli --raw keys '*#{hash}*'`.each_line do |key|
+    key.strip!
+    puts "removing #{key} from redis"
+    system('redis-cli', 'del', key) unless dry_run?
+  end
+
+  Dir.glob("#{registry}/repositories/*/_layers/sha256/#{hash}") do |layer_file|
+    rm(layer_file)
+  end
+
+  Dir.glob("#{registry}/repositories/*/_manifests/revisions/sha256/#{hash}") do |revision_file|
+    rm(revision_file)
+  end
+
+  Dir.glob("#{registry}/repositories/*/_manifests/tags/*/index/sha256/#{hash}") do |index_file|
+    rm(index_file)
+  end
+end
+
+def repair(desired)
+  blob_path = blob(desired)
+
+  unless File.file?(blob_path)
+    puts "missing file for #{blob_path}"
+    remove_layers(desired)
+    return
+  end
+
+  actual = Digest::SHA256.file(blob_path).hexdigest
+  return if desired == "sha256:#{actual}"
+
+  remove_layers(desired)
+  puts "removing #{blob(desired)}"
+  system('redis-cli', 'del', "blobs::sha256:#{desired}")
+  rm(blob(desired))
+end
+
+puts "verifying #{repo}:#{tag}"
+puts "--dry-run is enabled, will not actually delete anything" if dry_run?
+
+link = "#{registry}/repositories/#{repo}/_manifests/tags/#{tag}/current/link"
+unless File.file?(link)
+  puts "#{link} is missing, cannot read manifest"
+end
+
+link_hash = File.read(link).strip
+manifest = JSON.parse(File.read(blob(link_hash)))
+
+repair manifest['config']['digest']
+
+manifest['layers'].each do |layer|
+  repair layer['digest']
+end
+
+# The following code may actually be needed in more serious cases
+exit unless delete_tag?
+
+remove_layers(link_hash)
+
+puts "removing #{link}"
+rm("#{registry}/repositories/#{repo}/_manifests/tags/#{tag}")

pkgs/docker-registry/docker-registry-tail.rb

@@ -0,0 +1,59 @@
+require 'open3'
+require 'set'
+require 'optparse'
+
+OPTIONS = {
+  delay: 5,
+  delete_tag: false,
+  dry_run: false,
+  script: 'docker-registry-repair',
+  service: 'docker-registry.service',
+  since: '-1h',
+}
+
+op = OptionParser.new do |parser|
+  parser.banner = 'Usage: docker-registry-tail [options]'
+
+  parser.on '--repair-path PATH', "the repair script path to utilize; defaults to: #{OPTIONS[:script]}" do |v|
+    OPTIONS[:script] = v
+  end
+
+  parser.on '-s', '--since LOOKBACK', "the lookback period for journal history; defaults to: #{OPTIONS[:since]}" do |v|
+    OPTIONS[:since] = v
+  end
+
+  parser.on '-u', '--service SERVICE', "the systemd service to tail; defaults to: #{OPTIONS[:service]}" do |v|
+    OPTIONS[:service] = v
+  end
+
+  parser.on '-t', '--delay SEC', Integer, "the time delay in seconds between repair spawn jobs; defaults to: #{OPTIONS[:delay]}" do |v|
+    OPTIONS[:delay] = v
+  end
+
+  parser.on '-d', '--dry-run [FLAG]', TrueClass, 'avoid deleting anything' do |v|
+    OPTIONS[:dry_run] = v.nil? ? true : v
+  end
+
+  parser.on '--delete-tag [FLAG]', TrueClass, 'also delete all tag references' do |v|
+    OPTIONS[:delete_tag] = v.nil? ? true : v
+  end
+end
+
+op.parse!
+
+def dry_run?; OPTIONS[:dry_run] end
+def delete_tag?; OPTIONS[:delete_tag] end
+def delay; OPTIONS[:delay] end
+def script; OPTIONS[:script] end
+def service; OPTIONS[:service] end
+def since; OPTIONS[:since] end
+
+# Check back in systemd log history, and follow all newly pushed images
+Open3.popen2e('journalctl', '-S', since, '-f', '-u', service, '-g', '"PUT /v2/.+/manifests') do |_, out|
+  out.each_line do |line|
+    %r!/v2/(?<repo>[^/\s]+)/manifests/(?<tag>[^/\s]+)! =~ line
+    next unless repo && tag
+    system(script, '--repo', repo, '--tag', tag, '--dry-run', dry_run? ? "true" : "false", '--delete-tag', delete_tag? ? "true" : "false")
+    sleep delay
+  end
+end