|
| 1 | +# Invoked by the Clojure checker qualification. Load the actual harness without |
| 2 | +# starting its TCP server; no copied Owner/Driver logic lives in this probe. |
| 3 | +System.put_env("GROUP_JEPSEN_LIBRARY_ONLY", "1") |
| 4 | +Code.require_file("node.exs", __DIR__) |
| 5 | +Application.ensure_all_started(:group) |
| 6 | +Logger.configure(level: :emergency) |
| 7 | + |
| 8 | +defmodule Group.Jepsen.ConflictProbe do |
| 9 | + alias Group.Jepsen.{ConflictEvidence, Driver, EDN} |
| 10 | + |
| 11 | + def run do |
| 12 | + directory = Path.join(__DIR__, ".cache/conflict-probe-#{System.unique_integer([:positive])}") |
| 13 | + File.mkdir_p!(directory) |
| 14 | + |
| 15 | + try do |
| 16 | + {winner, rejected, winner_events, archive} = winner(directory) |
| 17 | + |
| 18 | + cases = [ |
| 19 | + {"historical winner subsequently unregistered and died", true, 1, "jepsen/registry/0", |
| 20 | + winner, false}, |
| 21 | + {"victim killed during register", true, 1, "jepsen/registry/0", winner, true}, |
| 22 | + {"forged key and rank", false, 99, "nonexistent", %{token: "invented", revision: -100}, |
| 23 | + false}, |
| 24 | + {"rightful winner killed", false, 99, "jepsen/registry/0", winner, false}, |
| 25 | + {"nonexistent winning claim", false, 1, "jepsen/registry/0", |
| 26 | + %{token: "invented", revision: 100}, false}, |
| 27 | + {"definitively rejected winning claim", false, 1, "jepsen/registry/0", rejected, false} |
| 28 | + ] |
| 29 | + |
| 30 | + scenarios = |
| 31 | + Enum.with_index(cases, fn {label, valid, revision, key, meta, pending}, index -> |
| 32 | + {events, reset_events} = victim(directory, index, revision, key, meta, pending) |
| 33 | + |
| 34 | + %{ |
| 35 | + label: label, |
| 36 | + valid: valid, |
| 37 | + archive: archive, |
| 38 | + reset_evidence: reset_events, |
| 39 | + snapshots: %{ |
| 40 | + "n1" => %{conflict_evidence: events}, |
| 41 | + "n2" => %{conflict_evidence: winner_events} |
| 42 | + } |
| 43 | + } |
| 44 | + end) |
| 45 | + |
| 46 | + IO.puts("CONFLICT-PROBE " <> EDN.encode(scenarios)) |
| 47 | + after |
| 48 | + File.rm_rf!(directory) |
| 49 | + end |
| 50 | + end |
| 51 | + |
| 52 | + defp start(directory, label, node_id) do |
| 53 | + {:ok, group} = Group.start_link(name: :jepsen_group, shards: 1, log: false) |
| 54 | + path = Path.join(directory, label) |
| 55 | + {:ok, evidence} = ConflictEvidence.start_link(conflict_evidence_path: path) |
| 56 | + {:ok, driver} = Driver.start_link(index: 0, node_id: node_id, boot_id: label) |
| 57 | + {group, evidence, driver, path} |
| 58 | + end |
| 59 | + |
| 60 | + defp mutate(driver, operation, revision) do |
| 61 | + GenServer.call(driver, {:mutate, operation, "owner", nil, 0, revision}) |
| 62 | + end |
| 63 | + |
| 64 | + defp winner(directory) do |
| 65 | + {group, evidence, driver, path} = start(directory, "winner", "n2") |
| 66 | + %{status: :ok, owner: %{token: token}} = mutate(driver, :register, 10) |
| 67 | + |
| 68 | + %{status: :fail, owner: %{token: rejected}} = |
| 69 | + GenServer.call(driver, {:mutate, :register, "rejected", nil, 0, 100}) |
| 70 | + |
| 71 | + %{status: :ok} = mutate(driver, :unregister, 0) |
| 72 | + %{status: :ok} = GenServer.call(driver, {:kill, "owner"}) |
| 73 | + %{status: :ok} = GenServer.call(driver, {:kill, "rejected"}) |
| 74 | + events = ConflictEvidence.snapshot() |
| 75 | + archive = File.read!(path) |
| 76 | + GenServer.stop(driver) |
| 77 | + GenServer.stop(evidence) |
| 78 | + Supervisor.stop(group) |
| 79 | + {%{token: token, revision: 10}, %{token: rejected, revision: 100}, events, archive} |
| 80 | + end |
| 81 | + |
| 82 | + defp victim(directory, index, revision, key, winner, pending) do |
| 83 | + {group, evidence, driver, path} = start(directory, "victim-#{index}", "n1") |
| 84 | + %{status: :ok} = mutate(driver, :join, 0) |
| 85 | + {pid, _token, _monitor, _cached} = :sys.get_state(driver).owners["owner"] |
| 86 | + shard = Group.Replica.shard_for(:jepsen_group, nil, "jepsen/registry/0") |
| 87 | + |
| 88 | + task = |
| 89 | + if pending do |
| 90 | + :ok = :sys.suspend(shard) |
| 91 | + task = Task.async(fn -> mutate(driver, :register, revision) end) |
| 92 | + wait(fn -> Enum.any?(ConflictEvidence.snapshot(), &(&1.kind == :register)) end) |
| 93 | + task |
| 94 | + else |
| 95 | + %{status: :ok} = mutate(driver, :register, revision) |
| 96 | + nil |
| 97 | + end |
| 98 | + |
| 99 | + Process.exit(pid, {:group_registry_conflict, key, winner}) |
| 100 | + if task, do: Task.await(task) |
| 101 | + wait(fn -> Enum.any?(ConflictEvidence.snapshot(), &(&1.kind == :death)) end) |
| 102 | + if pending, do: :sys.resume(shard) |
| 103 | + events = ConflictEvidence.snapshot() |
| 104 | + [] = GenServer.call(driver, :unexpected_deaths) |
| 105 | + {:ok, []} = GenServer.call(driver, :owner_snapshots) |
| 106 | + GenServer.stop(driver) |
| 107 | + GenServer.stop(evidence) |
| 108 | + |
| 109 | + # The exact registration/death obligations must survive a recorder restart. |
| 110 | + {:ok, evidence} = ConflictEvidence.start_link(conflict_evidence_path: path) |
| 111 | + ^events = ConflictEvidence.snapshot() |
| 112 | + :ok = ConflictEvidence.reset() |
| 113 | + [] = ConflictEvidence.snapshot() |
| 114 | + "" = File.read!(path) |
| 115 | + GenServer.stop(evidence) |
| 116 | + {:ok, evidence} = ConflictEvidence.start_link(conflict_evidence_path: path) |
| 117 | + [] = ConflictEvidence.snapshot() |
| 118 | + |
| 119 | + 1 = |
| 120 | + ConflictEvidence.record(%{ |
| 121 | + kind: :register, |
| 122 | + token: "next-history", |
| 123 | + key: 0, |
| 124 | + cluster: nil, |
| 125 | + revision: 1 |
| 126 | + }) |
| 127 | + |
| 128 | + :ok = ConflictEvidence.reset() |
| 129 | + reset_events = ConflictEvidence.snapshot() |
| 130 | + GenServer.stop(evidence) |
| 131 | + Supervisor.stop(group) |
| 132 | + {events, reset_events} |
| 133 | + end |
| 134 | + |
| 135 | + defp wait(fun, remaining \\ 200) |
| 136 | + defp wait(_fun, 0), do: raise("probe timed out") |
| 137 | + |
| 138 | + defp wait(fun, remaining) do |
| 139 | + unless fun.() do |
| 140 | + Process.sleep(5) |
| 141 | + wait(fun, remaining - 1) |
| 142 | + end |
| 143 | + end |
| 144 | +end |
| 145 | + |
| 146 | +Group.Jepsen.ConflictProbe.run() |
0 commit comments