Implementation of Handling Arbitrary Miscalibrations in Ratings

docs/requirements.txt

@@ -21,4 +21,5 @@ seaborn
 numba
 preflibtools
 prefsampling
-filelock
+filelock
+sortedcontainers

pref_voting/stochastic_methods.py

@@ -9,7 +9,11 @@
 from pref_voting.voting_method import *
 from pref_voting.iterative_methods import consensus_builder
 from pref_voting.probabilistic_methods import maximal_lottery, RaDiUS
+from pref_voting.grade_profiles import GradeProfile
+from networkx import topological_sort, is_directed_acyclic_graph, DiGraph, find_cycle
 import math
+import logging
+from sortedcontainers import SortedDict
 
 @vm(name="Random Consensus Builder (Stochastic)")
 def random_consensus_builder_st(profile, curr_cands=None, beta=0.5):
@@ -96,4 +100,151 @@ def sample_beta(B):
 
     else:
         beta = sample_beta(B)
-        return [RaDiUS.choose(profile, curr_cands=curr_cands, beta=beta)]
+        return [RaDiUS.choose(profile, curr_cands=curr_cands, beta=beta)]
+
+
+logger = logging.getLogger("RGCR")
+
+@vm(name="Randomized Grade Calibrated Ranking")
+def RGCR(gprofile:GradeProfile, w=(lambda x: x/(1+x)), curr_cands=None):
+
+    """
+    An implementation of the cardinal ranking estimator proposed by Wang and Shah (2018) in https://arxiv.org/abs/1806.05085.
+    by Avital Zar, 2026-04-21
+
+    Args:
+        gprofile: A profile of linear orders with associated cardinal scores (a GProfile).
+        curr_cands: A list of candidates to consider. Defaults to all candidates if not provided.
+
+    Returns:
+        A sorted list of candidates.
+
+    .. code block:: python
+        # Example usage:
+        from pref_voting.grade_profiles import GradeProfile
+        from pref_voting.stochastic_methods import RGCR
+
+        # Create a GProfile with 2 voters and 3 candidates
+        gprofile = GradeProfile([{1: 4, 2: 8}, {2: 6, 3: 2}], range(0, 10), candidates=[1, 2, 3])
+
+        # Get the ranking using RGCR
+        ranking = RGCR(gprofile)
+        print(ranking)
+        # Output should be either [1, 0, 2] or [1, 2, 0], with higher probability for [1, 0, 2].
+    """
+
+    w_results = SortedDict()
+
+    candidates = curr_cands if curr_cands is not None else gprofile.candidates
+    logger.info("Starting RGCR with candidates: %s", candidates)
+
+    def _ranking_graph(gprofile:GradeProfile):
+        # Helper function to create the ranking graph from the GProfile.
+        GB = DiGraph()
+        GB.add_nodes_from(gprofile.candidates)
+        gmap = [g.mapping for g in gprofile._grades]
+        for i in range(len(gmap)):
+            voter = gmap[i]
+            voter_sorted_cands = sorted(voter.keys(), key=lambda c: voter[c], reverse=True)
+            for j in range(len(voter_sorted_cands)-1):
+                for k in range(j+1, len(voter_sorted_cands)):
+                    c1 = voter_sorted_cands[j]
+                    c2 = voter_sorted_cands[k]
+                    if c1 in candidates and c2 in candidates and voter[c1] != voter[c2]:
+                        GB.add_edge(c1, c2)
+        return GB
+
+    # This part isn't in the paper, the contrary - the paper says that ties broken is in order of the indices of the items.
+    # However, such an arrangement creates a large bias in favor of the given order of candidates, which hurts the probability.
+    # Naturally, I preferred to fix the algorithm rather than change all my probability calculations.
+    gmap = [g.mapping for g in gprofile._grades]
+    random.shuffle(gmap)
+    Y = GradeProfile(gmap, gprofile.grades, candidates = candidates) # Create a copy of the evaluations to avoid modifying the original one.
+    B = Y.to_ranking_profile() # The ordinaly ranking
+    GB = _ranking_graph(Y) # The graph g(B) which represent the ordinal ranking.
+    if not is_directed_acyclic_graph(GB): # Then someone ranked a higher-ranked item lower, in contrast to the paper's assumption.
+        cycle = find_cycle(GB)
+        nodes = [u for u, v in cycle] + [cycle[-1][1]]
+        cycle_str = " -> ".join(str(node) for node in nodes)
+        logger.error("Cycle detected in majority graph: %s", cycle_str)
+        raise   ValueError("As the algorithm assumes, there can't be cycles in voting order.")
+    ordering = list(topological_sort(GB)) # Maybe the ties break could be more efficient
+    ordering = [c for c in ordering if c in set(candidates)] # Remove candidates not in curr_cands
+    logger.debug("Initial topological ordering: %s", ordering)
+
+    def _our_can(tuple):
+        # Helper random function which get two scores and return true if the first score probablistically beats the second.
+        w_result = check_w(abs(tuple[0]-tuple[1]))
+        prob = (1+w_result)/2 # The probability that the higher-ranked item is really better.
+        result = random.random() < prob # That is, if the first one is bigger then in probability prob we return true - the first beated the second.
+        if tuple[0] < tuple[1]: # If the second one is bigger, then in probability 1-prob we return true because in probability 1-prob the first beats the second.
+            result = not result
+
+        logger.debug("our_can: scores %s, prob %.4f -> flip: %g", tuple, round(prob, 4), result)
+        return result
+
+    def _find_reviewer(item):
+        # Helper function which finds a random voter who graded the given item.
+        reviewer = None
+        for voter in Y._grades:
+            if voter.has_grade(item) and voter.val(item) is not None:
+                reviewer = voter
+                break
+        return reviewer
+
+    def check_w(argument):
+        # Helper function to check that w is a valid function.
+        w_res = w(argument)
+        if not (0 <= w_res <= 1):
+            logger.error("Invalid w function: w(%g) = %g is not in [0, 1]", argument, w_res)
+            raise ValueError("w must return values in [0, 1]")
+        ind = w_results.bisect_left(argument)
+        if ind > 0:
+            k, prev = w_results.peekitem(ind-1)
+            if prev > w_res:
+                logger.error("Invalid w function: w is not non-decreasing. w(%g) = %g < w(%g) = %g", argument, w_res, w_results.keys()[ind-1], prev)
+                raise ValueError("w must be non-decreasing")
+        if ind < len(w_results):
+            k, next = w_results.peekitem(ind)
+            if next < w_res:
+                logger.error("Invalid w function: w is not non-decreasing. w(%g) = %g > w(%g) = %g", argument, w_res, w_results.keys()[ind], next)
+                raise ValueError("w must be non-decreasing")
+
+        w_results[argument] = w_res
+        logger.debug("Checked w(%g) = %g", argument, w_res)
+        return w_res
+
+    t = 0
+    while(t < len(ordering)-1):
+        t_th_item = ordering[t]
+        t_plus_1_th_item = ordering[t+1]
+
+        logger.info("Checking pair: (%s, %s) at index %g", t_th_item, t_plus_1_th_item, t)
+
+        t_reviewer = _find_reviewer(t_th_item)
+        t_plus_1_reviewer = _find_reviewer(t_plus_1_th_item)
+        # If the flipping of t and t+1 isn't a topological order, means there's no one who ranked t above t+1
+        # Also if there's a reviewing for both items
+        # Otherwise we continue
+        if t_reviewer and t_plus_1_reviewer and not (B.majority_prefers(t_th_item, t_plus_1_th_item)): # The order is opposite (compared to the paper) because if there's no reviewer then the item doesn't exist at all in B.
+                t_score = t_reviewer.val(t_th_item)
+                t_plus_1_score = t_plus_1_reviewer.val(t_plus_1_th_item)
+
+                logger.debug("Pair satisfies flip conditions. Scores: %s=%g, %s=%g", t_th_item, t_score, t_plus_1_th_item, t_plus_1_score)
+
+                Y._grades.remove(t_reviewer)
+                if(t_plus_1_reviewer in Y._grades): # In case we choose the same reviewer for both items.
+                    Y._grades.remove(t_plus_1_reviewer)
+                if(_our_can((t_plus_1_score, t_score))): # If the second item ranked higher, the we flip them.
+                    logger.info("Flipping %s and %s", t_th_item, t_plus_1_th_item)
+                    ordering[t], ordering[t+1] = ordering[t+1], ordering[t]
+                t = t+2
+        else:
+            if not t_reviewer or not t_plus_1_reviewer:
+                logger.debug("Skipping pair: missing reviewers.")
+            else:
+                logger.debug("Skipping pair: There's a reviewer prefers %s over %s", t_th_item, t_plus_1_th_item)
+            t=t+1
+
+    logger.info("Final RGCR ranking: %s", ordering)
+    return ordering

tests/test_rgcr_method.py

@@ -0,0 +1,205 @@
+'''
+Tests for the implementation of the main algorithm in:
+"Your 2 is My 1, Your 3 is My 9: Handling Arbitrary Miscalibrations in Ratings", by J, Wang and N. B. Shah (2018), https://arxiv.org/abs/1806.05085
+
+Programmer: Avital Zar.
+Date: 2026-06-01
+'''
+
+from pref_voting.stochastic_methods import RGCR
+from pref_voting.grade_profiles import GradeProfile
+import networkx as nx
+import pytest
+import numpy as np
+from scipy.stats import kendalltau
+import logging
+
+logging.getLogger("RGCR").setLevel(logging.NOTSET)
+logging.getLogger("test").setLevel(logging.INFO)
+
+def random_ordinal_ranking(gprofile:GradeProfile, curr_cands=None):
+	x = gprofile.to_ranking_profile().majority_graph().to_networkx()
+	return list(nx.topological_sort(x))
+
+def mean_estimator(gprofile:GradeProfile, curr_cands=None):
+	gprofile = GradeProfile([g.mapping for g in gprofile._grades], gprofile.grades.tolist(), candidates=gprofile.candidates)
+	if curr_cands is None:
+		curr_cands = gprofile.candidates
+	return sorted(curr_cands, key=lambda c: gprofile.avg(c) if gprofile.has_grade(c) else 0, reverse=True)
+
+def median_estimator(gprofile:GradeProfile, curr_cands=None):
+	if curr_cands is None:
+		curr_cands = gprofile.candidates
+	return sorted(curr_cands, key=lambda c: gprofile.median(c) if gprofile.has_grade(c) else 0, reverse=True)
+
+
+logger = logging.getLogger("test")
+
+def is_topological_order(profile, ranking):
+	G = profile.to_ranking_profile().majority_graph().to_networkx()
+	if len(ranking) != len(G.nodes) or set(ranking) != set(G.nodes):
+		logger.error("is_top_ord: Ranking does not contain the same candidates as the graph. len(ranking)=%g, len(G.nodes)=%g", len(ranking), len(G.nodes))
+		logger.error("is_top_ord: Ranking candidates: %s, all candidates: %s", ranking, list(G.nodes))
+		return False
+
+	# שמירת המיקום של כל צומת ברשימה
+	index_map = {node: i for i, node in enumerate(ranking)}
+
+	# בדיקה שעבור כל קשת, צומת המקור מופיע לפני צומת היעד
+	for u, v in G.edges():
+		if index_map[u] > index_map[v]:
+			logger.error("is_top_ord: Edge (%s, %s) violates the topological order.", u, v)
+			return False
+	logger.info("is_top_ord: Ranking is a valid topological order.")
+	return True
+
+def create_random_legal_gprofile(size=5, num_voters=10, rev_prob=0.3): # creates a random gprofile which is legal (i.e. does not contain cycles). the 'true' order is 0 < 1 < ... < size-1.
+	candidates = list(range(size))
+	voters = []
+	for _ in range(num_voters):
+		voter = {}
+		val = 0
+		for c in candidates:
+			if np.random.rand() < rev_prob:
+				voter[c] = val + np.random.randint(0, 10)+1
+				val = voter[c] # ensure that the scores are non-decreasing
+		voters.append(voter)
+	logger.debug("Created random legal graph with %s", voters)
+	return GradeProfile(voters, np.arange(0, 10*size+1, 1), candidates=candidates)
+
+
+def test_topological_order():
+	for i in range(1,100,10):
+		n = np.random.rand() * 5*i
+		gprofile = create_random_legal_gprofile(size=i, num_voters=int(n))
+		logger.info("Test topological order with %g candidates and %g voters", i, int(n))
+		ranking = RGCR(gprofile)
+		assert is_topological_order(gprofile, ranking)
+
+@pytest.mark.parametrize("profile, expected_sol, expected_prob", [
+	(GradeProfile([{1: 7}, {2: 3}], range(0, 10), candidates=[1, 2]), [1, 2], 0.9),
+	(GradeProfile([{1: 4, 2: 8}, {2: 6, 3: 2}], range(0, 10), candidates=[1, 2, 3]), [2, 1, 3], 5/6),
+	(GradeProfile([{1: 3, 2: 4, 5: 5}, {1: 5, 3: 6}, {4: 2, 5: 10}, {2: 7, 6: 10}], range(0, 11), candidates=[1,2,3,4,5,6]), [6,5,4,3,2,1], 7/198)
+])
+def test_probability(profile, expected_sol, expected_prob): #test approximation to the probability, only for small inputs.
+	prob = 0
+	trials = 10000
+	for _ in range(trials):
+		solution = RGCR(profile)
+		if solution == expected_sol:
+			prob += 1
+	assert abs(prob/trials - expected_prob) < 0.05
+
+@pytest.mark.parametrize("w, expected_prob", [
+	(lambda x: x/(1+x), 5/6), #the default w
+	(lambda x: 3*x/(1+3*x), 13/14),
+	(lambda x: 0.1*x/(1+0.1*x), 7/12)
+])
+def test_probability_with_diff_w(w, expected_prob):
+	profile = GradeProfile([{1: 4, 2: 8}, {2: 6, 3: 2}], np.arange(0, 10, 1), candidates=[1, 2, 3])
+	expected_sol = [2,1,3]
+	prob = 0
+	trials = 10000
+	for _ in range(trials):
+		if RGCR(profile, w = w) == expected_sol:
+			prob += 1
+	assert abs(prob/trials - expected_prob) < 0.05
+
+@pytest.mark.parametrize("gprofile", [
+	(GradeProfile([{1: 4, 2: 8}, {2: 6, 1: 7}], range(0, 10), candidates=[1, 2, 3])), # cycle
+	(GradeProfile([{1: 3, 2: 4, 5: 5}, {2: 6, 3: 7}, {3: 2, 1: 5}], range(0, 11), candidates=[1,2,3,4,5])) # cycle
+])
+def test_illegal_input(gprofile):
+	with pytest.raises(ValueError):
+		RGCR(gprofile)
+
+
+#This is no-use test because the paper assumes there are no ties. In any case it passes, in case we'd want to allow ties. 👑
+@pytest.mark.parametrize("gprofile", [
+	(GradeProfile([{1: 4, 2: 8}, {2: 8, 1: 8}], range(0, 10), candidates=[1, 2, 3])), # cycle
+	(GradeProfile([{2: 8, 8: 0, 10: 10}, {2: 10, 4: 7, 10: 10}, {2: 3, 6: 8, 7: 4, 9: 7}, { 4: 6, 8: 1}, {2: 2, 3: 2, 10: 7}, {1: 2, 2: 9, 7: 9}, {3: 0, 6: 5, 7: 5, 8: 4}], range(0, 11), candidates=[1,2,3,4,5,6,7,8,9,10])) # cycle
+])
+def test_legal_complex_input(gprofile):
+	RGCR(gprofile) # should not raise an error, even though the ties.
+
+
+@pytest.mark.parametrize("w", [lambda x: x, lambda x: x**2, lambda x: np.sqrt(x), lambda x: 1-x/(1+x)]) #w must be increasing and return value in [0,1].
+def test_illegal_w(w):
+	with pytest.raises(ValueError):
+		RGCR(GradeProfile([{1: 7}, {2: 3}, {3: 5}, {4: 3}], range(0, 10), candidates=[1, 2, 3, 4]), w=w)
+
+
+
+# The following test checks the strict uniform dominance as described in the paper.
+
+@pytest.mark.parametrize("estimator", [random_ordinal_ranking, mean_estimator, median_estimator])
+def test_strict_uniform_dominance(estimator):
+	rgcr_success = 0
+	another_estimator_success = 0
+	trials = 1000
+	for i in range(1, trials):
+		voters = 10
+		items = np.random.randint(voters, voters*3)
+		gprofile = create_random_legal_gprofile(size=items, num_voters=voters)
+		rgcr_ranking = RGCR(gprofile)
+		another_ranking = estimator(gprofile)
+		if rgcr_ranking == list(range(items))[::-1]: # the true order is always 0 < 1 < ...
+			logger.info("RGCR found the true order in trial %g", i)
+			rgcr_success += 1
+		else:
+			if i % 100 == 0: # log only every 100 trials to avoid cluttering the logs
+				logger.debug("RGCR did not find the true order in trial %g. RGCR ranking: %s", i, rgcr_ranking)
+		if another_ranking == list(range(items))[::-1]:
+			logger.info("Another estimator found the true order in trial %g", i)
+			another_estimator_success += 1
+		else:
+			if i % 100 == 0: # log only every 100 trials to avoid cluttering the logs
+				logger.debug("Another estimator did not find the true order in trial %g. Another ranking: %s", i, another_ranking)
+	assert rgcr_success > another_estimator_success
+
+
+# Another test for strict uniform dominance, this time using the Kendall tau correlation with the true order as a measure of success, instead of exact equality.
+# We could say we check strict uniform dominance with another loss function, as described in the paper.
+
+@pytest.mark.parametrize("estimator", [random_ordinal_ranking, mean_estimator, median_estimator])
+def test_strict_uniform_dominance_kendall_tau(estimator):
+    count = 0
+    trials = 1000
+    for i in range(1, trials):
+        voters = 10
+        items = np.random.randint(voters, voters*3)
+        gprofile = create_random_legal_gprofile(size=items, num_voters=voters)
+        rgcr_ranking = RGCR(gprofile)
+        another_ranking = estimator(gprofile)
+        true_order = list(range(items))[::-1] # the true order is always 0 < 1 < ...
+        rgcr_kendall = kendalltau(rgcr_ranking, true_order).correlation
+        another_estimator_kendall = kendalltau(another_ranking, true_order).correlation
+        diff = rgcr_kendall - another_estimator_kendall
+        if diff > 0:
+            count += 1 
+        if diff < 0:
+            count -= 1
+    assert count > 0
+
+# Another test for strict uniform dominance, this time using the recall of 10 first items.
+
+@pytest.mark.parametrize("estimator", [random_ordinal_ranking, mean_estimator, median_estimator])
+def test_strict_uniform_dominance_recall(estimator):
+	count = 0
+	trials = 1000
+	k = 10
+	for i in range(1, trials):
+		voters = 10
+		items = np.random.randint(voters, voters*3)
+		gprofile = create_random_legal_gprofile(size=items, num_voters=voters)
+		rgcr_ranking = RGCR(gprofile)
+		another_ranking = estimator(gprofile)
+		true_order = list(range(items))[::-1] # the true order is always 0 < 1 < ...
+		rgcr_recall = len(set(rgcr_ranking[:k]) & set(true_order[:k])) / len(set(true_order[:k]))
+		another_estimator_recall = len(set(another_ranking[:k]) & set(true_order[:k])) / len(set(true_order[:k]))
+		diff = rgcr_recall - another_estimator_recall
+		if diff > 0:
+			count += 1
+		if diff < 0:
+			count -= 1
+	assert count > 0