{% macro sort_link(label, field) -%} {%- if sort_field == field -%} {%- set next_dir = 'desc' if direction == 'asc' else 'asc' -%} {{ label }} {{ '▲' if direction == 'asc' else '▼' }} {%- else -%} {{ label }} {%- endif -%} {%- endmacro %} {{ benchmark_label }} Reviewer Analytics

{{ benchmark_label }} Reviewer Analytics

Aggregated statistics derived from reviewer exports.

Back to Queue Results Table Manage Question Pool
{% if keep_filter_enabled %}
{% endif %}
{{ overall.total }} Total Responses
{{ overall.matches }} Agreed
{{ overall.mismatches }} Disagreed
{{ (overall.accuracy * 100) | round(1) }}% Agreement Rate
{% if overall_kappa is not none %} {{ overall_kappa | round(3) }} {% else %} — {% endif %} Cohen's κ (avg)
{% if model_accuracy is not none %} {{ (model_accuracy * 100) | round(1) }}% {% else %} — {% endif %} Model Accuracy
{% if model_kappa_overall is not none %} {{ model_kappa_overall | round(3) }} {% else %} — {% endif %} Model Cohen's κ (avg)

Reviewer Consensus (incl. original user weight {{ original_user_weight }})

{% if avg_question_accuracy is not none %} {{ (avg_question_accuracy * 100) | round(1) }}% {% else %} — {% endif %} Avg Accuracy (per question)
{% if majority_metrics.accuracy is not none %} {{ (majority_metrics.accuracy * 100) | round(1) }}% {% else %} — {% endif %} Majority Vote Accuracy
{% if majority_metrics.consensus_strength is not none %} {{ (majority_metrics.consensus_strength * 100) | round(1) }}% {% else %} — {% endif %} Avg Consensus Strength
{% if majority_metrics.unanimous_rate is not none %} {{ (majority_metrics.unanimous_rate * 100) | round(1) }}% {% else %} — {% endif %} Unanimous Rate
{% if majority_metrics.tie_rate is not none %} {{ (majority_metrics.tie_rate * 100) | round(1) }}% {% else %} — {% endif %} Tie Rate
{% if majority_metrics.tie_count is not none %} {{ majority_metrics.tie_count }} {% else %} — {% endif %} Tie Count
{% if majority_metrics.avg_entropy is not none %} {{ majority_metrics.avg_entropy | round(3) }} {% else %} — {% endif %} Avg Vote Entropy (0-1)

Human Majority vs Model Majority

{% set original = model_majority_metrics.original %} {% set swapped = model_majority_metrics.swapped %} {% set combined = model_majority_metrics.combined %} {% if original.agreement is not none or swapped.agreement is not none or combined.agreement is not none %}
Prediction Set Agreement Matches Comparable Coverage Tie Rate
Original order {% if original.agreement is not none %} {{ (original.agreement * 100) | round(1) }}% {% else %} — {% endif %} {{ original.matches }} {{ original.total }} {% if original.coverage is not none %} {{ (original.coverage * 100) | round(1) }}% {% else %} — {% endif %} {% if original.tie_rate is not none %} {{ (original.tie_rate * 100) | round(1) }}% {% else %} — {% endif %}
Swapped order {% if swapped.agreement is not none %} {{ (swapped.agreement * 100) | round(1) }}% {% else %} — {% endif %} {{ swapped.matches }} {{ swapped.total }} {% if swapped.coverage is not none %} {{ (swapped.coverage * 100) | round(1) }}% {% else %} — {% endif %} {% if swapped.tie_rate is not none %} {{ (swapped.tie_rate * 100) | round(1) }}% {% else %} — {% endif %}
Aggregated {% if combined.agreement is not none %} {{ (combined.agreement * 100) | round(1) }}% {% else %} — {% endif %} {{ combined.matches }} {{ combined.total }} {% if combined.coverage is not none %} {{ (combined.coverage * 100) | round(1) }}% {% else %} — {% endif %} {% if combined.tie_rate is not none %} {{ (combined.tie_rate * 100) | round(1) }}% {% else %} — {% endif %}
{% else %}

No model predictions available for majority vote agreement.

{% endif %}

Winner vs Model Prediction (Order-Invariant analysis.py)

{% if order_invariant_groups %} {% for group in order_invariant_groups %} {% for item in group.rows %} {% set metrics = item.metrics %} {% endfor %} {% endfor %}
Model Correct / (Correct + Wrong) Correct / Examples
{{ group.title }}
{{ item.display }} {% if metrics and metrics.win_rate is not none %} {{ metrics.correct }} / {{ metrics.correct + metrics.wrong }}
{{ (metrics.win_rate * 100) | round(2) }}% when model answered
Blank: {{ metrics.blank }}, Inconsistent: {{ metrics.inconsistent }}
{% else %}
Insufficient data
{% endif %} 		{% if metrics and metrics.total %} {% set examples = metrics.total %} {% set pct = (metrics.correct / examples * 100) if examples else None %} {{ metrics.correct }} / {{ examples }}
{{ pct | round(2) if pct is not none else '—' }}% overall
{% else %}
Insufficient data
{% endif %}
{% else %}

No model predictions available for order-invariant scoring.

{% endif %}

Reviewer Majority vs Model Prediction

{% if majority_vs_model_groups %} {% for group in majority_vs_model_groups %} {% for item in group.rows %} {% set metrics = item.metrics %} {% endfor %} {% endfor %}
Model Correct / (Correct + Wrong) Correct / Examples
{{ group.title }}
{{ item.display }} {% if metrics and metrics.win_rate is not none %} {{ metrics.correct }} / {{ metrics.correct + metrics.wrong }}
{{ (metrics.win_rate * 100) | round(2) }}% when model answered
Blank: {{ metrics.not_follow or 0 }}, Inconsistent: {{ metrics.inconsistent or 0 }}
{% else %}
Insufficient data
{% endif %} 		{% if metrics and metrics.examples is not none %} {{ metrics.correct }} / {{ metrics.examples }}
{{ (metrics.overall * 100) | round(2) }}% overall
{% else %}
Insufficient data
{% endif %}
{% else %}

No model predictions available for reviewer-majority comparison.

{% endif %}

Model vs Reviewer Accuracy

Reviewers
{{ accuracy_chart.reviewer | round(1) }}%
{% if accuracy_chart.model is not none %}
Model
{{ accuracy_chart.model | round(1) }}%
{% else %}

Model accuracy unavailable for this benchmark.

{% endif %}

Reviewer vs Model Accuracy Trend

Comparison of reviewer accuracy per question versus the model prediction.

{% if accuracy_chart_image %}
Line chart comparing reviewer and model accuracy
{% else %}

Not enough data to draw this chart.

{% endif %}

Per Reviewer

{% for item in per_reviewer %} {% endfor %}
Reviewer Total Agree Disagree Missing Accuracy
{{ item.reviewer_id }} {{ item.stats.total }} {{ item.stats.matches }} {{ item.stats.mismatches }} {{ item.stats.missing }} {{ (item.stats.accuracy * 100) | round(1) }}%

Fleiss' Kappa (Reviewers)

{% if reviewer_fleiss_kappa is not none %} {{ reviewer_fleiss_kappa | round(3) }} {% else %} — {% endif %} Fleiss' κ
{{ reviewer_fleiss_stats["items"] }} Items
{{ reviewer_fleiss_stats.avg_ratings | round(2) }} Avg ratings per item

Fleiss' Kappa (Models)

{% if model_fleiss_kappa is not none %} {{ model_fleiss_kappa | round(3) }} {% else %} — {% endif %} Aggregated Fleiss' κ
{{ model_fleiss_stats["items"] }} Items
{{ model_fleiss_stats.avg_ratings | round(2) }} Avg ratings per item
{% if model_fleiss_kappa_orig is not none %} {{ model_fleiss_kappa_orig | round(3) }} {% else %} — {% endif %} Original Fleiss' κ
{{ model_fleiss_stats_orig["items"] }} Items
{{ model_fleiss_stats_orig.avg_ratings | round(2) }} Avg ratings per item
{% if model_fleiss_kappa_swap is not none %} {{ model_fleiss_kappa_swap | round(3) }} {% else %} — {% endif %} Swapped Fleiss' κ
{{ model_fleiss_stats_swap["items"] }} Items
{{ model_fleiss_stats_swap.avg_ratings | round(2) }} Avg ratings per item

Per Question

{% for item in per_question %} {% endfor %}
{{ sort_link('Question', 'question_id')|safe }} {{ sort_link('Model Prediction', 'model_prediction')|safe }} {{ sort_link('Total', 'total')|safe }} {{ sort_link('Agree', 'agree')|safe }} {{ sort_link('Disagree', 'disagree')|safe }} {{ sort_link('Option A', 'option_a')|safe }} {{ sort_link('Option B', 'option_b')|safe }} {{ sort_link('Accuracy', 'accuracy')|safe }} Reviewer vs Model Actions
{{ item.question_id }} {% if item.has_disagreement %} Dispute {% endif %} {% if item.model_prediction %}
{{ item.model_prediction }}
{% if item.model_prediction_text %} {{ item.model_prediction_text | truncate(80) }} {% endif %} {% else %} — {% endif %} 		{{ item.stats.total }} {{ item.stats.matches }} {{ item.stats.mismatches }} {{ item.choices.counts['A'] }} ({{ (item.choices.percentages['A'] * 100) | round(1) }}%) {{ item.choices.counts['B'] }} ({{ (item.choices.percentages['B'] * 100) | round(1) }}%) {{ (item.stats.accuracy * 100) | round(1) }}% {% set reviewer_pct = (item.stats.accuracy * 100) if item.stats.total else 0 %} {% set model_pct = (item.model_probability * 100) if item.model_probability is not none else none %}
Reviewers
{{ reviewer_pct | round(1) }}%
Model {% if model_pct is not none %}
{{ model_pct | round(1) }}% {% else %}
— {% endif %}
Open

Disagreements

{% if disagreements %}
{% for question_id, payload in disagreements.items() %}
{{ question_id }} A {{ (payload.percentages['A'] * 100) | round(1) }}% / B {{ (payload.percentages['B'] * 100) | round(1) }}% Open
    {% for entry in payload.entries %}
  • {{ entry.reviewer_id }}: chose {{ entry.selected }} {% if entry.comment %}- {{ entry.comment }}{% endif %}
    • {% endfor %}
{% endfor %}
{% else %}

No disagreements recorded.

{% endif %}