from rm_gallery.gallery.evaluation.judgebench import main

# Run evaluation with custom settings
main(
    data_path="data/benchmarks/JudgeBench/data/dataset=judgebench,response_model=gpt-4o-2024-05-13.jsonl",
    result_path="data/results/judgebench_results.json",
    judge_type="arena_hard",
    max_samples=50,
    model="deepseek-chat",
    max_workers=4
)

from rm_gallery.gallery.evaluation.judgebench import main

# Custom model configuration
model_config = {
    "model": "qwen3-32b",
    "temperature": 0.1,
    "max_tokens": 2048,
    "enable_thinking": True
}

main(
    data_path="data/benchmarks/JudgeBench/data/dataset=judgebench,response_model=gpt-4o-2024-05-13.jsonl",
    result_path="data/results/judgebench_custom.json",
    judge_type="prometheus_2",
    max_samples=100,
    model=model_config,
    max_workers=8
)

from rm_gallery.gallery.evaluation.judgebench import main

# Configuration for custom API endpoint
model_config = {
    "model": "qwen2.5-32b-instruct",
    "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
    "temperature": 0.1,
    "max_tokens": 2048
}

main(
    data_path="data/benchmarks/JudgeBench/data/dataset=judgebench,response_model=gpt-4o-2024-05-13.jsonl",
    result_path="data/results/judgebench_custom_api.json",
    judge_type="arena_hard",
    max_samples=50,
    model=model_config,
    max_workers=4
)

from rm_gallery.core.data.load.base import create_loader
from rm_gallery.core.model.openai_llm import OpenaiLLM
from rm_gallery.gallery.evaluation.judgebench import JudgeBenchEvaluator, JudgeBenchReward
from rm_gallery.core.utils.file import write_json

# 1. Set up data loading
config = {
    "path": "data/benchmarks/JudgeBench/data/dataset=judgebench,response_model=gpt-4o-2024-05-13.jsonl",
    "limit": 50,
}

load_module = create_loader(
    name="judgebench",
    load_strategy_type="local",
    data_source="judgebench",
    config=config,
)

# 2. Initialize language model
llm = OpenaiLLM(model="deepseek-chat", enable_thinking=True)

# 3. Load dataset
dataset = load_module.run()

# 4. Create evaluator with specific judge type
evaluator = JudgeBenchEvaluator(
    reward=JudgeBenchReward(
        name="judgebench_eval",
        judge_type="arena_hard",  # Choose your judge type
        llm=llm,
        max_workers=4,
    )
)

# 5. Run evaluation with optional response shuffling
results = evaluator.run(
    samples=dataset.get_data_samples(),
    shuffle_responses=True  # Enable to reduce position bias
)

# 6. Save results
write_json(results, "data/results/judgebench_detailed.json")

# 7. Print summary
print(f"Judge Type: {results['judge_type']}")
print(f"Overall Accuracy: {results['overall_accuracy']['accuracy']:.4f}")
print(f"Valid Samples: {results['overall_accuracy']['valid_samples']}")

from rm_gallery.gallery.evaluation.judgebench import JudgeBenchBaseTemplate, JudgeBenchReward, JudgeBenchEvaluator
from rm_gallery.core.model.openai_llm import OpenaiLLM
from pydantic import Field

class ResearchJudgeTemplate(JudgeBenchBaseTemplate):
    """Custom template for research purposes"""

    decision: str = Field(default="", description="Research decision format")
    confidence: float = Field(default=0.0, description="Confidence score")

    @classmethod
    def format(cls, question: str, answer_A: str, answer_B: str, **kwargs) -> str:
        # Use direct prompt instead of template file
        return f"""
As a research evaluator, compare these two responses:

Question: {question}

Response A: {answer_A}

Response B: {answer_B}

Instructions:
1. Evaluate based on accuracy, completeness, and clarity
2. Provide your decision as [[A>B]], [[B>A]], or [[A=B]]
3. Include confidence level (0.0-1.0)

Format your response as:
Decision: [[your_decision]]
Confidence: your_confidence_score
"""

    @classmethod
    def parse(cls, text: str) -> "ResearchJudgeTemplate":
        decision = ""
        confidence = 0.0

        # Parse decision - MUST return A>B, B>A, A=B, or empty string
        if "[[A>B]]" in text:
            decision = "A>B"  # ✅ Valid format
        elif "[[B>A]]" in text:
            decision = "B>A"  # ✅ Valid format
        elif "[[A=B]]" in text:
            decision = "A=B"  # ✅ Valid format
        else:
            decision = ""  # ✅ Valid - unable to parse

        # Parse confidence
        import re
        conf_match = re.search(r"Confidence:\s*([0-9.]+)", text)
        if conf_match:
            try:
                confidence = float(conf_match.group(1))
            except ValueError:
                confidence = 0.0

        return cls(decision=decision, confidence=confidence)

# Use the custom template
llm = OpenaiLLM(model="deepseek-chat")
evaluator = JudgeBenchEvaluator(
    reward=JudgeBenchReward(
        name="research_judge",
        judge_type="research",  # Custom type
        template=ResearchJudgeTemplate,  # Use custom template
        llm=llm,
        max_workers=4,
    )
)

# Run evaluation
results = evaluator.run(samples=dataset.get_data_samples())

import subprocess
import sys

judge_types = ["vanilla", "arena_hard", "auto_j", "prometheus_2", "skywork_critic"]

for judge_type in judge_types:
    print(f"Running {judge_type} evaluation...")

    cmd = [
        sys.executable, "rm_gallery/gallery/evaluation/judgebench.py",
        "--data_path", "data/benchmarks/JudgeBench/data/dataset=judgebench,response_model=gpt-4o-2024-05-13.jsonl",
        "--result_path", f"data/results/judgebench_{judge_type}.json",
        "--judge_type", judge_type,
        "--max_samples", "20",
        "--model", "deepseek-chat",
        "--max_workers", "4"
    ]

    result = subprocess.run(cmd, capture_output=True, text=True)
    if result.returncode == 0:
        print(f"✅ {judge_type} evaluation completed successfully")
    else:
        print(f"❌ {judge_type} evaluation failed: {result.stderr}")

from rm_gallery.gallery.evaluation.judgebench import main

# Full evaluation with custom model configuration
main(
    data_path="data/benchmarks/JudgeBench/data/dataset=judgebench,response_model=gpt-4o-2024-05-13.jsonl",
    result_path="data/results/judgebench_production.json",
    judge_type="arena_hard",
    max_samples=200,
    model={
        "model": "qwen3-32b",
        "temperature": 0.1,
        "enable_thinking": True
    },
    max_workers=8,
    shuffle_responses=True
)

from rm_gallery.gallery.evaluation.judgebench import JudgeBenchBaseTemplate, JudgeBenchReward
from rm_gallery.core.model.openai_llm import OpenaiLLM
from pydantic import Field

# Define and test a custom template
class TestCustomTemplate(JudgeBenchBaseTemplate):
    decision: str = Field(default="", description="Decision in A>B format")
    confidence: float = Field(default=0.0, description="Confidence score")

    @classmethod
    def format(cls, question: str, answer_A: str, answer_B: str, **kwargs) -> str:
        return f"""
Evaluate these responses:
Q: {question}
A: {answer_A}
B: {answer_B}

Which is better? Format: Decision: [[A>B/B>A/A=B]] Confidence: 0.X
"""

    @classmethod
    def parse(cls, text: str) -> "TestCustomTemplate":
        decision = ""
        confidence = 0.0

        # Test different parsing patterns
        test_patterns = [
            (r"\[\[A>B\]\]", "A>B"),
            (r"\[\[B>A\]\]", "B>A"),
            (r"\[\[A=B\]\]", "A=B"),
            (r"Decision:\s*A>B", "A>B"),
            (r"Decision:\s*B>A", "B>A"),
            (r"Decision:\s*A=B", "A=B"),
        ]

        import re
        for pattern, result in test_patterns:
            if re.search(pattern, text):
                decision = result
                break

        # Parse confidence
        conf_match = re.search(r"Confidence:\s*([0-9.]+)", text)
        if conf_match:
            try:
                confidence = float(conf_match.group(1))
            except ValueError:
                confidence = 0.0

        return cls(decision=decision, confidence=confidence)

# Test the template parsing
def test_template_parsing():
    template = TestCustomTemplate

    test_cases = [
        "Decision: [[A>B]] Confidence: 0.8",
        "I think [[B>A]] is better. Confidence: 0.9",
        "Both are equal [[A=B]] Confidence: 0.5",
        "Unclear response without clear format",
    ]

    print("Testing custom template parsing:")
    for i, test_input in enumerate(test_cases):
        result = template.parse(test_input)
        print(f"Test {i+1}: '{test_input[:30]}...'")
        print(f"  Decision: '{result.decision}' (Valid: {result.decision in ['A>B', 'B>A', 'A=B', '']})")
        print(f"  Confidence: {result.confidence}")
        print()

# Run the test
test_template_parsing()

# Use the tested template in evaluation
llm = OpenaiLLM(model="deepseek-chat")
reward = JudgeBenchReward(
    name="tested_custom_judge",
    template=TestCustomTemplate,
    llm=llm
)
print("Custom template ready for evaluation!")

from rm_gallery.core.data.load.base import create_loader
from rm_gallery.core.model.openai_llm import OpenaiLLM
from rm_gallery.gallery.evaluation.judgebench import JudgeBenchEvaluator, JudgeBenchReward
import json

# Load and process results
def analyze_judge_performance(results_path):
    with open(results_path, 'r') as f:
        results = json.load(f)

    print(f"Judge Type: {results['judge_type']}")
    print(f"Model: {results['model']}")
    print(f"Overall Accuracy: {results['overall_accuracy']['accuracy']:.2%}")

    # Analyze by source
    print("\nPerformance by Source:")
    for source, metrics in results['source_accuracy'].items():
        print(f"  {source}: {metrics['accuracy']:.2%} ({metrics['correct_count']}/{metrics['valid_samples']})")

# Run analysis
analyze_judge_performance("data/results/judgebench_production.json")

from rm_gallery.gallery.evaluation.judgebench import JudgeBenchReward, JudgeBenchBaseTemplate
from rm_gallery.core.model.openai_llm import OpenaiLLM
from pydantic import Field

class CustomJudgeTemplate(JudgeBenchBaseTemplate):
    decision: str = Field(default="", description="Custom decision format")

    @classmethod
    def format(cls, question: str, answer_A: str, answer_B: str, **kwargs) -> str:
        renderer = cls.get_renderer()
        return renderer.render_template(
            "custom_judge_prompt",  # Create this template file
            question=question,
            answer_a=answer_A,
            answer_b=answer_B
        )

    @classmethod
    def parse(cls, text: str) -> "CustomJudgeTemplate":
        # Implement custom parsing logic
        decision = ""  # Parse from text
        return cls(decision=decision)

# Use custom template
llm = OpenaiLLM(model="deepseek-chat")
reward = JudgeBenchReward(
    name="custom_judge",
    judge_type="custom",  # Can be any string when using custom template
    template=CustomJudgeTemplate,  # Pass your custom template class
    llm=llm
)

# Internal processing logic in _after_evaluate method
if decision == "A>B":
    scores = [1, 0]  # Response A gets score 1, Response B gets score 0
elif decision == "B>A":
    scores = [0, 1]  # Response A gets score 0, Response B gets score 1
elif decision == "A=B":
    scores = [0.5, 0.5]  # Both responses get equal scores
else:  # Empty string or invalid format
    scores = [0, 0]  # Both responses get zero scores (evaluation failed)

class ValidCustomTemplate(JudgeBenchBaseTemplate):
    decision: str = Field(default="", description="Must be A>B, B>A, A=B, or empty")
    reason: str = Field(default="", description="Optional explanation")

    @classmethod
    def parse(cls, text: str) -> "ValidCustomTemplate":
        decision = ""
        reason = ""

        # Your parsing logic MUST produce one of these decision values
        if "Response A is better" in text:
            decision = "A>B"
            reason = "Response A provided more accurate information"
        elif "Response B is better" in text:
            decision = "B>A"
            reason = "Response B was more comprehensive"
        elif "Both responses are equal" in text:
            decision = "A=B"
            reason = "Both responses have similar quality"
        else:
            decision = ""  # Unable to parse
            reason = "Could not determine preference"

        return cls(decision=decision, reason=reason)

# ❌ WRONG: Invalid decision format
class InvalidTemplate(JudgeBenchBaseTemplate):
    decision: str = Field(default="", description="Decision")

    @classmethod
    def parse(cls, text: str) -> "InvalidTemplate":
        # These values will NOT work with the evaluation system
        if "first" in text:
            decision = "first"  # ❌ Invalid - should be "A>B"
        elif "second" in text:
            decision = "second"  # ❌ Invalid - should be "B>A"
        elif "winner: A" in text:
            decision = "A"  # ❌ Invalid - should be "A>B"

        return cls(decision=decision)

# ✅ CORRECT: Valid decision format
class ValidTemplate(JudgeBenchBaseTemplate):
    decision: str = Field(default="", description="Decision")

    @classmethod
    def parse(cls, text: str) -> "ValidTemplate":
        decision = ""
        if "first" in text:
            decision = "A>B"  # ✅ Valid format
        elif "second" in text:
            decision = "B>A"  # ✅ Valid format
        elif "winner: A" in text:
            decision = "A>B"  # ✅ Valid format

        return cls(decision=decision)

# Using predefined judge types (backward compatible)
reward = JudgeBenchReward(
    name="arena_hard_judge",
    judge_type="arena_hard",  # Uses ArenaHardTemplate automatically
    llm=llm
)

# Custom template overrides judge_type mapping
reward = JudgeBenchReward(
    name="custom_judge",
    judge_type="arena_hard",  # This will be ignored
    template=CustomJudgeTemplate,  # This takes precedence
    llm=llm
)

import subprocess
import sys

# Example: Combining with RewardBench2
def run_evaluation(script_path, **kwargs):
    """Helper function to run evaluation scripts"""
    cmd = [sys.executable, script_path]
    for key, value in kwargs.items():
        cmd.extend([f"--{key}", str(value)])

    result = subprocess.run(cmd, capture_output=True, text=True)
    if result.returncode == 0:
        print(f"✅ {script_path} completed successfully")
    else:
        print(f"❌ {script_path} failed: {result.stderr}")
    return result.returncode == 0

# Run both evaluations
print("Running RewardBench2 evaluation...")
rewardbench2_success = run_evaluation(
    "rm_gallery/gallery/evaluation/rewardbench2.py",
    data_path="data/benchmarks/reward-bench-2/data/test-00000-of-00001.parquet",
    result_path="data/results/rewardbench2.json",
    max_samples=100,
    model="deepseek-chat"
)

print("Running JudgeBench evaluation...")
judgebench_success = run_evaluation(
    "rm_gallery/gallery/evaluation/judgebench.py",
    data_path="data/benchmarks/JudgeBench/data/dataset=judgebench,response_model=gpt-4o-2024-05-13.jsonl",
    result_path="data/results/judgebench.json",
    judge_type="arena_hard",
    max_samples=100,
    model="deepseek-chat"
)

if rewardbench2_success and judgebench_success:
    print("🎉 Both evaluations completed successfully!")
else:
    print("⚠️  Some evaluations failed. Check the logs above.")