Hallucination & Reliability

1. Hallucination Taxonomy

2. Self-Consistency Detection

from openai import OpenAI

client = OpenAI()

def self_consistency_check(question: str, n_samples: int = 5, temperature: float = 0.8):
    """Generate multiple answers and check agreement."""
    responses = []
    for _ in range(n_samples):
        r = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": question}],
            temperature=temperature,
        )
        responses.append(r.choices[0].message.content.strip())

    # Use LLM to check consistency across responses
    check_prompt = f"""Given these {n_samples} answers to the same question, are they
consistent with each other? Report any contradictions.

Question: {question}

Answers:
""" + "\n".join(f"{i+1}. {r}" for i, r in enumerate(responses))

    verdict = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": check_prompt}],
        temperature=0.0,
    )
    return {"responses": responses, "consistency_check": verdict.choices[0].message.content}

NLI-Based Hallucination Detection

from transformers import pipeline

nli = pipeline("text-classification", model="facebook/bart-large-mnli")

def check_faithfulness(source: str, claim: str) -> dict:
    """Check if a claim is supported by the source using NLI."""
    result = nli(f"{source}", candidate_labels=[
        "entailment", "contradiction", "neutral"
    ])
    return {
        "claim": claim,
        "supported": result["labels"][0] == "entailment",
        "label": result["labels"][0],
        "confidence": round(result["scores"][0], 3),
    }

source = "The Eiffel Tower was completed in 1889 and is 330 meters tall."
print(check_faithfulness(source, "The Eiffel Tower is 330 meters tall."))
print(check_faithfulness(source, "The Eiffel Tower was built in 1920."))

3. Mitigation Strategies

def calibrated_abstention(question: str, context: str, threshold: float = 0.7):
    """Generate answer with confidence score; abstain if below threshold."""
    from openai import OpenAI
    client = OpenAI()

    prompt = f"""Based ONLY on the context below, answer the question.
After your answer, rate your confidence from 0.0 to 1.0.

Context: {context}
Question: {question}

Format:
Answer: [your answer]
Confidence: [0.0 to 1.0]"""

    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": prompt}],
        temperature=0.0,
    )
    text = response.choices[0].message.content

    # Parse confidence
    import re
    conf_match = re.search(r"Confidence:\s*([\d.]+)", text)
    confidence = float(conf_match.group(1)) if conf_match else 0.0

    if confidence < threshold:
        return {"answer": "I don't have enough information to answer confidently.",
                "confidence": confidence, "abstained": True}
    return {"answer": text, "confidence": confidence, "abstained": False}