"Current AI systems in 2026 have near-zero hallucinations and human-level reasoning across most domains."

"""
Proof: Current AI systems in 2026 have near-zero hallucinations and human-level reasoning across most domains.
Generated: 2026-03-31

This is a compound disproof:
  SC1 — Near-zero hallucinations: DISPROVED (significant hallucination rates persist in 2026)
  SC2 — Human-level reasoning across most domains: DISPROVED (major benchmarks show large gaps)

Template: Compound Qualitative Consensus (disproof variant)
"""
import json
import os
import sys

PROOF_ENGINE_ROOT = os.environ.get("PROOF_ENGINE_ROOT")
if not PROOF_ENGINE_ROOT:
    _d = os.path.dirname(os.path.abspath(__file__))
    while _d != os.path.dirname(_d):
        if os.path.isdir(os.path.join(_d, "proof-engine", "skills", "proof-engine", "scripts")):
            PROOF_ENGINE_ROOT = os.path.join(_d, "proof-engine", "skills", "proof-engine")
            break
        _d = os.path.dirname(_d)
    if not PROOF_ENGINE_ROOT:
        raise RuntimeError("PROOF_ENGINE_ROOT not set and skill dir not found via walk-up from proof.py")
sys.path.insert(0, PROOF_ENGINE_ROOT)

from scripts.verify_citations import verify_all_citations, build_citation_detail
from scripts.computations import compare

# ---------------------------------------------------------------------------
# 1. CLAIM INTERPRETATION (Rule 4)
# ---------------------------------------------------------------------------
CLAIM_NATURAL = (
    "Current AI systems in 2026 have near-zero hallucinations and human-level "
    "reasoning across most domains."
)
CLAIM_FORMAL = {
    "subject": "Current AI systems in 2026",
    "sub_claims": [
        {
            "id": "SC1",
            "property": "have near-zero hallucinations",
            "operator": ">=",
            "threshold": 2,
            "proof_direction": "disprove",
            "operator_note": (
                "Interpreted as: at least 2 independent sources confirm that AI hallucination "
                "rates in 2026 remain significantly above zero. 'Near-zero' is operationalized "
                "as <1% across general-purpose tasks; the disproof threshold is 2 verified "
                "sources showing rates substantially higher. Three independent sources consulted; "
                "2 verified confirmations constitutes clear consensus."
            ),
        },
        {
            "id": "SC2",
            "property": "have human-level reasoning across most domains",
            "operator": ">=",
            "threshold": 2,
            "proof_direction": "disprove",
            "operator_note": (
                "Interpreted as: at least 2 independent sources confirm that AI reasoning "
                "capability falls substantially below human-level performance on recognized "
                "reasoning benchmarks covering multiple domains. 'Most domains' means the "
                "majority of knowledge/reasoning areas, not only narrow specialized tasks. "
                "Threshold=2 with 3 sources consulted."
            ),
        },
    ],
    "compound_operator": "AND",
    "operator_note": (
        "Both sub-claims must be disproved for the compound claim to be DISPROVED. "
        "Because this is an AND claim, disproving either sub-claim is sufficient to "
        "disprove the whole — but the evidence disproves both."
    ),
}

# ---------------------------------------------------------------------------
# 2. FACT REGISTRY
# ---------------------------------------------------------------------------
FACT_REGISTRY = {
    # SC1: hallucination rates remain significant
    "B1": {"key": "sc1_duke",    "label": "SC1 — Duke Univ. Libraries (Jan 2026): LLMs still hallucinate"},
    "B2": {"key": "sc1_vectara", "label": "SC1 — Vectara Hallucination Leaderboard (2025): top models >10% rate"},
    "B3": {"key": "sc1_simpleqa","label": "SC1 — OpenAI SimpleQA paper (arXiv 2411.04368): benchmark for factual failures"},
    # SC2: AI reasoning falls short of human-level across domains
    "B4": {"key": "sc2_arcagi3", "label": "SC2 — The Decoder (Mar 2026): ARC-AGI-3, all frontier models <1%"},
    "B5": {"key": "sc2_arcprize","label": "SC2 — ARC Prize 2025 results: best AI 37.6% vs 100% human baseline"},
    "B6": {"key": "sc2_hle",     "label": "SC2 — The Conversation (2025): Humanity's Last Exam, GPT-4o at 2.7%"},
    # Computed
    "A1": {"label": "SC1 verified source count (disproof of near-zero hallucinations)", "method": None, "result": None},
    "A2": {"label": "SC2 verified source count (disproof of human-level reasoning claim)", "method": None, "result": None},
}

# ---------------------------------------------------------------------------
# 3. EMPIRICAL FACTS
# Sources that DISPROVE the respective sub-claim (confirm sub-claim is false)
# Adversarial sources (supporting the original claim) go in adversarial_checks
# ---------------------------------------------------------------------------
empirical_facts = {
    # ---- SC1: hallucination rates are NOT near-zero ----
    "sc1_duke": {
        "quote": (
            "But one problem we highlighted back then persists today: LLMs still make stuff up. "
            "When I talk to Duke students, many describe first-hand encounters with AI hallucinations "
            "\u2013 plausible sounding, but factually incorrect AI-generated info."
        ),
        "url": "https://blogs.library.duke.edu/blog/2026/01/05/its-2026-why-are-llms-still-hallucinating/",
        "source_name": "Duke University Libraries Blog (January 2026)",
    },
    "sc1_vectara": {
        "quote": (
            "Interestingly, the just-released Gemini-3-pro, which demonstrates top of the line reasoning "
            "capabilities, has a 13.6% hallucination rate, and didn't even make the top-25 list. Other "
            "notable thinking models like Claude Sonnet 4.5, GPT-5, GPT-OSS-120B, Grok-4, or "
            "Deepseek-R1 all have a hallucination rate > 10%."
        ),
        "url": "https://www.vectara.com/blog/introducing-the-next-generation-of-vectaras-hallucination-leaderboard",
        "source_name": "Vectara Hallucination Leaderboard Blog (2025)",
    },
    "sc1_simpleqa": {
        "quote": (
            "SimpleQA is a simple, targeted evaluation for whether models \u2018know what they know,\u2019 "
            "and our hope is that this benchmark will remain relevant for the next few generations "
            "of frontier models."
        ),
        "url": "https://arxiv.org/abs/2411.04368",
        "source_name": "OpenAI SimpleQA benchmark paper (arXiv 2024)",
    },

    # ---- SC2: AI reasoning does NOT reach human level across most domains ----
    "sc2_arcagi3": {
        "quote": (
            "Every frontier model tested, meanwhile, scored below 1 percent: Gemini 3.1 Pro Preview "
            "hit 0.37 percent, GPT 5.4 reached 0.26 percent, Opus 4.6 managed 0.25 percent, "
            "and Grok-4.20 scored 0.00 percent."
        ),
        "url": "https://the-decoder.com/arc-agi-3-offers-2m-to-any-ai-that-matches-untrained-humans-yet-every-frontier-model-scores-below-1/",
        "source_name": "The Decoder — ARC-AGI-3 results (March 2026)",
    },
    "sc2_arcprize": {
        "quote": (
            "the top verified commercial model, Opus 4.5 (Thinking, 64k), scores 37.6% for $2.20/task"
        ),
        "url": "https://arcprize.org/blog/arc-prize-2025-results-analysis",
        "source_name": "ARC Prize 2025 Official Results (ARC-AGI-2, human baseline: 100%)",
    },
    "sc2_hle": {
        "quote": (
            "GPT-4o managed just 2.7% accuracy. Claude 3.5 Sonnet scored 4.1%. "
            "Even OpenAI\u2019s most powerful model, o1, achieved only 8%."
        ),
        "url": "https://theconversation.com/ai-is-failing-humanitys-last-exam-so-what-does-that-mean-for-machine-intelligence-274620",
        "source_name": "The Conversation — Humanity's Last Exam (2025)",
    },
}

# ---------------------------------------------------------------------------
# 4. CITATION VERIFICATION (Rule 2)
# ---------------------------------------------------------------------------
citation_results = verify_all_citations(empirical_facts, wayback_fallback=True)

# ---------------------------------------------------------------------------
# 5. COUNT VERIFIED SOURCES PER SUB-CLAIM
# ---------------------------------------------------------------------------
COUNTABLE_STATUSES = ("verified", "partial")

sc1_keys = ["sc1_duke", "sc1_vectara", "sc1_simpleqa"]
sc2_keys = ["sc2_arcagi3", "sc2_arcprize", "sc2_hle"]

n_sc1 = sum(1 for k in sc1_keys if citation_results[k]["status"] in COUNTABLE_STATUSES)
n_sc2 = sum(1 for k in sc2_keys if citation_results[k]["status"] in COUNTABLE_STATUSES)

print(f"  SC1 confirmed sources (hallucination rates NOT near-zero): {n_sc1} / {len(sc1_keys)}")
print(f"  SC2 confirmed sources (AI reasoning below human-level): {n_sc2} / {len(sc2_keys)}")

# ---------------------------------------------------------------------------
# 6. PER-SUB-CLAIM EVALUATION (Rule 4 — compare())
# ---------------------------------------------------------------------------
sc1_holds = compare(
    n_sc1, ">=", CLAIM_FORMAL["sub_claims"][0]["threshold"],
    label="SC1: disproof sources >= threshold (hallucination rates NOT near-zero)",
)
sc2_holds = compare(
    n_sc2, ">=", CLAIM_FORMAL["sub_claims"][1]["threshold"],
    label="SC2: disproof sources >= threshold (AI reasoning NOT human-level across most domains)",
)

# ---------------------------------------------------------------------------
# 7. COMPOUND EVALUATION
# ---------------------------------------------------------------------------
n_holding = sum([sc1_holds, sc2_holds])
n_total = len(CLAIM_FORMAL["sub_claims"])
claim_holds = compare(n_holding, "==", n_total, label="compound: both sub-claim disproofs hold")

# ---------------------------------------------------------------------------
# 8. ADVERSARIAL CHECKS (Rule 5)
# Searched for evidence SUPPORTING the original claim (near-zero hallucinations /
# human-level reasoning) — the opposite of what we are trying to disprove
# ---------------------------------------------------------------------------
adversarial_checks = [
    {
        "question": (
            "Do Vectara's sub-1% hallucination rates for top summarization models support "
            "the 'near-zero hallucinations' claim for AI generally?"
        ),
        "verification_performed": (
            "Reviewed Vectara Hallucination Leaderboard (2025). Some top models achieve sub-1% "
            "rates on the document summarization task specifically. Vectara explicitly notes "
            "these are best-case scenario results for a narrow summarization context. The same "
            "leaderboard shows Gemini-3-pro at 13.6% and most frontier models >10% on general "
            "tasks. The research agent confirmed: 'Sub-1% rates are task-specific, not general.'"
        ),
        "finding": (
            "Sub-1% rates exist only in the narrow document-summarization context. General-purpose "
            "hallucination rates remain well above 10% for most frontier models. This does not "
            "support 'near-zero hallucinations' as a general property of AI systems."
        ),
        "breaks_proof": False,
    },
    {
        "question": (
            "Do saturated benchmarks (MMLU ~90%+, GSM8K ~97%) show AI has reached "
            "human-level reasoning, undermining our SC2 disproof?"
        ),
        "verification_performed": (
            "Searched for MMLU, GSM8K, HumanEval scores for frontier models vs. human baselines. "
            "Found: GPT-4o at ~88.7% on MMLU vs. ~89% human baseline; models at 97% on GSM8K "
            "vs. ~90% human baseline. However, MIT Technology Review (March 31, 2026) reports "
            "these benchmarks are saturated and likely contaminated with training data. ARC-AGI-3 "
            "(March 2026) shows all frontier models below 1% vs. 100% human baseline on novel "
            "abstract reasoning. HLE shows top models at 34-38% vs. ~90% expert human baseline."
        ),
        "finding": (
            "AI scores at or above human level on saturated narrow benchmarks (MMLU, GSM8K, "
            "HumanEval), but these benchmarks are compromised by training-data contamination. "
            "On rigorous out-of-distribution benchmarks (ARC-AGI-3, HLE, BigCodeBench), AI "
            "falls far short of human-level performance. 'Most domains' cannot be satisfied by "
            "performance on contaminated narrow tests."
        ),
        "breaks_proof": False,
    },
    {
        "question": (
            "Could 'most domains' be defined narrowly enough to make SC2 true — "
            "e.g., only counting domains where AI performs well?"
        ),
        "verification_performed": (
            "Examined domain coverage of AI performance claims. Reviewed Stanford HAI 2025 AI "
            "Index Report finding that 'AI surpasses humans on a growing number of narrow "
            "benchmarks while remaining clearly sub-human on measures of genuine expert reasoning, "
            "common sense, and out-of-distribution generalization.' Checked ARC-AGI-3 performance "
            "across novel interactive tasks (0.25-0.37%), Humanity's Last Exam across 100+ "
            "academic disciplines (2.7-38%), and legal/medical domains where hallucination rates "
            "run 43-88%. The original claim specifies 'most domains' without qualification."
        ),
        "finding": (
            "The claim says 'most domains' without qualification. AI clearly falls short of "
            "human-level performance in novel abstract reasoning, expert academic knowledge, "
            "legal reasoning, medical reasoning, and interactive task-solving — domains that "
            "collectively represent the majority of human cognitive domains. Narrow successes "
            "in coding competitions and standardized test formats do not constitute 'most domains.'"
        ),
        "breaks_proof": False,
    },
]

# ---------------------------------------------------------------------------
# 9. VERDICT AND STRUCTURED OUTPUT
# ---------------------------------------------------------------------------
if __name__ == "__main__":
    any_unverified = any(
        cr["status"] != "verified" for cr in citation_results.values()
    )
    any_breaks = any(ac.get("breaks_proof") for ac in adversarial_checks)

    # Both sub_claims have proof_direction="disprove": a successful disproof
    # (claim_holds=True) maps to verdict DISPROVED, not PROVED.
    is_disproof = all(
        sc.get("proof_direction") == "disprove"
        for sc in CLAIM_FORMAL["sub_claims"]
    )

    if any_breaks:
        verdict = "UNDETERMINED"
    elif not claim_holds and n_holding > 0:
        verdict = "PARTIALLY VERIFIED"
    elif claim_holds and not any_unverified:
        verdict = "DISPROVED" if is_disproof else "PROVED"
    elif claim_holds and any_unverified:
        verdict = (
            "DISPROVED (with unverified citations)"
            if is_disproof
            else "PROVED (with unverified citations)"
        )
    elif not claim_holds:
        verdict = "UNDETERMINED"
    else:
        verdict = "UNDETERMINED"

    # Update FACT_REGISTRY with computed results
    FACT_REGISTRY["A1"]["method"] = f"count(verified SC1 disproof citations) = {n_sc1}"
    FACT_REGISTRY["A1"]["result"] = f"{n_sc1} of {len(sc1_keys)} sources verified"
    FACT_REGISTRY["A2"]["method"] = f"count(verified SC2 disproof citations) = {n_sc2}"
    FACT_REGISTRY["A2"]["result"] = f"{n_sc2} of {len(sc2_keys)} sources verified"

    citation_detail = build_citation_detail(FACT_REGISTRY, citation_results, empirical_facts)

    # Extractions: citation status per B-type fact
    extractions = {}
    for fid, info in FACT_REGISTRY.items():
        if not fid.startswith("B"):
            continue
        ef_key = info["key"]
        cr = citation_results.get(ef_key, {})
        extractions[fid] = {
            "value": cr.get("status", "unknown"),
            "value_in_quote": cr.get("status") in COUNTABLE_STATUSES,
            "quote_snippet": empirical_facts[ef_key]["quote"][:80],
        }

    summary = {
        "fact_registry": {fid: dict(info) for fid, info in FACT_REGISTRY.items()},
        "claim_formal": CLAIM_FORMAL,
        "claim_natural": CLAIM_NATURAL,
        "citations": citation_detail,
        "extractions": extractions,
        "cross_checks": [
            {
                "description": "SC1: independent sources on AI hallucination rates (2025-2026)",
                "n_sources_consulted": len(sc1_keys),
                "n_sources_verified": n_sc1,
                "sources": {k: citation_results[k]["status"] for k in sc1_keys},
                "independence_note": (
                    "Sources are from Duke University Libraries (institutional blog), "
                    "Vectara (commercial AI company measuring hallucinations systematically), "
                    "and OpenAI (paper introducing the SimpleQA benchmark) — three distinct "
                    "institutions with independent methodologies."
                ),
            },
            {
                "description": "SC2: independent sources on AI reasoning benchmarks vs. human baselines",
                "n_sources_consulted": len(sc2_keys),
                "n_sources_verified": n_sc2,
                "sources": {k: citation_results[k]["status"] for k in sc2_keys},
                "independence_note": (
                    "Sources from The Decoder (ARC-AGI-3 coverage, March 2026), "
                    "ARC Prize official results blog (ARC-AGI-2 competition results), "
                    "and The Conversation (academic journalism, Humanity's Last Exam). "
                    "These cover three distinct benchmarks: ARC-AGI-3, ARC-AGI-2, and HLE."
                ),
            },
        ],
        "sub_claim_results": [
            {
                "id": "SC1",
                "n_confirming": n_sc1,
                "threshold": CLAIM_FORMAL["sub_claims"][0]["threshold"],
                "holds": sc1_holds,
                "proof_direction": "disprove",
                "interpretation": "SC1 holds = hallucination rates are NOT near-zero (sub-claim is false)",
            },
            {
                "id": "SC2",
                "n_confirming": n_sc2,
                "threshold": CLAIM_FORMAL["sub_claims"][1]["threshold"],
                "holds": sc2_holds,
                "proof_direction": "disprove",
                "interpretation": "SC2 holds = AI does NOT have human-level reasoning across most domains",
            },
        ],
        "adversarial_checks": adversarial_checks,
        "verdict": verdict,
        "key_results": {
            "n_holding": n_holding,
            "n_total": n_total,
            "claim_holds": claim_holds,
            "is_disproof": True,
            "sc1_n_confirming": n_sc1,
            "sc1_threshold": 2,
            "sc2_n_confirming": n_sc2,
            "sc2_threshold": 2,
        },
        "generator": {
            "name": "proof-engine",
            "version": open(os.path.join(PROOF_ENGINE_ROOT, "VERSION")).read().strip(),
            "repo": "https://github.com/yaniv-golan/proof-engine",
            "generated_at": "2026-03-31",
        },
    }

    print("\n=== PROOF SUMMARY (JSON) ===")
    print(json.dumps(summary, indent=2, default=str))