#!/usr/bin/env python3
"""Rebuild a small HN evidence audit from inputs and FROZEN human/model decisions.
No network calls, language-model calls, fresh classification, or playback tests.
Requires Python 3.9+, pandas, matplotlib, Pillow.
"""
import argparse, collections, hashlib, html, json, platform, re
from pathlib import Path
from html.parser import HTMLParser
from urllib.parse import urlsplit, parse_qs
import pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from PIL import Image

ALIASES = {
 "hn_comments.jsonl":"hn_comments-78c23616d7b5.jsonl",
 "comment_video_links.csv":"comment_video_links-f9ff64257870.csv",
 "video_metadata.csv":"video_metadata-dcc11572ab87.csv",
 "analysis_notes.md":"analysis_notes-f3f21ece6fce.md",
 "source_brief.md":"source_brief-5c883d784200.md"}
DECISIONS = ["final_judgments.csv","final_selection_audited.csv","revision_log.csv",
             "final_omissions.csv","evidence_audit_and_revisions.csv"]
STANCES = ["recommend","neutral_or_reference","critical","unclear"]
TOPICS = ["yes","no","unclear"]
COLORS = ["#287D8E","#A9B1BC","#D59A2B"]
class Parser(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.text=[]; self.links=[]
    def handle_starttag(self,tag,attrs):
        if tag in ("p","br","li"): self.text.append("\n")
        if tag=="a": self.links.append(dict(attrs).get("href",""))
    def handle_data(self,data): self.text.append(data)
def parse(s):
    p=Parser(); p.feed(s or ""); return p
def vid(url):
    u=urlsplit(url); host=(u.hostname or "").lower()
    if host in ("youtube.com","www.youtube.com","m.youtube.com") and u.path=="/watch":
        value=parse_qs(u.query).get("v",[""])[0]
    elif host in ("youtu.be","www.youtu.be"): value=u.path.strip("/").split("/")[0]
    else: return None
    return value if re.fullmatch(r"[A-Za-z0-9_-]{11}",value) else None
def wc(s): return len(re.findall(r"\b\w+(?:['’-]\w+)*\b",s))
def require(condition,message):
    if not condition: raise ValueError(message)
def csv(path): return pd.read_csv(path,dtype=str,keep_default_na=False)
def pairs(df): return set(zip(df.comment_id,df.video_id))
def main():
    ap=argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--input-dir",type=Path,default=Path("."))
    ap.add_argument("--decisions-dir",type=Path,default=Path("."))
    ap.add_argument("--out",type=Path,default=Path("rebuilt"))
    args=ap.parse_args()
    out=args.out
    require(not out.exists() or not any(out.iterdir()),"Output directory must be new or empty.")
    paths={}
    for name,alias in ALIASES.items():
        p=args.input_dir/name
        if not p.exists(): p=args.input_dir/alias
        require(p.is_file(),"Missing input: "+name); paths[name]=p
    for name in DECISIONS:
        p=args.decisions_dir/name
        require(p.is_file(),"Missing frozen decision file: "+name); paths[name]=p
    with paths["hn_comments.jsonl"].open() as f: comments=[json.loads(l) for l in f if l.strip()]
    byid={str(c["comment_id"]):c for c in comments}
    links=csv(paths["comment_video_links.csv"]); meta=csv(paths["video_metadata.csv"])
    j=csv(paths["final_judgments.csv"]); selection=csv(paths["final_selection_audited.csv"])
    revisions=csv(paths["revision_log.csv"]); omitted=csv(paths["final_omissions.csv"])
    audit=csv(paths["evidence_audit_and_revisions.csv"])
    selection=selection.sort_values("rank",key=lambda s:s.astype(int))
    rows=[]; source_text={}
    for c in comments:
        cid=str(c["comment_id"]); p=parse(c["comment_html"])
        source_text[cid]="".join(p.text)
        for par in c["parent_chain_nearest_first"]:
            source_text[str(par["id"])]="".join(parse(par.get("text_html")).text)
        urls=collections.defaultdict(set)
        for url in p.links+re.findall(r'https?://[^\s<>"\x27]+'," ".join(p.text)):
            url=url.rstrip(".,;!?)"); v=vid(url)
            if v: urls[v].add(url)
        for v,originals in sorted(urls.items()):
            rows.append(dict(comment_id=cid,video_id=v,comment_url=c["comment_url"],
                video_url="https://www.youtube.com/watch?v="+v,
                original_urls_json=json.dumps(sorted(originals)),
                counting_unit="distinct_comment_video_pair"))
    mapping=pd.DataFrame(rows)
    require(pairs(mapping)==pairs(links)==pairs(j),"Original, extracted and judgment pairs differ.")
    require(len(j)==len(pairs(j))==len(mapping),"Duplicate judgment or mapping pairs.")
    require(meta.video_id.is_unique and set(meta.video_id)==set(j.video_id),"Metadata ID coverage mismatch.")
    require(set(j.stance)<=set(STANCES) and set(j.topic_fit)<=set(TOPICS),"Invalid semantic labels.")
    require(selection.video_id.is_unique and len(selection)<=5,"Selection must contain up to five unique IDs.")
    require(set(selection.video_id).isdisjoint(omitted.video_id)
        and set(selection.video_id)|set(omitted.video_id)==set(meta.video_id),"Selection/omission coverage mismatch.")
    require(pairs(revisions)==pairs(j),"Revision log must retain every pair.")
    totals=collections.Counter()
    for r in j.itertuples():
        require(r.evidence_comment_id in source_text,"Unknown evidence source.")
        require(not r.evidence_quote or r.evidence_quote in source_text[r.evidence_comment_id],"Non-exact quote.")
        totals[r.evidence_comment_id]+=wc(r.evidence_quote)
        require(r.comment_url==byid[r.comment_id]["comment_url"],"Source URL mismatch.")
        require(json.loads(r.context_item_ids)==[p["id"] for p in byid[r.comment_id]["parent_chain_nearest_first"]],"Context ID mismatch.")
    require(max(totals.values(),default=0)<=20,"Quotation budget exceeded.")
    selected_pairs=set()
    mi=meta.set_index("video_id")
    for r in selection.itertuples():
        require(r.title==mi.loc[r.video_id,"title"] and r.author_name==mi.loc[r.video_id,"author_name"],"Selected metadata altered.")
        ids=r.source_comment_ids.split(";")
        require(r.source_comment_urls.split(";")==[byid[c]["comment_url"] for c in ids],"Selection source URL mismatch.")
        for cid in ids:
            x=j[(j.comment_id==cid)&(j.video_id==r.video_id)]
            require(len(x)==1 and x.iloc[0].stance=="recommend" and x.iloc[0].topic_fit=="yes","Unsupported selected pair.")
            selected_pairs.add((cid,r.video_id))
    require(pairs(audit)==selected_pairs,"Selected evidence audit coverage mismatch.")
    for r in revisions.itertuples():
        x=j[(j.comment_id==r.comment_id)&(j.video_id==r.video_id)].iloc[0]
        require(r.new_stance==x.stance and r.new_topic_fit==x.topic_fit,"Revision/final labels differ.")
    cross=pd.crosstab(j.stance,j.topic_fit).reindex(index=STANCES,columns=TOPICS,fill_value=0)
    ncomments=len(comments); npairs=len(j); nvideos=meta.video_id.nunique()
    degree=mapping.groupby("video_id").size()
    summary=dict(comments=ncomments,pairs=npairs,unique_videos=nvideos,
        stance={s:int((j.stance==s).sum()) for s in STANCES},
        topic={t:int((j.topic_fit==t).sum()) for t in TOPICS},
        stance_by_topic={s:{t:int(cross.loc[s,t]) for t in TOPICS} for s in STANCES},
        http_status=meta.http_status.value_counts().to_dict(),
        playback_not_tested=int(meta.playability_checked.str.lower().eq("false").sum()),
        content_not_viewed=int(meta.content_viewed.str.lower().eq("false").sum()),
        parent_chains_truncated=sum(c["parent_chain_truncated"] for c in comments),
        selected_videos=len(selection),selected_source_pairs=len(selected_pairs),
        remaining_videos=len(omitted),clarified_rows=int(revisions.status.eq("clarified").sum()),
        label_changes=int(((revisions.old_stance!=revisions.new_stance)|(revisions.old_topic_fit!=revisions.new_topic_fit)).sum()),
        max_quote_words_per_source=max(totals.values(),default=0))
    out.mkdir(parents=True,exist_ok=True)
    mapping.to_csv(out/"rebuilt_mapping.csv",index=False)
    (out/"numerical_summary.json").write_text(json.dumps(summary,indent=2),encoding="utf-8")
    plt.rcParams.update({"font.family":"DejaVu Sans","font.size":12})
    def save(fig,name):
        fig.savefig(out/name,dpi=150); plt.close(fig)
        with Image.open(out/name) as im: require(max(im.size)<=2400,"Image exceeds 2400 pixels.")
    # 1: explicitly different counting units.
    fig,ax=plt.subplots(figsize=(10,6))
    vals=[ncomments,npairs,nvideos]
    ax.barh(range(3),vals,color=["#226B9A","#A9B1BC","#287D8E"])
    ax.set_yticks(range(3),["Selected comments","Comment–video pairs","Unique video IDs"])
    ax.invert_yaxis(); ax.set_xlim(0,max(vals)*1.2)
    for i,n in enumerate(vals): ax.text(n+.5,i,str(n),va="center",weight="bold")
    ax.set_title("1 | Frozen sample: different counting units")
    ax.set_xlabel("Count — these units are not additive")
    fig.subplots_adjust(left=.34,bottom=.2,top=.85)
    save(fig,"sample_counts.png")
    # 2: source graph, no duplicate video nodes.
    adj=collections.defaultdict(set)
    for r in mapping.itertuples():
        a="c:"+r.comment_id; b="v:"+r.video_id; adj[a].add(b); adj[b].add(a)
    remaining=set(adj); components=[]
    while remaining:
        todo=[min(remaining)]; comp=set()
        while todo:
            node=todo.pop()
            if node in comp: continue
            comp.add(node); todo.extend(adj[node]-comp)
        remaining-=comp; components.append(comp)
    cols=[[],[],[]]; heights=[0,0,0]
    for comp in sorted(components,key=lambda c:(-max(sum(n.startswith("c:") for n in c),sum(n.startswith("v:") for n in c)),min(c))):
        weight=max(sum(n.startswith("c:") for n in comp),sum(n.startswith("v:") for n in comp))+1.2
        k=min(range(3),key=lambda k:heights[k]); cols[k].append((comp,weight)); heights[k]+=weight
    fig,axes=plt.subplots(1,3,figsize=(16,13))
    for ax,col in zip(axes,cols):
        ax.set_xlim(-.7,1.9); ax.set_ylim(max(heights)+.3,-1.5); ax.axis("off"); top=0
        ax.text(0,-1,"Comment ID",ha="center",weight="bold")
        ax.text(1,-1,"Video ID",ha="center",weight="bold")
        for comp,weight in col:
            cs=sorted(n for n in comp if n.startswith("c:")); vs=sorted(n for n in comp if n.startswith("v:"))
            pos={}; span=weight-2.2
            for group,x in [(cs,0),(vs,1)]:
                for i,n in enumerate(group): pos[n]=(x,top+span/2 if len(group)==1 else top+i*span/(len(group)-1))
            for c in cs:
                for v in sorted(adj[c]): ax.plot([0,1],[pos[c][1],pos[v][1]],color="#D59A2B" if degree[v[2:]]>1 else "#CBD5E1")
            for n,(x,y) in pos.items():
                isc=n.startswith("c:"); ax.scatter(x,y,s=35,marker="o" if isc else "s",color="#226B9A" if isc else "#D59A2B" if degree[n[2:]]>1 else "#64748B")
                ax.text(x-.08 if isc else x+.08,y,n[2:],ha="right" if isc else "left",va="center",fontsize=10,fontfamily="monospace")
            top+=weight
    fig.suptitle(f"2 | Source network: {ncomments} comments, {nvideos} videos, {npairs} mention edges",fontsize=18,y=.98)
    fig.text(.04,.04,"Each node appears once. Gold marks shared video IDs. Edges represent mentions, not votes or recommendations.",fontsize=11)
    fig.subplots_adjust(left=.025,right=.98,top=.91,bottom=.09,wspace=.15)
    save(fig,"source_network.png")
    # 3: stance distribution from frozen final decisions.
    fig,ax=plt.subplots(figsize=(12,7)); left=[0]*len(STANCES)
    for topic,color in zip(TOPICS,COLORS):
        values=cross[topic].tolist()
        ax.barh(range(4),values,left=left,label="Topic: "+topic,color=color,hatch="///" if topic=="unclear" else None)
        for i,n in enumerate(values):
            if n: ax.text(left[i]+n/2,i,str(n),ha="center",va="center",weight="bold")
        left=[a+b for a,b in zip(left,values)]
    ax.set_yticks(range(4),["Recommend","Neutral / reference","Critical","Unclear stance"]); ax.invert_yaxis()
    ax.set_xlim(0,max(left)+5)
    for i,n in enumerate(left): ax.text(n+.3,i,str(n)+" pairs",va="center")
    ax.set_title("3 | Stance and topic fit are separate judgments")
    ax.set_xlabel(f"Distinct comment–video pairs (n={npairs})"); ax.legend()
    fig.text(.03,.04,"Zero categories remain visible. Unclear topic fit is retained. No videos were viewed.",fontsize=11)
    fig.subplots_adjust(left=.24,bottom=.20,top=.88)
    save(fig,"stance_distribution.png")
    # 4: categorical evidence matrix; no invented numerical confidence scores.
    rankmap=selection.set_index("video_id")["rank"].to_dict()
    audit=audit.assign(_rank=audit.video_id.map(rankmap).astype(int)).sort_values(["_rank","comment_id"])
    cellrows=[]
    for r in audit.itertuples():
        cellrows.append([rankmap[r.video_id]+" / "+r.comment_id,
            r.recommendation_scope.replace("_"," "),r.reason_supported.replace("_"," "),
            "truncated" if str(r.parent_chain_truncated).lower()=="true" else "root supplied",
            r.revision_status])
    fig,ax=plt.subplots(figsize=(15,7)); ax.axis("off")
    table=ax.table(cellText=cellrows,colLabels=["Rank / source ID","Endorsement","Reason support","Context","Initial selection audit"],cellLoc="left",loc="center",colWidths=[.19,.19,.24,.15,.23])
    table.auto_set_font_size(False); table.set_fontsize(11); table.scale(1,2.4)
    for (r,c),cell in table.get_celld().items():
        cell.set_edgecolor("#D1DAE2")
        cell.set_facecolor("#DCEAEF" if r==0 else "#FFF0CE" if c==3 and cell.get_text().get_text()=="truncated" else "#F5F8FA")
    ax.set_title(f"4 | Evidence matrix: {len(audit)} source pairs support {len(selection)} selected videos",fontsize=18,pad=25)
    fig.text(.03,.05,"The later full audit clarified wording/extracts in 4 rows; no stance/topic labels or selections changed.\nNon-blind model review; not human accuracy. Context gaps and title differences remain documented.",fontsize=11)
    fig.subplots_adjust(left=.03,right=.97,top=.85,bottom=.20)
    save(fig,"evidence_matrix.png")
    # 5: request outcomes versus unperformed tests.
    fig,axes=plt.subplots(1,2,figsize=(14,6))
    status=meta.groupby(["metadata_status","http_status"]).size()
    labels=[f"{a} / HTTP {b}" for (a,b) in status.index]; vals=status.tolist()
    axes[0].barh(range(len(vals)),vals,color=["#D59A2B" if a!="metadata_available" else "#287D8E" for a,b in status.index])
    axes[0].set_yticks(range(len(vals)),labels)
    for i,n in enumerate(vals): axes[0].text(n+.3,i,str(n),va="center")
    axes[0].set_title("Recorded oEmbed outcomes")
    testvals=[summary["playback_not_tested"],summary["content_not_viewed"]]
    axes[1].barh([0,1],testvals,color="#A9B1BC",hatch="///")
    axes[1].set_yticks([0,1],["Playback not tested","Content not viewed"])
    for i,n in enumerate(testvals): axes[1].text(n+.3,i,str(n),va="center")
    axes[1].set_title("Separate, overlapping checks")
    for ax in axes: ax.set_xlim(0,nvideos+4); ax.set_xlabel("Unique videos")
    fig.suptitle("5 | Metadata success does not establish playback",fontsize=18)
    fig.text(.03,.04,"All 36 metadata requests have recorded timestamps. HTTP errors do not prove deletion.\nThe same videos appear in both panels. No new requests, playback tests or content viewing were performed.",fontsize=11)
    fig.subplots_adjust(left=.23,right=.97,wspace=.95,top=.82,bottom=.22)
    save(fig,"metadata_request_statuses.png")
    # 6: readable synthesis, computed counts and frozen selected themes.
    fig,axes=plt.subplots(3,2,figsize=(16,14))
    panels=[
      ("1 | Sample",f"{ncomments} comments\n{npairs} distinct comment–video pairs\n{nvideos} unique videos\nDifferent units; not additive."),
      ("2 | Source relationships",f"{len(adj)} nodes; {npairs} mention edges\n{int(degree.gt(1).sum())} videos shared across comments\nAll input pairs recovered from HTML\nMentions are not quality votes."),
      ("3 | Stance and topic",f"{summary['stance']['recommend']} recommend; {summary['stance']['neutral_or_reference']} neutral/reference\n{summary['stance']['critical']} critical; {summary['stance']['unclear']} unclear stance\nTopic fit: {summary['topic']['yes']} yes / {summary['topic']['no']} no / {summary['topic']['unclear']} unclear\nUnit: comment–video pair"),
      ("4 | Evidence review",f"{len(selection)} selected videos; {len(audit)} source pairs\n{int(audit.recommendation_scope.eq('direct_video').sum())} direct and {int(audit.recommendation_scope.eq('favorable_list').sum())} list-level endorsements\n{summary['clarified_rows']} wording/extract clarifications\n{summary['label_changes']} label changes; no selection changes"),
      ("5 | Metadata and limits",f"{int(meta.http_status.eq('200').sum())} HTTP 200; {int(meta.http_status.eq('403').sum())} HTTP 403; {int(meta.http_status.eq('404').sum())} HTTP 404\n{summary['playback_not_tested']}/{nvideos} playback tests not performed\n{summary['content_not_viewed']}/{nvideos} videos not viewed\nFailed requests do not prove deletion."),
      ("6 | Selected list coverage","\n".join(f"{r.rank}. {r.theme} — {len(r.source_comment_ids.split(';'))} source(s)" for r in selection.itertuples())+"\nAll five: topic fit yes; metadata retrieved.")]
    for ax,(title,text) in zip(axes.flat,panels):
        ax.set_facecolor("#F1F6F9"); ax.set_xticks([]); ax.set_yticks([])
        for spine in ax.spines.values(): spine.set_color("#D4DFE7")
        ax.text(.04,.92,title,transform=ax.transAxes,va="top",fontsize=17,weight="bold")
        ax.text(.04,.73,text,transform=ax.transAxes,va="top",fontsize=14,linespacing=1.65)
    fig.suptitle("6 | Turning comments into a bounded recommendation list",x=.04,ha="left",fontsize=23,y=.97)
    fig.text(.04,.09,f"Unresolved: {summary['parent_chains_truncated']}/{ncomments} parent chains truncated; {summary['topic']['unclear']}/{npairs} topic fits unclear; selected Starlink title differs between snapshots.",fontsize=12)
    fig.text(.04,.055,"Small non-blind evidence audit only. HackerTube's full system was not tested; no other model's performance was measured.",fontsize=12)
    fig.text(.04,.025,"Comment endorsement and metadata do not establish content accuracy, educational quality, regional availability or playback.",fontsize=12)
    fig.subplots_adjust(left=.04,right=.97,top=.91,bottom=.14,hspace=.23,wspace=.13)
    save(fig,"figure_6_audit_synthesis.png")
    # Final HTML uses frozen reasons only and no source quotations.
    esc=lambda s:html.escape(str(s),quote=True)
    def anchor(url,label):
        require(url.startswith("https://"),"Unsafe link scheme.")
        return '<a href="'+esc(url)+'" rel="noopener noreferrer">'+esc(label)+'</a>'
    cards=[]
    for r in selection.itertuples():
        ids=r.source_comment_ids.split(";")
        refs="; ".join(anchor(byid[c]["comment_url"],"HN "+c) for c in ids)
        cards.append("<li><h2>"+anchor(r.video_url,r.title)+"</h2><p>"+esc(r.author_name)+
            " — supplied oEmbed author attribution</p><p>"+esc(r.selection_reason)+
            "</p><p><strong>Comment evidence:</strong> "+esc(r.evidence_summary)+
            "</p><p>"+refs+"</p><p><strong>Unresolved limits:</strong> "+esc(r.unresolved_limits)+
            "</p><p>Recorded metadata: HTTP "+esc(mi.loc[r.video_id,"http_status"])+
            ". Playback not tested; content not viewed. Duration, full description and transcript not supplied.</p></li>")
    page='<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Audited astronomy watchlist</title><style>body{max-width:940px;margin:auto;padding:28px;font:17px/1.6 system-ui;color:#203244}li{margin:24px 0;padding:18px;background:#f2f6f9}a{color:#075b96}h2{font-size:1.3rem}</style><h1>Five evidence-backed astronomy picks</h1><p>Small non-blind audit of a frozen HN sample, not a benchmark of HackerTube or another model. Ranks suggest viewing order, not quality. Endorsements and topic fit are frozen model judgments. No videos were watched or playback-tested; metadata success is not proof of availability.</p><ol>'+''.join(cards)+'</ol><p>33 metadata successes and 3 request errors among 36 videos. Errors do not prove deletion. No thumbnails, durations or video content summaries have been invented. The evidence summaries describe comments, not inspected videos.</p></html>'
    (out/"final_watchlist.html").write_text(page,encoding="utf-8")
    validation=dict(pair_sets_equal=True,source_ids_valid=True,quotes_exact=True,
        quote_words=dict(totals),selected_pairs_valid=True,metadata_preserved=True,
        all_cases_retained=True,no_network_or_model_calls=True,
        semantic_input_files=DECISIONS,
        note="This validates recorded inputs and rendering, not correctness of semantic judgments.")
    (out/"rebuild_checks.json").write_text(json.dumps(validation,indent=2),encoding="utf-8")
    manifest=dict(python=platform.python_version(),pandas=pd.__version__,matplotlib=matplotlib.__version__,
        inputs={n:hashlib.sha256(p.read_bytes()).hexdigest() for n,p in paths.items()},
        execution_model_record=j.execution_model.unique().tolist(),
        semantic_boundary="Judgments are recorded decisions. Python does not reproduce semantic reasoning or call a model.")
    (out/"rebuild_manifest.json").write_text(json.dumps(manifest,indent=2),encoding="utf-8")
    print(json.dumps(summary,indent=2))
if __name__=="__main__": main()
