from pathlib import Path
import io, json, zipfile, hashlib, platform, warnings
import numpy as np
import pandas as pd
import scipy
from scipy import stats
OUTPUT_DIR=Path('r2_results')
OUTPUT_DIR.mkdir(exist_ok=True)
assert hashlib.sha256(Path('r2_inputs/proteinGroupss_LFQ.txt').read_bytes()).hexdigest()=='0d4ca19f8ec6c086cb352a892e90f81c2ad11c5d93b46268b6146083d5ded160'
assert hashlib.sha256(Path('r2_inputs/recovery_pmc_supplements.zip').read_bytes()).hexdigest()=='b463126a1680871ce8ac5539cd9cd862d17e0342e2b262abe8be1df7d342cfbe'
p=pd.read_csv('r2_inputs/proteinGroupss_LFQ.txt',sep='\t',low_memory=False)
assert hashlib.sha1(Path('r2_inputs/proteinGroupss_LFQ.txt').read_bytes()).hexdigest()=='02149d5c49eea651694c4f2207a650aa92f452cc'
with zipfile.ZipFile('r2_inputs/recovery_pmc_supplements.zip') as z:
    tab_name=next(n for n in z.namelist() if 'Table S5' in n)
    s5=pd.read_excel(io.BytesIO(z.read(tab_name)),sheet_name='MatrixForLFQ_RawValues',header=1)
s5.columns=s5.columns.str.strip()
sample_labels=[c for c in s5 if c.startswith(('ICM ','NF '))]
sample_ids=[c.split()[1] for c in sample_labels]
icm=np.array([c.startswith('ICM ') for c in sample_labels])
assert icm.sum()==12 and (~icm).sum()==10 and len(set(sample_ids))==22
excluded=np.zeros(len(p),dtype=bool)
for flag in ['Potential contaminant','Reverse','Only identified by site']:
    excluded |= p[flag].eq('+').to_numpy()
protein=p.loc[~excluded].reset_index(drop=True)
ann=protein[['Protein IDs','Majority protein IDs','Gene names','Protein names']].copy()
ann.columns=['protein_group','majority_protein_ids','gene_names','protein_names']
assert ann.protein_group.is_unique
raw=protein[['LFQ intensity '+s for s in sample_ids]].astype(float)
raw.columns=sample_labels
assert not (raw<0).any().any()
observed=raw.mask(raw.eq(0))
Y=np.log2(observed).to_numpy()
ni=np.isfinite(Y[:,icm]).sum(axis=1); nn=np.isfinite(Y[:,~icm]).sum(axis=1)
eligible=(ni>=9)&(nn>=7)
m=int(eligible.sum())
assert m==1327 and len(protein)==2700
def bh(pvalues):
    pv=np.asarray(pvalues,dtype=float)
    assert np.all(np.isfinite(pv)) and np.all((pv>=0)&(pv<=1))
    order=np.argsort(pv); ranks=np.arange(1,len(pv)+1)
    adjusted=np.minimum.accumulate((pv[order]*len(pv)/ranks)[::-1])[::-1]
    result=np.empty_like(pv); result[order]=np.minimum(adjusted,1)
    return result
def welch(matrix,groups):
    a=matrix[:,groups]; b=matrix[:,~groups]
    n1=np.isfinite(a).sum(axis=1); n0=np.isfinite(b).sum(axis=1)
    mean1=np.nanmean(a,axis=1); mean0=np.nanmean(b,axis=1)
    v1=np.nanvar(a,axis=1,ddof=1); v0=np.nanvar(b,axis=1,ddof=1)
    se=np.sqrt(v1/n1+v0/n0)
    df=(v1/n1+v0/n0)**2/((v1/n1)**2/(n1-1)+(v0/n0)**2/(n0-1))
    effect=mean1-mean0
    t=effect/se; pv=2*stats.t.sf(np.abs(t),df)
    crit=stats.t.ppf(.975,df)
    return pd.DataFrame({'n_icm':n1,'n_nf':n0,'mean_log2_icm':mean1,'mean_log2_nf':mean0,
         'log2fc_icm_minus_nf':effect,'se':se,'df':df,'t':t,'p':pv,
         'ci95_low':effect-crit*se,'ci95_high':effect+crit*se,
         'variance_icm':v1,'variance_nf':v0})
primary=welch(Y[eligible],icm)
primary['q']=bh(primary.p)
primary=pd.concat([ann.loc[eligible].reset_index(drop=True),primary],axis=1)
primary['discovery']=primary.q<.05
assert np.linalg.matrix_rank(np.column_stack([np.ones(22),icm.astype(float)]))==2
ref=stats.ttest_ind(Y[eligible][:,icm],Y[eligible][:,~icm],axis=1,equal_var=False,nan_policy='omit')
assert np.allclose(primary.p,ref.pvalue,rtol=1e-10,atol=1e-14)
# Donor influence: refit on the same eligible universe, without changing filtering.
loo_effect=[]; loo_q=[]
for j in range(22):
    keep=np.arange(22)!=j
    fit=welch(Y[eligible][:,keep],icm[keep])
    loo_effect.append(fit.log2fc_icm_minus_nf.to_numpy()); loo_q.append(bh(fit.p))
loo_effect=np.stack(loo_effect); loo_q=np.stack(loo_q)
primary['loo_direction_agreement']=(np.sign(loo_effect)==np.sign(primary.log2fc_icm_minus_nf.to_numpy())).mean(axis=0)
primary['loo_discovery_fraction']=(loo_q<.05).mean(axis=0)
primary['loo_min_log2fc']=loo_effect.min(axis=0); primary['loo_max_log2fc']=loo_effect.max(axis=0)
# Diagnostics remain descriptive; no data-driven sample exclusions.
zres=np.full_like(Y[eligible],np.nan)
for g in [icm,~icm]:
    sub=Y[eligible][:,g]
    zres[:,g]=(sub-np.nanmean(sub,axis=1,keepdims=True))/np.nanstd(sub,axis=1,ddof=1,keepdims=True)
primary['max_abs_standardized_residual']=np.nanmax(np.abs(zres),axis=1)
primary['variance_ratio_max_min']=np.maximum(primary.variance_icm,primary.variance_nf)/np.minimum(primary.variance_icm,primary.variance_nf)
sample_qc=pd.DataFrame({'sample_label':sample_labels,'experiment_id':sample_ids,'group':np.where(icm,'ICM','NF'),
    'observed_protein_groups':np.isfinite(Y).sum(axis=0),'missing_fraction':np.isnan(Y).mean(axis=0),
    'median_log2_lfq':np.nanmedian(Y,axis=0),'batch':'unavailable'})
sample_qc.to_csv(OUTPUT_DIR/'sample_qc.csv',index=False)
pd.concat([ann,observed],axis=1).to_csv(OUTPUT_DIR/'observed_lfq.csv',index=False)
pd.concat([ann,pd.DataFrame(np.isnan(Y).astype(int),columns=sample_labels)],axis=1).to_csv(OUTPUT_DIR/'missingness_mask.csv',index=False)
pd.concat([ann,pd.DataFrame(Y,columns=sample_labels)],axis=1).to_csv(OUTPUT_DIR/'observed_log2_lfq.csv',index=False)
primary.to_csv(OUTPUT_DIR/'primary_results.csv',index=False)
overview={'retained_protein_groups':len(protein),'n_icm':int(icm.sum()),'n_nf':int((~icm).sum()),
    'primary_testable':m,'complete_groups':int(np.isfinite(Y).all(axis=1).sum()),
    'all_matrix_missing_fraction':float(np.isnan(Y).mean()),'primary_missing_fraction':float(np.isnan(Y[eligible]).mean()),
    'primary_discoveries':int(primary.discovery.sum()),'primary_higher_icm':int(((primary.q<.05)&(primary.log2fc_icm_minus_nf>0)).sum()),
    'primary_lower_icm':int(((primary.q<.05)&(primary.log2fc_icm_minus_nf<0)).sum()),
    'max_standardized_residual':float(np.nanmax(np.abs(zres)))}
print(json.dumps(overview,indent=2))
print(primary.sort_values('q')[['gene_names','log2fc_icm_minus_nf','ci95_low','ci95_high','q','n_icm','n_nf','loo_direction_agreement']].head(12).to_string(index=False))


# Sensitivity analysis; all runs retain the primary universe and donor count.
Yp=Y[eligible].copy()
complete=np.isfinite(Yp).all(axis=1)
complete_fit=welch(Yp[complete],icm)
complete_padded=np.ones(m); complete_padded[complete]=complete_fit.p
complete_qfixed=bh(complete_padded)
complete_qsubset=bh(complete_fit.p)
sens=primary.copy()
sens['complete_observation_testable']=complete
sens['complete_q_fixed_universe']=np.where(complete,complete_qfixed,np.nan)
sens['complete_q_subset']=np.nan
sens.loc[complete,'complete_q_subset']=complete_qsubset
# Deterministic seed per scenario; draws are conditional modeling assumptions.
parameters=[(1.0,.3),(1.8,.3),(2.5,.3),(1.8,.2),(1.8,.5)]
sample_mean=np.nanmean(Y,axis=0)
sample_sd=np.nanstd(Y,axis=0,ddof=1)
upper=np.nanquantile(Y,.05,axis=0)
imputed_effect=[]; imputed_q=[]; imputed_p=[]; run_rows=[]; long_rows=[]
for shift,width in parameters:
    scenario=f'shift_{shift:.1f}_width_{width:.1f}'
    loc=sample_mean-shift*sample_sd; scale=width*sample_sd
    trunc_upper=(upper-loc)/scale
    for seed in range(20260928,20260948):
        rng=np.random.default_rng(seed)
        filled=Yp.copy()
        for j in range(22):
            absent=np.isnan(filled[:,j])
            filled[absent,j]=stats.truncnorm.rvs(-np.inf,trunc_upper[j],loc=loc[j],scale=scale[j],size=int(absent.sum()),random_state=rng)
        assert np.isfinite(filled).all()
        assert np.array_equal(filled[np.isfinite(Yp)],Yp[np.isfinite(Yp)])
        fit=welch(filled,icm)
        q=bh(fit.p); effect=fit.log2fc_icm_minus_nf.to_numpy()
        imputed_effect.append(effect); imputed_q.append(q); imputed_p.append(fit.p.to_numpy())
        sig=q<.05; primary_sig=primary.q.to_numpy()<.05
        run_rows.append({'scenario':scenario,'downshift_sd':shift,'width_sd':width,'seed':seed,
            'discoveries':int(sig.sum()),'primary_retained':int((sig&primary_sig).sum()),
            'additional_discoveries':int((sig&~primary_sig).sum()),
            'effect_spearman':float(stats.spearmanr(effect,primary.log2fc_icm_minus_nf).statistic),
            'direction_agreement_fraction':float((np.sign(effect)==np.sign(primary.log2fc_icm_minus_nf)).mean()),
            'jaccard_discoveries':float((sig&primary_sig).sum()/(sig|primary_sig).sum()) if (sig|primary_sig).any() else 1.0})
        long_rows.append(pd.DataFrame({'protein_group':primary.protein_group,'scenario':scenario,'seed':seed,
                         'log2fc':effect,'p':fit.p.to_numpy(),'q':q}))
imputed_effect=np.stack(imputed_effect); imputed_q=np.stack(imputed_q); imputed_p=np.stack(imputed_p)
sens['imputation_log2fc_median']=np.median(imputed_effect,axis=0)
sens['imputation_log2fc_min']=np.min(imputed_effect,axis=0)
sens['imputation_log2fc_max']=np.max(imputed_effect,axis=0)
sens['imputation_q_min']=np.min(imputed_q,axis=0)
sens['imputation_q_max']=np.max(imputed_q,axis=0)
sens['imputation_discovery_fraction']=(imputed_q<.05).mean(axis=0)
sens['imputation_direction_agreement']=(np.sign(imputed_effect)==np.sign(primary.log2fc_icm_minus_nf.to_numpy())).mean(axis=0)
sens['stable_under_missing_data_scenarios']=sens.discovery & (sens.imputation_discovery_fraction==1)&(sens.imputation_direction_agreement==1)
# Complete observations must have exactly unchanged estimates and p-values under fills.
assert np.allclose(imputed_effect[:,complete],primary.log2fc_icm_minus_nf.to_numpy()[complete][None,:])
assert np.allclose(imputed_p[:,complete],primary.p.to_numpy()[complete][None,:])
center_offsets=np.nanmedian(Y,axis=0)-np.median(np.nanmedian(Y,axis=0))
Ycentered=Y-center_offsets
normalized=welch(Ycentered[eligible],icm)
sens['median_centered_log2fc']=normalized.log2fc_icm_minus_nf
sens['median_centered_q']=bh(normalized.p)
sens['median_centered_discovery']=sens.median_centered_q<.05
sens['primary_and_normalization_supported']=sens.discovery & sens.median_centered_discovery & (np.sign(sens.log2fc_icm_minus_nf)==np.sign(sens.median_centered_log2fc))
sens['classification']=np.where(sens.stable_under_missing_data_scenarios,'primary discovery stable under missing-data scenarios',
    np.where(sens.discovery,'primary discovery sensitive to missing-data scenarios',
    np.where(sens.imputation_discovery_fraction>0,'imputation-only discovery in some scenarios','not discovered')))
# Exploratory detection tests use a separate, declared family across all retained groups.
det=ann.copy()
det['detected_icm']=ni; det['detected_nf']=nn
det['missing_icm']=12-ni; det['missing_nf']=10-nn
det['detection_rate_difference']=ni/12-nn/10
det['detection_p']=[stats.fisher_exact([[int(a),12-int(a)],[int(b),10-int(b)]],alternative='two-sided').pvalue for a,b in zip(ni,nn)]
det['detection_q']=bh(det.detection_p)
det['primary_testable']=eligible
det['abundance_effect_identifiable_in_both_groups']=(ni>=2)&(nn>=2)
det['entire_group_unobserved']=(ni==0)|(nn==0)
all_results=det.merge(sens.drop(columns=['majority_protein_ids','gene_names','protein_names']),on='protein_group',how='left',validate='one_to_one')
all_results['classification']=all_results.classification.fillna('insufficient observations for primary test')
all_results.to_csv(OUTPUT_DIR/'all_protein_results.csv',index=False)
sens.to_csv(OUTPUT_DIR/'sensitivity_results.csv',index=False)
det.to_csv(OUTPUT_DIR/'detection_results.csv',index=False)
run_summary=pd.DataFrame(run_rows)
run_summary.to_csv(OUTPUT_DIR/'imputation_run_summary.csv',index=False)
pd.concat(long_rows,ignore_index=True).to_csv(OUTPUT_DIR/'imputation_all_runs.csv.gz',index=False,compression={'method':'gzip','mtime':0})
overview.update({'imputation_runs':len(run_summary),'imputation_discoveries_min':int(run_summary.discoveries.min()),
    'imputation_discoveries_max':int(run_summary.discoveries.max()),
    'missingness_stable_primary_discoveries':int(sens.stable_under_missing_data_scenarios.sum()),
    'imputation_only_ever_discovered':int(((sens.imputation_discovery_fraction>0)&~sens.discovery).sum()),
    'complete_fixed_universe_discoveries':int((sens.complete_q_fixed_universe<.05).sum()),
    'complete_subset_discoveries':int((sens.complete_q_subset<.05).sum()),
    'median_centered_discoveries':int(sens.median_centered_discovery.sum()),
    'primary_and_normalization_supported':int(sens.primary_and_normalization_supported.sum()),
    'detection_discoveries':int((det.detection_q<.05).sum()),
    'groups_unobserved_in_entire_group':int(det.entire_group_unobserved.sum())})
(OUTPUT_DIR/'analysis_summary.json').write_text(json.dumps(overview,indent=2))
print(json.dumps(overview,indent=2))
print(sens.loc[sens.discovery,['protein_group','gene_names','log2fc_icm_minus_nf','q','imputation_q_min','imputation_q_max','imputation_discovery_fraction','median_centered_log2fc','median_centered_q','loo_discovery_fraction']].to_string(index=False))
print(run_summary.groupby('scenario').agg(discoveries_min=('discoveries','min'),discoveries_max=('discoveries','max'),effect_rho_min=('effect_spearman','min')).to_string())
print('Detection leading:',det.sort_values('detection_q')[['gene_names','detected_icm','detected_nf','detection_q']].head(8).to_string(index=False))


# Post-result protein-identity QC: report, do not silently alter the frozen primary analysis.
contains_contaminant_id=ann.protein_group.str.contains('CON__',regex=False)
sens['contains_contaminant_database_id']=sens.protein_group.str.contains('CON__',regex=False)
strict=~sens.contains_contaminant_database_id.to_numpy()
sens['posthoc_no_contaminant_id_q']=np.nan
sens.loc[strict,'posthoc_no_contaminant_id_q']=bh(sens.loc[strict,'p'])
all_results['contains_contaminant_database_id']=all_results.protein_group.str.contains('CON__',regex=False)
all_results=all_results.merge(sens[['protein_group','posthoc_no_contaminant_id_q']],on='protein_group',how='left',validate='one_to_one')
sens.to_csv(OUTPUT_DIR/'sensitivity_results.csv',index=False)
all_results.to_csv(OUTPUT_DIR/'all_protein_results.csv',index=False)
overview.update({'retained_groups_with_contaminant_id':int(contains_contaminant_id.sum()),
    'primary_groups_with_contaminant_id':int((~strict).sum()),
    'posthoc_strict_id_discoveries':int((sens.posthoc_no_contaminant_id_q<.05).sum()),
    'min_imputation_effect_spearman':float(run_summary.effect_spearman.min()),
    'max_imputation_effect_spearman':float(run_summary.effect_spearman.max()),
    'min_imputation_direction_agreement':float(run_summary.direction_agreement_fraction.min()),
    'max_imputation_direction_agreement':float(run_summary.direction_agreement_fraction.max())})
(OUTPUT_DIR/'analysis_summary.json').write_text(json.dumps(overview,indent=2))
print(sens.loc[sens.discovery,['protein_group','majority_protein_ids','protein_names','posthoc_no_contaminant_id_q','loo_min_log2fc','loo_max_log2fc']].to_string(index=False))
print('Annotation sensitivity', {k:v for k,v in overview.items() if 'contaminant' in k or 'strict_id' in k})
print('Strongest detection p',det.detection_p.min(),'q',det.detection_q.min())
print('Largest imputation changes',sens.assign(effect_shift=lambda d:abs(d.imputation_log2fc_median-d.log2fc_icm_minus_nf)).sort_values('effect_shift',ascending=False)[['gene_names','log2fc_icm_minus_nf','imputation_log2fc_median','imputation_q_min','imputation_q_max']].head(5).to_string(index=False))

# Producer metadata only; native environment locks are supplied by the application, not this JSON.
import importlib.metadata
producer_record={
'analysis':'unchanged approved ICM-versus-NF LFQ analysis',
'input_versions':['0bd0eaef-4e35-4f23-b990-225a7368ea84','2706f4dd-7269-4f36-a828-7be5ca19e35d'],
'inputs':{
'r2_inputs/proteinGroupss_LFQ.txt':'0d4ca19f8ec6c086cb352a892e90f81c2ad11c5d93b46268b6146083d5ded160',
'r2_inputs/recovery_pmc_supplements.zip':'b463126a1680871ce8ac5539cd9cd862d17e0342e2b262abe8be1df7d342cfbe'},
'python_version':platform.python_version(),
'package_versions':{name:importlib.metadata.version(name) for name in ['numpy','pandas','scipy','openpyxl']},
'seeds':list(range(20260928,20260948)),
'parameters':parameters,
'output_hashes':{
'primary_results.csv':hashlib.sha256(Path('r2_results/primary_results.csv').read_bytes()).hexdigest(),
'sensitivity_results.csv':hashlib.sha256(Path('r2_results/sensitivity_results.csv').read_bytes()).hexdigest()},
'producer':'single self-contained Notebook cell after named-environment restart',
'old_results_used_for_computation':False,
'lock_statement':'This record is not an environment lock; inspect native lineage capture.'}
Path('r2_results/producer_provenance.json').write_text(json.dumps(producer_record,indent=2))
print('Standalone recomputation completed:',len(primary),'primary rows;',len(sens),'sensitivity rows.')
