#!/usr/bin/env python3 """Exact published-coefficient audit, with Gaussian random-intercept REML.""" from pathlib import Path import json, os import numpy as np import pandas as pd from scipy.optimize import minimize_scalar ROOT=Path(__file__).resolve().parents[1] CACHE=Path(os.environ.get('BIO_DISCOVERY_CACHE','/mnt/data/research/bioinformatics-discovery/cache/round-004'))/'great_tit_clock' def random_intercept(y,X,groups,names): y=np.asarray(y,float);X=np.asarray(X,float);codes=pd.factorize(groups)[0];Z=np.eye(codes.max()+1)[codes];n,k=X.shape;counts=Z.sum(0) def evaluate(logratio,ret=False): r=np.exp(logratio);Vinv=np.eye(n)-(Z*(r/(1+r*counts)))@Z.T bread=np.linalg.inv(X.T@Vinv@X);beta=bread@X.T@Vinv@y;e=y-X@beta;ss=float(e@Vinv@e);s2=ss/(n-k) obj=(n-k)*np.log(s2)+np.log1p(r*counts).sum()+np.linalg.slogdet(X.T@Vinv@X)[1] if not ret:return obj se=np.sqrt(np.diag(s2*bread)) return {'n':n,'groups':Z.shape[1],'residual_variance':s2,'group_variance':r*s2,'terms':[{'term':name,'beta':float(b),'se':float(s),'z':float(b/s)} for name,b,s in zip(names,beta,se)]} o=minimize_scalar(evaluate,bounds=(-20,15),method='bounded',options={'xatol':1e-10});return evaluate(o.x,True) def load_fixed(): t=pd.read_csv(CACHE/'Extern_MethData_Train_DevelopClock.txt',sep='\t');m=pd.read_csv(CACHE/'Extern_Samples_DevelopClock.txt',sep='\t').set_index('Sample_ID') z=pd.read_csv(CACHE/'Extern_MethData_Brood_Test_DevelopClock.txt',sep='\t').T coef=pd.read_excel(CACHE/'clock_coefficients.xlsx',sheet_name='Pre-fleding (Development) Clock',header=3).iloc[:,:2];coef.columns=['CpG','coefficient'];coef=coef.set_index('CpG').coefficient assert set(coef.index)-{'(Intercept)'}==set(z.columns) d=m.loc[z.index].copy();d['Sample_ID']=d.index;d['in_training']=d.index.isin(t.Sample_ID);d['predicted_age']=coef['(Intercept)']+z@coef.drop('(Intercept)');d['error']=d.predicted_age-d.Age_in_Days d=d[d.Age_in_Days==14].copy(); assert len(d)==96 return d def treatment_model(d): d=d.dropna(subset=['Genetic_ID']);X=np.column_stack([np.ones(len(d)),(d.Treatment=='Enlarged').astype(int),(d.Treatment=='Reduced').astype(int)]) return random_intercept(d.predicted_age,X,d.Genetic_ID,['intercept','Enlarged-Control','Reduced-Control']) if __name__=='__main__': d=load_fixed();full=treatment_model(d);held=treatment_model(d[~d.in_training]);means=d.groupby('in_training').predicted_age.mean();delta=float(means[True]-means[False]) by=d.groupby(['Treatment','in_training']).agg(n=('predicted_age','size'),mean=('predicted_age','mean'),sd=('predicted_age','std')).reset_index() a={'full_model':full,'heldout_model':held,'prediction_mean_by_membership':{str(k):float(v) for k,v in means.items()},'training_minus_heldout_days':delta,'training_n':int(d.in_training.sum()),'heldout_n':int((~d.in_training).sum()),'reproduces_original':bool(abs(full['terms'][2]['beta']-.5823)<.01 and abs(full['residual_variance']-.5611)<.02 and abs(full['group_variance']-.3594)<.02)} a['advance']=bool(a['reproduces_original'] and a['heldout_n']>=50 and delta>.5) d.to_csv(ROOT/'results/029_fixed_predictions.csv',index=False,float_format='%.12g');by.to_csv(ROOT/'results/029_membership_means.csv',index=False,float_format='%.12g');(ROOT/'results/029_pilot_summary.json').write_text(json.dumps(a,indent=2)+'\n');print(json.dumps(a,indent=2));print(by.to_string(index=False))