#!/usr/bin/env python3 """Frozen December pilot. Does not examine January/February synchrony outcomes.""" import json, os from pathlib import Path import numpy as np import pandas as pd from scipy.stats import t as tdist ROOT=Path(__file__).resolve().parents[1] CACHE=Path(os.environ.get('BIO_DISCOVERY_CACHE','/mnt/data/research/bioinformatics-discovery/cache/round-004'))/'finch_water' def ols_cluster(df,y,xcols,cluster): X=np.column_stack([np.ones(len(df)),df[xcols].astype(float).values]);Y=df[y].to_numpy(float) bread=np.linalg.pinv(X.T@X); beta=np.linalg.lstsq(X,Y,rcond=None)[0]; e=Y-X@beta scores=pd.DataFrame(X*e[:,None]).groupby(df[cluster].to_numpy()).sum().to_numpy() n,k=X.shape;g=len(scores);cov=bread@scores.T@scores@bread*(g/(g-1))*((n-1)/(n-k)); se=np.sqrt(np.diag(cov)) return {'n':n,'clusters':g,'rank':int(np.linalg.matrix_rank(X)),'terms':[{'term':term,'beta':float(b),'se':float(s),'t':float(b/s),'p':float(2*tdist.sf(abs(b/s),g-1))} for term,b,s in zip(['intercept']+xcols,beta,se)]} def read_pilot(): d=pd.read_csv(CACHE/'All_Birds_all_days.csv'); d['date']=pd.to_datetime(d.DATE,dayfirst=True,format='mixed') d=d[d.date.between('2017-12-16','2017-12-31')].copy() d['minute']=(pd.to_timedelta(d.R_TIME).dt.total_seconds()/60).astype(int); d['hour']=d.minute//60;d['bin']=d.minute//5 raw=len(d);d=d.drop_duplicates(['date','ID','Trough','bin']) w=pd.read_csv(CACHE/'weather_data.csv');w['date']=pd.to_datetime(w.DATE,format='%m/%d/%Y') w=w.rename(columns={'Air Temperature in degrees C':'temp','Relative humidity in percentage %':'rh','Wind speed in km/h':'wind','Precipitation since 9am local time in mm':'rain'}) out=[] for (date,trough),day in d.groupby(['date','Trough']): lo,hi=day.hour.min(),day.hour.max() for hour,h in day.groupby('hour'): if hour<=lo or hour>=hi or h.ID.nunique()<10:continue c=h.groupby('ID').size().to_numpy();bins=h.groupby('bin').ID.nunique().to_numpy() observed=float(np.sum(bins*(bins-1)/2));expected=float((c.sum()**2-np.sum(c*c))/24) out.append({'date':date,'trough':trough,'hour':hour,'visits':len(h),'birds':h.ID.nunique(),'observed':observed,'expected':expected,'log_ratio':np.log((observed+.5)/(expected+.5))}) h=pd.DataFrame(out).merge(w[['date','hour','temp','rh','wind','rain']],on=['date','hour'],how='left',validate='many_to_one') return h,{'raw_pilot_rows':raw,'unique_bird_bins':len(d),'pilot_birds':d.ID.nunique(),'pilot_dates':d.date.nunique(),'minute_mod5':d.minute.mod(5).value_counts().to_dict()} def fit(h): f=h.dropna(subset=['temp']).copy(); f['temp10']=f.temp/10;f['log_visits']=np.log(f.visits) X=pd.get_dummies(f[['hour','trough']].astype(str),drop_first=True,dtype=float) f=pd.concat([f,X],axis=1) return ols_cluster(f,'log_ratio',['temp10','log_visits']+list(X.columns),'date') if __name__=='__main__': h,a=read_pilot(); m=fit(h);effect=m['terms'][1] a.update({'eligible_hours':len(h),'eligible_dates':h.date.nunique(),'temperature_range':[h.temp.min(),h.temp.max()],'primary_model':m,'advance':bool(h.date.nunique()>=10 and len(h)>=100 and effect['beta']>=.1 and effect['t']>=1.5),'provenance_caveat':'No timezone label in deposited CSVs. Nominal date/hour match used; clock alignment would require sensitivity checks before advancement.'}) h.to_csv(ROOT/'results/027_pilot_hours.csv',index=False,float_format='%.10g') (ROOT/'results/027_pilot_summary.json').write_text(json.dumps(a,indent=2)+'\n');print(json.dumps(a,indent=2))