#!/usr/bin/env python3
"""Offline subgroup reproduction: NumPy/SciPy, pinned input files, fixed phase3 weights."""
import argparse,csv,hashlib,json,pathlib
import numpy as np
from scipy.stats import norm,chi2
ROOT=pathlib.Path(__file__).resolve().parents[1]
def digest(p):return hashlib.sha256(p.read_bytes()).hexdigest()
def hac_scores(scores,t,lag=12):
 s=scores.T@scores;ix={v:i for i,v in enumerate(t)}
 for lag_i in range(1,lag+1):
  pairs=[(i,ix[v-lag_i]) for i,v in enumerate(t) if v-lag_i in ix]
  if pairs:
   p,q=np.array(pairs).T;c=scores[p].T@scores[q];s+=(1-lag_i/(lag+1))*(c+c.T)
 return s
def design(t,post,trend=True):
 return np.column_stack([np.ones(len(t))]+[(t%12==m).astype(float) for m in range(1,12)]+([t/12] if trend else [])+[post.astype(float)])
def fit(a,pre,post,w,pooled=False):
 if np.any(a[:12].mean(axis=0)<=0):return None
 index=a/a[:12].mean(axis=0);comparison=index[:,1:]@w
 if pooled:comparison=a[:,1:].sum(axis=1)/a[:12,1:].sum(axis=1).mean()
 select=pre|post;t=np.where(select)[0]
 if np.any(a[select,0]<=0) or np.any(comparison[select]<=0):return None
 y=np.log(index[select,0]/comparison[select]);x=design(t,post[select]);pinv=np.linalg.pinv(x);b=pinv@y;e=y-x@b;score=e[:,None]*pinv.T;cov=hac_scores(score,t)*len(t)/(len(t)-x.shape[1]);se=float(np.sqrt(max(0,cov[-1,-1])));
 tp=np.where(pre)[0];xp=design(tp,np.zeros(len(tp),bool))[:,:-1];yp=np.log(index[pre,0]/comparison[pre]);pp=np.linalg.pinv(xp);bp=pp@yp;ep=yp-xp@bp;pc=hac_scores(ep[:,None]*pp.T,tp)*len(tp)/(len(tp)-xp.shape[1]);ps=float(np.sqrt(max(0,pc[-1,-1])))
 return {'coefficient':float(b[-1]),'se':se,'pct':float(100*np.expm1(b[-1])),'nominalCI':[float(100*np.expm1(b[-1]+v*1.96*se)) for v in (-1,1)],'conditionalMdePct':float(100*(-np.expm1(-(1.96+.841621)*se))),'preTrendPct':float(100*np.expm1(bp[-1])),'preTrendCI':[float(100*np.expm1(bp[-1]+v*1.96*ps)) for v in (-1,1)],'score':score[:,-1],'comparisonIndex':(100*comparison).tolist(),'walesIndex':(100*index[:,0]).tolist()}
def holm(pvalues):
 order=np.argsort(pvalues);out=np.ones(len(order));last=0
 for j,i in enumerate(order):last=max(last,min(1,float(pvalues[i])*(len(order)-j)));out[i]=last
 return out.tolist()
def run(inp,out):
 pins=json.loads((inp/'input-pins.json').read_text())
 for name,pin in pins.items():assert digest(inp/name)==pin,('Input hash mismatch',name)
 spec=json.loads((inp/'analysis-specification.json').read_text());raw=list(csv.DictReader((inp/'monthly-subgroups.csv').open()));weight=json.loads((inp/'comparison-weights.json').read_text());regions=['Wales']+weight['regions'];w=np.array(weight['weights']);assert abs(w.sum()-1)<1e-8
 months=sorted({r['month'] for r in raw});tt=np.arange(len(months));pre=np.array([m<'2020-01' or '2022-01'<=m<'2023-09' for m in months]);post=np.array([m>='2023-10' for m in months]);assert len(months)==132 and pre.sum()==80 and post.sum()==27
 families=list(dict.fromkeys(r['family'] for r in raw));keys=list(dict.fromkeys((r['family'],r['group']) for r in raw));lookup={(r['month'],r['region'],r['family'],r['group']):int(r['casualties']) for r in raw};result=[];fits={};annual=[];gate=spec['supportGate'];panels={}
 for fam,g in keys:
  a=np.array([[lookup[(m,r,fam,g)] for r in regions] for m in months],dtype=float);panels[(fam,g)]=a
  for y in range(2015,2026):
   mask=np.array([m.startswith(str(y)) for m in months]);annual.append({'family':fam,'group':g,'year':y,'wales':int(a[mask,0].sum()),'england':int(a[mask,1:].sum())})
  main=fit(a,pre,post,w);pooled=fit(a,pre,post,w,True);reasons=[];pr=int(a[pre,0].sum());po=int(a[post,0].sum());minimum=int(a[pre|post,0].min())
  if g in ('Unknown','Unmatched / excluded','Other / unknown'):reasons.append('Unknown / unmatched category: descriptive only')
  if pr<gate['minPreCount']:reasons.append('Fewer than100 pre-period casualties')
  if po<gate['minPostCount']:reasons.append('Fewer than50 post-period casualties')
  if minimum<gate['minWelshCountEveryIncludedMonth']:reasons.append('Fewer than5 Welsh casualties in at least one included month')
  if main is None or pooled is None:reasons.append('Nonpositive baseline or monthly series')
  elif max(main['conditionalMdePct'],pooled['conditionalMdePct'])>gate['maxConditionalMdeReductionPct']:reasons.append('Conditional detectable reduction exceeds30% in at least one comparison')
  item={'family':fam,'group':g,'preCount':pr,'postCount':po,'preMonthlyMean':pr/80,'postMonthlyMean':po/27,'minimumIncludedMonth':minimum,'monthsBelowFive':int((a[pre|post,0]<5).sum()),'conditionalMdePct':None if main is None else main['conditionalMdePct'],'eligible':not reasons,'reasons':reasons,'main':None,'pooled':None}
  if not reasons:fits[(fam,g)]=(main,pooled)
  result.append(item)
 # Freeze multiplicity over all eligible subgroup estimates, excluding the overall reference.
 k=sum(r['eligible'] and r['family']!='all' for r in result);critical=float(norm.ppf(1-.05/(2*k)))
 for item in result:
  key=(item['family'],item['group'])
  if key not in fits:continue
  for label,fitvalue in zip(('main','pooled'),fits[key]):
   z=1.96 if item['family']=='all' else critical
   item[label]={k:v for k,v in fitvalue.items() if k not in ('score','comparisonIndex','walesIndex')}
   item[label]['simultaneousCI']=[float(100*np.expm1(fitvalue['coefficient']+d*z*fitvalue['se'])) for d in (-1,1)]
 heterogeneity=[]
 for model_index,model_name in enumerate(('main','pooled')):
  batch=[]
  for fam in families:
   eligible=[(key,v[model_index]) for key,v in fits.items() if key[0]==fam]
   if fam=='all' or len(eligible)<2:continue
   b=np.array([r['coefficient'] for key,r in eligible]);scores=np.column_stack([r['score'] for key,r in eligible]);cov=hac_scores(scores,np.where(pre|post)[0])*107/(107-14);h=np.column_stack([-np.ones(len(b)-1),np.eye(len(b)-1)]);v=h@cov@h.T;delta=h@b;rank=np.linalg.matrix_rank(v);stat=float(delta@np.linalg.pinv(v)@delta)
   batch.append({'model':model_name,'family':fam,'groups':[key[1] for key,r in eligible],'statistic':stat,'df':int(rank),'p':float(chi2.sf(stat,rank)),'holmP':None})
  for r,p in zip(batch,holm([r['p'] for r in batch])):r['holmP']=p
  heterogeneity+=batch
 trends=[]
 for fam,g in keys:
  a=panels[(fam,g)];fm=fit(a,pre,post,w)
  for q in range(44):
   sl=slice(q*3,q*3+3);trends.append({'family':fam,'group':g,'period':f'{2015+q//4} Q{q%4+1}','walesCount':int(a[sl,0].sum()),'englandCount':int(a[sl,1:].sum()),'walesIndex':None if fm is None else float(np.mean(fm['walesIndex'][sl])),'comparisonIndex':None if fm is None else float(np.mean(fm['comparisonIndex'][sl]))})
 missing=[]
 for fam in families:
  unknown=[g for f,g in keys if f==fam and g in ('Unknown','Unmatched / excluded','Other / unknown')]
  if unknown:
   counts=sum((panels[(fam,g)] for g in unknown),np.zeros((132,10)));total=panels[('all','All casualties')]
   for country,col in [('Wales',[0]),('England',list(range(1,10)))]:
    pr=counts[pre][:,col].sum()/total[pre][:,col].sum()*100;po=counts[post][:,col].sum()/total[post][:,col].sum()*100
    missing.append({'family':fam,'country':country,'prePct':float(pr),'postPct':float(po),'changePp':float(po-pr)})
 output={'version':'20mph-phase4-v4','analysisDate':'2026-11-04','causalGate':'Not met','preMonths':80,'postMonths':27,'multiplicityCount':k,'criticalValue':critical,'groups':result,'heterogeneity':heterogeneity,'missingness':missing,'annual':annual,'quarterly':trends,'inputSha256':pins}
 out.mkdir(parents=True,exist_ok=True);(out/'results.json').write_text(json.dumps(output,separators=(',',':'),allow_nan=False)+'\n')
 flat=[]
 for r in result:
  flat.append({**{key:r[key] for key in ('family','group','preCount','postCount','minimumIncludedMonth','monthsBelowFive','conditionalMdePct','eligible')},'reason':'; '.join(r['reasons']),**{f'{m}_{key}':r[m][key] if r[m] else '' for m in ('main','pooled') for key in ('pct','simultaneousCI','preTrendPct')}})
 for name,rows in [('estimates.csv',flat),('annual-counts.csv',annual),('heterogeneity.csv',heterogeneity),('missingness.csv',missing)]:
  with (out/name).open('w',newline='') as f:writer=csv.DictWriter(f,list(rows[0]));writer.writeheader();writer.writerows(rows)
 print(json.dumps({'eligible':k,'groups':[{k:r[k] for k in ('family','group','eligible','reasons')}|{'main':r['main']['pct'] if r['main'] else None,'pooled':r['pooled']['pct'] if r['pooled'] else None} for r in result],'missingness':missing,'heterogeneity':heterogeneity},indent=2))
if __name__=='__main__':
 p=argparse.ArgumentParser();p.add_argument('--input-dir',type=pathlib.Path,default=ROOT/'public/research/20mph/v4');p.add_argument('--output-dir',type=pathlib.Path,default=ROOT/'public/research/20mph/v4');a=p.parse_args();run(a.input_dir,a.output_dir)
