"""Declared finite analytical controls; no neural training or source benchmark reproduction."""
import json,math,itertools,hashlib
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
checks=[]
cases=0
def check(ok,label):
 global cases
 assert ok,label
 cases+=1
 if label not in checks: checks.append(label)
def fig(fid,title,xlabel,ylabel,x,series,caption):
 return dict(id=fid,title=title,kind='analytical_curve',x_label=xlabel,y_label=ylabel,x={'name':xlabel,'values':x},series=[{'name':n,'values':v}for n,v in series],caption=caption)
qs=[0,.05,.1,.2,.3,.4,.5,.6]
gaps=[]
for q in qs:
 outcomes=[(('AB',),.3),(('A','B'),q),(('X',),.7-q)]
 decode=lambda t:''.join({'AB':'ab','A':'a','B':'b','X':'x'}[v]for v in t)
 masses={}
 for t,p in outcomes:masses[decode(t)]=masses.get(decode(t),0)+p
 check(abs(sum(masses.values())-1)<1e-12,'decoded probability normalized')
 check(masses['ab']>=.3,'canonical probability bounded by string probability')
 gap=-math.log2(.3)+math.log2(masses['ab'])
 check(abs(gap-math.log2((.3+q)/.3))<1e-12,'code-length gap identity')
 gaps.append(gap)
ns=[10,20,40,80,100];bits=80;bytecount=100
ppl=[2**(bits/n)for n in ns]
check(all(a>b for a,b in zip(ppl,ppl[1:])),'token perplexity changes with denominator')
for n in ns:check(abs(math.log2(2**(bits/n))*n/bytecount-.8)<1e-12,'aligned bits per byte unchanged')
figures=[fig('segmentation-mass','Canonical and string probabilities differ','Alternative segmentation probability q','Code length (bits)',qs,[('Canonical AB',[-math.log2(.3)]*len(qs)),('Decoded string ab',[-math.log2(.3+q)for q in qs])],'Finite terminated generator: P(AB)=0.3, P(A,B)=q, remaining mass0.7-q produces x. Both AB and A,B decode to ab; no real tokenizer measurement.'),
fig('perplexity-units','Per-token perplexity is unit dependent','Number of tokens N for the same string','Per-token perplexity',ns,[('Fixed total80 bits',ppl)],'Constructed fixed100-byte string with total code length80 bits. Bits per byte remains0.8 for every point; no language model comparison is measured.')]
out={'article_slug':'tokenization-probability-accounting','figures':figures}
(ROOT/'figures').mkdir(exist_ok=True)
(ROOT/'figures/tokenization-probability-accounting.json').write_text(json.dumps(out,indent=2)+'\n',encoding='utf-8')
receipt={'status':'PASS','checks':checks,'cases':cases,'empirical_benchmark':False,'source_proofs_verified':False,'figure_sha256':hashlib.sha256((ROOT/'figures/tokenization-probability-accounting.json').read_bytes()).hexdigest()}
(ROOT/'analysis/tokenization-probability-accounting.receipt.json').write_text(json.dumps(receipt,indent=2)+'\n',encoding='utf-8')
print(json.dumps(receipt))