{
  "article_slug": "tokenization-probability-accounting",
  "figures": [
    {
      "id": "segmentation-mass",
      "title": "Canonical and string probabilities differ",
      "kind": "analytical_curve",
      "x_label": "Alternative segmentation probability q",
      "y_label": "Code length (bits)",
      "x": {
        "name": "Alternative segmentation probability q",
        "values": [
          0,
          0.05,
          0.1,
          0.2,
          0.3,
          0.4,
          0.5,
          0.6
        ]
      },
      "series": [
        {
          "name": "Canonical AB",
          "values": [
            1.7369655941662063,
            1.7369655941662063,
            1.7369655941662063,
            1.7369655941662063,
            1.7369655941662063,
            1.7369655941662063,
            1.7369655941662063,
            1.7369655941662063
          ]
        },
        {
          "name": "Decoded string ab",
          "values": [
            1.7369655941662063,
            1.5145731728297582,
            1.3219280948873622,
            1.0,
            0.7369655941662062,
            0.5145731728297583,
            0.3219280948873623,
            0.1520030934450501
          ]
        }
      ],
      "caption": "Finite terminated generator: P(AB)=0.3, P(A,B)=q, remaining mass0.7-q produces x. Both AB and A,B decode to ab; no real tokenizer measurement."
    },
    {
      "id": "perplexity-units",
      "title": "Per-token perplexity is unit dependent",
      "kind": "analytical_curve",
      "x_label": "Number of tokens N for the same string",
      "y_label": "Per-token perplexity",
      "x": {
        "name": "Number of tokens N for the same string",
        "values": [
          10,
          20,
          40,
          80,
          100
        ]
      },
      "series": [
        {
          "name": "Fixed total80 bits",
          "values": [
            256.0,
            16.0,
            4.0,
            2.0,
            1.7411011265922482
          ]
        }
      ],
      "caption": "Constructed fixed100-byte string with total code length80 bits. Bits per byte remains0.8 for every point; no language model comparison is measured."
    }
  ]
}