{
  "analytic": "mamba-state",
  "kind": "declared_analytical_or_constructed_example",
  "experimental_model_result": false,
  "panels": [
    {
      "kind": "line",
      "x": [
        128,
        512,
        2048,
        8192,
        32768
      ],
      "series": [
        {
          "label": "Fixed state,128 scalars",
          "y": [
            128,
            128,
            128,
            128,
            128
          ]
        },
        {
          "label": "KV, one head d=128",
          "y": [
            32768,
            131072,
            524288,
            2097152,
            8388608
          ]
        }
      ],
      "xlabel": "Decoded sequence length",
      "ylabel": "Stored scalar count",
      "logx": true,
      "logy": true
    }
  ],
  "title": "Bounded recurrent state is not bounded training memory",
  "assumptions": "One abstract recurrent state128 vs one attention head dimension128, key and value. Excludes layers, channels, weights and training activations.",
  "worked": "在8192位置时，此限定例子的KV为2097152标量，递推状态为128标量。两者表达和架构不同，不能把该比例当作全模型内存或能力比较。",
  "source_ids": [
    "arxiv-2312.00752",
    "arxiv-2405.21060",
    "attention"
  ],
  "formula_source": "Article equations and this versioned script",
  "diagram_nodes": [
    "连续系统及离散化",
    "输入依赖选择参数",
    "有限维状态更新",
    "扫描与IO组织",
    "在线递推 / 训练分项",
    "长度、质量与记忆验证"
  ]
}