{
  "analytic": "alignment-proxy",
  "kind": "declared_analytical_or_constructed_example",
  "experimental_model_result": false,
  "panels": [
    {
      "kind": "line",
      "x": [
        0.0,
        0.05,
        0.1,
        0.15000000000000002,
        0.2,
        0.25,
        0.30000000000000004,
        0.35000000000000003,
        0.4,
        0.45,
        0.5,
        0.55,
        0.6000000000000001,
        0.65,
        0.7000000000000001,
        0.75,
        0.8,
        0.8500000000000001,
        0.9,
        0.9500000000000001,
        1.0,
        1.05,
        1.1,
        1.1500000000000001,
        1.2000000000000002,
        1.25,
        1.3,
        1.35,
        1.4000000000000001,
        1.4500000000000002,
        1.5,
        1.55,
        1.6,
        1.6500000000000001,
        1.7000000000000002,
        1.75,
        1.8,
        1.85,
        1.9000000000000001,
        1.9500000000000002,
        2.0,
        2.0500000000000003,
        2.1,
        2.15,
        2.2,
        2.25,
        2.3000000000000003,
        2.35,
        2.4000000000000004,
        2.45,
        2.5,
        2.5500000000000003,
        2.6,
        2.6500000000000004,
        2.7,
        2.75,
        2.8000000000000003,
        2.85,
        2.9000000000000004,
        2.95,
        3.0,
        3.0500000000000003,
        3.1,
        3.1500000000000004,
        3.2,
        3.25,
        3.3000000000000003,
        3.35,
        3.4000000000000004,
        3.45,
        3.5,
        3.5500000000000003,
        3.6,
        3.6500000000000004,
        3.7,
        3.75,
        3.8000000000000003,
        3.85,
        3.9000000000000004,
        3.95,
        4.0,
        4.05,
        4.1000000000000005,
        4.15,
        4.2,
        4.25,
        4.3,
        4.3500000000000005,
        4.4,
        4.45,
        4.5,
        4.55,
        4.6000000000000005,
        4.65,
        4.7,
        4.75,
        4.800000000000001,
        4.8500000000000005,
        4.9,
        4.95,
        5.0,
        5.050000000000001,
        5.1000000000000005,
        5.15,
        5.2,
        5.25,
        5.300000000000001,
        5.3500000000000005,
        5.4,
        5.45,
        5.5,
        5.550000000000001,
        5.6000000000000005,
        5.65,
        5.7,
        5.75,
        5.800000000000001,
        5.8500000000000005,
        5.9,
        5.95,
        6.0
      ],
      "series": [
        {
          "label": "Proxy reward",
          "y": [
            0.0,
            0.05,
            0.1,
            0.15000000000000002,
            0.2,
            0.25,
            0.30000000000000004,
            0.35000000000000003,
            0.4,
            0.45,
            0.5,
            0.55,
            0.6000000000000001,
            0.65,
            0.7000000000000001,
            0.75,
            0.8,
            0.8500000000000001,
            0.9,
            0.9500000000000001,
            1.0,
            1.05,
            1.1,
            1.1500000000000001,
            1.2000000000000002,
            1.25,
            1.3,
            1.35,
            1.4000000000000001,
            1.4500000000000002,
            1.5,
            1.55,
            1.6,
            1.6500000000000001,
            1.7000000000000002,
            1.75,
            1.8,
            1.85,
            1.9000000000000001,
            1.9500000000000002,
            2.0,
            2.0500000000000003,
            2.1,
            2.15,
            2.2,
            2.25,
            2.3000000000000003,
            2.35,
            2.4000000000000004,
            2.45,
            2.5,
            2.5500000000000003,
            2.6,
            2.6500000000000004,
            2.7,
            2.75,
            2.8000000000000003,
            2.85,
            2.9000000000000004,
            2.95,
            3.0,
            3.0500000000000003,
            3.1,
            3.1500000000000004,
            3.2,
            3.25,
            3.3000000000000003,
            3.35,
            3.4000000000000004,
            3.45,
            3.5,
            3.5500000000000003,
            3.6,
            3.6500000000000004,
            3.7,
            3.75,
            3.8000000000000003,
            3.85,
            3.9000000000000004,
            3.95,
            4.0,
            4.05,
            4.1000000000000005,
            4.15,
            4.2,
            4.25,
            4.3,
            4.3500000000000005,
            4.4,
            4.45,
            4.5,
            4.55,
            4.6000000000000005,
            4.65,
            4.7,
            4.75,
            4.800000000000001,
            4.8500000000000005,
            4.9,
            4.95,
            5.0,
            5.050000000000001,
            5.1000000000000005,
            5.15,
            5.2,
            5.25,
            5.300000000000001,
            5.3500000000000005,
            5.4,
            5.45,
            5.5,
            5.550000000000001,
            5.6000000000000005,
            5.65,
            5.7,
            5.75,
            5.800000000000001,
            5.8500000000000005,
            5.9,
            5.95,
            6.0
          ]
        },
        {
          "label": "Constructed true utility",
          "y": [
            0.0,
            0.19750000000000023,
            0.3900000000000001,
            0.5774999999999997,
            0.7599999999999998,
            0.9375,
            1.1100000000000003,
            1.2775000000000003,
            1.4399999999999995,
            1.5974999999999997,
            1.75,
            1.8975,
            2.04,
            2.1774999999999998,
            2.3100000000000005,
            2.4375,
            2.56,
            2.6775,
            2.79,
            2.8975000000000004,
            3.0,
            3.0975,
            3.1900000000000004,
            3.2775000000000003,
            3.3600000000000003,
            3.4375,
            3.5100000000000002,
            3.5775,
            3.64,
            3.6975000000000002,
            3.75,
            3.7975,
            3.84,
            3.8775,
            3.91,
            3.9375,
            3.96,
            3.9775,
            3.99,
            3.9975,
            4.0,
            3.9975,
            3.9899999999999998,
            3.9775,
            3.96,
            3.9375,
            3.9099999999999997,
            3.8775,
            3.84,
            3.7975,
            3.75,
            3.6975,
            3.6399999999999997,
            3.5774999999999997,
            3.51,
            3.4375,
            3.3599999999999994,
            3.2775,
            3.1899999999999995,
            3.0974999999999997,
            3.0,
            2.8974999999999995,
            2.79,
            2.6774999999999993,
            2.5599999999999996,
            2.4375,
            2.3099999999999996,
            2.1774999999999998,
            2.039999999999999,
            1.8974999999999995,
            1.75,
            1.5974999999999993,
            1.4399999999999995,
            1.277499999999999,
            1.1099999999999994,
            0.9375,
            0.7599999999999989,
            0.5774999999999997,
            0.3899999999999988,
            0.19749999999999934,
            0.0,
            -0.20249999999999968,
            -0.4100000000000019,
            -0.6225000000000014,
            -0.8400000000000007,
            -1.0625,
            -1.2899999999999991,
            -1.5225000000000026,
            -1.7600000000000016,
            -2.0025000000000013,
            -2.25,
            -2.5024999999999995,
            -2.7600000000000025,
            -3.0225000000000017,
            -3.290000000000001,
            -3.5625,
            -3.8400000000000043,
            -4.122500000000002,
            -4.410000000000002,
            -4.702500000000001,
            -5.0,
            -5.302500000000004,
            -5.610000000000003,
            -5.922500000000003,
            -6.240000000000002,
            -6.5625,
            -6.890000000000004,
            -7.222500000000004,
            -7.560000000000002,
            -7.902500000000002,
            -8.25,
            -8.602500000000004,
            -8.960000000000004,
            -9.322500000000003,
            -9.690000000000001,
            -10.0625,
            -10.440000000000005,
            -10.822500000000003,
            -11.210000000000003,
            -11.602500000000001,
            -12.0
          ]
        },
        {
          "label": "Proxy minus quadratic penalty",
          "y": [
            0.0,
            0.04975,
            0.099,
            0.14775000000000002,
            0.196,
            0.24375,
            0.29100000000000004,
            0.33775000000000005,
            0.384,
            0.42975,
            0.475,
            0.51975,
            0.5640000000000001,
            0.60775,
            0.651,
            0.69375,
            0.736,
            0.77775,
            0.819,
            0.85975,
            0.9,
            0.9397500000000001,
            0.9790000000000001,
            1.0177500000000002,
            1.056,
            1.09375,
            1.131,
            1.16775,
            1.2040000000000002,
            1.2397500000000001,
            1.275,
            1.30975,
            1.344,
            1.37775,
            1.411,
            1.44375,
            1.476,
            1.5077500000000001,
            1.5390000000000001,
            1.56975,
            1.6,
            1.62975,
            1.659,
            1.6877499999999999,
            1.7160000000000002,
            1.74375,
            1.7710000000000001,
            1.7977500000000002,
            1.8240000000000003,
            1.8497500000000002,
            1.875,
            1.89975,
            1.924,
            1.94775,
            1.971,
            1.99375,
            2.016,
            2.03775,
            2.059,
            2.0797499999999998,
            2.0999999999999996,
            2.11975,
            2.139,
            2.15775,
            2.176,
            2.1937499999999996,
            2.211,
            2.22775,
            2.2439999999999998,
            2.25975,
            2.275,
            2.2897499999999997,
            2.304,
            2.31775,
            2.331,
            2.34375,
            2.356,
            2.36775,
            2.379,
            2.3897500000000003,
            2.4,
            2.40975,
            2.419,
            2.42775,
            2.436,
            2.4437499999999996,
            2.4509999999999996,
            2.45775,
            2.464,
            2.46975,
            2.475,
            2.4797499999999997,
            2.484,
            2.4877499999999997,
            2.491,
            2.49375,
            2.496,
            2.49775,
            2.499,
            2.4997499999999997,
            2.5,
            2.4997499999999997,
            2.4989999999999997,
            2.49775,
            2.496,
            2.49375,
            2.4909999999999997,
            2.48775,
            2.484,
            2.4797499999999997,
            2.4749999999999996,
            2.46975,
            2.464,
            2.45775,
            2.4509999999999996,
            2.4437499999999996,
            2.436,
            2.4277499999999996,
            2.4189999999999996,
            2.4097499999999994,
            2.3999999999999995
          ]
        }
      ],
      "xlabel": "Constructed behavior coordinate",
      "ylabel": "Arbitrary units",
      "logx": false,
      "logy": false
    }
  ],
  "title": "Optimizing a proxy need not optimize the intended utility",
  "assumptions": "Proxy=x, intended utility=4-(x-2)^2, penalty=0.1*x^2; all constructed, no model outputs.",
  "worked": "构造的真实效用在x=2取最大；代理减惩罚在x=5取最大，此处真实效用反而为−5。它只提供反例，不估计RLHF或DPO的真实退化幅度。",
  "source_ids": [
    "rlhf",
    "dpo",
    "know",
    "arxiv-1503.02531"
  ],
  "formula_source": "Article equations and this versioned script",
  "diagram_nodes": [
    "候选与标注规则",
    "偏好对 / 分歧记录",
    "参考策略与概率比",
    "受约束策略优化",
    "独立及域外评价",
    "能力 / 校准 / 风险分项"
  ]
}