AGILLM-4 / local_verify_m_fold_agillm4.json
OpenTransformer's picture
Harvest exact M-fold attention from n1
3b8aefc
Raw
History Blame Contribute Delete
3.55 kB
[
{
"backend": "manual",
"d": 32,
"dk": 16,
"expected_k_width": 16,
"heads": 2,
"ok": true,
"preset": "pico_1x",
"rank": 16,
"rows": {
"cache_k_width": 16.0,
"cache_v_width": 16.0,
"cached_append_forward": 5.960464477539063e-08,
"causal_alibi_forward": 0.0,
"causal_alibi_loss": 0.0,
"causal_alibi_param_grad": 0.0,
"causal_alibi_x_grad": 0.0,
"none_forward": 0.0,
"none_loss": 0.0,
"none_param_grad": 0.0,
"none_x_grad": 0.0,
"sat_alibi_forward": 0.0,
"sat_alibi_loss": 0.0,
"sat_alibi_param_grad": 0.0,
"sat_alibi_x_grad": 0.0
},
"tol": 0.0002
},
{
"backend": "sdpa",
"d": 32,
"dk": 16,
"expected_k_width": 16,
"heads": 2,
"ok": true,
"preset": "pico_1x",
"rank": 16,
"rows": {
"cache_k_width": 16.0,
"cache_v_width": 16.0,
"cached_append_forward": 5.960464477539063e-08,
"causal_alibi_forward": 7.450580596923828e-08,
"causal_alibi_loss": 0.0,
"causal_alibi_param_grad": 1.862645149230957e-09,
"causal_alibi_x_grad": 2.3283064365386963e-10,
"none_forward": 8.940696716308594e-08,
"none_loss": 0.0,
"none_param_grad": 9.313225746154785e-10,
"none_x_grad": 6.548361852765083e-11,
"sat_alibi_forward": 1.1920928955078125e-07,
"sat_alibi_loss": 1.862645149230957e-09,
"sat_alibi_param_grad": 9.313225746154785e-10,
"sat_alibi_x_grad": 3.4924596548080444e-10
},
"tol": 0.0002
},
{
"backend": "manual",
"d": 128,
"dk": 16,
"expected_k_width": 16,
"heads": 8,
"ok": true,
"preset": "micro_3x",
"rank": 48,
"rows": {
"cache_k_width": 16.0,
"cache_v_width": 16.0,
"cached_append_forward": 6.51925802230835e-08,
"causal_alibi_forward": 5.960464477539063e-08,
"causal_alibi_loss": 0.0,
"causal_alibi_param_grad": 4.656612873077393e-10,
"causal_alibi_x_grad": 5.820766091346741e-11,
"metric_cache_cleared_on_train": 0.0,
"metric_cache_reused": 0.0,
"none_forward": 5.960464477539063e-08,
"none_loss": 0.0,
"none_param_grad": 2.3283064365386963e-10,
"none_x_grad": 2.1827872842550278e-11,
"sat_alibi_forward": 1.1920928955078125e-07,
"sat_alibi_loss": 1.862645149230957e-09,
"sat_alibi_param_grad": 5.820766091346741e-10,
"sat_alibi_x_grad": 7.275957614183426e-11
},
"tol": 0.0002
},
{
"backend": "sdpa",
"d": 128,
"dk": 16,
"expected_k_width": 16,
"heads": 8,
"ok": true,
"preset": "micro_3x",
"rank": 48,
"rows": {
"cache_k_width": 16.0,
"cache_v_width": 16.0,
"cached_append_forward": 7.450580596923828e-08,
"causal_alibi_forward": 1.043081283569336e-07,
"causal_alibi_loss": 0.0,
"causal_alibi_param_grad": 4.656612873077393e-10,
"causal_alibi_x_grad": 9.458744898438454e-11,
"metric_cache_cleared_on_train": 0.0,
"metric_cache_reused": 0.0,
"none_forward": 8.940696716308594e-08,
"none_loss": 0.0,
"none_param_grad": 2.3283064365386963e-10,
"none_x_grad": 2.9103830456733704e-11,
"sat_alibi_forward": 1.1920928955078125e-07,
"sat_alibi_loss": 0.0,
"sat_alibi_param_grad": 6.984919309616089e-10,
"sat_alibi_x_grad": 1.1641532182693481e-10
},
"tol": 0.0002
}
]