Fam Zheng
d923aa1e31
Hopfield + Hebbian hybrid memory system for LLMs. Two nights of experiments (16 iterations), validated on LongMemEval (ICLR 2025). Architecture: - Single-hop: Two-Stage Hopfield (NN top-20 → softmax settle) - Multi-hop: Hebbian W matrix with WTA pattern separation - 64% on LongMemEval (500 questions), retrieval-only, no LLM dependency - 4ms latency @ 20K memories, ~1GB VRAM Key findings: - Hopfield attention solved noise tolerance (20% → 100% vs flat Hebbian) - WTA pattern separation enables 20K+ capacity - Multi-hop associative chains (6 hops, CosSim=1.0) — RAG can't do this - MiniLM-L6 is optimal (discrimination gap > absolute similarity) - Paraphrase cue augmentation: 55% → 100% on synthetic, 36% → 64% on benchmark - SNN encoder viable (CosSim 0.99) but not needed for current architecture
326 lines
6.7 KiB
JSON
326 lines
6.7 KiB
JSON
[
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 2048,
|
|
"k": 20,
|
|
"mean_overlap": 0.010202019593932412,
|
|
"max_overlap": 0.14999999105930328
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 2048,
|
|
"k": 50,
|
|
"mean_overlap": 0.024711112705520306,
|
|
"max_overlap": 0.12000001221895218
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 2048,
|
|
"k": 100,
|
|
"mean_overlap": 0.04898586208941509,
|
|
"max_overlap": 0.15000000596046448
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 4096,
|
|
"k": 20,
|
|
"mean_overlap": 0.005979797623374246,
|
|
"max_overlap": 0.09999999403953552
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 4096,
|
|
"k": 50,
|
|
"mean_overlap": 0.012800000862717027,
|
|
"max_overlap": 0.12000000476837158
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 4096,
|
|
"k": 100,
|
|
"mean_overlap": 0.024448486435405835,
|
|
"max_overlap": 0.11000000685453415
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 8192,
|
|
"k": 20,
|
|
"mean_overlap": 0.0025757574222304604,
|
|
"max_overlap": 0.09999999403953552
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 8192,
|
|
"k": 50,
|
|
"mean_overlap": 0.006472727722968116,
|
|
"max_overlap": 0.06000000238418579
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 8192,
|
|
"k": 100,
|
|
"mean_overlap": 0.012430303828659083,
|
|
"max_overlap": 0.06000000610947609
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 16384,
|
|
"k": 20,
|
|
"mean_overlap": 0.0012222221493721009,
|
|
"max_overlap": 0.09999999403953552
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 16384,
|
|
"k": 50,
|
|
"mean_overlap": 0.003167676991133979,
|
|
"max_overlap": 0.06000000238418579
|
|
},
|
|
{
|
|
"test": "overlap",
|
|
"code_dim": 16384,
|
|
"k": 100,
|
|
"mean_overlap": 0.006484848919202282,
|
|
"max_overlap": 0.05000000447034836
|
|
},
|
|
{
|
|
"test": "sep_pairs_1",
|
|
"correct": 1.0,
|
|
"wrong": 0,
|
|
"disc": 1.0,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 1,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_pairs_5",
|
|
"correct": 1.0000000476837159,
|
|
"wrong": 0.010000000707805157,
|
|
"disc": 0.9900000469759107,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 5,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_pairs_10",
|
|
"correct": 1.0000000715255737,
|
|
"wrong": 0.007111111614439222,
|
|
"disc": 0.9928889599111345,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 10,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_pairs_20",
|
|
"correct": 1.0000000715255737,
|
|
"wrong": 0.007473684719910747,
|
|
"disc": 0.992526386805663,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 20,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_pairs_50",
|
|
"correct": 1.0000000524520873,
|
|
"wrong": 0.006183673899468719,
|
|
"disc": 0.9938163785526186,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 50,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_pairs_100",
|
|
"correct": 1.0000000536441802,
|
|
"wrong": 0.005727273127948395,
|
|
"disc": 0.9942727805162318,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_pairs_200",
|
|
"correct": 1.0000000607967376,
|
|
"wrong": 0.00616582957512919,
|
|
"disc": 0.9938342312216084,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 200,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_pairs_500",
|
|
"correct": 1.0000000553131103,
|
|
"wrong": 0.006348697836501506,
|
|
"disc": 0.9936513574766088,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 500,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_codedim_2048",
|
|
"correct": 1.0000000619888305,
|
|
"wrong": 0.0245959611731873,
|
|
"disc": 0.9754041008156432,
|
|
"code_dim": 2048,
|
|
"k_active": 50,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_codedim_4096",
|
|
"correct": 1.0000000619888305,
|
|
"wrong": 0.01184848565608263,
|
|
"disc": 0.9881515763327479,
|
|
"code_dim": 4096,
|
|
"k_active": 50,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_codedim_8192",
|
|
"correct": 1.0000000548362733,
|
|
"wrong": 0.006383838832867567,
|
|
"disc": 0.9936162160034058,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_codedim_16384",
|
|
"correct": 1.0000000536441802,
|
|
"wrong": 0.003434343673665114,
|
|
"disc": 0.9965657099705151,
|
|
"code_dim": 16384,
|
|
"k_active": 50,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_k_10",
|
|
"correct": 1.0,
|
|
"wrong": 0.0013131313326984946,
|
|
"disc": 0.9986868686673015,
|
|
"code_dim": 8192,
|
|
"k_active": 10,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_k_20",
|
|
"correct": 0.9999999302625656,
|
|
"wrong": 0.002348484708504243,
|
|
"disc": 0.9976514455540614,
|
|
"code_dim": 8192,
|
|
"k_active": 20,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_k_50",
|
|
"correct": 1.000000058412552,
|
|
"wrong": 0.0059797983936438655,
|
|
"disc": 0.9940202600189081,
|
|
"code_dim": 8192,
|
|
"k_active": 50,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_k_100",
|
|
"correct": 1.0000000667572022,
|
|
"wrong": 0.012792930120338846,
|
|
"disc": 0.9872071366368633,
|
|
"code_dim": 8192,
|
|
"k_active": 100,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "sep_k_200",
|
|
"correct": 1.000000069141388,
|
|
"wrong": 0.025040405879098206,
|
|
"disc": 0.9749596632622898,
|
|
"code_dim": 8192,
|
|
"k_active": 200,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "cap_10",
|
|
"correct": 0.9999999344348908,
|
|
"wrong": 0.001111111044883728,
|
|
"disc": 0.9988888233900071,
|
|
"code_dim": 16384,
|
|
"k_active": 20,
|
|
"num_pairs": 10,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "cap_50",
|
|
"correct": 0.9999999284744263,
|
|
"wrong": 0.001836734584399632,
|
|
"disc": 0.9981631938900267,
|
|
"code_dim": 16384,
|
|
"k_active": 20,
|
|
"num_pairs": 50,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "cap_100",
|
|
"correct": 0.9999999344348908,
|
|
"wrong": 0.0014141413298520175,
|
|
"disc": 0.9985857931050387,
|
|
"code_dim": 16384,
|
|
"k_active": 20,
|
|
"num_pairs": 100,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "cap_200",
|
|
"correct": 0.9999999329447746,
|
|
"wrong": 0.0011055275722963726,
|
|
"disc": 0.9988944053724782,
|
|
"code_dim": 16384,
|
|
"k_active": 20,
|
|
"num_pairs": 200,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "cap_500",
|
|
"correct": 0.9999999303817749,
|
|
"wrong": 0.001167334599760109,
|
|
"disc": 0.9988325957820148,
|
|
"code_dim": 16384,
|
|
"k_active": 20,
|
|
"num_pairs": 500,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "cap_1000",
|
|
"correct": 0.9999999321103096,
|
|
"wrong": 0.0012262261531374476,
|
|
"disc": 0.9987737059571721,
|
|
"code_dim": 16384,
|
|
"k_active": 20,
|
|
"num_pairs": 1000,
|
|
"lr": 1.0
|
|
},
|
|
{
|
|
"test": "cap_2000",
|
|
"correct": 0.999999930858612,
|
|
"wrong": 0.0013319158785906119,
|
|
"disc": 0.9986680149800214,
|
|
"code_dim": 16384,
|
|
"k_active": 20,
|
|
"num_pairs": 2000,
|
|
"lr": 1.0
|
|
}
|
|
] |