{"id":10020187,"date":"2024-07-09T00:00:00","date_gmt":"2024-07-09T00:00:00","guid":{"rendered":"https:\/\/itmax.cn\/?p=10020187"},"modified":"-0001-11-30T00:00:00","modified_gmt":"-0001-11-29T16:00:00","slug":"","status":"publish","type":"post","link":"https:\/\/itmax.cn\/?p=10020187","title":{"rendered":"\u5927\u6a21\u578b\u6700\u5f3a\u67b6\u6784TTT\u95ee\u4e16\uff01\u65af\u5766\u798fUCSD\u7b495\u5e74\u78e8\u4e00\u5251\uff0c \u4e00\u591c\u63a8\u7ffbTransformer"},"content":{"rendered":"<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/06\/01\/75d31620-202e-11ef-86d7-00163e142b65.png\"><\/p>\n<p>\u4e00\u89c9\u9192\u6765\uff0c\u8d85\u8d8aTransformer\u548cMamba\u7684\u65b0\u67b6\u6784\u8bde\u751f\u4e86\uff1f<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e54e01aa-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u65af\u5766\u798f\u3001UCSD\u3001UC\u4f2f\u514b\u5229\u548cMeta\u7684\u7814\u7a76\u4eba\u5458\u63d0\u51fa\u4e86\u4e00\u79cd\u5168\u65b0\u67b6\u6784\uff0c\u7528\u673a\u5668\u5b66\u4e60\u6a21\u578b\u53d6\u4ee3RNN\u7684\u9690\u85cf\u72b6\u6001\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e5a8e354-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u8bba\u6587\u5730\u5740\uff1ahttps:\/\/arxiv.org\/abs\/2407.04620<\/p>\n<p>\u8fd9\u4e2a\u6a21\u578b\u901a\u8fc7\u5bf9\u8f93\u5165token\u8fdb\u884c\u68af\u5ea6\u4e0b\u964d\u6765\u538b\u7f29\u4e0a\u4e0b\u6587\uff0c\u8fd9\u79cd\u65b9\u6cd5\u88ab\u79f0\u4e3a\u300c\u6d4b\u8bd5\u65f6\u95f4\u8bad\u7ec3\u5c42\uff08Test-Time-Training layers\uff0cTTT\uff09\u300d\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e60d9196-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>TTT\u5c42\u76f4\u63a5\u66ff\u4ee3\u4e86\u6ce8\u610f\u529b\u673a\u5236\uff0c\u89e3\u9501\u4e86\u5177\u6709\u8868\u73b0\u529b\u8bb0\u5fc6\u7684\u7ebf\u6027\u590d\u6742\u5ea6\u67b6\u6784\uff0c\u4f7f\u6211\u4eec\u80fd\u591f\u5728\u4e0a\u4e0b\u6587\u4e2d\u8bad\u7ec3\u5305\u542b\u6570\u767e\u4e07\uff08\u672a\u6765\u53ef\u80fd\u662f\u6570\u5341\u4ebf\uff09\u4e2atoken\u7684LLM\u3002<\/p>\n<p>\u4f5c\u8005\u76f8\u4fe1\uff0c\u8fd9\u4e2a\u7814\u7a76\u4e86\u4e00\u5e74\u591a\u7684\u9879\u76ee\uff0c\u5c06\u4ece\u6839\u672c\u4e0a\u6539\u53d8\u6211\u4eec\u7684\u8bed\u8a00\u6a21\u578b\u65b9\u6cd5\u3002<\/p>\n<div class=\"js-star yyp--fancyPost\" data-id=\"45\"><\/div>\n<p>\u800c\u7ed3\u679c\u8bc1\u660e\uff0cTTT-Linear\u548cTTT-MLP\u76f4\u63a5\u8d76\u8d85\u6216\u51fb\u8d25\u4e86\u6700\u5f3a\u7684Transformer\u548cMamba\uff01<\/p>\n<p>\u4f5c\u8005\u4e4b\u4e00\u7684Xiaolong Wang\u60ca\u559c\u5730\u8868\u793a\uff1a\u4e0d\u6562\u76f8\u4fe1\uff0c\u6211\u4eec\u771f\u7684\u505a\u5230\u4e86\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e6c3beee-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u66f4\u4ee4\u4eba\u5174\u594b\u7684\u662f\uff0c\u867d\u7136\u76ee\u524dTTT\u53ea\u5e94\u7528\u4e8e\u8bed\u8a00\u5efa\u6a21\uff0c\u4f46\u5728\u672a\u6765\uff0c\u5b83\u4e5f\u53ef\u4ee5\u7528\u5728\u957f\u89c6\u9891\u4e0a\uff0c\u53ef\u8c13\u524d\u666f\u8fdc\u5927\u3002<\/p>\n<p>\u5728\u5c06\u6765\uff0c\u5f53\u6211\u4eec\u5bf9\u957f\u89c6\u9891\u8fdb\u884c\u5efa\u6a21\u65f6\uff0c\u5c31\u53ef\u4ee5\u5bf9\u5e27\u8fdb\u884c\u5bc6\u96c6\u91c7\u6837\uff0c\u800c\u4e0d\u662f\u91c7\u68371FPS\u4e86\u3002\u8fd9\u4e9b\u5bc6\u96c6\u5e27\u5bf9Transformer\u662f\u4e00\u79cd\u8d1f\u62c5\uff0c\u4f46\u5bf9\u4e8eTTT\u5c42\u6765\u8bf4\uff0c\u8fd9\u5374\u662f\u4e00\u79cd\u798f\u97f3\uff01<\/p>\n<h2  id=\"toc-1\" class=\"jltoc--item\">01 \u4e00\u4e2a5\u5e74\u591a\u7684\u60f3\u6cd5\uff0c\u7ec8\u4e8e\u5b9e\u73b0\u4e86<\/h2>\n<p>\u4f5c\u8005\u8868\u793a\uff0c\u5728\u8fc7\u53bb\u76841.5\u5e74\u91cc\uff0c\u56e2\u961f\u4e00\u76f4\u5728\u5f00\u53d1\u4e00\u79cd\u65b0\u7684LLM\u67b6\u6784\uff0c\u53ef\u4ee5\u5177\u6709\u7ebf\u6027\u590d\u6742\u5ea6\u548c\u66f4\u5f3a\u7684\u9690\u85cf\u72b6\u6001\uff0c\u7528\u4e8e\u957f\u4e0a\u4e0b\u6587\u5efa\u6a21\u3002<\/p>\n<p>\u800c\u8fd9\u4e2a\u6d4b\u8bd5\u65f6\u8bad\u7ec3\uff08TTT\uff09\u7684\u60f3\u6cd5\uff0c\u5df2\u7ecf\u7814\u7a76\u4e86\u8d85\u8fc75\u5e74\u3002<\/p>\n<p>Xiaolong\u6e05\u6670\u8bb0\u5f97\uff0c\u5728\u521a\u5f00\u59cb\u505a\u535a\u58eb\u540e\u65f6\uff0cAlyosha\u66fe\u8ba9\u81ea\u5df1\u53bb\u627eYu Sun\u8ba8\u8bbaTTT\u3002<\/p>\n<p>\u8fd9\u6b21\u4f1a\u9762\uff0c\u5c31\u662f\u8fd9\u9879\u7814\u7a76\u7684\u8d77\u70b9\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e7845848-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u5e8f\u5217\u6a21\u578b\u4f1a\u628a\u5386\u53f2\u4e0a\u4e0b\u6587\u5b58\u50a8\u5728\u4e00\u4e2a\u9690\u85cf\u72b6\u6001\u4e2d\u3002<\/p>\n<p>\u50cfMamba\u8fd9\u6837\u7684RNN\u5c42\uff0c\u4f1a\u968f\u7740\u65f6\u95f4\u7684\u63a8\u79fb\u538b\u7f29\u6210\u4e00\u4e2a\u56fa\u5b9a\u5927\u5c0f\u7684\u72b6\u6001\uff0c\u5b83\u4eec\u867d\u7136\u6548\u7387\u5f88\u9ad8\uff0c\u4f46\u6027\u80fd\u53d7\u9650\u4e8e\u5176\u8868\u8fbe\u80fd\u529b\u3002<\/p>\n<p>\u6ce8\u610f\u529b\u673a\u5236\u6709\u4e00\u4e2aKV\u7f13\u5b58\uff0c\u5b83\u4f1a\u968f\u7740\u65f6\u95f4\u7684\u63a8\u79fb\u4e0d\u65ad\u589e\u957f\u3002\u8fd9\u4e2a\u72b6\u6001\u4e0d\u4f1a\u538b\u7f29\u4efb\u4f55\u5386\u53f2\u4e0a\u4e0b\u6587\uff0c\u4f46\u968f\u7740\u4e0a\u4e0b\u6587\u957f\u5ea6\u7684\u589e\u52a0\uff0c\u6210\u672c\u4e5f\u4f1a\u8d8a\u6765\u8d8a\u9ad8\u3002<\/p>\n<p>\u56e2\u961f\u6210\u5458\u60f3\uff1a\u65e2\u7136\u8fd9\u6837\uff0c\u4e3a\u4ec0\u4e48\u4e0d\u628a\u4e0a\u4e0b\u6587\u538b\u7f29\u5230\u6a21\u578b\u7684\u6743\u91cd\u4e2d\u2014\u2014\u5c31\u50cfLLM\u5904\u7406\u4e92\u8054\u7f51\u6570\u636e\u90a3\u6837\u5462\uff1f<\/p>\n<p>\u8fd9\u79cd\u300c\u9690\u85cf\u72b6\u6001\u6a21\u578b\u300d\u65e2\u80fd\u5728\u65f6\u95f4\u4e0a\u4fdd\u6301\u56fa\u5b9a\u5927\u5c0f\uff0c\u53c8\u80fd\u5927\u5927\u589e\u5f3a\u8868\u8fbe\u80fd\u529b\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e7e5aecc-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u7814\u7a76\u4eba\u5458\u4f7f\u7528\u4e86\u81ea\u76d1\u7763\u5b66\u4e60\u6765\u66f4\u65b0\u9690\u85cf\u72b6\u6001\u7684\u6743\u91cd\uff0c\u5bf9\u6bcf\u4e2atoken\u8fdb\u884c\u4e00\u6b21\u68af\u5ea6\u4e0b\u964d\u3002\u5728\u5904\u7406\u4e00\u4e2a\u5e8f\u5217\u65f6\uff0c\u8be5\u72b6\u6001\u5df2\u7ecf\u5728\u5176\u4e0a\u4e0b\u6587\u7a97\u53e3\u4e2d\u7684token\u4e0a\u300c\u8bad\u7ec3\u300d\u8fc7\u4e86\u3002<\/p>\n<p>\u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0c\u9690\u85cf\u72b6\u6001\u53ea\u5b58\u5728\u4e8e\u7aef\u5230\u7aef\u67b6\u6784\u4e2d\u7684\u4e00\u5c42\u3002\u5176\u4ed6\u7ec4\u4ef6\uff0c\u6bd4\u5982QKV\u6295\u5f71\u77e9\u9635\uff0c\u662f\u5728\u9884\u8bad\u7ec3\u671f\u95f4\u901a\u8fc7\u6807\u51c6\u7684\u4ea4\u53c9\u71b5\u76ee\u6807\u51fd\u6570\u5b66\u4e60\u7684\u3002<\/p>\n<p>\u56e0\u6b64\uff0c\u7aef\u5230\u7aef\u67b6\u6784\u5b9e\u9645\u4e0a\u662f\u5728\u8fdb\u884c\u5143\u5b66\u4e60\uff0c\u5bfb\u627e\u538b\u7f29\u4e0a\u4e0b\u6587\u7684\u6700\u4f73\u65b9\u5f0f\uff0c\u4ee5\u4fbf\u66f4\u597d\u5730\u9884\u6d4b\u4e0b\u4e00\u4e2atoken\uff0c\u4e5f\u5c31\u662f\u5728\u300c\u5b66\u4e60\u5982\u4f55\u5728\u6d4b\u8bd5\u65f6\u5b66\u4e60\u300d\u3002<\/p>\n<p>\u7ed3\u679c\u663e\u793a\uff0c\u4e0eMamba\u76f8\u6bd4\uff0cTTT-Linear\u5177\u6709\u66f4\u597d\u7684\u56f0\u60d1\u5ea6\u548c\u66f4\u5c11\u7684FLOP\uff08\u5de6\uff09\uff0c\u5e76\u4e14\u66f4\u597d\u5730\u5229\u7528\u4e86\u957f\u4e0a\u4e0b\u6587\uff08\u53f3\uff09\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e842e3c6-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u4e0b\u56fe\u663e\u793a\u4e86\u6279\u5927\u5c0f\u4e3a16\u7684\u60c5\u51b5\u4e0b\uff0c\u968f\u7740\u4e0a\u4e0b\u6587\u957f\u5ea6\u7684\u53d8\u5316\uff0c\u6bcf\u4e2atoken\u7684\u524d\u5411\u65f6\u95f4\uff08\u5ef6\u8fdf\uff09\u3002\u6240\u6709\u6a21\u578b\u7684\u53c2\u6570\u90fd\u662f1.3B\uff08Mamba\u4e3a1.4B\uff09\u3002<\/p>\n<p>\u53ef\u4ee5\u770b\u5230\uff0c\u968f\u7740\u4e0a\u4e0b\u6587\u957f\u5ea6\u7684\u589e\u52a0\uff0cTransformer\u6bcf\u4e2atoken\u7684\u524d\u5411\u65f6\u95f4\u5448\u7ebf\u6027\u589e\u957f\uff0c\u4f46\u5176\u4ed6\u4e24\u79cd\u65b9\u6cd5\u7684\u524d\u5411\u65f6\u95f4\u57fa\u672c\u4fdd\u6301\u4e0d\u53d8\u3002<\/p>\n<p>\u57288k\u4e0a\u4e0b\u6587\u65f6\uff0cTTT-Linear\u6bd4Transformer\u66f4\u5feb\uff0c\u4e0eMamba\u76f8\u5f53\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e8a1843a-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<h2  id=\"toc-2\" class=\"jltoc--item\">02 RNN\u7684\u5c34\u5c2c\u73b0\u5b9e<\/h2>\n<p>2020\u5e74\uff0cOpenAI\u7f29\u653e\u5b9a\u5f8b\u8bba\u6587\u8868\u660eLSTM\uff08RNN\u7684\u4e00\u79cd\uff09\u65e0\u6cd5\u50cfTransformer\u90a3\u6837\u8fdb\u884c\u7f29\u653e\uff0c\u6216\u6709\u6548\u5730\u4f7f\u7528\u957f\u4e0a\u4e0b\u6587\u3002<\/p>\n<p>\u771f\u7684\u662f\u8fd9\u6837\u5417\uff1f<\/p>\n<p>\u5728\u8fd9\u4e2a\u9879\u76ee\u4e2d\uff0c\u7814\u7a76\u4eba\u5458\u91cd\u65b0\u8bc4\u4f30\u4e86\u56fe2\u4e2d\u7684\u8fd9\u4e9b\u53d1\u73b0\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e8f64b6e-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u5728\u5de6\u4fa7\uff0c\u53ef\u4ee5\u89c2\u5bdf\u5230Mamba\uff08\u5f53\u4eca\u6700\u6d41\u884c\u7684RNN\u4e4b\u4e00\uff09\u7684\u6269\u5c55\u6027\u4e0e\u5f3a\u5927\u7684Transformer\u7c7b\u4f3c\uff0c\u8fd9\u662f\u81ea2020\u5e74\u7684LSTM\u4ee5\u6765\u663e\u793a\u51fa\u7684\u5de8\u5927\u8fdb\u6b65\u3002<\/p>\n<p>\u7136\u800c\uff0c\u5728\u53f3\u4fa7\uff0c\u53ef\u4ee5\u89c2\u5bdf\u5230\u4e0eOpenAI\u76f8\u540c\u7684Mamba\u95ee\u9898\u3002<\/p>\n<p>\u5e73\u5747\u800c\u8a00\uff0c\u5e8f\u5217\u4e2d\u9760\u540e\u7684token\u5e94\u8be5\u66f4\u5bb9\u6613\u9884\u6d4b\uff0c\u56e0\u4e3a\u5b83\u4eec\u4ee5\u66f4\u591a\u4fe1\u606f\u4e3a\u6761\u4ef6\u3002<\/p>\n<p>\u5bf9Transformer\u6765\u8bf4\u786e\u5b9e\u5982\u6b64\uff0c\u6bcf\u4e2atoken\u7d22\u5f15\u7684\u5e73\u5747\u590d\u6742\u5ea6\u5728\u517632k\u4e0a\u4e0b\u6587\u4e2d\u4e0d\u65ad\u51cf\u5c11\u3002\u76f8\u6bd4\u4e4b\u4e0b\uff0cMamba\u572816k\u540e\u5c31\u51fa\u73b0\u4e86\u540c\u6837\u7684\u60c5\u51b5\u3002<\/p>\n<p>\u5bf9\u4e8e\u73b0\u6709\u7684RNN\u6765\u8bf4\uff0c\u8fd9\u4e2a\u7ed3\u679c\u4ee3\u8868\u4e86\u4e00\u4e2a\u5c34\u5c2c\u7684\u73b0\u5b9e\u2014\u2014<\/p>\n<p>\u4e00\u65b9\u9762\uff0cRNN\uff08\u76f8\u5bf9\u4e8eTransformer\uff09\u7684\u4e3b\u8981\u4f18\u52bf\u5c31\u662f\u5b83\u4eec\u7684\u7ebf\u6027\uff08\u76f8\u5bf9\u4e8e\u4e8c\u6b21\uff09\u590d\u6742\u6027\u3002\u8fd9\u79cd\u6e10\u8fdb\u4f18\u52bf\u5b9e\u9645\u4e0a\u53ea\u4f1a\u5728\u957f\u4e0a\u4e0b\u6587\u4e2d\u5b9e\u73b0\u3002<\/p>\n<p>\u53e6\u4e00\u65b9\u9762\uff0c\u4e00\u65e6\u4e0a\u4e0b\u6587\u8db3\u591f\u957f\uff0c\u73b0\u6709\u7684RNN\uff08\u5982Mamba\uff09\u5c31\u5f88\u96be\u771f\u6b63\u5229\u7528\u989d\u5916\u7684\u6761\u4ef6\u4fe1\u606f\u3002<\/p>\n<p>\u957f\u4e0a\u4e0b\u6587\u7684\u56f0\u96be\u662fRNN\u5c42\u672c\u8d28\u4e0a\u7684\u95ee\u9898\uff1a\u4e0e\u81ea\u6ce8\u610f\u529b\u673a\u5236\u4e0d\u540c\uff0cRNN\u5c42\u5fc5\u987b\u5c06\u4e0a\u4e0b\u6587\u538b\u7f29\u4e3a\u56fa\u5b9a\u5927\u5c0f\u7684\u9690\u85cf\u72b6\u6001\u3002<\/p>\n<p>\u4f5c\u4e3a\u4e00\u79cd\u538b\u7f29\u542f\u53d1\u5f0f\uff0c\u66f4\u65b0\u89c4\u5219\u9700\u8981\u53d1\u73b0\u6210\u5343\u4e0a\u4e07\u751a\u81f3\u6570\u767e\u4e07\u4e2atoken\u4e4b\u95f4\u7684\u5e95\u5c42\u7ed3\u6784\u548c\u5173\u7cfb\u3002<\/p>\n<p>\u7814\u7a76\u4eba\u5458\u9996\u5148\u89c2\u5bdf\u5230\uff0c\u81ea\u76d1\u7763\u5b66\u4e60\u53ef\u4ee5\u5c06\u5927\u91cf\u8bad\u7ec3\u96c6\u538b\u7f29\u4e3aLLM\u7b49\u6a21\u578b\u7684\u6743\u91cd\uff0c\u8be5\u6a21\u578b\u901a\u5e38\u8868\u73b0\u51fa\u5bf9\u5176\u8bad\u7ec3\u6570\u636e\u4e4b\u95f4\u8bed\u4e49\u8054\u7cfb\u7684\u6df1\u523b\u7406\u89e3\uff0c\u800c\u8fd9\uff0c\u6070\u6070\u662f\u4ed6\u4eec\u6240\u9700\u8981\u7684\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/e96e510e-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<h3>1. TTT\u5c42<\/h3>\n<p>\u53d7\u6b64\u542f\u53d1\uff0c\u7814\u7a76\u4eba\u5458\u8bbe\u8ba1\u4e86\u4e00\u7c7b\u65b0\u7684\u5e8f\u5217\u5efa\u6a21\u5c42\uff0c\u5176\u4e2d\u9690\u85cf\u72b6\u6001\u662f\u6a21\u578b\uff0c\u66f4\u65b0\u89c4\u5219\u662f\u81ea\u76d1\u7763\u5b66\u4e60\u7684\u4e00\u4e2a\u6b65\u9aa4\u3002<\/p>\n<p>\u7531\u4e8e\u66f4\u65b0\u6d4b\u8bd5\u5e8f\u5217\u4e0a\u9690\u85cf\u72b6\u6001\u7684\u8fc7\u7a0b\uff0c\u76f8\u5f53\u4e8e\u5728\u6d4b\u8bd5\u65f6\u8bad\u7ec3\u6a21\u578b\uff0c\u56e0\u6b64\u6b64\u7c7b\u65b0\u5c42\u79f0\u4e3a\u6d4b\u8bd5\u65f6\u8bad\u7ec3\uff08TTT\uff09\u5c42\u3002<\/p>\n<p>\u7814\u7a76\u4eba\u5458\u5f15\u5165\u4e24\u4e2a\u7b80\u5355\u7684\u5b9e\u4f8b\uff1aTTT-Linear\u548cTTT-MLP\uff0c\u5176\u4e2d\u9690\u85cf\u72b6\u6001\u5206\u522b\u662f\u7ebf\u6027\u6a21\u578b\u548c\u4e24\u5c42MLP\u3002TTT\u5c42\u53ef\u4ee5\u96c6\u6210\u5230\u4efb\u4f55\u7f51\u7edc\u67b6\u6784\u4e2d\u5e76\u8fdb\u884c\u7aef\u5230\u7aef\u4f18\u5316\uff0c\u7c7b\u4f3c\u4e8eRNN\u5c42\u548c\u81ea\u6ce8\u610f\u529b\u3002<\/p>\n<h3>2. \u5b9e\u9645\u8fd0\u884c\u65f6\u95f4<\/h3>\n<p>TTT\u5c42\u5728FLOP\u65b9\u9762\u5df2\u7ecf\u975e\u5e38\u9ad8\u6548\uff0c\u7814\u7a76\u4eba\u5458\u5219\u66f4\u8fdb\u4e00\u6b65\u5730\u63d0\u51fa\u4e86\u4e24\u9879\u521b\u65b0\uff0c\u4f7f\u5176\u5728\u5b9e\u9645\u8fd0\u884c\u65f6\u95f4\u5185\u4e5f\u80fd\u4fdd\u6301\u9ad8\u6548\u3002<\/p>\n<p>\u9996\u5148\uff0c\u4e0e\u5728\u5e38\u89c4\u8bad\u7ec3\u4e2d\u5bf9mini-batch\u5e8f\u5217\u91c7\u53d6\u68af\u5ea6\u6b65\u8fdb\u4ee5\u5b9e\u73b0\u66f4\u597d\u7684\u5e76\u884c\u6027\u7c7b\u4f3c\uff0c\u4ed6\u4eec\u4e5f\u5728TTT\u4e2d\u4f7f\u7528\u4e86mini-batch\u7684token\u3002<\/p>\n<p>\u5176\u6b21\uff0c\u7814\u7a76\u4eba\u5458\u4e3a\u6bcf\u4e2aTTT mini-batch\u5185\u7684\u64cd\u4f5c\u5f00\u53d1\u4e86\u4e00\u79cd\u5bf9\u5076\u5f62\u5f0f\uff0c\u4ee5\u66f4\u597d\u5730\u5229\u7528\u73b0\u4ee3GPU\u548cTPU\u3002\u8fd9\u79cd\u5bf9\u5076\u5f62\u5f0f\u7684\u8f93\u51fa\u4e0e\u539f\u59cb\u5b9e\u73b0\u76f8\u5f53\uff0c\u4f46\u8bad\u7ec3\u901f\u5ea6\u5374\u5feb\u4e865\u500d\u4ee5\u4e0a\u3002<\/p>\n<p>\u6b63\u5982\u56fe3\u6240\u793a\uff0cTTT-Linear\u57288k\u4e0a\u4e0b\u6587\u4e2d\u6bd4Transformer\u66f4\u5feb\uff0c\u5e76\u4e14\u4e0eMamba\u76f8\u5f53\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/ea166150-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<h2  id=\"toc-3\" class=\"jltoc--item\">03 Transformer\u6740\u624b\u2014\u2014TTT<\/h2>\n<p>\u5982\u56fe4\u6240\u793a\uff0c\u6240\u6709\u7684\u5e8f\u5217\u5efa\u6a21\u5c42\uff0c\u90fd\u53ef\u4ee5\u4ece\u5c06\u5386\u53f2\u4e0a\u4e0b\u6587\u5b58\u50a8\u5230\u9690\u85cf\u72b6\u6001\u7684\u89d2\u5ea6\u6765\u770b\u5f85\u3002<\/p>\n<p>\u6bd4\u5982\uff0cRNN\u5c42\u2014\u2014\u5982LSTM\u3001RWKV\u548cMamba\u5c42\u2014\u2014\u5c06\u4e0a\u4e0b\u6587\u538b\u7f29\u6210\u4e00\u4e2a\u56fa\u5b9a\u5927\u5c0f\u7684\u72b6\u6001\uff0c\u8fd9\u4e2a\u72b6\u6001\u968f\u65f6\u95f4\u53d8\u5316\u3002<\/p>\n<p>\u8fd9\u79cd\u538b\u7f29\u5e26\u6765\u4e86\u4e24\u79cd\u7ed3\u679c\uff1a\u4f18\u52bf\u662f\u5904\u7406\u6548\u7387\u9ad8\uff0c\u56e0\u4e3a\u6bcf\u4e2atoken\u7684\u5904\u7406\u65f6\u95f4\u662f\u6052\u5b9a\u7684\u3002\u52a3\u52bf\u662f\u5728\u5904\u7406\u957f\u4e0a\u4e0b\u6587\u65f6\uff0cRNN\u6027\u80fd\u53d7\u9650\u4e8e\u9690\u85cf\u72b6\u6001\u7684\u300c\u8868\u8fbe\u80fd\u529b\u300d\u3002<\/p>\n<p>\u81ea\u6ce8\u610f\u529b\u673a\u5236\uff08Self-attention\uff09\u4e5f\u53ef\u4ee5\u4ece\u5982\u4e0a\u89d2\u5ea6\u6765\u7406\u89e3\u3002<\/p>\n<p>\u4e0d\u540c\u4e4b\u5904\u5728\u4e8e\uff0c\u5b83\u7684\u9690\u85cf\u72b6\u6001\uff0c\u901a\u5e38\u79f0\u4e3a\u952e\u503c\uff08KV\uff09\u7f13\u5b58\u662f\u4e00\u4e2a\u968ft\u589e\u957f\u7684\u7ebf\u6027list\u3002<\/p>\n<p>\u5b83\u53ef\u4ee5\u5b58\u50a8\u6240\u6709\u7684\u4e0a\u4e0b\u6587\uff0c\u5e76\u4e14\u4e0d\u4f1a\u8fdb\u884c\u538b\u7f29\uff0c\u5177\u6709\u5f88\u597d\u7684\u8868\u8fbe\u80fd\u529b\uff0c\u4e0d\u8fc7\u5176\u5904\u7406\u65f6\u95f4\u968f\u4e0a\u4e0b\u6587\u957f\u5ea6\u7ebf\u6027\u589e\u957f\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/ea747074-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u56e0\u6b64\uff0c\u4e3a\u4e86\u5728\u957f\u4e0a\u4e0b\u6587\u4e2d\u65e2\u4fdd\u6301\u6548\u7387\uff0c\u53c8\u5177\u6709\u8868\u8fbe\u80fd\u529b\uff0c\u9700\u8981\u4e00\u4e2a\u66f4\u597d\u7684\u201c\u538b\u7f29\u542f\u53d1\u5f0f\u201d\uff08compression heuristic\uff09\u65b9\u6cd5\u3002<\/p>\n<p>\u5177\u4f53\u6765\u8bf4\uff0c\u5c31\u9700\u8981\u5c06\u6570\u767e\u4e07\u4e2atoken\u538b\u7f29\u6210\u4e00\u4e2a\u80fd\u6709\u6548\u6355\u6349\u5176\u5e95\u5c42\u7ed3\u6784\u548c\u5173\u7cfb\u7684\u9690\u85cf\u72b6\u6001\u3002<\/p>\n<h3>1. TTT\u9690\u85cf\u72b6\u6001<\/h3>\n<p>\u7814\u7a76\u4eba\u5458\u7684\u5173\u952e\u601d\u60f3\u662f\uff0c\u4f7f\u7528\u81ea\u76d1\u7763\u5b66\u4e60\u6765\u5c06\u5386\u53f2\u4e0a\u4e0b\u6587x1,&#8230;,xt\u538b\u7f29\u6210\u4e00\u4e2a\u9690\u85cf\u72b6\u6001St\u3002<\/p>\n<p>\u65b9\u6cd5\u662f\u5c06\u4e0a\u4e0b\u6587\u89c6\u4e3a\u4e00\u4e2a\u65e0\u6807\u7b7e\u6570\u636e\u96c6\uff0c\u800c\u5c06\u72b6\u6001\u89c6\u4e3a\u4e00\u4e2a\u6a21\u578b\u3002<\/p>\n<p>\u5177\u4f53\u6765\u8bf4\uff0c\u9690\u85cf\u72b6\u6001St\u73b0\u5728\u7b49\u540c\u4e8e\u4e00\u4e2a\u6a21\u578bf\u7684\u6743\u91cdWt\uff0c\u8fd9\u4e2a\u6a21\u578bf\u53ef\u4ee5\u662f\u7ebf\u6027\u6a21\u578b\u3001\u5c0f\u578b\u795e\u7ecf\u7f51\u7edc\u6216\u5176\u4ed6\u4efb\u4f55\u5f62\u5f0f\u3002\u8f93\u51fa\u89c4\u5219\u7b80\u5355\u5730\u8868\u793a\u4e3a\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/ec8d408e-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u76f4\u89c2\u6765\u8bb2\uff0c\u8f93\u51fatoken\u5c31\u662f\u7531\u66f4\u65b0\u540e\u6743\u91cdWt\u7684\u6a21\u578bf\u5bf9xt\u6240\u505a\u7684\u9884\u6d4b\u3002\u66f4\u65b0\u89c4\u5219\u662f\u5728\u67d0\u4e2a\u81ea\u76d1\u7763\u635f\u5931\u2113\u4e0a\u8fdb\u884c\u7684\u4e00\u6b65\u68af\u5ea6\u4e0b\u964d\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/ed2dece6-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u5176\u4e2d\u5b66\u4e60\u7387\u4e3a\u03b7\u3002\u4ece\u538b\u7f29\u7684\u89d2\u5ea6\u6765\u770b\uff0c\u6bcf\u79cd\u542f\u53d1\u5f0f\u65b9\u6cd5\u90fd\u9700\u8981\u51b3\u5b9a\u8bb0\u4f4f\/\u5fd8\u8bb0\u54ea\u4e9b\u8f93\u5165\u3002W\u4f1a\u8bb0\u4f4f\u90a3\u4e9b\u4ea7\u751f\u5927\u68af\u5ea6\u7684\u8f93\u5165\u2014\u2014\u76f4\u89c2\u5730\u8bf4\uff0c\u5c31\u662f\u90a3\u4e9b\u4f7fW\u5b66\u4e60\u5f88\u591a\u7684\u8f93\u5165\u3002<\/p>\n<p>\u2113\u7684\u4e00\u79cd\u9009\u62e9\u662f\u91cd\u6784xt\u672c\u8eab\u3002\u4e3a\u4e86\u4f7f\u5b66\u4e60\u95ee\u9898\u53d8\u5f97\u975e\u5e73\u51e1\uff0c\u4f5c\u8005\u9996\u5148\u5c06xt\u5904\u7406\u6210\u4e00\u4e2a\u88ab\u7834\u574f\u7684\u8f93\u5165x\u0304t\uff0c\u7136\u540e\u4f18\u5316\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/edd19e0e-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u7c7b\u4f3c\u4e8e\u53bb\u566a\u81ea\u7f16\u7801\u5668\uff0cf\u9700\u8981\u53d1\u73b0xt\u5404\u7ef4\u5ea6\u4e4b\u95f4\u7684\u76f8\u5173\u6027\uff0c\u4ee5\u4fbf\u4ece\u90e8\u5206\u4fe1\u606fx\u0304t\u4e2d\u91cd\u6784\u51faxt\u3002<\/p>\n<p>\u5982\u56fe5\u6240\u793a\uff0c\u68af\u5ea6\u4e0b\u964d\u80fd\u591f\u51cf\u5c11\u2113\uff0c\u4f46\u65e0\u6cd5\u5c06\u5176\u964d\u81f3\u96f6\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/ee5157fc-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u4e0e\u5176\u4ed6RNN\u5c42\u548c\u81ea\u6ce8\u610f\u529b\u673a\u5236\u4e00\u6837\uff0c\u7814\u7a76\u4eba\u5458\u5c06\u8f93\u5165\u5e8f\u5217x1,&#8230;,xT\u6620\u5c04\u5230\u8f93\u51fa\u5e8f\u5217Z1,&#8230;,ZT\u7684\u7b97\u6cd5\u53ef\u4ee5\u88ab\u7f16\u7a0b\u5230\u5e8f\u5217\u5efa\u6a21\u5c42\u7684\u524d\u5411\u4f20\u64ad\u4e2d\uff0c\u4f7f\u7528\u4e0a\u8ff0\u7684\u9690\u85cf\u72b6\u6001\u3001\u66f4\u65b0\u89c4\u5219\u548c\u8f93\u51fa\u89c4\u5219\u3002<\/p>\n<p>\u5373\u4f7f\u5728\u6d4b\u8bd5\u65f6\uff0c\u65b0\u5c42\u4ecd\u7136\u4e3a\u6bcf\u4e2a\u8f93\u5165\u5e8f\u5217\u8bad\u7ec3\u4e00\u4e2a\u4e0d\u540c\u7684\u6743\u91cd\u5e8f\u5217W1,&#8230;,WT\u3002<\/p>\n<p>\u56e0\u6b64\uff0c\u7814\u7a76\u4eba\u5458\u5c06\u5176\u79f0\u4e4b\u4e3a\u6d4b\u8bd5-\u65f6\u95f4\u8bad\u7ec3\u5c42\uff08TTT\uff09\u3002<\/p>\n<h3>4. \u4f7f\u7528TTT\u5c42\u8bad\u7ec3\u795e\u7ecf\u7f51\u7edc<\/h3>\n<p>TTT\u5c42\u7684\u524d\u5411\u4f20\u64ad\uff0c\u4e5f\u6709\u76f8\u5e94\u7684\u540e\u5411\u4f20\u64ad\u3002<\/p>\n<p>TTT\u5c42\u4e0eRNN\u5c42\u3001\u81ea\u6ce8\u610f\u529b\u673a\u5236\u6709\u7740\u76f8\u540c\u7684\u63a5\u53e3\uff0c\u56e0\u6b64\u53ef\u4ee5\u5728\u4efb\u4f55\u66f4\u5927\u7684\u795e\u7ecf\u7f51\u7edc\u67b6\u6784\u4e2d\u66ff\u6362\u5b83\u4eec\u3002<\/p>\n<p>\u503c\u5f97\u4e00\u63d0\u7684\u662f\uff0c\u8bad\u7ec3\u5e26\u6709TTT\u5c42\u795e\u7ecf\u7f51\u7edc\u7684\u65b9\u5f0f\uff0c\u4e0e\u8bad\u7ec3\u4efb\u4f55\u5176\u4ed6Transformer\u6a21\u578b\u76f8\u540c\u3002<\/p>\n<p>\u53ef\u4ee5\u4f7f\u7528\u76f8\u540c\u7684\u6570\u636e\u3001\u65b9\u6cd5\u548c\u76ee\u6807\uff08\u5982\u4e0b\u4e00\u4e2atoken\u9884\u6d4b\uff09\u6765\u4f18\u5316\u7f51\u7edc\u5176\u4f59\u90e8\u5206\u7684\u53c2\u6570\u3002<\/p>\n<p>\u5728\u6b64\uff0c\u7814\u7a76\u4eba\u5458\u5c06\u8bad\u7ec3\u66f4\u5927\u7684\u795e\u7ecf\u7f51\u7edc\u79f0\u4e3a\u5916\u5faa\u73af\uff08outer loop\uff09\uff0c\u800c\u5728\u6bcf\u4e2aTTT\u5c42\u5185\u8bad\u7ec3W\u79f0\u4e3a\u5185\u5faa\u73af\uff08inner loop\uff09\u3002<\/p>\n<p>\u5b83\u4eec\u4e4b\u95f4\u68af\u5ea6\u8ba1\u7b97\u7684\u533a\u522b\u662f\uff0c\u5185\u5faa\u73af\u9488\u5bf9\u7684\u662fW\uff08\u5373\u6a21\u578bf\u7684\u53c2\u6570\uff09\uff0c\u5916\u5faa\u73af\u9488\u5bf9\u7684\u662f\u7f51\u7edc\u5176\u4f59\u90e8\u5206\u7684\u53c2\u6570\u03b8rest\u3002<\/p>\n<h3>5. TTT\u5b66\u4e60\u81ea\u76d1\u7763\u4efb\u52a1<\/h3>\n<p>\u53ef\u4ee5\u8bf4\uff0cTTT\u6700\u91cd\u8981\u7684\u90e8\u5206\u662f\u81ea\u76d1\u7763\u4efb\u52a1\uff0c\u56e0\u4e3a\u5b83\u51b3\u5b9a\u4e86W\u4ece\u6d4b\u8bd5\u5e8f\u5217\u4e2d\u5b66\u4e60\u7684\u7279\u5f81\u7c7b\u578b\u3002<\/p>\n<p>\u5728\u8fd9\u4e2a\u4efb\u52a1\u7684\u8bbe\u8ba1\u4e0a\uff0c\u7814\u7a76\u4eba\u5458\u91c7\u53d6\u4e86\u66f4\u52a0\u7aef\u5230\u7aef\u7684\u65b9\u6cd5\u2014\u2014\u76f4\u63a5\u4f18\u5316\u81ea\u76d1\u7763\u4efb\u52a1\u4ee5\u5b9e\u73b0\u4e0b\u4e00\u4e2atoken\u9884\u6d4b\u7684\u6700\u7ec8\u76ee\u6807\u3002<\/p>\n<p>\u5177\u4f53\u6765\u8bf4\uff0c\u7814\u7a76\u8005\u5c06\u81ea\u76d1\u7763\u4efb\u52a1\u7684\u5b66\u4e60\uff0c\u4f5c\u4e3a\u5916\u5faa\u73af\u7684\u4e00\u90e8\u5206\u3002<\/p>\n<p>\u4ece\u5982\u4e0a\u516c\u5f0f3\u4e2d\u7684\u7b80\u5355\u91cd\u6784\u4efb\u52a1\u5f00\u59cb\uff0c\u6dfb\u52a0\u4e86\u4e00\u4e9b\u5916\u5faa\u73af\u53c2\u6570\u6765\u8ba9\u8fd9\u4e2a\u4efb\u52a1\u53ef\u5b66\u4e60\u3002\u6700\u65b0\u7684\u81ea\u76d1\u7763\u635f\u5931\u662f\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f113b5c0-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u5728\u5185\u5faa\u73af\u4e2d\uff0c\u53ea\u6709W\u88ab\u4f18\u5316\uff0c\u56e0\u6b64\u4f5c\u4e3a\u2113\u7684\u53c2\u6570\u5199\u51fa\uff1b\u03b8\u4eec\u662f\u8fd9\u4e2a\u635f\u5931\u51fd\u6570\u7684\u201c\u8d85\u53c2\u6570\u201d\u3002\u5728\u5916\u5faa\u73af\u4e2d\uff0c\u03b8K,\u03b8V,\u03b8Q\u4e0e\u03b8rest\u4e00\u8d77\u88ab\u4f18\u5316\uff0c\u800cW\u4ec5\u4ec5\u662f\u4e00\u4e2a\u9690\u85cf\u72b6\u6001\uff0c\u4e0d\u662f\u53c2\u6570\u3002<\/p>\n<p>\u56fe6\u7528\u4ee3\u7801\u8bf4\u660e\u4e86\u8fd9\u79cd\u533a\u522b\uff0c\u5176\u4e2d\u03b8K\u548c\u03b8Q\u88ab\u5b9e\u73b0\u4e3aTTT\u5c42\u7684\u53c2\u6570\uff0c\u7c7b\u4f3c\u4e8e\u81ea\u6ce8\u610f\u529b\u4e2d\u7684KV\u53c2\u6570\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f260c6de-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u603b\u7684\u6765\u8bf4\uff0c\u03b8K,\u03b8V,\u03b8Q\u6240\u6709\u53ef\u80fd\u7684\u9009\u62e9\u6784\u6210\u4e86\u4e00\u7cfb\u5217\u591a\u89c6\u56fe\u91cd\u6784\u4efb\u52a1\uff0c\u5916\u5faa\u73af\u53ef\u4ee5\u88ab\u7406\u89e3\u4e3a\u4ece\u8fd9\u4e2a\u4efb\u52a1\u7ec4\u4e2d\u9009\u62e9\u4e00\u4e2a\u5177\u4f53\u4efb\u52a1\u3002\u4e3a\u4e86\u7b80\u5355\u8d77\u89c1\uff0c\u7814\u7a76\u4eba\u5458\u5728\u8fd9\u91cc\u5c06\u6240\u6709\u89c6\u56fe\u8bbe\u8ba1\u4e3a\u7ebf\u6027\u6295\u5f71\u3002<\/p>\n<h3>6. mini-batch TTT\u5e76\u884c\u5316<\/h3>\n<p>\u76ee\u524d\uff0c\u5f00\u53d1\u7684\u539f\u751fTTT\u5c42\u5728\u6d6e\u70b9\u8fd0\u7b97\uff08FLOP\uff09\u6b21\u6570\u65b9\u9762\u5df2\u7ecf\u975e\u5e38\u9ad8\u6548\u3002<\/p>\n<p>\u7136\u800c\uff0c\u5176\u66f4\u65b0\u89c4\u5219\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f2c23ac2-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u65e0\u6cd5\u5b9e\u73b0\u5e76\u884c\u5316\uff0c\u56e0\u4e3aWt\u5728\u4e24\u4e2a\u4f4d\u7f6e\u4e0a\u4f9d\u8d56\u4e8eWt-1\uff1a\u8d1f\u53f7\u548c\u25bdl\u3002<\/p>\n<p>\u5bf9\u6b64\uff0c\u7814\u7a76\u4eba\u5458\u63d0\u51fa\u4e86mini-batch\u68af\u5ea6\u4e0b\u964d\uff0c\u7528b\u8868\u793aTTT\u6279\u5927\u5c0f\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f4628d64-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u7814\u7a76\u4e2d\u4f7f\u7528Gt = \u25bdl(Wt&#8217;;xt)\uff0c\u5176\u4e2dt&#8217; = t &#8211; mod(t,b)\uff0c\u5176\u4e2d\u4ee3\u8868\u7740\u524d\u4e00\u4e2amini-batch\u7684\u6700\u540e\u4e00\u4e2a\u65f6\u95f4\u6b65\uff08\u6216\u8005\u7b2c\u4e00\u4e2amini-batch 0\uff09\uff0c\u56e0\u6b64\uff0c\u53ef\u4ee5\u4e00\u6b21\u5e76\u884cb\u4e2a\u68af\u5ea6\u8ba1\u7b97\u3002<\/p>\n<h3>7. \u5bf9\u5076\u5f62\u5f0f<\/h3>\n<p>\u4e0a\u9762\u4ecb\u7ecd\u7684\u5e76\u884c\u5316\u662f\u5fc5\u8981\u7684\uff0c\u4f46\u5bf9\u4e8e\u201c\u5b9e\u9645\u8fd0\u884c\u65f6\u95f4\u201d\uff08wall-clock time\uff09\u7684\u6548\u7387\u6765\u8bf4\u8fd8\u4e0d\u591f\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f5710ea6-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u7136\u800c\uff0c\u73b0\u5b9e\u4e2d\uff0c\u662f\u65e0\u6cd5\u5bf9\u5355\u4e2amatmul\u6765\u8ba1\u7b97GtS\u6240\u6709\u7684b\u3002\u76f8\u53cd\uff0c\u9700\u8981b\u4e2a\u5916\u79ef\u6765\u5bf9\u5176\u8fdb\u884c\u4e00\u4e00\u8ba1\u7b97\u3002\u66f4\u7cdf\u7cd5\u7684\u662f\uff0c\u5bf9\u4e8e\u6bcf\u4e2a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f60e06a2-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>Gt\u662fd\u00d7d\uff0c\u8fd9\u4f1a\u6bd4\u5927dXt\u4ea7\u751f\u66f4\u5927\u7684\u5185\u5b58\u5360\u7528\u548cI\/O\u6210\u672c\u3002<\/p>\n<p>\u4e3a\u4e86\u89e3\u51b3\u8fd9\u4e24\u4e2a\u95ee\u9898\uff0c\u7814\u7a76\u4eba\u5458\u89c2\u5bdf\u5230\uff1a\u6211\u4eec\u5b9e\u9645\u4e0a\u5e76\u4e0d\u9700\u8981\u5177\u4f53\u5316G1, . . . , Gb\uff0c\u53ea\u8981\u8981\u6211\u4eec\u53ef\u4ee5\u5728mini-batch\u7ed3\u675f\u65f6\u8ba1\u7b97Wb\uff0c\u5e76\u4e14\u8f93\u51fatoken z1, . . . , zb\uff08\u5982\u4e0a\u56fe7\u6240\u793a\uff09\u3002<\/p>\n<p>\u73b0\u5728\uff0c\u5c31\u53ef\u4ee5\u7528\u4e0a\u9762\u7b80\u5316\u7684TTT-Linear\u60c5\u51b5\u6765\u6f14\u793a\u8fd9\u4e9b\u8ba1\u7b97\uff0c\u8868\u793aX = [x1, . . . , xb]\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f75066d6-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u6240\u4ee5Wb\u53ef\u4ee5\u7528matmul\u65b9\u4fbf\u5730\u8ba1\u7b97\u51fa\u6765\u3002\u4e3a\u4e86\u8ba1\u7b97Z = [z1, . . . , zb]\uff0c\u6211\u4eec\u77e5\u9053\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f7eab43e-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u8868\u793a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f83d47ee-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u548c\u77e9\u9635<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f88c19c8-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u53ef\u4ee5\u5f97\u51fa\uff1a<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f8d3e442-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u5982\u4e0a\u8fc7\u7a0b\uff0c\u7814\u7a76\u4eba\u5458\u5c06\u5176\u79f0\u4e3a\u300c\u5bf9\u5076\u5f62\u5f0f\u300d\u3002<\/p>\n<h3>8. \u7406\u8bba\u7b49\u4ef7<\/h3>\n<p>\u524d\u9762\u5df2\u7ecf\u63d0\u5230f\u53ef\u4ee5\u662f\u7ebf\u6027\u6a21\u578b\uff0c\u4e5f\u53ef\u4ee5\u662f\u795e\u7ecf\u7f51\u7edc\u3002\u8fd8\u6709\u66f4\u65b0\u89c4\u5219\u7684\u4e09\u79cd\u53d8\u4f53\uff1aonline GD\u3001batch GD\u548cmini-batch GD\u3002<\/p>\n<p>\u5982\u4e0b\u56fe\u6240\u793a\uff0c\u5728\u8fd9\u4e9b2\u00d73\u7ec4\u5408\u4e2d\uff0c\u6bcf\u4e00\u79cd\u90fd\u4f1a\u5f15\u8d77TTT\u5c42\u7684\u4e0d\u540c\u5b9e\u4f8b\u5316\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f93008f8-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u7814\u7a76\u4e2d\uff0c\u4f5c\u8005\u5206\u522b\u4ece2\u4e2a\u5b9a\u7406\u8bc1\u660e\u4e86\u5728\u8fd9\u4e9b\u8bf1\u5bfc\u5b9e\u4f8b\u4e2d\uff0c\u5177\u6709\u7ebf\u6027\u6a21\u578b\u548cbatch GD\u7684TTT\u5c42\u7b49\u540c\u4e8e\u7ebf\u6027\u6ce8\u610f\u529b\u2014\u2014\u4e00\u4e2a\u5e7f\u4e3a\u4eba\u77e5\u7684RNN\u5c42\u3002<\/p>\n<p>\u56fe10\u603b\u7ed3\u4e86\u6240\u6709\u5e8f\u5217\u5efa\u6a21\u5c42\u7684\u66f4\u5e7f\u6cdb\u8303\u56f4\u5185TTT\u5c42\u7684\u4e00\u822c\u5b9a\u4e49\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f98764e0-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<h3>9. \u4e24\u79cd\u53d8\u4f53<\/h3>\n<p>\u7814\u7a76\u4e2d\uff0c\u4f5c\u8005\u63d0\u51fa\u4e86TTT\u5c42\u7684\u4e24\u79cd\u53d8\u4f53TTT-Linear\u548cTTT-MLP\uff0c\u4ec5\u5728f\u7684\u5b9e\u4f8b\u5316\u65b9\u9762\u6709\u6240\u4e0d\u540c\u3002<\/p>\n<p>\u5bf9\u4e8eTTT-Linear\uff0c<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/f9d39f68-3dbf-11ef-89ea-00163e142b65.png\">\uff0c\u5176\u4e2dW\u662f\u5e73\u65b9\u3002\u5bf9\u4e8eTTT-MLP\uff0c<img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fa205b14-3dbf-11ef-89ea-00163e142b65.png\">\u6709\u4e24\u5c42\uff0c\u7c7b\u4f3c\u4e8eTransfomer\u7684MLP\u3002<\/p>\n<p>\u5177\u4f53\u6765\u8bf4\uff0c\u9690\u85cf\u7ef4\u5ea6\u662f4\u00d7\u8f93\u5165\u7ef4\u5ea6\uff0c\u7136\u540e\u662fGELU\u6fc0\u6d3b\u3002\u4e3a\u4e86\u5728TTT\u671f\u95f4\u83b7\u5f97\u66f4\u597d\u7684\u7a33\u5b9a\u6027\uff0cf\u59cb\u7ec8\u5305\u542b\u5c42\u5f52\u4e00\u5316 (LN) \u548c\u6b8b\u5dee\u8fde\u63a5\u3002<\/p>\n<p>\u5373\uff0c<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fa6597ce-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\uff0c\u5176\u4e2d\uff0c<img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fab7d28c-3dbf-11ef-89ea-00163e142b65.png\">\u53ef\u4ee5\u662f<img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fb3c9e9a-3dbf-11ef-89ea-00163e142b65.png\">\u6216<img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fb8ad740-3dbf-11ef-89ea-00163e142b65.png\">\u3002<\/p>\n<\/p>\n<h2  id=\"toc-4\" class=\"jltoc--item\">04 \u5b9e\u9a8c<\/h2>\n<p>\u901a\u8fc7\u4e0e\u4e24\u4e2a\u57fa\u7ebfTransformer\u548cMamba\uff08\u73b0\u4ee3RNN\uff09\u6bd4\u8f83\uff0c\u7814\u7a76\u4eba\u5458\u8bc4\u4f30\u4e86TTT-Linear\u548cTTT-MLP\u3002<\/p>\n<p><strong>\u6570\u636e\u96c6<\/strong><\/p>\n<p>\u7ee7\u7eedMamba\u8bba\u6587\u4e4b\u540e\uff0c\u7814\u7a76\u4eba\u5458\u5728Pile\u4e0a\u6267\u884c\u4e862k\u548c8k\u4e0a\u4e0b\u6587\u957f\u5ea6\u7684\u6807\u51c6\u5b9e\u9a8c\uff0cPile\u662f\u4e00\u4e2a\u7528\u4e8e\u8bad\u7ec3\u5f00\u6e90LLM\u7684\u6d41\u884c\u6587\u6863\u6570\u636e\u96c6\u3002<\/p>\n<p><strong>\u4e3b\u67b6\u6784<\/strong><\/p>\n<p>Transformer\u548cMamba\u4f7f\u7528\u4e0d\u540c\u7684\uff0c\u9664\u975e\u53e6\u6709\u8bf4\u660e\uff0cTTT-Linear\u548cTTT-MLP\u59cb\u7ec8\u4f7f\u7528Mamba\u67b6\u6784\u3002<\/p>\n<h3>1. \u77ed\u4e0a\u4e0b\u6587\uff1athe Pile<\/h3>\n<p>\u57282k\u4e0a\u4e0b\u6587\u4e2d\uff0cTTT-Linear\uff08M\uff09\u3001Mamba\u548cTransformer\u5177\u6709\u76f8\u5f53\u7684\u6027\u80fd\uff0c\u7ebf\u6761\u5927\u90e8\u5206\u91cd\u53e0\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fbd51de6-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>TTT-MLP\uff08M\uff09\u5728\u8f83\u5927\u7684FLOP\u9884\u7b97\u4e0b\u8868\u73b0\u7a0d\u5dee\u3002\u5c3d\u7ba1TTT-MLP\u5728\u6bcf\u4e2a\u6a21\u578b\u5927\u5c0f\u4e0a\uff0c\u90fd\u6bd4TTT-Linear\u5177\u6709\u66f4\u597d\u7684\u590d\u6742\u5ea6\uff0c\u4f46FLOP\u7684\u989d\u5916\u6210\u672c\u62b5\u6d88\u4e86\u8fd9\u79cd\u4f18\u52bf\u3002<\/p>\n<p>\u57288k\u4e0a\u4e0b\u6587\u4e2d\uff0cTTT-Linear\uff08M\uff09\u548cTTT-MLP\uff08M\uff09\u7684\u8868\u73b0\u5747\u660e\u663e\u4f18\u4e8eMamba\u3002\u5373\u4f7f\u662f\u5177\u6709Transformer\u67b6\u6784\u7684TTT-MLP\uff08T\uff09\uff0c\u6027\u80fd\u4e5f\u6bd4Mamba\u7565\u597d\u3002<\/p>\n<p>\u53e6\u5916\uff0c\u7814\u7a76\u4eba\u5458\u8fd8\u89c2\u5bdf\u5230\u4e86\u4e00\u4e2a\u975e\u5e38\u660e\u663e\u7684\u73b0\u8c61\uff1a\u968f\u7740\u4e0a\u4e0b\u6587\u957f\u5ea6\u53d8\u957f\uff0cTTT\u5c42\u76f8\u5bf9\u4e8eMamba\u7684\u4f18\u52bf\u5c31\u66f4\u5927\u4e86\u3002<\/p>\n<h3>2. \u957f\u4e0a\u4e0b\u6587\uff1aBooks<\/h3>\n<p>\u4e3a\u4e86\u8bc4\u4f30\u957f\u4e0a\u4e0b\u6587\u4e2d\u7684\u529f\u80fd\uff0c\u7814\u7a76\u4eba\u5458\u4f7f\u7528\u4e86Pile\u7684\u4e00\u4e2a\u6d41\u884c\u5b50\u96c6\u2014\u2014Books\uff0c\u5bf9\u4ece1k\u523032k\u4ee52\u4e2a\u589e\u91cf\u7684\u4e0a\u4e0b\u6587\u957f\u5ea6\u8fdb\u884c\u4e86\u5b9e\u9a8c\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fc2a8420-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u6839\u636e\u4e0a\u56fe\uff0c\u53ef\u4ee5\u89c2\u5bdf\u5230\u2014\u2014<\/p>\n<p>\u5728Books\u76842k\u4e0a\u4e0b\u6587\u4e2d\uff0cPile 2k\u7684\u6240\u6709\u89c2\u5bdf\u7ed3\u679c\u4ecd\u7136\u6210\u7acb\uff0c\u552f\u4e00\u7684\u4f8b\u5916\u662fMamba\u7684\u8868\u73b0\u7565\u597d\u4e8eTTT-Linear\u3002<\/p>\n<p>\u572832k\u4e0a\u4e0b\u6587\u4e2d\uff0cTTT-Linear\uff08M\uff09\u548cTTT-MLP\uff08M\uff09\u7684\u6027\u80fd\u5747\u4f18\u4e8eMamba\uff0c\u4e0ePile 8k\u7684\u89c2\u5bdf\u7ed3\u679c\u7c7b\u4f3c\u3002\u5373\u4f7f\u5177\u6709Transformer\u67b6\u6784\u7684TTT-MLP\uff08T\uff09\uff0c\u572832k\u4e0a\u4e0b\u6587\u4e2d\u7684\u8868\u73b0\u4e5f\u6bd4Mamba\u7a0d\u597d\u3002<\/p>\n<p>\u57281.3B\u5c3a\u5ea6\u4e0a\uff0cTTT-MLP\uff08T\uff09\u4ec5\u6bd4TTT-MLP\uff08M\uff09\u7a0d\u5dee\u3002\u7531\u4e8e\u7f3a\u4e4b\u6e05\u6670\u7684\u7ebf\u6027\u62df\u5408\uff0c\u5f88\u96be\u63a8\u5bfc\u51fa\u7ecf\u9a8c\u7f29\u653e\u5b9a\u5f8b\u3002\u7136\u800c\uff0cTTT-MLP\uff08T\uff09\u7684\u5f3a\u52b2\u8d8b\u52bf\u8868\u660e\uff0cTransformer\u67b6\u6784\u53ef\u80fd\u66f4\u9002\u5408\u8d85\u51fa\u8bc4\u4f30\u7684\u66f4\u5927\u6a21\u578b\u548c\u66f4\u957f\u4e0a\u4e0b\u6587\u3002<\/p>\n<p><strong>\u4e0a\u4e0b\u6587\u957f\u5ea6\u4f5c\u4e3a\u8d85\u53c2\u6570<\/strong><\/p>\n<p>\u867d\u7136\u8f93\u5165\u5e8f\u5217\u7684\u957f\u5ea6\u7531\u7528\u6237\u786e\u5b9a\uff0c\u4f46\u8bed\u8a00\u6a21\u578b\u5904\u7406\u8f93\u5165\u7684\u4e0a\u4e0b\u6587\u957f\u5ea6\u53ef\u4ee5\u7531\u5de5\u7a0b\u5e08\u786e\u5b9a\u3002\u56e0\u6b64\uff0c\u4e0a\u4e0b\u6587\u957f\u5ea6\u4e5f\u662f\u4e00\u4e2a\u53ef\u4ee5\u9009\u62e9\u7684\u8d85\u53c2\u6570\u3002<\/p>\n<p>\u5bf9\u4e8e\u5177\u6709\u7ebf\u6027\u590d\u6742\u5ea6\u7684LLM\uff0c\u7814\u7a76\u4eba\u5458\u9009\u62e9\u4e86\u56f0\u60d1\u5ea6\u4e2d\u7684argmin\uff0c\u56e0\u4e3a\u6bcf\u4e2a\u4e0a\u4e0b\u6587\u957f\u5ea6\u90fd\u6709\u76f8\u540c\u7684FLOP\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fc7f24f8-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u4ece\u56fe13\u4e2d\uff0c\u53ef\u4ee5\u89c2\u5bdf\u5230\u4ee5\u4e0b\u7ed3\u679c\u2014\u2014<\/p>\n<ul>\n<li>\u6027\u80fd\u6700\u597d\u7684\u65b9\u6cd5TTT-Linear\u548cTTT-MLP\u7684\u7ebf\u51e0\u4e4e\u5b8c\u5168\u91cd\u53e0\u3002Mamba\u548cTF Finetune\u7684\u7ebf\u572810^20 FLOP\u540e\u4e5f\u5927\u90e8\u5206\u91cd\u53e0\u3002<\/li>\n<li>TF Finetune\u7684\u6027\u80fd\u660e\u663e\u4f18\u4e8eTF Pretrain\uff0c\u56e0\u4e3a\u5b83\u53d7\u76ca\u4e8e\u957f\u4e0a\u4e0b\u6587\uff0c\u800c\u4e0d\u4f1a\u5728\u8bad\u7ec3FLOP\u4e2d\u4ea7\u751f\u6781\u5927\u7684\u6210\u672c\u3002<\/li>\n<li>\u5bf9\u4e8e\u6240\u6709\u4ece\u5934\u5f00\u59cb\u8bad\u7ec3\u7684\u65b9\u6cd5\uff08\u5305\u62ecTF\u9884\u8bad\u7ec3\uff09\uff0c\u4e00\u65e6\u4e0a\u4e0b\u6587\u957f\u5ea6\u53d8\u5f97\u592a\u5927\uff0c\u56f0\u60d1\u5ea6\u5c31\u4f1a\u53d8\u5f97\u66f4\u7cdf\u3002<\/li>\n<\/ul>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fcd05cec-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u4ece\u4e0a\u56fe\u53ef\u89c1\uff0c\u4e0eTTT-Linear\u76f8\u6bd4\uff0cTTT-MLP\u5728\u77ed\u4e0a\u4e0b\u6587\u4e2d\u8868\u73b0\u7a0d\u5dee\uff0c\u4f46\u5728\u957f\u4e0a\u4e0b\u6587\u4e2d\u8868\u73b0\u66f4\u597d\u3002<\/p>\n<p>\u8fd9\u4e00\u89c2\u5bdf\u7ed3\u679c\u6b63\u7b26\u5408\u7814\u7a76\u4eba\u5458\u7684\u9884\u671f\uff0c\u5373\u4f5c\u4e3a\u9690\u85cf\u72b6\u6001\u7684MLP\u6bd4\u7ebf\u6027\u6a21\u578b\u66f4\u5177\u8868\u73b0\u529b\u3002\u540c\u6837\uff0c\u6240\u6709\u65b9\u6cd5\u90fd\u5177\u6709\u4e0eMamba 1.4B\u76f8\u540c\u7684\u8bad\u7ec3FLOP\u3002<\/p>\n<h3>3. \u5b9e\u9645\u8fd0\u884c\u65f6\u95f4<\/h3>\n<p>LLM\u8bad\u7ec3\u548c\u63a8\u7406\u53ef\u4ee5\u5206\u89e3\u4e3a\u524d\u5411\u3001\u540e\u5411\u548c\u751f\u6210\u3002<\/p>\n<p>\u7531\u4e8e\u524d\u5411\uff08\u5728\u8bad\u7ec3\u548c\u63a8\u7406\u671f\u95f4\uff09\u548c\u540e\u5411\u90fd\u53ef\u4ee5\u5e76\u884c\u5316\uff0c\u56e0\u6b64\u7814\u7a76\u4eba\u5458\u4f7f\u7528\u5bf9\u5076\u5f62\u5f0f\u3002\u751f\u6210\u65b0token\uff08\u4e5f\u79f0\u4e3a\u89e3\u7801\uff09\u672c\u8d28\u4e0a\u662f\u987a\u5e8f\u7684\uff0c\u56e0\u6b64\u7814\u7a76\u4eba\u5458\u4f7f\u7528\u539f\u59cb\u5f62\u5f0f\u3002<\/p>\n<p>\u7531\u4e8e\u8d44\u6e90\u9650\u5236\uff0c\u8fd9\u9879\u5b9e\u9a8c\u662f\u7528JAX\u7f16\u5199\u5e76\u5728TPU\u4e0a\u8fd0\u884c\u7684\u3002<\/p>\n<p>\u7136\u800c\uff0c\u7531\u4e8eMamba\uff08\u5728PyTorch\u3001Triton\u548cCUDA\u4e2d\u5b9e\u73b0\uff09\u53ea\u80fd\u5728GPU\u4e0a\u8fd0\u884c\uff0c\u56e0\u6b64\u4e3a\u4e86\u516c\u5e73\u6bd4\u8f83\uff0c\u7814\u7a76\u4eba\u5458\u8fd8\u91cd\u5199\u4e86\u65b9\u6cd5\uff0c\u4ee5\u5728GPU\u4e0a\u8fd0\u884c\u3002<\/p>\n<p>\u5177\u4f53\u6765\u8bf4\uff0c\u7814\u7a76\u4eba\u5458\u5728ThunderKittens\u4e2d\u7f16\u5199\u4e86\u4e00\u4e2a\u7528\u4e8e\u524d\u5411\u7684GPU\u5185\u6838\u3002\u4ece\u5386\u53f2\u4e0a\u770b\uff0c\u7531\u4e8e\u5e76\u884c\u6027\u548c\u77e9\u9635\u76f8\u4e58\u7684\u4f7f\u7528\u4e0d\u5f53\uff0cRNN\u5728\u524d\u5411\u548c\u540e\u5411\u8fc7\u7a0b\u4e2d\u6548\u7387\u4f4e\u4e0b\u3002<\/p>\n<p>\u8fd9\u4e2a\u524d\u5411\u5185\u6838\u7684\u76ee\u6807\uff0c\u662f\u8bc1\u660emini-batch TTT\u548c\u8fd9\u4e9b\u95ee\u9898\u5bf9\u5076\u5f62\u5f0f\u7684\u6709\u6548\u6027\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fd46d3b8-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u56fe15\u7684\u5de6\u56fe\u663e\u793a\u4e86\u524d\u5411\u5185\u6838\u6279\u5927\u5c0f\u4e3a16\u7684\u5ef6\u8fdf\u3002\u6240\u6709\u6a21\u578b\u53c2\u6570\u5747\u4e3a1.3B\uff08Mamba\u4e3a 1.4B\uff09\u3002<\/p>\n<p>\u5bf9\u4e8eTransformer\uff0c\u6bcf\u4e2atoken\u7684\u65f6\u95f4\u968f\u7740\u4e0a\u4e0b\u6587\u957f\u5ea6\u7684\u589e\u52a0\u800c\u7ebf\u6027\u589e\u957f\uff0c\u4f46\u5bf9\u4e8e\u5176\u4ed6\u65b9\u6cd5\u5219\u5927\u81f4\u4fdd\u6301\u4e0d\u53d8\u3002<\/p>\n<p>\u6b64\u5916\uff0c\u7814\u7a76\u4eba\u5458\u5728Triton\u4e2d\u7f16\u5199\u4e86\u53e6\u4e00\u4e2a\u7528\u4e8e\u751f\u6210\u7684GPU\u5185\u6838\uff0c\u5e76\u5728\u56fe15\u7684\u53f3\u56fe\u4e2d\u5bf9\u6279\u5927\u5c0f\u4e3a512\u7684\u901f\u5ea6\u8fdb\u884c\u4e86\u57fa\u51c6\u6d4b\u8bd5\u3002<\/p>\n<p>\u53ef\u4ee5\u770b\u51fa\uff0cTTT-Linear\u548cMamba\u7684\u5ef6\u8fdf\u51e0\u4e4e\u76f8\u540c\uff0c\u660e\u663e\u5c0f\u4e8eTransformer\u548cTTT-MLP\u3002<\/p>\n<p>Mamba\u4e4b\u540e\uff0c\u53c8\u770b\u5230TTT\u8fd9\u4e48\u80fd\u6253\u7684\u65b0\u67b6\u6784\u8bde\u751f\uff0c\u5c11\u4e0d\u4e86AI\u793e\u533a\u7684\u70ed\u8bae\u3002<\/p>\n<p>\u6709\u7f51\u53cb\u79f0\uff0c\u8fd9\u4f1a\u4e0d\u4f1a\u662f\u6700\u63a5\u8fd1\u5b9e\u65f6\u4e0a\u4e0b\u6587\u7684\u65b9\u6cd5\uff1f\u5f88\u60f3\u542c\u542c\u5927\u5bb6\u7684\u60f3\u6cd5\u3002\u8fd9\u610f\u5473\u7740TTT\u751a\u81f3\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\uff0c\u4e5f\u80fd\u591f\u5b66\u4e60\u548c\u9002\u5e94\uff0c\u4e3a\u957f\u4e0a\u4e0b\u6587\u63d0\u4f9b\u66f4\u597d\u7684\u6027\u80fd\uff0c\u800c\u4e0d\u4f1a\u4ea7\u751f\u901a\u5e38\u4e0eTransformer\u76f8\u5173\u7684\u9ad8\u6602\u8ba1\u7b97\u6210\u672c\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fda3a192-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>OpenAI\u89c6\u9891\u751f\u6210\u7814\u7a76\u4eba\u5458\u5bf9\u6b64\u8868\u793a\uff0c\u8fd9\u9879\u7814\u7a76\u770b\u8d77\u6765\u5f88\u6709\u8da3\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fdf79e6e-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u5982\u679cscaling law\u4f9d\u7136\u5b58\u5728\uff0cTTT\u5c06\u5e26\u6765\u96be\u4ee5\u7f6e\u4fe1\u7684\u5f71\u54cd\u3002\u5bf9\u4e8e\u957f\u5e8f\u5217\uff0cTransformer\u7684\u8ba1\u7b97\u6210\u672c\u5f80\u5f80\u5f88\u9ad8\uff0c\u5f53\u957f\u5e8f\u5217\u53d8\u5f97\u66f4\u957f\u65f6\uff0cRNN\u4f1a\u9057\u5fd8\u3002TTT\u8bad\u7ec3\u5de7\u5999\u5730\u5229\u7528\u795e\u7ecf\u7f51\u7edc\u89e3\u51b3RNN\u7684\u4e0d\u8db3\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fe46b058-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<h2  id=\"toc-5\" class=\"jltoc--item\">\u4f5c\u8005\u4ecb\u7ecd<\/h2>\n<p>\u8bba\u6587\u6700\u540e\uff0c\u5206\u522b\u5217\u51fa\u4e86\u8fd9\u7bc7\u7814\u7a76\u7684\u4f5c\u8005\u8d21\u732e\u3002<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/fe9a3f84-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>\u5176\u4e2d\u7684\u6838\u5fc3\u4f5c\u8005\u662f\uff0cYu Sun\u3001Xinhao Li\u548cKaran Dalal\u3002<\/p>\n<p>Yu Sun<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/ff180d24-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>Yu Sun\u662f\u65af\u5766\u798f\u5927\u5b66\u8ba1\u7b97\u673a\u4e13\u4e1a\u7684\u535a\u58eb\u540e\uff0c\u5bfc\u5e08\u662fCarlos Guestrin\u3001Tatsu Hashimoto\u548cSanmi Koyejo\u3002<\/p>\n<p>\u6b64\u524d\uff0c\u4ed6\u66fe\u5728\u52a0\u5dde\u5927\u5b66\u4f2f\u514b\u5229\u5206\u6821\u5b8c\u6210\u4e86\u7535\u5b50\u5de5\u7a0b\u79d1\u5b66\u535a\u58eb\u5b66\u4f4d\uff0c\u5bfc\u5e08\u662fAlyosha Efros\u548cMoritz Hardt\u3002\u4ed6\u8fd8\u5728\u5eb7\u5948\u5c14\u5927\u5b66\u62ff\u5230\u4e86\u5b66\u58eb\u5b66\u4f4d\u3002<\/p>\n<p>\u4e2a\u4eba\u4e3b\u9875\u4e2d\uff0c\u4ed6\u4ecb\u7ecd\u81ea\u5df1\u7684\u7814\u7a76\u91cd\u70b9\u662f\u4e00\u79cd\u540d\u4e3a\u6d4b\u8bd5\u65f6\u95f4\u8bad\u7ec3\uff08test-time training\uff09\u7684\u7b97\u6cd5\u6846\u67b6\u3002\u5176\u6838\u5fc3\u601d\u60f3\u662f\uff0c\u6bcf\u4e2a\u6d4b\u8bd5\u5b9e\u4f8b\u90fd\u5b9a\u4e49\u4e86\u81ea\u5df1\u7684\u5b66\u4e60\u95ee\u9898\uff0c\u90fd\u6709\u81ea\u5df1\u7684\u6cdb\u5316\u76ee\u6807\u3002\u8fd9\u901a\u5e38\u4f7f\u7528\u81ea\u76d1\u7763\u5b66\u4e60\uff0c\u4e3a\u6bcf\u4e2a\u5b9e\u4f8b\u5373\u65f6\u8bad\u7ec3\u4e00\u4e2a\u4e0d\u540c\u7684\u6a21\u578b\u6765\u5b9e\u73b0\u7684\u3002<\/p>\n<p>\u5728\u6700\u65b0\u7814\u7a76\u4e2d\uff0cYu Sun\u4e0eXinhao Li\u57282022\u5e7411\u6708\u5171\u540c\u542f\u52a8\u4e86\u8fd9\u4e00\u9879\u76ee\u3002\u81ea2023\u5e746\u6708\u8d77\uff0cYu Sun\u4e13\u804c\u8d1f\u8d23\u8be5\u9879\u76ee\u3002<\/p>\n<p>\u4ed6\u63d0\u51fa\u4e86\u9879\u76ee\u7684\u6982\u5ff5\u6846\u67b6\uff0c\u8bbe\u8ba1\u4e86mini-batch TTT\u548c\u5bf9\u5076\u5f62\u5f0f\uff08dual form\uff09\u3002<\/p>\n<p>Xinhao Li<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/ffaa64ee-3dbf-11ef-89ea-00163e142b65.png\"><\/p>\n<p>Xinhao Li\u662fUC San Diego\u7814\u4e8c\u7684\u5b66\u751f\uff0c\u5bfc\u5e08\u662fXiaolong Wang\u6559\u6388\u3002\u4ed6\u672c\u4eba\u7684\u7814\u7a76\u5174\u8da3\u4e3b\u8981\u662f\u6df1\u5ea6\u5b66\u4e60\u548c\u8ba1\u7b97\u673a\u89c6\u89c9\u3002<\/p>\n<p>\u4ed6\u5728\u65af\u5766\u798f\u5927\u5b66Tatsunori Hashimoto\u6559\u6388\u7684\u56e2\u961f\u4e2d\u4f5c\u4e3a\u8bbf\u95ee\u5b66\u751f\uff0c\u4e0eYu Sun\u535a\u58eb\u548c\u5176\u4ed6\u5bfc\u5e08\u670b\u53cb\u4e00\u8d77\u5de5\u4f5c\u3002\u5728\u6b64\u4e4b\u524d\uff0c\u4ed6\u66fe\u5728\u7535\u5b50\u79d1\u6280\u5927\u5b66\u83b7\u5f97\u4e86\u5b66\u58eb\u5b66\u4f4d\u3002<\/p>\n<p>\u57282024\u5e743\u6708\u4e4b\u524d\uff0cXinhao Li\u662fTTT\u65e9\u671f\u4ee3\u7801\u5e93\u7684\u4e3b\u8981\u8d21\u732e\u8005\uff0c\u8fd9\u4e9b\u4ee3\u7801\u5e93\u5851\u9020\u4e86\u6700\u65b0\u9879\u76ee\u3002<\/p>\n<p>Karan Dalal<\/p>\n<p><img decoding=\"async\" data-action=\"zoom\" class=\"aligncenter\" src=\"https:\/\/image.woshipm.com\/2024\/07\/09\/002458bc-3dc0-11ef-89ea-00163e142b65.png\"><\/p>\n<p>Karan Dalal\u662fUC Berkeley\u7535\u5b50\u5de5\u7a0b\u79d1\u5b66\u7cfb\u7684\u672c\u79d1\u751f\u3002\u4ed6\u4e8e2023\u5e746\u6708\u5168\u804c\u52a0\u5165\u8be5\u9879\u76ee\uff0c\u4e0eXinhao Li\u5408\u4f5c\u5171\u540c\u9886\u5bfc\u4e86\u5f53\u524d\u4ee3\u7801\u5e93\u7684\u5f00\u53d1\u5de5\u4f5c\u3002<\/p>\n<p>\u53c2\u8003\u8d44\u6599\uff1a<\/p>\n<p>https:\/\/x.com\/karansdalal\/status\/1810338845659131940<\/p>\n<p>https:\/\/x.com\/xiaolonw\/status\/1810387662060269668<\/p>\n<p>https:\/\/arxiv.org\/abs\/2407.04620<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u8d85\u8d8aTransformer\u548cMamba\u7684\u65b0\u67b6\u6784\uff0c\u521a\u521a\u8bde\u751f\u4e86\u3002\u65af\u5766\u798fUCSD\u7b49\u673a\u6784\u7814\u7a76\u8005\u63d0\u51fa\u7684TTT\u65b9\u6cd5\uff0c\u76f4\u63a5\u66ff\u4ee3\u4e86\u6ce8\u610f\u529b\u673a\u5236\uff0c\u8bed\u8a00\u6a21\u578b\u65b9\u6cd5\u4ece\u6b64\u6216\u5c06\u5f7b\u5e95\u6539\u53d8\u3002<\/p>\n","protected":false},"author":8,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[8],"tags":[],"class_list":["post-10020187","post","type-post","status-publish","format-standard","hentry","category-aigc"],"jetpack_sharing_enabled":true,"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10020187","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/users\/8"}],"replies":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=10020187"}],"version-history":[{"count":0,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10020187\/revisions"}],"wp:attachment":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=10020187"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=10020187"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=10020187"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}