{"id":10006059,"date":"2022-07-29T18:53:54","date_gmt":"2022-07-29T10:53:54","guid":{"rendered":"https:\/\/ityet.com\/?p=10006059"},"modified":"2022-07-29T18:53:54","modified_gmt":"2022-07-29T10:53:54","slug":"easynlp%e4%b8%ad%e6%96%87%e6%96%87%e5%9b%be%e7%94%9f%e6%88%90%e6%a8%a1%e5%9e%8b%e5%b8%a6%e4%bd%a0%e7%a7%92%e5%8f%98%e8%89%ba%e6%9c%af%e5%ae%b6","status":"publish","type":"post","link":"https:\/\/itmax.cn\/?p=10006059","title":{"rendered":"EasyNLP\u4e2d\u6587\u6587\u56fe\u751f\u6210\u6a21\u578b\u5e26\u4f60\u79d2\u53d8\u827a\u672f\u5bb6"},"content":{"rendered":"<p>\u7b80\u4ecb\uff1a\u00a0\u6211\u4eec\u5728EasyNLP\u6846\u67b6\u4e2d\u96c6\u6210\u4e86\u4e2d\u6587\u6587\u56fe\u751f\u6210\u529f\u80fd\uff0c\u540c\u65f6\u5f00\u653e\u4e86\u6a21\u578b\u7684Checkpoint\u3002\u5bfc\u8bfb<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240415\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240416\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240417\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240418\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0\u00a0\u5ba3\u7269\u83ab\u5927\u4e8e\u8a00\uff0c\u5b58\u5f62\u83ab\u5584\u4e8e\u753b\u3002&#8211;\u3010\u664b\u3011\u9646\u673a\u591a\u6a21\u6001\u6570\u636e\uff08\u6587\u672c\u3001\u56fe\u50cf\u3001\u58f0\u97f3\uff09\u662f\u4eba\u7c7b\u8ba4\u8bc6\u3001\u7406\u89e3\u548c\u8868\u8fbe\u4e16\u95f4\u4e07\u7269\u7684\u91cd\u8981\u8f7d\u4f53\u3002\u8fd1\u5e74\u6765\uff0c\u591a\u6a21\u6001\u6570\u636e\u7684\u7206\u70b8\u6027\u589e\u957f\u4fc3\u8fdb\u4e86\u5185\u5bb9\u4e92\u8054\u7f51\u7684\u7e41\u8363\uff0c\u4e5f\u5e26\u6765\u4e86\u5927\u91cf\u591a\u6a21\u6001\u5185\u5bb9\u7406\u89e3\u548c\u751f\u6210\u7684\u9700\u6c42\u3002\u4e0e\u5e38\u89c1\u7684\u8de8\u6a21\u6001\u7406\u89e3\u4efb\u52a1\u4e0d\u540c\uff0c\u6587\u5230\u56fe\u7684\u751f\u6210\u4efb\u52a1\u662f\u6d41\u884c\u7684\u8de8\u6a21\u6001\u751f\u6210\u4efb\u52a1\uff0c\u65e8\u5728\u751f\u6210\u4e0e\u7ed9\u5b9a\u6587\u672c\u5bf9\u5e94\u7684\u56fe\u50cf\u3002\u8fd9\u4e00\u6587\u56fe\u751f\u6210\u7684\u4efb\u52a1\uff0c\u6781\u5927\u5730\u91ca\u653e\u4e86AI\u7684\u60f3\u8c61\u529b\uff0c\u4e5f\u6fc0\u53d1\u4e86\u4eba\u7c7b\u7684\u521b\u610f\u3002\u5178\u578b\u7684\u6a21\u578b\u4f8b\u5982OpenAI\u5f00\u53d1\u7684DALL-E\u548cDALL-E2\u3002\u8fd1\u671f\uff0c\u4e1a\u754c\u4e5f\u8bad\u7ec3\u51fa\u4e86\u66f4\u5927\u3001\u66f4\u65b0\u7684\u6587\u56fe\u751f\u6210\u6a21\u578b\uff0c\u4f8b\u5982Google\u63d0\u51fa\u7684Parti\u548cImagen\u3002\u7136\u800c\uff0c\u4e0a\u8ff0\u6a21\u578b\u4e00\u822c\u4e0d\u80fd\u7528\u4e8e\u5904\u7406\u4e2d\u6587\u7684\u9700\u6c42\uff0c\u800c\u4e14\u4e0a\u8ff0\u6a21\u578b\u7684\u53c2\u6570\u91cf\u5e9e\u5927\uff0c\u5f88\u96be\u88ab\u5f00\u6e90\u793e\u533a\u7684\u5e7f\u5927\u7528\u6237\u76f4\u63a5\u7528\u6765Fine-tune\u548c\u63a8\u7406\u3002\u672c\u6b21\uff0cEasyNLP\u5f00\u6e90\u6846\u67b6\u518d\u6b21\u8fce\u6765\u5927\u5347\u7ea7\uff0c\u96c6\u6210\u4e86\u5148\u8fdb\u7684\u6587\u56fe\u751f\u6210\u67b6\u6784Transformer+VQGAN\uff0c\u540c\u65f6\uff0c\u5411\u5f00\u6e90\u793e\u533a\u514d\u8d39\u5f00\u653e\u4e0d\u540c\u53c2\u6570\u91cf\u7684\u4e2d\u6587\u6587\u56fe\u751f\u6210\u6a21\u578b\u7684Checkpoint\uff0c\u4ee5\u53ca\u76f8\u5e94Fine-tune\u548c\u63a8\u7406\u63a5\u53e3\u3002\u7528\u6237\u53ef\u4ee5\u5728\u6211\u4eec\u5f00\u653e\u7684Checkpoint\u57fa\u7840\u4e0a\u8fdb\u884c\u5c11\u91cf\u9886\u57df\u76f8\u5173\u7684\u5fae\u8c03\uff0c\u5728\u4e0d\u6d88\u8017\u5927\u91cf\u8ba1\u7b97\u8d44\u6e90\u7684\u60c5\u51b5\u4e0b\uff0c\u5c31\u80fd\u4e00\u952e\u8fdb\u884c\u5404\u79cd\u827a\u672f\u521b\u4f5c\u3002EasyNLP\u662f\u963f\u91cc\u4e91\u673a\u5668\u5b66\u4e60PAI \u56e2\u961f\u57fa\u4e8e PyTorch \u5f00\u53d1\u7684\u6613\u7528\u4e14\u4e30\u5bcc\u7684\u4e2d\u6587NLP\u7b97\u6cd5\u6846\u67b6\uff0c\u5e76\u4e14\u63d0\u4f9b\u4e86\u4ece\u8bad\u7ec3\u5230\u90e8\u7f72\u7684\u4e00\u7ad9\u5f0f NLP \u5f00\u53d1\u4f53\u9a8c\u3002EasyNLP \u63d0\u4f9b\u4e86\u7b80\u6d01\u7684\u63a5\u53e3\u4f9b\u7528\u6237\u5f00\u53d1 NLP \u6a21\u578b\uff0c\u5305\u62ecNLP\u5e94\u7528 AppZoo \u3001\u9884\u8bad\u7ec3\u6a21\u578b ModelZoo\u3001\u6570\u636e\u4ed3\u5e93DataHub\u7b49\u7279\u6027\u3002\u7531\u4e8e\u8de8\u6a21\u6001\u7406\u89e3\u548c\u751f\u6210\u9700\u6c42\u7684\u4e0d\u65ad\u589e\u52a0\uff0cEasyNLP\u4e5f\u652f\u6301\u5404\u79cd\u8de8\u6a21\u6001\u6a21\u578b\uff0c\u7279\u522b\u662f\u4e2d\u6587\u9886\u57df\u7684\u8de8\u6a21\u6001\u6a21\u578b\uff0c\u63a8\u5411\u5f00\u6e90\u793e\u533a\u3002\u4f8b\u5982\uff0c\u5728\u5148\u524d\u7684\u5de5\u4f5c\u4e2d\uff0cEasyNLP\u5df2\u7ecf\u5bf9\u4e2d\u6587\u56fe\u6587\u68c0\u7d22CLIP\u6a21\u578b\u8fdb\u884c\u4e86\u652f\u6301\uff08\u770b\u8fd9\u91cc\uff09\u3002\u6211\u4eec\u5e0c\u671b\u80fd\u591f\u670d\u52a1\u66f4\u591a\u7684 NLP \u548c\u591a\u6a21\u6001\u7b97\u6cd5\u5f00\u53d1\u8005\u548c\u7814\u7a76\u8005\uff0c\u4e5f\u5e0c\u671b\u548c\u793e\u533a\u4e00\u8d77\u63a8\u52a8 NLP \/\u591a\u6a21\u6001\u6280\u672f\u7684\u53d1\u5c55\u548c\u6a21\u578b\u843d\u5730\u3002\u672c\u6587\u7b80\u8981\u4ecb\u7ecd\u6587\u56fe\u751f\u6210\u7684\u6280\u672f\uff0c\u4ee5\u53ca\u5982\u4f55\u5728EasyNLP\u6846\u67b6\u4e2d\u5982\u4f55\u8f7b\u677e\u5b9e\u73b0\u6587\u56fe\u751f\u6210\uff0c\u5e26\u4f60\u79d2\u53d8\u827a\u672f\u5bb6\u3002\u672c\u6587\u5f00\u5934\u7684\u5c55\u793a\u56fe\u7247\u5373\u4e3a\u6211\u4eec\u6a21\u578b\u521b\u4f5c\u7684\u4f5c\u54c1\u3002\u6587\u56fe\u751f\u6210\u6a21\u578b\u7b80\u8ff0\u4e0b\u9762\u4ee5\u51e0\u4e2a\u7ecf\u5178\u7684\u57fa\u4e8eTransformer\u7684\u5de5\u4f5c\u4e3a\u4f8b\uff0c\u7b80\u5355\u4ecb\u7ecd\u6587\u56fe\u751f\u6210\u6a21\u578b\u7684\u6280\u672f\u3002DALL-E\u7531OpenAI\u63d0\u51fa\uff0c\u91c7\u53d6\u4e24\u9636\u6bb5\u7684\u65b9\u6cd5\u751f\u6210\u56fe\u50cf\u3002\u5728\u7b2c\u4e00\u9636\u6bb5\uff0c\u8bad\u7ec3\u4e00\u4e2adVAE\uff08discrete variational autoencoder\uff09\u7684\u6a21\u578b\u5c06256\u00d7256\u7684RGB\u56fe\u7247\u8f6c\u5316\u4e3a32\u00d732\u7684image token\uff0c\u8fd9\u4e00\u6b65\u9aa4\u5c06\u56fe\u7247\u8fdb\u884c\u4fe1\u606f\u538b\u7f29\u548c\u79bb\u6563\u5316\uff0c\u65b9\u4fbf\u8fdb\u884c\u6587\u672c\u5230\u56fe\u50cf\u7684\u751f\u6210\u3002\u7b2c\u4e8c\u9636\u6bb5\uff0cDALL-E\u8bad\u7ec3\u4e00\u4e2a\u81ea\u56de\u5f52\u7684Transformer\u6a21\u578b\uff0c\u5c06\u6587\u672c\u8f93\u5165\u8f6c\u5316\u4e3a\u4e0a\u8ff01024\u4e2aimage token\u3002\u7531\u6e05\u534e\u5927\u5b66\u7b49\u5355\u4f4d\u63d0\u51fa\u7684CogView\u6a21\u578b\u5bf9\u4e0a\u8ff0\u4e24\u9636\u6bb5\u6587\u56fe\u751f\u6210\u7684\u8fc7\u7a0b\u8fdb\u884c\u4e86\u8fdb\u4e00\u6b65\u7684\u4f18\u5316\u3002\u5728\u4e0b\u56fe\u4e2d\uff0cCogView\u91c7\u7528\u4e86sentence piece\u4f5c\u4e3atext tokenizer\u4f7f\u5f97\u8f93\u5165\u6587\u672c\u7684\u7a7a\u95f4\u8868\u8fbe\u66f4\u52a0\u4e30\u5bcc\uff0c\u5e76\u4e14\u5728\u6a21\u578b\u7684Fine-tune\u8fc7\u7a0b\u4e2d\u91c7\u7528\u4e86\u591a\u79cd\u6280\u672f\uff0c\u4f8b\u5982\u56fe\u50cf\u7684\u8d85\u5206\u3001\u98ce\u683c\u8fc1\u79fb\u7b49\u3002<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240419\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>ERNIE-ViLG\u6a21\u578b\u8003\u8651\u8fdb\u4e00\u6b65\u8003\u8651\u4e86Transformer\u6a21\u578b\u5b66\u4e60\u77e5\u8bc6\u7684\u53ef\u8fc1\u79fb\u6027\uff0c\u540c\u65f6\u5b66\u4e60\u4e86\u4ece\u6587\u672c\u751f\u6210\u56fe\u50cf\u548c\u4ece\u56fe\u50cf\u751f\u6210\u6587\u672c\u8fd9\u4e24\u79cd\u4efb\u52a1\u3002\u5176\u67b6\u6784\u56fe\u5982\u4e0b\u6240\u793a\uff1a<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240420\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u968f\u7740\u6587\u56fe\u751f\u6210\u6280\u672f\u7684\u4e0d\u65ad\u53d1\u5c55\uff0c\u65b0\u7684\u6a21\u578b\u548c\u6280\u672f\u4e0d\u65ad\u6d8c\u73b0\u3002\u4e3e\u4f8b\u6765\u8bf4\uff0cOFA\u5c06\u591a\u79cd\u8de8\u6a21\u6001\u7684\u751f\u6210\u4efb\u52a1\u7edf\u4e00\u5728\u540c\u4e00\u4e2a\u6a21\u578b\u67b6\u6784\u4e2d\u3002DALL-E 2\u540c\u6837\u7531OpenAI\u63d0\u51fa\uff0c\u662fDALL-E\u6a21\u578b\u7684\u5347\u7ea7\u7248\uff0c\u8003\u8651\u4e86\u5c42\u6b21\u5316\u7684\u56fe\u50cf\u751f\u6210\u6280\u672f\uff0c\u6a21\u578b\u5229\u7528CLIP encoder\u4f5c\u4e3a\u7f16\u7801\u5668\uff0c\u66f4\u597d\u5730\u878d\u5165\u4e86CLIP\u9884\u8bad\u7ec3\u7684\u8de8\u6a21\u6001\u8868\u5f81\u3002Google\u8fdb\u4e00\u6b65\u63d0\u51fa\u4e86Diffusion Model\u7684\u67b6\u6784\uff0c\u80fd\u6709\u6548\u751f\u6210\u9ad8\u6e05\u5927\u56fe\uff0c\u5982\u4e0b\u6240\u793a\uff1a<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240421\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u5728\u672c\u6587\u4e2d\uff0c\u6211\u4eec\u4e0d\u518d\u5bf9\u8fd9\u4e9b\u7ec6\u8282\u8fdb\u884c\u8d58\u8ff0\u3002\u611f\u5174\u8da3\u7684\u8bfb\u8005\u53ef\u4ee5\u8fdb\u4e00\u6b65\u67e5\u9605\u53c2\u8003\u6587\u732e\u3002EasyNLP\u6587\u56fe\u751f\u6210\u6a21\u578b\u7531\u4e8e\u524d\u8ff0\u6a21\u578b\u7684\u89c4\u6a21\u5f80\u5f80\u5728\u6570\u5341\u4ebf\u3001\u767e\u4ebf\u53c2\u6570\u7ea7\u522b\uff0c\u5e9e\u5927\u7684\u6a21\u578b\u867d\u7136\u80fd\u751f\u6210\u8d28\u91cf\u8f83\u5927\u7684\u56fe\u7247\uff0c\u7136\u540e\u5bf9\u8ba1\u7b97\u8d44\u6e90\u548c\u9884\u8bad\u7ec3\u6570\u636e\u7684\u8981\u6c42\u4f7f\u5f97\u8fd9\u4e9b\u6a21\u578b\u5f88\u96be\u5728\u5f00\u6e90\u793e\u533a\u5e7f\u6cdb\u5e94\u7528\uff0c\u5c24\u5176\u5728\u9700\u8981\u9762\u5411\u5782\u76f4\u9886\u57df\u7684\u60c5\u51b5\u4e0b\u3002\u5728\u672c\u8282\u4e2d\uff0c\u6211\u4eec\u8be6\u7ec6\u4ecb\u7ecdEasyNLP\u63d0\u4f9b\u7684\u4e2d\u6587\u6587\u56fe\u751f\u6210\u6a21\u578b\uff0c\u5b83\u5728\u8f83\u5c0f\u53c2\u6570\u91cf\u7684\u60c5\u51b5\u4e0b\uff0c\u4f9d\u7136\u5177\u6709\u826f\u597d\u7684\u6587\u56fe\u751f\u6210\u6548\u679c\u3002\u6a21\u578b\u67b6\u6784\u6a21\u578b\u6846\u67b6\u56fe\u5982\u4e0b\u56fe\u6240\u793a\uff1a<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240422\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u8003\u8651\u5230Transformer\u6a21\u578b\u590d\u6742\u5ea6\u968f\u5e8f\u5217\u957f\u5ea6\u5448\u4e8c\u6b21\u65b9\u589e\u957f\uff0c\u6587\u56fe\u751f\u6210\u6a21\u578b\u7684\u8bad\u7ec3\u4e00\u822c\u4ee5\u56fe\u50cf\u77e2\u91cf\u91cf\u5316\u548c\u81ea\u56de\u5f52\u8bad\u7ec3\u4e24\u9636\u6bb5\u7ed3\u5408\u7684\u65b9\u5f0f\u8fdb\u884c\u3002\u56fe\u50cf\u77e2\u91cf\u91cf\u5316\u662f\u6307\u5c06\u56fe\u50cf\u8fdb\u884c\u79bb\u6563\u5316\u7f16\u7801\uff0c\u5982\u5c06256\u00d7256\u7684RGB\u56fe\u50cf\u8fdb\u884c16\u500d\u964d\u91c7\u6837\uff0c\u5f97\u523016\u00d716\u7684\u79bb\u6563\u5316\u5e8f\u5217\uff0c\u5e8f\u5217\u4e2d\u7684\u6bcf\u4e2aimage token\u5bf9\u5e94\u4e8ecodebook\u4e2d\u7684\u8868\u793a\u3002\u5e38\u89c1\u7684\u56fe\u50cf\u77e2\u91cf\u91cf\u5316\u65b9\u6cd5\u5305\u62ec\uff1aVQVAE\u3001VQVAE-2\u548cVQGAN\u7b49\u3002\u6211\u4eec\u91c7\u7528VQGAN\u5728ImageNet\u4e0a\u8bad\u7ec3\u7684f16_16384\uff0816\u500d\u964d\u91c7\u6837\uff0c\u8bcd\u8868\u5927\u5c0f\u4e3a16384\uff09\u7684\u6a21\u578b\u6743\u91cd\u6765\u751f\u6210\u56fe\u50cf\u7684\u79bb\u6563\u5316\u5e8f\u5217\u3002\u81ea\u56de\u5f52\u8bad\u7ec3\u662f\u6307\u5c06\u6587\u672c\u5e8f\u5217\u548c\u56fe\u50cf\u5e8f\u5217\u4f5c\u4e3a\u8f93\u5165\uff0c\u5728\u56fe\u50cf\u90e8\u5206\uff0c\u6bcf\u4e2aimage token\u4ec5\u4e0e\u6587\u672c\u5e8f\u5217\u7684tokens\u548c\u5176\u4e4b\u524d\u7684image tokens\u8fdb\u884cattention\u8ba1\u7b97\u3002\u6211\u4eec\u91c7\u7528GPT\u4f5c\u4e3abackbone\uff0c\u80fd\u591f\u9002\u5e94\u4e0d\u540c\u6a21\u578b\u89c4\u6a21\u7684\u751f\u6210\u4efb\u52a1\u3002\u5728\u6a21\u578b\u9884\u6d4b\u9636\u6bb5\uff0c\u8f93\u5165\u6587\u672c\u5e8f\u5217\uff0c\u6a21\u578b\u4ee5\u81ea\u56de\u5f52\u7684\u65b9\u5f0f\u9010\u6b65\u751f\u6210\u5b9a\u957f\u7684\u56fe\u50cf\u5e8f\u5217\uff0c\u518d\u901a\u8fc7VQGAN decoder\u91cd\u6784\u4e3a\u56fe\u50cf\u3002\u5f00\u6e90\u6a21\u578b\u53c2\u6570\u8bbe\u7f6e\u5728EasyNLP\u4e2d\uff0c\u6211\u4eec\u63d0\u4f9b\u4e24\u4e2a\u7248\u672c\u7684\u4e2d\u6587\u6587\u56fe\u751f\u6210\u6a21\u578b\uff0c\u6a21\u578b\u53c2\u6570\u914d\u7f6e\u5982\u4e0b\u8868\uff1a\u6a21\u578b\u914d\u7f6epai-painter-base-zhpai-painter-large-zh\u53c2\u6570\u91cf\uff08Parameters\uff09202M433M\u5c42\u6570\uff08Number of Layers\uff091224\u6ce8\u610f\u529b\u5934\u6570\uff08Attention Heads\uff091216\u9690\u5411\u91cf\u7ef4\u5ea6\uff08Hidden Size\uff097681024\u6587\u672c\u957f\u5ea6\uff08Text Length\uff093232\u56fe\u50cf\u5e8f\u5217\u957f\u5ea6\uff08Image Length\uff0916 x 1616 x 16\u56fe\u50cf\u5c3a\u5bf8\uff08Image Size\uff09256 x 256256 x 256VQGAN\u8bcd\u8868\u5927\u5c0f\uff08Codebook Size\uff091638416384\u6a21\u578b\u5b9e\u73b0\u5728EasyNLP\u6846\u67b6\u4e2d\uff0c\u6211\u4eec\u5728\u6a21\u578b\u5c42\u6784\u5efa\u57fa\u4e8eminGPT\u7684backbone\u6784\u5efa\u6a21\u578b\uff0c\u6838\u5fc3\u90e8\u5206\u5982\u4e0b\u6240\u793a\uff1aself.first_stage_model = VQModel(ckpt_path=vqgan_ckpt_path).eval()<br \/>self.transformer = GPT(self.config)VQModel\u7684Encoding\u9636\u6bb5\u8fc7\u7a0b\u4e3a\uff1a# in easynlp\/appzoo\/text2image_generation\/model.py<\/p>\n<p>@torch.no_grad()<br \/>def encode_to_z(self, x):<\/p>\n<pre><code>quant_z, _, info = self.first_stage_model.encode(x)\nindices = info[2].view(quant_z.shape[0], -1)\nreturn quant_z, indices\n<\/code><\/pre>\n<p>x = inputs[&#8216;image&#8217;]<br \/>x = x.permute(0, 3, 1, 2).to(memory_format=torch.contiguous_format)<\/p>\n<h2>one step to produce the logits<\/h2>\n<p>_, z_indices = self.encode_to_z(x)  # z_indice: torch.Size([batch_size, 256]) VQModel\u7684Decoding\u9636\u6bb5\u8fc7\u7a0b\u4e3a\uff1a# in easynlp\/appzoo\/text2image_generation\/model.py<\/p>\n<p>@torch.no_grad()<br \/>def decode_to_img(self, index, zshape):<\/p>\n<pre><code>bhwc = (zshape[0],zshape[2],zshape[3],zshape[1])\nquant_z = self.first_stage_model.quantize.get_codebook_entry(\n    index.reshape(-1), shape=bhwc)\nx = self.first_stage_model.decode(quant_z)\nreturn x\n<\/code><\/pre>\n<h2>sample\u4e3a\u8bad\u7ec3\u9636\u6bb5\u7684\u7ed3\u679c\u751f\u6210\uff0c\u4e0e\u9884\u6d4b\u9636\u6bb5\u7684generate\u7c7b\u4f3c\uff0c\u8be6\u89e3\u89c1\u4e0b\u6587generate<\/h2>\n<p>index_sample = self.sample(z_start_indices, c_indices,<\/p>\n<pre><code>                       steps=z_indices.shape[1],\n                       ...)<\/code><\/pre>\n<p>x_sample = self.decode_to_img(index_sample, quant_z.shape)Transformer\u91c7\u7528minGPT\u8fdb\u884c\u6784\u5efa\uff0c\u8f93\u5165\u56fe\u50cf\u7684\u79bb\u6563\u7f16\u7801\uff0c\u8f93\u51fa\u6587\u672ctoken\u3002\u524d\u5411\u4f20\u64ad\u8fc7\u7a0b\u4e3a\uff1a# in easynlp\/appzoo\/text2image_generation\/model.py<\/p>\n<p>def forward(self, inputs):<\/p>\n<pre><code>x = inputs['image']\nc = inputs['text']\nx = x.permute(0, 3, 1, 2).to(memory_format=torch.contiguous_format)\n# one step to produce the logits\n_, z_indices = self.encode_to_z(x)  # z_indice: torch.Size([batch_size, 256]) \nc_indices = c\n\nif self.training and self.pkeep &lt; 1.0:\n    mask = torch.bernoulli(self.pkeep*torch.ones(z_indices.shape,\n                                                 device=z_indices.device))\n    mask = mask.round().to(dtype=torch.int64)\n    r_indices = torch.randint_like(z_indices, self.transformer.config.vocab_size)\n    a_indices = mask*z_indices+(1-mask)*r_indices\n\nelse:\n    a_indices = z_indices\n    cz_indices = torch.cat((c_indices, a_indices), dim=1)\n    # target includes all sequence elements (no need to handle first one\n    # differently because we are conditioning)\n    target = z_indices\n    # make the prediction\n    logits, _ = self.transformer(cz_indices[:, :-1])\n    # cut off conditioning outputs - output i corresponds to p(z_i | z_{&lt;i}, c)\n    logits = logits[:, c_indices.shape[1]-1:]\nreturn logits, target\u5728\u9884\u6d4b\u9636\u6bb5\uff0c\u8f93\u5165\u4e3a\u6587\u672ctoken\uff0c \u8f93\u51fa\u4e3a256*256\u7684\u56fe\u50cf\u3002\u9996\u5148\uff0c\u5c06\u8f93\u5165\u6587\u672c\u9884\u5904\u7406\u4e3atoken\u5e8f\u5217\uff1a# in easynlp\/appzoo\/text2image_generation\/predictor.py\n<\/code><\/pre>\n<p>def preprocess(self, in_data):<\/p>\n<pre><code>if not in_data:\n    raise RuntimeError(&quot;Input data should not be None.&quot;)\n\nif not isinstance(in_data, list):\n    in_data = [in_data]\nrst = {&quot;idx&quot;: [], &quot;input_ids&quot;: []}\nmax_seq_length = -1\nfor record in in_data:\n    if &quot;sequence_length&quot; not in record:\n        break\n    max_seq_length = max(max_seq_length, record[&quot;sequence_length&quot;])\nmax_seq_length = self.sequence_length if (max_seq_length == -1) else max_seq_length\n\nfor record in in_data:\n    text= record[self.first_sequence]\n    try:\n        self.MUTEX.acquire()\n        text_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(text))\n        text_ids = text_ids[: self.text_len]\n        n_pad = self.text_len - len(text_ids)\n        text_ids += [self.pad_id] * n_pad\n        text_ids = np.array(text_ids) + self.img_vocab_size\n\n    finally:\n        self.MUTEX.release()\n\n    rst[&quot;idx&quot;].append(record[&quot;idx&quot;]) \n    rst[&quot;input_ids&quot;].append(text_ids)\nreturn rst\u9010\u6b65\u751f\u6210\u957f\u5ea6\u4e3a16*16\u7684\u56fe\u50cf\u79bb\u6563token\u5e8f\u5217\uff1a# in easynlp\/appzoo\/text2image_generation\/model.py\n<\/code><\/pre>\n<p>def generate(self, inputs, top_k=100, temperature=1.0):<\/p>\n<pre><code>cidx = inputs\nsample = True\nsteps = 256\nfor k in range(steps):\n    x_cond = cidx\n    logits, _ = self.transformer(x_cond)\n    # pluck the logits at the final step and scale by temperature\n    logits = logits[:, -1, :] \/ temperature\n    # optionally crop probabilities to only the top k options\n    if top_k is not None:\n        logits = self.top_k_logits(logits, top_k)\n    # apply softmax to convert to probabilities\n    probs = torch.nn.functional.softmax(logits, dim=-1)\n    # sample from the distribution or take the most likely\n    if sample:\n        ix = torch.multinomial(probs, num_samples=1)\n    else:\n        _, ix = torch.topk(probs, k=1, dim=-1)\n    # append to the sequence and continue\n    cidx = torch.cat((cidx, ix), dim=1)\nimg_idx = cidx[:, 32:]\nreturn img_idx\u6700\u540e\uff0c\u6211\u4eec\u8c03\u7528VQModel\u7684Decoding\u8fc7\u7a0b\u5c06\u8fd9\u4e9b\u56fe\u50cf\u79bb\u6563token\u5e8f\u5217\u8f6c\u6362\u4e3a\u56fe\u50cf\u3002\u6a21\u578b\u6548\u679c\u6211\u4eec\u5728\u56db\u4e2a\u4e2d\u6587\u7684\u516c\u5f00\u6570\u636e\u96c6COCO-CN\u3001MUGE\u3001Flickr8k-CN\u3001Flickr30k-CN\u4e0a\u9a8c\u8bc1\u4e86EasyNLP\u6846\u67b6\u4e2d\u6587\u56fe\u751f\u6210\u6a21\u578b\u7684\u6548\u679c\u3002\u540c\u65f6\uff0c\u6211\u4eec\u5bf9\u6bd4\u4e86\u8fd9\u4e2a\u6a21\u578b\u548cCogView\u3001DALL-E\u7684\u6548\u679c\uff0c\u5982\u4e0b\u6240\u793a\uff1aParadigmModel#Param.COCO-CNMUGEFlickr8k-CNFlickr30k-CNFID\u2193IS\u2191FID\u2193IS\u2191FID\u2193IS\u2191FID\u2193IS\u2191Zero-shotCogView4B102.3011.81\u00b10.8429.0810.71\u00b10.40102.0111.58\u00b10.66103.3410.50\u00b10.35DALL-E209M89.7310.32\u00b10.6440.289.90\u00b10.4877.8410.57\u00b10.3777.0810.03\u00b10.60Fine-tuningDALL-E209M84.7311.08\u00b10.8922.4210.28\u00b10.4472.179.89\u00b10.4168.759.86\u00b10.40pai-painter-base-zh202M76.8911.65\u00b10.8913.3111.91\u00b10.3655.5612.54\u00b10.48\u00a055.6610.19\u00b10.30\u00a0\u5176\u4e2d\uff0c1\uff09MUGE\u662f\u5929\u6c60\u5e73\u53f0\u516c\u5e03\u7684\u7535\u5546\u573a\u666f\u7684\u4e2d\u6587\u5927\u89c4\u6a21\u591a\u6a21\u6001\u8bc4\u6d4b\u57fa\u51c6\uff08http:\/\/tianchi.aliyun.com\/muge\uff09\u3002\u4e3a\u4e86\u65b9\u4fbf\u8ba1\u7b97\u6307\u6807\uff0cMUGE\u6211\u4eec\u91c7\u7528valid\u6570\u636e\u96c6\u7684\u7ed3\u679c\uff0c\u5176\u4ed6\u6570\u636e\u96c6\u91c7\u7528test\u6570\u636e\u96c6\u7684\u7ed3\u679c\u30022\uff09CogView\u6e90\u81eahttps:\/\/github.com\/THUDM\/CogView3\uff09DALL-E\u6a21\u578b\u6ca1\u6709\u516c\u5f00\u7684\u5b98\u65b9\u4ee3\u7801\u3002\u5df2\u7ecf\u516c\u5f00\u7684\u90e8\u5206\u53ea\u5305\u542bVQVAE\u7684\u4ee3\u7801\uff0c\u4e0d\u5305\u62ecTransformer\u90e8\u5206\u3002\u6211\u4eec\u57fa\u4e8e\u5e7f\u53d7\u5173\u6ce8\u7684https:\/\/github.com\/lucidrains\/DALLE-pytorch\u7248\u672c\u7684\u4ee3\u7801\u548c\u8be5\u7248\u672c\u63a8\u8350\u7684checkpoits\u8fdb\u884c\u590d\u73b0\uff0ccheckpoints\u4e3a2.09\u4ebf\u53c2\u6570\uff0c\u4e3aOpenAI\u7684DALL-E\u6a21\u578b\u53c2\u6570\u91cf\u76841\/100\u3002\uff08OpenAI\u7248\u672cDALL-E\u4e3a120\u4ebf\u53c2\u6570\uff0c\u5176\u4e2dCLIP\u4e3a4\u4ebf\u53c2\u6570\uff09\u3002\u7ecf\u5178\u6848\u4f8b\u6211\u4eec\u5206\u522b\u5728\u81ea\u7136\u98ce\u666f\u6570\u636e\u96c6COCO-CN\u4e0aFine-tune\u4e86base\u548clarge\u7ea7\u522b\u7684\u6a21\u578b\uff0c\u5982\u4e0b\u5c55\u793a\u4e86\u6a21\u578b\u7684\u6548\u679c\uff1a\u793a\u4f8b1\uff1a\u4e00\u53ea\u4fcf\u76ae\u7684\u72d7\u6b63\u8dd1\u8fc7\u8349\u5730pai-painter-base-zhpai-painter-large-zh<\/code><\/pre>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240423\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240424\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240425\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240426\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0\u793a\u4f8b2\uff1a\u4e00\u7247\u6c34\u57df\u7684\u666f\u8272\u4ee5\u65e5\u843d\u4e3a\u80cc\u666fpai-painter-base-zhpai-painter-large-zh<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240427\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240428\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240429\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240430\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0\u6211\u4eec\u4e5f\u79ef\u7d2f\u4e86\u963f\u91cc\u96c6\u56e2\u7684\u6d77\u91cf\u7535\u5546\u5546\u54c1\u6570\u636e\uff0c\u5fae\u8c03\u5f97\u5230\u4e86\u9762\u5411\u7535\u5546\u5546\u54c1\u7684\u6587\u56fe\u751f\u6210\u6a21\u578b\u3002\u6548\u679c\u5982\u4e0b\uff1a\u793a\u4f8b3\uff1a\u5973\u7ae5\u5957\u5934\u6bdb\u8863\u6253\u5e95\u886b\u79cb\u51ac\u9488\u7ec7\u886b\u7ae5\u88c5\u513f\u7ae5\u5185\u642d\u4e0a\u8863pai-painter-base-zhpai-painter-large-zh<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240431\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240432\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240433\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240434\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0\u793a\u4f8b4\uff1a\u6625\u590f\u771f\u76ae\u5de5\u4f5c\u978b\u5973\u6df1\u8272\u8f6f\u76ae\u4e45\u7ad9\u8212\u9002\u4e0a\u73ed\u9762\u8bd5\u804c\u4e1a\u76ae\u978bpai-painter-base-zhpai-painter-large-zh<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240435\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240436\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240437\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240438\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u00a0\u9664\u4e86\u652f\u6301\u7279\u5b9a\u9886\u57df\u7684\u5e94\u7528\uff0c\u6587\u56fe\u751f\u6210\u4e5f\u6781\u5927\u5730\u8f85\u52a9\u4e86\u4eba\u7c7b\u7684\u827a\u672f\u521b\u4f5c\u3002\u4f7f\u7528\u8bad\u7ec3\u5f97\u5230\u7684\u6a21\u578b\uff0c\u6211\u4eec\u53ef\u4ee5\u79d2\u53d8\u201c\u4e2d\u56fd\u56fd\u753b\u827a\u672f\u5927\u5e08\u201d\uff0c\u793a\u4f8b\u5982\u4e0b\u6240\u793a\uff1a<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240439\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u9759\u591c\u6c89\u6c89\uff0c\u6d6e\u5149\u972d\u972d<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240440\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u9065\u671b\u5434\u5c71\u4e3a\u8c01\u597d\uff0c\u5ffd\u95fb\u695a\u4e9b\u4ee4\u4eba\u4f24<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240441\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u98ce\u9601\u6c34\u5e18\u4eca\u5728\u773c\uff0c\u4e14\u6765\u5148\u770b\u65e9\u6885\u7ea2<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240442\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u89c1\u8bf4\u6625\u98ce\u504f\u6709\u8d3a\uff0c\u9732\u82b1\u5343\u6735\u7167\u5ead\u95f1\u00a0\u66f4\u591a\u7684\u793a\u4f8b\u8bf7\u6b23\u8d4f\uff1a<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042240443\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\"><br \/>\u4f7f\u7528\u6559\u7a0b\u6b23\u8d4f\u4e86\u6a21\u578b\u751f\u6210\u7684\u4f5c\u54c1\u4e4b\u540e\uff0c\u5982\u679c\u6211\u4eec\u60f3DIY\uff0c\u8bad\u7ec3\u81ea\u5df1\u7684\u6587\u56fe\u751f\u6210\u6a21\u578b\uff0c\u5e94\u8be5\u5982\u4f55\u8fdb\u884c\u5462\uff1f\u4ee5\u4e0b\u6211\u4eec\u7b80\u8981\u4ecb\u7ecd\u5728EasyNLP\u6846\u67b6\u5bf9\u9884\u8bad\u7ec3\u7684\u6587\u56fe\u751f\u6210\u6a21\u578b\u8fdb\u884cFine-tune\u548c\u63a8\u7406\u3002\u5b89\u88c5EasyNLP\u7528\u6237\u53ef\u4ee5\u76f4\u63a5\u53c2\u8003\u94fe\u63a5\u7684\u8bf4\u660e\u5b89\u88c5EasyNLP\u7b97\u6cd5\u6846\u67b6\u3002\u6570\u636e\u51c6\u5907\u9996\u5148\u51c6\u5907\u8bad\u7ec3\u6570\u636e\u4e0e\u9a8c\u8bc1\u6570\u636e\uff0c\u4e3atsv\u6587\u4ef6\u3002\u8fd9\u4e00\u6587\u4ef6\u5305\u542b\u4ee5\u5236\u8868\u7b26\/t\u5206\u9694\u7684\u4e24\u5217\uff0c\u7b2c\u4e00\u5217\u4e3a\u7d22\u5f15\u53f7\uff0c\u7b2c\u4e8c\u5217\u4e3a\u6587\u672c\uff0c\u7b2c\u4e09\u5217\u4e3a\u56fe\u7247\u7684base64\u7f16\u7801\u3002\u7528\u4e8e\u6d4b\u8bd5\u7684\u8f93\u5165\u6587\u4ef6\u4e3a\u4e24\u5217\uff0c\u4ec5\u5305\u542b\u7d22\u5f15\u53f7\u548c\u6587\u672c\u3002\u4e3a\u4e86\u65b9\u4fbf\u5f00\u53d1\u8005\uff0c\u6211\u4eec\u4e5f\u63d0\u4f9b\u4e86\u8f6c\u6362\u56fe\u7247\u5230base64\u7f16\u7801\u7684\u793a\u4f8b\u4ee3\u7801\uff1aimport base64<br \/>from io import BytesIO<br \/>from PIL import Image<\/p>\n<p>img = Image.open(fn)<br \/>img_buffer = BytesIO()<br \/>img.save(img_buffer, format=img.format)<br \/>byte_data = img_buffer.getvalue()<br \/>base64_str = base64.b64encode(byte_data) # bytes\u4e0b\u5217\u6587\u4ef6\u5df2\u7ecf\u5b8c\u6210\u9884\u5904\u7406\uff0c\u53ef\u7528\u4e8e\u6d4b\u8bd5\uff1a# train<br \/><a href=\"https:\/\/link.segmentfault.com\/?enc=ok03DZx9KTYZGnqYadi4ng%3D%3D.i7dp8%2F4E6%2B2gD%2F239DD29v21%2BFfGisqjHifhHTp4HCDdIjAfW5VbOhYAMWGrRwbz2eTPSBqMKqUAx1EuclCndYESdXV%2FU3jWHpfXsczHY%2Fqqx8ZgtPGycq2QqRcpVCT6gIOyDfMcui72VIzNim6VbX0QESXzisVE2JQrs7GreSo%3D\" rel=\"nofollow\">https:\/\/atp-modelzoo-sh.oss-c&#8230;<\/a><\/p>\n<h2>valid<\/h2>\n<p><a href=\"https:\/\/link.segmentfault.com\/?enc=rXuoSrq2EM%2BZBca1w3TS%2BQ%3D%3D.VVGrpFiaNsbLj1mDV5o1A3pYxQ%2BKW7i%2BBcjQPxM2es9cs510d76C3GCR7z1yfra9qlz7pdWkIgwDcajH%2FJyi%2BgOXxlW8DPSfp6G0AjUdkqyEiSM5cLxgj2PqEJwSoChJfSuSXFTUBS1idBBba%2Fg9%2B1Jm6lMjKeeKI8wycl2KO6I%3D\" rel=\"nofollow\">https:\/\/atp-modelzoo-sh.oss-c&#8230;<\/a><\/p>\n<h2>test<\/h2>\n<p><a href=\"https:\/\/link.segmentfault.com\/?enc=q3Q0ljx3cCruEHUa5b%2BZzg%3D%3D.Ahn7hXfRguJjwY2hdQk9ld7A%2F6qtAh0pjiqel6UexOu1eowGjG0g7MqD0rHk4iJHdE4H0%2FSQ2UzcBB%2BKpW9x5aDxc5PIqHPnN7Br3qwdrmuiqm0x8Sums4L7%2FQ3ITSwc2LnAPcS3G1oD%2BSqRBGpAGg%3D%3D\" rel=\"nofollow\">https:\/\/atp-modelzoo-sh.oss-c&#8230;<\/a>\u6a21\u578b\u8bad\u7ec3\u6211\u4eec\u91c7\u7528\u4ee5\u4e0b\u547d\u4ee4\u5bf9\u6a21\u578b\u8fdb\u884cfine-tune\uff1aeasynlp \/<\/p>\n<pre><code>--mode=train \/\n--worker_gpu=1 \/\n--tables=MUGE_val_text_imgbase64.tsv,MUGE_val_text_imgbase64.tsv \/\n--input_schema=idx:str:1,text:str:1,imgbase64:str:1 \/\n--first_sequence=text \/\n--second_sequence=imgbase64 \/\n--checkpoint_dir=.\/finetuned_model\/ \/\n--learning_rate=4e-5 \/\n--epoch_num=1 \/\n--random_seed=42 \/\n--logging_steps=100 \/\n--save_checkpoint_steps=1000 \/\n--sequence_length=288 \/\n--micro_batch_size=16 \/\n--app_name=text2image_generation \/\n--user_defined_parameters='\n    pretrain_model_name_or_path=alibaba-pai\/pai-painter-large-zh\n    size=256\n    text_len=32\n    img_len=256\n    img_vocab_size=16384\n' \u6211\u4eec\u63d0\u4f9bbase\u548clarge\u4e24\u4e2a\u7248\u672c\u7684\u9884\u8bad\u7ec3\u6a21\u578b\uff0cpretrain_model_name_or_path\u5206\u522b\u4e3aalibaba-pai\/pai-painter-base-zh\u548calibaba-pai\/pai-painter-large-zh\u3002\u8bad\u7ec3\u5b8c\u6210\u540e\u6a21\u578b\u88ab\u4fdd\u5b58\u5230.\/finetuned_model\/\u3002\u6a21\u578b\u6279\u91cf\u63a8\u7406\u6a21\u578b\u8bad\u7ec3\u5b8c\u6bd5\u540e\uff0c\u6211\u4eec\u53ef\u4ee5\u5c06\u5176\u7528\u4e8e\u56fe\u50cf\u751f\u6210\uff0c\u5176\u793a\u4f8b\u5982\u4e0b\uff1aeasynlp \/\n--mode=predict \/\n--worker_gpu=1 \/\n--tables=MUGE_test.text.tsv \/\n--input_schema=idx:str:1,text:str:1 \/\n--first_sequence=text \/\n--outputs=.\/T2I_outputs.tsv \/\n--output_schema=idx,text,gen_imgbase64 \/\n--checkpoint_dir=.\/finetuned_model\/ \/\n--sequence_length=288 \/\n--micro_batch_size=8 \/\n--app_name=text2image_generation \/\n--user_defined_parameters='\n    size=256\n    text_len=32\n    img_len=256\n    img_vocab_size=16384\n'\u7ed3\u679c\u5b58\u50a8\u5728\u4e00\u4e2atsv\u6587\u4ef6\u4e2d\uff0c\u6bcf\u884c\u5bf9\u5e94\u8f93\u5165\u4e2d\u7684\u4e00\u4e2a\u6587\u672c\uff0c\u8f93\u51fa\u7684\u56fe\u50cf\u4ee5base64\u7f16\u7801\u3002\u4f7f\u7528Pipeline\u63a5\u53e3\u5feb\u901f\u4f53\u9a8c\u6587\u56fe\u751f\u6210\u6548\u679c\u4e3a\u4e86\u8fdb\u4e00\u6b65\u65b9\u4fbf\u5f00\u53d1\u8005\u4f7f\u7528\uff0c\u6211\u4eec\u5728EasyNLP\u6846\u67b6\u5185\u4e5f\u5b9e\u73b0\u4e86Inference Pipeline\u529f\u80fd\u3002\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u8c03\u7528Fine-tune\u8fc7\u7684\u7535\u5546\u573a\u666f\u4e0b\u7684\u6587\u56fe\u751f\u6210\u6a21\u578b\uff1a# \u76f4\u63a5\u6784\u5efapipeline<\/code><\/pre>\n<p>default_ecommercial_pipeline = pipeline(&#8220;pai-painter-commercial-base-zh&#8221;)<\/p>\n<h2>\u6a21\u578b\u9884\u6d4b<\/h2>\n<p>data = [&#8220;\u5bbd\u677eT\u6064&#8221;]<br \/>results = default_ecommercial_pipeline(data)  # results\u7684\u6bcf\u4e00\u6761\u662f\u751f\u6210\u56fe\u50cf\u7684base64\u7f16\u7801<\/p>\n<h2>base64\u8f6c\u6362\u4e3a\u56fe\u50cf<\/h2>\n<p>def base64_to_image(imgbase64_str):<\/p>\n<pre><code>image = Image.open(BytesIO(base64.urlsafe_b64decode(imgbase64_str)))\nreturn image\n<\/code><\/pre>\n<h2>\u4fdd\u5b58\u4ee5\u6587\u672c\u547d\u540d\u7684\u56fe\u50cf<\/h2>\n<p>for text, result in zip(data, results):<\/p>\n<pre><code>imgpath = '{}.png'.format(text)\nimgbase64_str = result['gen_imgbase64']\nimage = base64_to_image(imgbase64_str)\nimage.save(imgpath)\nprint('text: {}, save generated image: {}'.format(text, imgpath))\u9664\u4e86\u7535\u5546\u573a\u666f\uff0c\u6211\u4eec\u8fd8\u63d0\u4f9b\u4e86\u4ee5\u4e0b\u573a\u666f\u7684\u6a21\u578b\uff1a\u81ea\u7136\u98ce\u5149\u573a\u666f\uff1a\u201cpai-painter-scenery-base-zh\u201d\u4e2d\u56fd\u5c71\u6c34\u753b\u573a\u666f\uff1a\u201cpai-painter-painting-base-zh\u201d\u5728\u4e0a\u9762\u7684\u4ee3\u7801\u5f53\u4e2d\u66ff\u6362\u201cpai-painter-commercial-base-zh\u201d\uff0c\u5c31\u53ef\u4ee5\u76f4\u63a5\u4f53\u9a8c\uff0c\u6b22\u8fce\u8bd5\u7528\u3002\u5bf9\u4e8e\u7528\u6237Fine-tune\u7684\u6587\u56fe\u751f\u6210\u6a21\u578b\uff0c\u6211\u4eec\u4e5f\u5f00\u653e\u4e86\u81ea\u5b9a\u4e49\u6a21\u578b\u52a0\u8f7d\u7684Pipeline\u63a5\u53e3\uff1a# \u52a0\u8f7d\u6a21\u578b\uff0c\u6784\u5efapipeline<\/code><\/pre>\n<p>local_model_path = &#8230;<br \/>text_to_image_pipeline = pipeline(&#8220;text2image_generation&#8221;, local_model_path)<\/p>\n<h2>\u6a21\u578b\u9884\u6d4b<\/h2>\n<p>data = [&#8220;xxxx&#8221;]<br \/>results = text_to_image_pipeline(data)  # results\u7684\u6bcf\u4e00\u6761\u662f\u751f\u6210\u56fe\u50cf\u7684base64\u7f16\u7801\u672a\u6765\u5c55\u671b\u5728\u8fd9\u4e00\u671f\u7684\u5de5\u4f5c\u4e2d\uff0c\u6211\u4eec\u5728EasyNLP\u6846\u67b6\u4e2d\u96c6\u6210\u4e86\u4e2d\u6587\u6587\u56fe\u751f\u6210\u529f\u80fd\uff0c\u540c\u65f6\u5f00\u653e\u4e86\u6a21\u578b\u7684Checkpoint\uff0c\u65b9\u4fbf\u5f00\u6e90\u793e\u533a\u7528\u6237\u5728\u8d44\u6e90\u6709\u9650\u60c5\u51b5\u4e0b\u8fdb\u884c\u5c11\u91cf\u9886\u57df\u76f8\u5173\u7684\u5fae\u8c03\uff0c\u8fdb\u884c\u5404\u79cd\u827a\u672f\u521b\u4f5c\u3002\u5728\u672a\u6765\uff0c\u6211\u4eec\u8ba1\u5212\u5728EasyNLP\u6846\u67b6\u4e2d\u63a8\u51fa\u66f4\u591a\u76f8\u5173\u6a21\u578b\uff0c\u656c\u8bf7\u671f\u5f85\u3002\u6211\u4eec\u4e5f\u5c06\u5728EasyNLP\u6846\u67b6\u4e2d\u96c6\u6210\u66f4\u591aSOTA\u6a21\u578b\uff08\u7279\u522b\u662f\u4e2d\u6587\u6a21\u578b\uff09\uff0c\u6765\u652f\u6301\u5404\u79cdNLP\u548c\u591a\u6a21\u6001\u4efb\u52a1\u3002\u6b64\u5916\uff0c\u963f\u91cc\u4e91\u673a\u5668\u5b66\u4e60PAI\u56e2\u961f\u4e5f\u5728\u6301\u7eed\u63a8\u8fdb\u4e2d\u6587\u591a\u6a21\u6001\u6a21\u578b\u7684\u81ea\u7814\u5de5\u4f5c\uff0c\u6b22\u8fce\u7528\u6237\u6301\u7eed\u5173\u6ce8\u6211\u4eec\uff0c\u4e5f\u6b22\u8fce\u52a0\u5165\u6211\u4eec\u7684\u5f00\u6e90\u793e\u533a\uff0c\u5171\u5efa\u4e2d\u6587NLP\u548c\u591a\u6a21\u6001\u7b97\u6cd5\u5e93\uff01Github\u5730\u5740\uff1a<a href=\"https:\/\/link.segmentfault.com\/?enc=nLP93OtETIix%2FwAFHedM6A%3D%3D.vRT0ZJT58m9RTbjyj1CqqQtG7h5CTh%2FEg0RVyabmM90mJbQR%2B2zSXBjt1mcgy53br5ZB8SAW3cLZ46cdF81rYA%3D%3D\" rel=\"nofollow\">https:\/\/github.com\/alibaba\/Ea&#8230;<\/a> Wang, Minghui Qiu, Taolin Zhang, Tingting Liu, Lei Li, Jianing Wang, Ming Wang, Jun Huang, Wei Lin. EasyNLP: A Comprehensive and Easy-to-use Toolkit for Natural Language Processing. arXivAditya Ramesh, Mikhail Pavlov, Gabriel Goh, Scott Gray, Chelsea Voss, Alec Radford, Mark Chen, Ilya Sutskever. Zero-Shot Text-to-Image Generation. ICML 2021: 8821-8831Ming Ding, Zhuoyi Yang, Wenyi Hong, Wendi Zheng, Chang Zhou, Da Yin, Junyang Lin, Xu Zou, Zhou Shao, Hongxia Yang, Jie Tang. CogView: Mastering Text-to-Image Generation via Transformers. NeurIPS 2021: 19822-19835Han Zhang, Weichong Yin, Yewei Fang, Lanxin Li, Boqiang Duan, Zhihua Wu, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. ERNIE-ViLG: Unified Generative Pre-training for Bidirectional Vision-Language Generation. arXivPeng Wang, An Yang, Rui Men, Junyang Lin, Shuai Bai, Zhikang Li, Jianxin Ma, Chang Zhou, Jingren Zhou, Hongxia Yang. Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework. ICML 2022Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, Mark Chen. Hierarchical Text-Conditional Image Generation with CLIP Latents. arXivVan Den Oord A, Vinyals O. Neural discrete representation learning. NIPS 2017Esser P, Rombach R, Ommer B. Taming transformers for high-resolution image synthesis. CVPR 2021: 12873-12883.Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar Seyed Ghasemipour, Burcu Karagol Ayan, S. Sara Mahdavi, Rapha Gontijo Lopes, Tim Salimans, Jonathan Ho, David J. Fleet, Mohammad Norouzi: Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding. arXivJiahui Yu, Yuanzhong Xu, Jing Yu Koh, Thang Luong, Gunjan Baid, Zirui Wang, Vijay Vasudevan, Alexander Ku, Yinfei Yang, Burcu Karagol Ayan, Ben Hutchinson, Wei Han, Zarana Parekh, Xin Li, Han Zhang, Jason Baldridge, Yonghui Wu. Scaling Autoregressive Models for Content-Rich Text-to-Image Generation. arXiv\u963f\u91cc\u7075\u6770\u56de\u987e\u963f\u91cc\u7075\u6770\uff1a\u963f\u91cc\u4e91\u673a\u5668\u5b66\u4e60PAI\u5f00\u6e90\u4e2d\u6587NLP\u7b97\u6cd5\u6846\u67b6EasyNLP\uff0c\u52a9\u529bNLP\u5927\u6a21\u578b\u843d\u5730\u963f\u91cc\u7075\u6770\uff1a\u9884\u8bad\u7ec3\u77e5\u8bc6\u5ea6\u91cf\u6bd4\u8d5b\u593a\u51a0\uff01\u963f\u91cc\u4e91PAI\u53d1\u5e03\u77e5\u8bc6\u9884\u8bad\u7ec3\u5de5\u5177\u963f\u91cc\u7075\u6770\uff1aEasyNLP\u5e26\u4f60\u73a9\u8f6cCLIP\u56fe\u6587\u68c0\u7d22\u539f\u6587\u94fe\u63a5\uff1a<a href=\"https:\/\/link.segmentfault.com\/?enc=tc8d4UXTra44K%2Be4vAHuDw%3D%3D.wGI%2BN0QSMBskYY2yilAPLy7jyFbh6epYhBHLJL5XDn5ojttw9cC7hoVDECvxwj7M\" rel=\"nofollow\">http:\/\/click.aliyun.com\/m\/100&#8230;<\/a>\u672c\u6587\u4e3a\u963f\u91cc\u4e91\u539f\u521b\u5185\u5bb9\uff0c\u672a\u7ecf\u5141\u8bb8\u4e0d\u5f97\u8f6c\u8f7d\u3002<\/p>\n<p><a href=\"https:\/\/segmentfault.com\/a\/1190000042240413\">https:\/\/segmentfault.com\/a\/1190000042240413<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u7b80\u4ecb\uff1a\u00a0\u6211\u4eec\u5728EasyNLP\u6846\u67b6\u4e2d\u96c6\u6210\u4e86\u4e2d\u6587\u6587\u56fe\u751f\u6210\u529f\u80fd\uff0c\u540c\u65f6\u5f00\u653e\u4e86\u6a21\u578b\u7684Checkpoint\u3002\u5bfc\u8bfb \u00a0\u00a0\u5ba3\u7269 [&hellip;]<\/p>\n","protected":false},"author":0,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[],"tags":[],"class_list":["post-10006059","post","type-post","status-publish","format-standard","hentry"],"jetpack_sharing_enabled":true,"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10006059","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"replies":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=10006059"}],"version-history":[{"count":0,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10006059\/revisions"}],"wp:attachment":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=10006059"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=10006059"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=10006059"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}