{"id":10008163,"date":"2023-08-25T14:59:29","date_gmt":"2023-08-25T06:59:29","guid":{"rendered":"https:\/\/blog.itmax.cn\/?p=10008163"},"modified":"2023-08-25T14:59:29","modified_gmt":"2023-08-25T06:59:29","slug":"20000-%e5%ad%97%ef%bc%8c%e5%bd%bb%e5%ba%95%e6%90%9e%e6%87%82-kafka%ef%bc%81","status":"publish","type":"post","link":"https:\/\/itmax.cn\/?p=10008163","title":{"rendered":"20,000+ \u5b57\uff0c\u5f7b\u5e95\u641e\u61c2 Kafka\uff01"},"content":{"rendered":"<h2><strong>1\u3001\u4e3a\u4ec0\u4e48\u6709\u6d88\u606f\u7cfb\u7edf<\/strong><\/h2>\n<h5>1\u3001\u89e3\u8026\u5408<\/h5>\n<h5>2\u3001\u5f02\u6b65\u5904\u7406<\/h5>\n<p>\u4f8b\u5982\u7535\u5546\u5e73\u53f0\uff0c\u79d2\u6740\u6d3b\u52a8\u3002<\/p>\n<p>\u4e00\u822c\u6d41\u7a0b\u4f1a\u5206\u4e3a\uff1a<\/p>\n<ol>\n<li>\u98ce\u9669\u63a7\u5236<\/li>\n<li>\u5e93\u5b58\u9501\u5b9a<\/li>\n<li>\u751f\u6210\u8ba2\u5355<\/li>\n<li>\u77ed\u4fe1\u901a\u77e5<\/li>\n<li>\u66f4\u65b0\u6570\u636e<\/li>\n<\/ol>\n<p>\u901a\u8fc7\u6d88\u606f\u7cfb\u7edf\u5c06\u79d2\u6740\u6d3b\u52a8\u4e1a\u52a1\u62c6\u5206\u5f00\uff0c\u5c06\u4e0d\u6025\u9700\u5904\u7406\u7684\u4e1a\u52a1\u653e\u5728\u540e\u9762\u6162\u6162\u5904\u7406\uff1b<\/p>\n<p>\u6d41\u7a0b\u6539\u4e3a\uff1a<\/p>\n<ol>\n<li>\u98ce\u9669\u63a7\u5236<\/li>\n<li>\u5e93\u5b58\u9501\u5b9a<\/li>\n<li>\u6d88\u606f\u7cfb\u7edf<\/li>\n<li>\u751f\u6210\u8ba2\u5355<\/li>\n<li>\u77ed\u4fe1\u901a\u77e5<\/li>\n<li>\u66f4\u65b0\u6570\u636e<\/li>\n<\/ol>\n<h5>3\u3001\u6d41\u91cf\u7684\u63a7\u5236<\/h5>\n<p>3.1 \u7f51\u5173\u5728\u63a5\u53d7\u5230\u8bf7\u6c42\u540e\uff0c\u5c31\u628a\u8bf7\u6c42\u653e\u5165\u5230\u6d88\u606f\u961f\u5217\u91cc\u9762<\/p>\n<p>3.2 \u540e\u7aef\u7684\u670d\u52a1\u4ece\u6d88\u606f\u961f\u5217\u91cc\u9762\u83b7\u53d6\u5230\u8bf7\u6c42\uff0c\u5b8c\u6210\u540e\u7eed\u7684\u79d2\u6740\u5904\u7406\u6d41\u7a0b\u3002\u7136\u540e\u518d\u7ed9\u7528\u6237\u8fd4\u56de\u7ed3\u679c\u3002<\/p>\n<ul>\n<li>\u4f18\u70b9\uff1a\u63a7\u5236\u4e86\u6d41\u91cf<\/li>\n<li>\u7f3a\u70b9\uff1a\u4f1a\u8ba9\u6d41\u7a0b\u53d8\u6162<\/li>\n<\/ul>\n<p>\u63a8\u8350\u4e00\u4e2a\u5f00\u6e90\u514d\u8d39\u7684 Spring Boot \u5b9e\u6218\u9879\u76ee\uff1a<\/p>\n<blockquote><p><a target=\"_blank\" href=\"https:\/\/link.segmentfault.com\/?enc=WjHoIe6oYYd1tEG7aJ5SIQ%3D%3D.qJKXI%2Bw1HPh1QUYa2awW3v0g2H8ffDiMVejdOS3vLEqNMLXqPVYDhy2cE6X0qTuiNNrXQGyh4AeXZmV%2BnS3sJg%3D%3D\" rel=\"noopener noreferrer\">https:\/\/github.com\/javastacks\/spring-boot-best-practice<\/a><\/p><\/blockquote>\n<h2><strong>2\u3001Kafka\u6838\u5fc3\u6982\u5ff5<\/strong><\/h2>\n<ul>\n<li>\u751f\u4ea7\u8005\uff1aProducer \u5f80Kafka\u96c6\u7fa4\u751f\u6210\u6570\u636e<\/li>\n<li>\u6d88\u8d39\u8005\uff1aConsumer \u5f80Kafka\u91cc\u9762\u53bb\u83b7\u53d6\u6570\u636e\uff0c\u5904\u7406\u6570\u636e\u3001\u6d88\u8d39\u6570\u636e<\/li>\n<\/ul>\n<p>Kafka\u7684\u6570\u636e\u662f\u7531\u6d88\u8d39\u8005\u81ea\u5df1\u53bb\u62c9\u53bbKafka\u91cc\u9762\u7684\u6570\u636e<\/p>\n<ul>\n<li>\u4e3b\u9898\uff1atopic<\/li>\n<li>\u5206\u533a\uff1apartition<\/li>\n<\/ul>\n<p>\u9ed8\u8ba4\u4e00\u4e2atopic\u6709\u4e00\u4e2a\u5206\u533a\uff08partition\uff09\uff0c\u81ea\u5df1\u53ef\u8bbe\u7f6e\u591a\u4e2a\u5206\u533a\uff08\u5206\u533a\u5206\u6563\u5b58\u50a8\u5728\u670d\u52a1\u5668\u4e0d\u540c\u8282\u70b9\u4e0a\uff09<\/p>\n<p>\u89e3\u51b3\u4e86\u4e00\u4e2a\u6d77\u91cf\u6570\u636e\u5982\u4f55\u5b58\u50a8\u7684\u95ee\u9898<\/p>\n<p>\u4f8b\u5982\uff1a\u67092T\u7684\u6570\u636e\uff0c\u4e00\u53f0\u670d\u52a1\u5668\u67091T\uff0c\u4e00\u4e2atopic\u53ef\u4ee5\u5206\u591a\u4e2a\u533a\uff0c\u5206\u522b\u5b58\u50a8\u5728\u591a\u53f0\u670d\u52a1\u5668\u4e0a\uff0c\u89e3\u51b3\u6d77\u91cf\u6570\u636e\u5b58\u50a8\u95ee\u9898<\/p>\n<h2><strong>3\u3001Kafka\u7684\u96c6\u7fa4\u67b6\u6784<\/strong><\/h2>\n<p>Kafka\u96c6\u7fa4\u4e2d\uff0c\u4e00\u4e2akafka\u670d\u52a1\u5668\u5c31\u662f\u4e00\u4e2abroker\uff0cTopic\u53ea\u662f\u903b\u8f91\u4e0a\u7684\u6982\u5ff5\uff0cpartition\u5728\u78c1\u76d8\u4e0a\u5c31\u4f53\u73b0\u4e3a\u4e00\u4e2a\u76ee\u5f55\u3002<\/p>\n<p>Consumer Group\uff1a\u6d88\u8d39\u7ec4\uff0c\u6d88\u8d39\u6570\u636e\u7684\u65f6\u5019\uff0c\u90fd\u5fc5\u987b\u6307\u5b9a\u4e00\u4e2agroup id\uff0c\u6307\u5b9a\u4e00\u4e2a\u7ec4\u7684id<\/p>\n<p>\u5047\u5b9a\u7a0b\u5e8fA\u548c\u7a0b\u5e8fB\u6307\u5b9a\u7684group id\u53f7\u4e00\u6837\uff0c\u90a3\u4e48\u4e24\u4e2a\u7a0b\u5e8f\u5c31\u5c5e\u4e8e\u540c\u4e00\u4e2a\u6d88\u8d39\u7ec4<\/p>\n<p>\u7279\u6b8a\uff1a<\/p>\n<ul>\n<li>\u6bd4\u5982\uff0c\u6709\u4e00\u4e2a\u4e3b\u9898topicA\uff0c \u7a0b\u5e8fA\u53bb\u6d88\u8d39\u4e86\u8fd9\u4e2atopicA\uff0c\u90a3\u4e48\u7a0b\u5e8fB\u5c31\u4e0d\u80fd\u518d\u53bb\u6d88\u8d39topicA\uff08\u7a0b\u5e8fA\u548c\u7a0b\u5e8fB\u5c5e\u4e8e\u4e00\u4e2a\u6d88\u8d39\u7ec4\uff09<\/li>\n<li>\u518d\u6bd4\u5982\u7a0b\u5e8fA\u5df2\u7ecf\u6d88\u8d39\u4e86topicA\u91cc\u9762\u7684\u6570\u636e\uff0c\u73b0\u5728\u8fd8\u662f\u91cd\u65b0\u518d\u6b21\u6d88\u8d39topicA\u7684\u6570\u636e\uff0c\u662f\u4e0d\u53ef\u4ee5\u7684\uff0c\u4f46\u662f\u91cd\u65b0\u6307\u5b9a\u4e00\u4e2agroup id\u53f7\u4ee5\u540e\uff0c\u53ef\u4ee5\u6d88\u8d39\u3002<\/li>\n<\/ul>\n<p>\u4e0d\u540c\u6d88\u8d39\u7ec4\u4e4b\u95f4\u6ca1\u6709\u5f71\u54cd\u3002\u6d88\u8d39\u7ec4\u9700\u81ea\u5b9a\u4e49\uff0c\u6d88\u8d39\u8005\u540d\u79f0\u7a0b\u5e8f\u81ea\u52a8\u751f\u6210\uff08\u72ec\u4e00\u65e0\u4e8c\uff09\u3002<\/p>\n<p>Controller\uff1aKafka\u8282\u70b9\u91cc\u9762\u7684\u4e00\u4e2a\u4e3b\u8282\u70b9\u3002\u501f\u52a9zookeeper<\/p>\n<h2><strong>4\u3001Kafka\u78c1\u76d8\u987a\u5e8f\u5199\u4fdd\u8bc1\u5199\u6570\u636e\u6027\u80fd<\/strong><\/h2>\n<p><strong>kafka\u5199\u6570\u636e\uff1a<\/strong><\/p>\n<p>\u987a\u5e8f\u5199\uff0c\u5f80\u78c1\u76d8\u4e0a\u5199\u6570\u636e\u65f6\uff0c\u5c31\u662f\u8ffd\u52a0\u6570\u636e\uff0c\u6ca1\u6709\u968f\u673a\u5199\u7684\u64cd\u4f5c\u3002<\/p>\n<p><strong>\u7ecf\u9a8c\uff1a<\/strong><\/p>\n<p>\u5982\u679c\u4e00\u4e2a\u670d\u52a1\u5668\u78c1\u76d8\u8fbe\u5230\u4e00\u5b9a\u7684\u4e2a\u6570\uff0c\u78c1\u76d8\u4e5f\u8fbe\u5230\u4e00\u5b9a\u8f6c\u6570\uff0c\u5f80\u78c1\u76d8\u91cc\u9762\u987a\u5e8f\u5199\uff08\u8ffd\u52a0\u5199\uff09\u6570\u636e\u7684\u901f\u5ea6\u548c\u5199\u5185\u5b58\u7684\u901f\u5ea6\u5dee\u4e0d\u591a\u3002<\/p>\n<p>\u751f\u4ea7\u8005\u751f\u4ea7\u6d88\u606f\uff0c\u7ecf\u8fc7kafka\u670d\u52a1\u5148\u5199\u5230os cache \u5185\u5b58\u4e2d\uff0c\u7136\u540e\u7ecf\u8fc7sync\u987a\u5e8f\u5199\u5230\u78c1\u76d8\u4e0a<\/p>\n<h2><strong>5\u3001Kafka\u96f6\u62f7\u8d1d\u673a\u5236\u4fdd\u8bc1\u8bfb\u6570\u636e\u9ad8\u6027\u80fd<\/strong><\/h2>\n<p>\u6d88\u8d39\u8005\u8bfb\u53d6\u6570\u636e\u6d41\u7a0b\uff1a<\/p>\n<ol>\n<li>\u6d88\u8d39\u8005\u53d1\u9001\u8bf7\u6c42\u7ed9kafka\u670d\u52a1<\/li>\n<li>kafka\u670d\u52a1\u53bbos cache\u7f13\u5b58\u8bfb\u53d6\u6570\u636e\uff08\u7f13\u5b58\u6ca1\u6709\u5c31\u53bb\u78c1\u76d8\u8bfb\u53d6\u6570\u636e\uff09<\/li>\n<li>\u4ece\u78c1\u76d8\u8bfb\u53d6\u4e86\u6570\u636e\u5230os cache\u7f13\u5b58\u4e2d<\/li>\n<li>os cache\u590d\u5236\u6570\u636e\u5230kafka\u5e94\u7528\u7a0b\u5e8f\u4e2d<\/li>\n<li>kafka\u5c06\u6570\u636e\uff08\u590d\u5236\uff09\u53d1\u9001\u5230socket cache\u4e2d<\/li>\n<li>socket cache\u901a\u8fc7\u7f51\u5361\u4f20\u8f93\u7ed9\u6d88\u8d39\u8005<\/li>\n<\/ol>\n<p><img loading=\"lazy\" decoding=\"async\" width=\"723\" height=\"433\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxJ\" alt=\"\" title=\"\"><\/p>\n<p>kafka linux sendfile\u6280\u672f \u2014 \u96f6\u62f7\u8d1d<\/p>\n<ol>\n<li>\u6d88\u8d39\u8005\u53d1\u9001\u8bf7\u6c42\u7ed9kafka\u670d\u52a1<\/li>\n<li>kafka\u670d\u52a1\u53bbos cache\u7f13\u5b58\u8bfb\u53d6\u6570\u636e\uff08\u7f13\u5b58\u6ca1\u6709\u5c31\u53bb\u78c1\u76d8\u8bfb\u53d6\u6570\u636e\uff09<\/li>\n<li>\u4ece\u78c1\u76d8\u8bfb\u53d6\u4e86\u6570\u636e\u5230os cache\u7f13\u5b58\u4e2d<\/li>\n<li>os cache\u76f4\u63a5\u5c06\u6570\u636e\u53d1\u9001\u7ed9\u7f51\u5361<\/li>\n<li>\u901a\u8fc7\u7f51\u5361\u5c06\u6570\u636e\u4f20\u8f93\u7ed9\u6d88\u8d39\u8005<\/li>\n<\/ol>\n<p><img decoding=\"async\" width=\"723\" height=\"431\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxK\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h2><strong>6\u3001Kafka\u65e5\u5fd7\u5206\u6bb5\u4fdd\u5b58<\/strong><\/h2>\n<p>Kafka\u4e2d\u4e00\u4e2a\u4e3b\u9898\uff0c\u4e00\u822c\u4f1a\u8bbe\u7f6e\u5206\u533a\uff1b\u6bd4\u5982\u521b\u5efa\u4e86\u4e00\u4e2atopic_a\uff0c\u7136\u540e\u521b\u5efa\u7684\u65f6\u5019\u6307\u5b9a\u4e86\u8fd9\u4e2a\u4e3b\u9898\u6709\u4e09\u4e2a\u5206\u533a\u3002<\/p>\n<p>\u5176\u5b9e\u5728\u4e09\u53f0\u670d\u52a1\u5668\u4e0a\uff0c\u4f1a\u521b\u5efa\u4e09\u4e2a\u76ee\u5f55\u3002<\/p>\n<p><strong>\u670d\u52a1\u56681\uff08kafka1\uff09\uff1a<\/strong><\/p>\n<ul>\n<li>\u521b\u5efa\u76ee\u5f55topic_a-0:<\/li>\n<li>\u76ee\u5f55\u4e0b\u9762\u662f\u6211\u4eec\u6587\u4ef6\uff08\u5b58\u50a8\u6570\u636e\uff09\uff0ckafka\u6570\u636e\u5c31\u662fmessage\uff0c\u6570\u636e\u5b58\u50a8\u5728log\u6587\u4ef6\u91cc<\/li>\n<li>.log\u7ed3\u5c3e\u7684\u5c31\u662f\u65e5\u5fd7\u6587\u4ef6\uff0c\u5728kafka\u4e2d\u628a\u6570\u636e\u6587\u4ef6\u5c31\u53eb\u505a\u65e5\u5fd7\u6587\u4ef6\u3002<\/li>\n<\/ul>\n<p>\u4e00\u4e2a\u5206\u533a\u4e0b\u9762\u9ed8\u8ba4\u6709n\u591a\u4e2a\u65e5\u5fd7\u6587\u4ef6\uff08\u5206\u6bb5\u5b58\u50a8\uff09\uff0c\u4e00\u4e2a\u65e5\u5fd7\u6587\u4ef6\u9ed8\u8ba41G<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"425\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxL\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<p><strong>\u670d\u52a1\u56682\uff08kafka2\uff09\uff1a<\/strong><\/p>\n<ul>\n<li>\u521b\u5efa\u76ee\u5f55topic_a-1:<\/li>\n<\/ul>\n<p><strong>\u670d\u52a1\u56683\uff08kafka3\uff09\uff1a<\/strong><\/p>\n<ul>\n<li>\u521b\u5efa\u76ee\u5f55topic_a-2:<\/li>\n<\/ul>\n<h2><strong>7\u3001Kafka\u4e8c\u5206\u67e5\u627e\u5b9a\u4f4d\u6570\u636e<\/strong><\/h2>\n<p>Kafka\u91cc\u9762\u6bcf\u4e00\u6761\u6d88\u606f\uff0c\u90fd\u6709\u81ea\u5df1\u7684offset\uff08\u76f8\u5bf9\u504f\u79fb\u91cf\uff09\uff0c\u5b58\u5728\u7269\u7406\u78c1\u76d8\u4e0a\u9762\uff0c\u5728position<\/p>\n<p>Position\uff1a\u7269\u7406\u4f4d\u7f6e\uff08\u78c1\u76d8\u4e0a\u9762\u90a3\u4e2a\u5730\u65b9\uff09<\/p>\n<p>\u4e5f\u5c31\u662f\u8bf4\u4e00\u6761\u6d88\u606f\u5c31\u6709\u4e24\u4e2a\u4f4d\u7f6e\uff1a<\/p>\n<ul>\n<li>offset\uff1a\u76f8\u5bf9\u504f\u79fb\u91cf\uff08\u76f8\u5bf9\u4f4d\u7f6e\uff09<\/li>\n<li>position\uff1a\u78c1\u76d8\u7269\u7406\u4f4d\u7f6e<\/li>\n<\/ul>\n<p>\u7a00\u758f\u7d22\u5f15\uff1a<\/p>\n<ul>\n<li>Kafka\u4e2d\u91c7\u7528\u4e86\u7a00\u758f\u7d22\u5f15\u7684\u65b9\u5f0f\u8bfb\u53d6\u7d22\u5f15\uff0ckafka\u6bcf\u5f53\u5199\u5165\u4e864k\u5927\u5c0f\u7684\u65e5\u5fd7\uff08.log\uff09\uff0c\u5c31\u5f80index\u91cc\u5199\u5165\u4e00\u4e2a\u8bb0\u5f55\u7d22\u5f15\u3002<\/li>\n<\/ul>\n<p>\u5176\u4e2d\u4f1a\u91c7\u7528\u4e8c\u5206\u67e5\u627e<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"566\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxM\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h2><strong>8\u3001\u9ad8\u5e76\u53d1\u7f51\u7edc\u8bbe\u8ba1\uff08\u5148\u4e86\u89e3NIO\uff09<\/strong><\/h2>\n<p>\u7f51\u7edc\u8bbe\u8ba1\u90e8\u5206\u662fkafka\u4e2d\u8bbe\u8ba1\u6700\u597d\u7684\u4e00\u4e2a\u90e8\u5206\uff0c\u8fd9\u4e5f\u662f\u4fdd\u8bc1Kafka\u9ad8\u5e76\u53d1\u3001\u9ad8\u6027\u80fd\u7684\u539f\u56e0<\/p>\n<p>\u5bf9kafka\u8fdb\u884c\u8c03\u4f18\uff0c\u5c31\u5f97\u5bf9kafka\u539f\u7406\u6bd4\u8f83\u4e86\u89e3\uff0c\u5c24\u5176\u662f\u7f51\u7edc\u8bbe\u8ba1\u90e8\u5206<\/p>\n<p>Reactor\u7f51\u7edc\u8bbe\u8ba1\u6a21\u5f0f1\uff1a<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"395\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxN\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<p>Reactor\u7f51\u7edc\u8bbe\u8ba1\u6a21\u5f0f2\uff1a<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"421\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxO\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<p>Reactor\u7f51\u7edc\u8bbe\u8ba1\u6a21\u5f0f3\uff1a<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"371\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxP\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<p>Kafka\u8d85\u9ad8\u5e76\u53d1\u7f51\u7edc\u8bbe\u8ba1\uff1a<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"401\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxQ\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"264\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxR\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h2><strong>9\u3001Kafka\u5197\u4f59\u526f\u672c\u4fdd\u8bc1\u9ad8\u53ef\u7528<\/strong><\/h2>\n<p>\u5728kafka\u91cc\u9762\u5206\u533a\u662f\u6709\u526f\u672c\u7684\uff0c\u6ce8\uff1a0.8\u4ee5\u524d\u662f\u6ca1\u6709\u526f\u672c\u673a\u5236\u7684\u3002\u521b\u5efa\u4e3b\u9898\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a\u5206\u533a\uff0c\u4e5f\u53ef\u4ee5\u6307\u5b9a\u526f\u672c\u4e2a\u6570\u3002\u526f\u672c\u662f\u6709\u89d2\u8272\u7684\uff1a<\/p>\n<p>leader partition\uff1a<\/p>\n<ul>\n<li>\u5199\u6570\u636e\u3001\u8bfb\u6570\u636e\u64cd\u4f5c\u90fd\u662f\u4eceleader partition\u53bb\u64cd\u4f5c\u7684\u3002<\/li>\n<li>\u4f1a\u7ef4\u62a4\u4e00\u4e2aISR\uff08in-sync- replica \uff09\u5217\u8868\uff0c\u4f46\u662f\u4f1a\u6839\u636e\u4e00\u5b9a\u7684\u89c4\u5219\u5220\u9664ISR\u5217\u8868\u91cc\u9762\u7684\u503c<\/li>\n<\/ul>\n<p>\u751f\u4ea7\u8005\u53d1\u9001\u6765\u4e00\u4e2a\u6d88\u606f\uff0c\u6d88\u606f\u9996\u5148\u8981\u5199\u5165\u5230leader partition\u4e2d<\/p>\n<p>\u5199\u5b8c\u4e86\u4ee5\u540e\uff0c\u8fd8\u8981\u628a\u6d88\u606f\u5199\u5165\u5230ISR\u5217\u8868\u91cc\u9762\u7684\u5176\u5b83\u5206\u533a\uff0c\u5199\u5b8c\u540e\u624d\u7b97\u8fd9\u4e2a\u6d88\u606f\u63d0\u4ea4<\/p>\n<p>follower partition\uff1a\u4eceleader partition\u540c\u6b65\u6570\u636e\u3002<\/p>\n<h2><strong>10\u3001\u4f18\u79c0\u67b6\u6784\u601d\u8003-\u603b\u7ed3<\/strong><\/h2>\n<p>Kafka \u2014 \u9ad8\u5e76\u53d1\u3001\u9ad8\u53ef\u7528\u3001\u9ad8\u6027\u80fd<\/p>\n<ul>\n<li>\u9ad8\u53ef\u7528\uff1a\u591a\u526f\u672c\u673a\u5236<\/li>\n<li>\u9ad8\u5e76\u53d1\uff1a\u7f51\u7edc\u67b6\u6784\u8bbe\u8ba1 \u4e09\u5c42\u67b6\u6784\uff1a\u591aselector -&gt; \u591a\u7ebf\u7a0b -&gt; \u961f\u5217\u7684\u8bbe\u8ba1\uff08NIO\uff09<\/li>\n<li>\u9ad8\u6027\u80fd\uff1a<\/li>\n<\/ul>\n<p>\u5199\u6570\u636e\uff1a<\/p>\n<ol>\n<li>\u628a\u6570\u636e\u5148\u5199\u5165\u5230OS Cache<\/li>\n<li>\u5199\u5230\u78c1\u76d8\u4e0a\u9762\u662f\u987a\u5e8f\u5199\uff0c\u6027\u80fd\u5f88\u9ad8<\/li>\n<\/ol>\n<p>\u8bfb\u6570\u636e\uff1a<\/p>\n<ol>\n<li>\u6839\u636e\u7a00\u758f\u7d22\u5f15\uff0c\u5feb\u901f\u5b9a\u4f4d\u5230\u8981\u6d88\u8d39\u7684\u6570\u636e<\/li>\n<li>\u96f6\u62f7\u8d1d\u673a\u5236<\/li>\n<li>\n<ul>\n<li>\u51cf\u5c11\u6570\u636e\u7684\u62f7\u8d1d<\/li>\n<li>\u51cf\u5c11\u4e86\u5e94\u7528\u7a0b\u5e8f\u4e0e\u64cd\u4f5c\u7cfb\u7edf\u4e0a\u4e0b\u6587\u5207\u6362<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<h2><strong>11\u3001Kafka\u751f\u4ea7\u73af\u5883\u642d\u5efa<\/strong><\/h2>\n<h4>11.1 \u9700\u6c42\u573a\u666f\u5206\u6790<\/h4>\n<p>\u7535\u5546\u5e73\u53f0\uff0c\u9700\u8981\u6bcf\u592910\u4ebf\u8bf7\u6c42\u90fd\u8981\u53d1\u9001\u5230Kafka\u96c6\u7fa4\u4e0a\u9762\u3002\u4e8c\u516b\u53cd\u6b63\uff0c\u4e00\u822c\u8bc4\u4f30\u51fa\u6765\u95ee\u9898\u90fd\u4e0d\u5927\u3002<\/p>\n<p>10\u4ebf\u8bf7\u6c42 -&gt; 24 \u8fc7\u6765\u7684\uff0c\u4e00\u822c\u60c5\u51b5\u4e0b\uff0c\u6bcf\u5929\u768412:00 \u5230\u65e9\u4e0a8:00 \u8fd9\u6bb5\u65f6\u95f4\u5176\u5b9e\u662f\u6ca1\u6709\u591a\u5927\u7684\u6570\u636e\u91cf\u7684\u300280%\u7684\u8bf7\u6c42\u662f\u7528\u7684\u53e6\u591616\u5c0f\u65f6\u7684\u5904\u7406\u7684\u300216\u4e2a\u5c0f\u65f6\u5904\u7406 -&gt; 8\u4ebf\u7684\u8bf7\u6c42\u300216 * 0.2 = 3\u4e2a\u5c0f\u65f6 \u5904\u7406\u4e868\u4ebf\u8bf7\u6c42\u768480%\u7684\u6570\u636e<\/p>\n<p>\u4e5f\u5c31\u662f\u8bf46\u4ebf\u7684\u6570\u636e\u662f\u97603\u4e2a\u5c0f\u65f6\u5904\u7406\u5b8c\u7684\u3002\u6211\u4eec\u7b80\u5355\u7684\u7b97\u4e00\u4e0b\u9ad8\u5cf0\u671f\u65f6\u5019\u7684qps<\/p>\n<pre><code>6\u4ebf\/3\u5c0f\u65f6 =5.5\u4e07\/s qps=5.5\u4e07<\/code><\/pre>\n<p><code>10\u4ebf\u8bf7\u6c42 * 50kb = 46T<\/code> \u6bcf\u5929\u9700\u8981\u5b58\u50a846T\u7684\u6570\u636e<\/p>\n<p>\u4e00\u822c\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u90fd\u4f1a\u8bbe\u7f6e\u4e24\u4e2a\u526f\u672c <code>46T * 2 = 92T<\/code>\uff0cKafka\u91cc\u9762\u7684\u6570\u636e\u662f\u6709\u4fdd\u7559\u7684\u65f6\u95f4\u5468\u671f\uff0c\u4fdd\u7559\u6700\u8fd13\u5929\u7684\u6570\u636e\u3002<\/p>\n<pre><code>92T * 3\u5929 = 276T<\/code><\/pre>\n<p>\u6211\u8fd9\u513f\u8bf4\u7684\u662f50kb\u4e0d\u662f\u8bf4\u4e00\u6761\u6d88\u606f\u5c31\u662f50kb\u4e0d\u662f\uff08\u628a\u65e5\u5fd7\u5408\u5e76\u4e86\uff0c\u591a\u6761\u65e5\u5fd7\u5408\u5e76\u5728\u4e00\u8d77\uff09\uff0c\u901a\u5e38\u60c5\u51b5\u4e0b\uff0c\u4e00\u6761\u6d88\u606f\u5c31\u51e0b\uff0c\u4e5f\u6709\u53ef\u80fd\u5c31\u662f\u51e0\u767e\u5b57\u8282\u3002<\/p>\n<h4>11.2 \u7269\u7406\u673a\u6570\u91cf\u8bc4\u4f30<\/h4>\n<p><strong>1\uff09\u9996\u5148\u5206\u6790\u4e00\u4e0b\u662f\u9700\u8981\u865a\u62df\u673a\u8fd8\u662f\u7269\u7406\u673a<\/strong><\/p>\n<p>\u50cfKafka mysql hadoop\u8fd9\u4e9b\u96c6\u7fa4\u642d\u5efa\u7684\u65f6\u5019\uff0c\u6211\u4eec\u751f\u4ea7\u91cc\u9762\u90fd\u662f\u4f7f\u7528\u7269\u7406\u673a\u3002<\/p>\n<p><strong>2\uff09\u9ad8\u5cf0\u671f\u9700\u8981\u5904\u7406\u7684\u8bf7\u6c42\u603b\u7684\u8bf7\u6c42\u6bcf\u79d25.5\u4e07\u4e2a\uff0c\u5176\u5b9e\u4e00\u4e24\u53f0\u7269\u7406\u673a\u7edd\u5bf9\u662f\u53ef\u4ee5\u6297\u4f4f\u7684\u3002\u4e00\u822c\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u8bc4\u4f30\u673a\u5668\u7684\u65f6\u5019\uff0c\u662f\u6309\u7167\u9ad8\u5cf0\u671f\u76844\u500d\u7684\u53bb\u8bc4\u4f30\u3002<\/strong><\/p>\n<p>\u5982\u679c\u662f4\u500d\u7684\u8bdd\uff0c\u5927\u6982\u6211\u4eec\u96c6\u7fa4\u7684\u80fd\u529b\u8981\u51c6\u5907\u5230 20\u4e07qps\u3002\u8fd9\u6837\u5b50\u7684\u96c6\u7fa4\u624d\u662f\u6bd4\u8f83\u5b89\u5168\u7684\u96c6\u7fa4\u3002\u5927\u6982\u5c31\u9700\u89815\u53f0\u7269\u7406\u673a\u3002\u6bcf\u53f0\u627f\u53d74\u4e07\u8bf7\u6c42\u3002<\/p>\n<p>\u573a\u666f\u603b\u7ed3\uff1a<\/p>\n<ul>\n<li>\u641e\u5b9a10\u4ebf\u8bf7\u6c42\uff0c\u9ad8\u5cf0\u671f5.5\u4e07\u7684qps,276T\u7684\u6570\u636e\uff0c\u9700\u89815\u53f0\u7269\u7406\u673a\u3002<\/li>\n<\/ul>\n<h4>11.3 \u78c1\u76d8\u9009\u62e9<\/h4>\n<p>\u641e\u5b9a10\u4ebf\u8bf7\u6c42\uff0c\u9ad8\u5cf0\u671f5.5\u4e07\u7684qps,276T\u7684\u6570\u636e\uff0c\u9700\u89815\u53f0\u7269\u7406\u673a\u3002<\/p>\n<p><strong>1\uff09SSD\u56fa\u6001\u786c\u76d8\uff0c\u8fd8\u662f\u9700\u8981\u666e\u901a\u7684\u673a\u68b0\u786c\u76d8<\/strong><\/p>\n<ul>\n<li>SSD\u786c\u76d8\uff1a\u6027\u80fd\u6bd4\u8f83\u597d\uff0c\u4f46\u662f\u4ef7\u683c\u8d35<\/li>\n<li>SAS\u76d8\uff1a\u67d0\u65b9\u9762\u6027\u80fd\u4e0d\u662f\u5f88\u597d\uff0c\u4f46\u662f\u6bd4\u8f83\u4fbf\u5b9c\u3002<\/li>\n<\/ul>\n<p>SSD\u786c\u76d8\u6027\u80fd\u6bd4\u8f83\u597d\uff0c\u6307\u7684\u662f\u5b83\u968f\u673a\u8bfb\u5199\u7684\u6027\u80fd\u6bd4\u8f83\u597d\u3002\u9002\u5408MySQL\u8fd9\u6837\u96c6\u7fa4\u3002<\/p>\n<p>\u4f46\u662f\u5176\u5b9e\u4ed6\u7684\u987a\u5e8f\u5199\u7684\u6027\u80fd\u8ddfSAS\u76d8\u5dee\u4e0d\u591a\u3002<\/p>\n<p>kafka\u7684\u7406\u89e3\uff1a\u5c31\u662f\u7528\u7684\u987a\u5e8f\u5199\u3002\u6240\u4ee5\u6211\u4eec\u5c31\u7528\u666e\u901a\u7684\u3010\u673a\u68b0\u786c\u76d8\u3011\u5c31\u53ef\u4ee5\u4e86\u3002<\/p>\n<p><strong>2\uff09\u9700\u8981\u6211\u4eec\u8bc4\u4f30\u6bcf\u53f0\u670d\u52a1\u5668\u9700\u8981\u591a\u5c11\u5757\u78c1\u76d8<\/strong><\/p>\n<p>5\u53f0\u670d\u52a1\u5668\uff0c\u4e00\u5171\u9700\u8981276T \uff0c\u5927\u7ea6\u6bcf\u53f0\u670d\u52a1\u5668 \u9700\u8981\u5b58\u50a860T\u7684\u6570\u636e\u3002\u6211\u4eec\u516c\u53f8\u91cc\u9762\u670d\u52a1\u5668\u7684\u914d\u7f6e\u7528\u7684\u662f 11\u5757\u786c\u76d8\uff0c\u6bcf\u4e2a\u786c\u76d8 7T\u3002<code>11 * 7T = 77T<\/code><\/p>\n<pre><code>77T * 5 \u53f0\u670d\u52a1\u5668 = 385T<\/code><\/pre>\n<p>\u573a\u666f\u603b\u7ed3\uff1a<\/p>\n<ul>\n<li>\u641e\u5b9a10\u4ebf\u8bf7\u6c42\uff0c\u9700\u89815\u53f0\u7269\u7406\u673a\uff0c11\uff08SAS\uff09 * 7T<\/li>\n<\/ul>\n<p>11.4 \u5185\u5b58\u8bc4\u4f30 \u641e\u5b9a10\u4ebf\u8bf7\u6c42\uff0c\u9700\u89815\u53f0\u7269\u7406\u673a\uff0c<code>11\uff08SAS\uff09 * 7T<\/code><\/p>\n<p>\u6211\u4eec\u53d1\u73b0kafka\u8bfb\u5199\u6570\u636e\u7684\u6d41\u7a0b \u90fd\u662f\u57fa\u4e8eos cache,\u6362\u53e5\u8bdd\u8bf4\u5047\u8bbe\u54b1\u4eec\u7684os cashe\u65e0\u9650\u5927\u90a3\u4e48\u6574\u4e2akafka\u662f\u4e0d\u662f\u76f8\u5f53\u4e8e\u5c31\u662f\u57fa\u4e8e\u5185\u5b58\u53bb\u64cd\u4f5c\uff0c\u5982\u679c\u662f\u57fa\u4e8e\u5185\u5b58\u53bb\u64cd\u4f5c\uff0c\u6027\u80fd\u80af\u5b9a\u5f88\u597d\u3002\u5185\u5b58\u662f\u6709\u9650\u7684\u3002<\/p>\n<ul>\n<li>\u5c3d\u53ef\u80fd\u591a\u7684\u5185\u5b58\u8d44\u6e90\u8981\u7ed9 os cache<\/li>\n<li>Kafka\u7684\u4ee3\u7801\u7528 \u6838\u5fc3\u7684\u4ee3\u7801\u7528\u7684\u662fscala\u5199\u7684\uff0c\u5ba2\u6237\u7aef\u7684\u4ee3\u7801java\u5199\u7684\u3002\u90fd\u662f\u57fa\u4e8ejvm\u3002\u6240\u4ee5\u6211\u4eec\u8fd8\u8981\u7ed9\u4e00\u90e8\u5206\u7684\u5185\u5b58\u7ed9jvm\u3002<\/li>\n<\/ul>\n<p>Kafka\u7684\u8bbe\u8ba1\uff0c\u6ca1\u6709\u628a\u5f88\u591a\u6570\u636e\u7ed3\u6784\u90fd\u653e\u5728jvm\u91cc\u9762\u3002\u6240\u4ee5\u6211\u4eec\u7684\u8fd9\u4e2ajvm\u4e0d\u9700\u8981\u592a\u5927\u7684\u5185\u5b58\u3002\u6839\u636e\u7ecf\u9a8c\uff0c\u7ed9\u4e2a10G\u5c31\u53ef\u4ee5\u4e86\u3002<\/p>\n<p>NameNode:jvm\u91cc\u9762\u8fd8\u653e\u4e86\u5143\u6570\u636e\uff08\u51e0\u5341G\uff09\uff0cJVM\u4e00\u5b9a\u8981\u7ed9\u5f97\u5f88\u5927\u3002\u6bd4\u5982\u7ed9\u4e2a100G\u3002<\/p>\n<p>\u5047\u8bbe\u6211\u4eec\u8fd9\u4e2a10\u8bf7\u6c42\u7684\u8fd9\u4e2a\u9879\u76ee\uff0c\u4e00\u5171\u4f1a\u6709100\u4e2atopic\u3002<code>100 topic * 5 partition * 2 = 1000 partition<\/code><\/p>\n<p>\u4e00\u4e2apartition\u5176\u5b9e\u5c31\u662f\u7269\u7406\u673a\u4e0a\u9762\u7684\u4e00\u4e2a\u76ee\u5f55\uff0c\u8fd9\u4e2a\u76ee\u5f55\u4e0b\u9762\u4f1a\u6709\u5f88\u591a\u4e2a.log\u7684\u6587\u4ef6\u3002<\/p>\n<ul>\n<li>.log\u5c31\u662f\u5b58\u50a8\u6570\u636e\u6587\u4ef6\uff0c\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u4e00\u4e2a.log\u6587\u4ef6\u7684\u5927\u5c0f\u662f1G\u3002<\/li>\n<\/ul>\n<p>\u6211\u4eec\u5982\u679c\u8981\u4fdd\u8bc1 1000\u4e2apartition \u7684\u6700\u65b0\u7684.log \u6587\u4ef6\u7684\u6570\u636e \u5982\u679c\u90fd\u5728\u5185\u5b58\u91cc\u9762\uff0c\u8fd9\u4e2a\u65f6\u5019\u6027\u80fd\u5c31\u662f\u6700\u597d\u3002<code>1000 * 1G = 1000G<\/code>\u5185\u5b58.<\/p>\n<p>\u6211\u4eec\u53ea\u9700\u8981\u628a\u5f53\u524d\u6700\u65b0\u7684\u8fd9\u4e2alog \u4fdd\u8bc1\u91cc\u9762\u768425%\u7684\u6700\u65b0\u7684\u6570\u636e\u5728\u5185\u5b58\u91cc\u9762\u3002<code>250M * 1000 = 0.25 G* 1000 =250G<\/code>\u7684\u5185\u5b58\u3002<\/p>\n<ul>\n<li><code>250\u5185\u5b58 \/ 5 = 50G<\/code>\u5185\u5b58<\/li>\n<li><code>50G+10G = 60G<\/code>\u5185\u5b58<\/li>\n<\/ul>\n<p>64G\u7684\u5185\u5b58\uff0c\u53e6\u5916\u76844G\uff0c\u64cd\u4f5c\u7cfb\u7edf\u672c\u751f\u662f\u4e0d\u662f\u4e5f\u9700\u8981\u5185\u5b58\u3002\u5176\u5b9eKafka\u7684jvm\u4e5f\u53ef\u4ee5\u4e0d\u7528\u7ed9\u523010G\u8fd9\u4e48\u591a\u3002\u8bc4\u4f30\u51fa\u676564G\u662f\u53ef\u4ee5\u7684\u3002\u5f53\u7136\u5982\u679c\u80fd\u7ed9\u5230128G\u7684\u5185\u5b58\u7684\u670d\u52a1\u5668\uff0c\u90a3\u5c31\u6700\u597d\u3002<\/p>\n<p>\u6211\u521a\u521a\u8bc4\u4f30\u7684\u65f6\u5019\u7528\u7684\u90fd\u662f\u4e00\u4e2atopic\u662f5\u4e2apartition\uff0c\u4f46\u662f\u5982\u679c\u662f\u6570\u636e\u91cf\u6bd4\u8f83\u5927\u7684topic\uff0c\u53ef\u80fd\u4f1a\u670910\u4e2apartition\u3002<\/p>\n<p>\u603b\u7ed3\uff1a<\/p>\n<ul>\n<li>\u641e\u5b9a10\u4ebf\u8bf7\u6c42\uff0c\u9700\u89815\u53f0\u7269\u7406\u673a\uff0c<code>11\uff08SAS\uff09 * 7T<\/code> \uff0c\u9700\u898164G\u7684\u5185\u5b58\uff08128G\u66f4\u597d\uff09<\/li>\n<\/ul>\n<h4>11.5 CPU\u538b\u529b\u8bc4\u4f30<\/h4>\n<p>\u8bc4\u4f30\u4e00\u4e0b\u6bcf\u53f0\u670d\u52a1\u5668\u9700\u8981\u591a\u5c11cpu core(\u8d44\u6e90\u5f88\u6709\u9650)<\/p>\n<p>\u6211\u4eec\u8bc4\u4f30\u9700\u8981\u591a\u5c11\u4e2acpu \uff0c\u4f9d\u636e\u5c31\u662f\u770b\u6211\u4eec\u7684\u670d\u52a1\u91cc\u9762\u6709\u591a\u5c11\u7ebf\u7a0b\u53bb\u8dd1\u3002\u7ebf\u7a0b\u5c31\u662f\u4f9d\u6258cpu \u53bb\u8fd0\u884c\u7684\u3002\u5982\u679c\u6211\u4eec\u7684\u7ebf\u7a0b\u6bd4\u8f83\u591a\uff0c\u4f46\u662fcpu core\u6bd4\u8f83\u5c11\uff0c\u8fd9\u6837\u7684\u8bdd\uff0c\u6211\u4eec\u7684\u673a\u5668\u8d1f\u8f7d\u5c31\u4f1a\u5f88\u9ad8\uff0c\u6027\u80fd\u4e0d\u5c31\u4e0d\u597d\u3002<\/p>\n<p>\u8bc4\u4f30\u4e00\u4e0b\uff0ckafka\u7684\u4e00\u53f0\u670d\u52a1\u5668 \u542f\u52a8\u4ee5\u540e\u4f1a\u6709\u591a\u5c11\u7ebf\u7a0b\uff1f<\/p>\n<ul>\n<li>Acceptor\u7ebf\u7a0b 1<\/li>\n<li>processor\u7ebf\u7a0b 3 6~9\u4e2a\u7ebf\u7a0b<\/li>\n<li>\u5904\u7406\u8bf7\u6c42\u7ebf\u7a0b 8\u4e2a 32\u4e2a\u7ebf\u7a0b<\/li>\n<li>\u5b9a\u65f6\u6e05\u7406\u7684\u7ebf\u7a0b\uff0c\u62c9\u53d6\u6570\u636e\u7684\u7ebf\u7a0b\uff0c\u5b9a\u65f6\u68c0\u67e5ISR\u5217\u8868\u7684\u673a\u5236 \u7b49\u7b49\u3002<\/li>\n<\/ul>\n<p>\u6240\u4ee5\u5927\u6982\u4e00\u4e2aKafka\u7684\u670d\u52a1\u542f\u52a8\u8d77\u6765\u4ee5\u540e\uff0c\u4f1a\u6709\u4e00\u767e\u591a\u4e2a\u7ebf\u7a0b\u3002<\/p>\n<ul>\n<li>cpu core = 4\u4e2a\uff0c\u4e00\u904d\u6765\u8bf4\uff0c\u51e0\u5341\u4e2a\u7ebf\u7a0b\uff0c\u5c31\u80af\u5b9a\u628acpu \u6253\u6ee1\u4e86\u3002<\/li>\n<li>cpu core = 8\u4e2a\uff0c\u5e94\u8be5\u5f88\u8f7b\u677e\u7684\u80fd\u652f\u6301\u51e0\u5341\u4e2a\u7ebf\u7a0b\u3002<\/li>\n<\/ul>\n<p>\u5982\u679c\u6211\u4eec\u7684\u7ebf\u7a0b\u662f100\u591a\u4e2a\uff0c\u6216\u8005\u5dee\u4e0d\u591a200\u4e2a\uff0c\u90a3\u4e488 \u4e2a cpu core\u662f\u641e\u4e0d\u5b9a\u7684\u3002<\/p>\n<p>\u6240\u4ee5\u6211\u4eec\u8fd9\u513f\u5efa\u8bae\uff1a<\/p>\n<ul>\n<li>CPU core = 16\u4e2a\u3002\u5982\u679c\u53ef\u4ee5\u7684\u8bdd\uff0c\u80fd\u670932\u4e2acpu core \u90a3\u5c31\u6700\u597d\u3002<\/li>\n<\/ul>\n<p>\u7ed3\u8bba\uff1a<\/p>\n<ul>\n<li>kafka\u96c6\u7fa4\uff0c\u6700\u4f4e\u4e5f\u8981\u7ed916\u4e2acpu core\uff0c\u5982\u679c\u80fd\u7ed9\u523032 cpu core\u90a3\u5c31\u66f4\u597d\u3002<\/li>\n<li><code>2cpu * 8 =16<\/code> cpu core<\/li>\n<li><code>4cpu * 8 = 32<\/code> cpu core<\/li>\n<\/ul>\n<p>\u603b\u7ed3\uff1a<\/p>\n<ul>\n<li>\u641e\u5b9a10\u4ebf\u8bf7\u6c42\uff0c\u9700\u89815\u53f0\u7269\u7406\u673a\uff0c <code>11\uff08SAS\uff09 * 7T<\/code> \uff0c\u9700\u898164G\u7684\u5185\u5b58\uff08128G\u66f4\u597d\uff09\uff0c\u9700\u898116\u4e2acpu core\uff0832\u4e2a\u66f4\u597d\uff09<\/li>\n<\/ul>\n<h3>11.6 \u7f51\u7edc\u9700\u6c42\u8bc4\u4f30<\/h3>\n<p>\u8bc4\u4f30\u6211\u4eec\u9700\u8981\u4ec0\u4e48\u6837\u7f51\u5361\uff1f<\/p>\n<p>\u4e00\u822c\u8981\u4e48\u662f\u5343\u5146\u7684\u7f51\u5361\uff081G\/s\uff09\uff0c\u8fd8\u6709\u7684\u5c31\u662f\u4e07\u5146\u7684\u7f51\u5361\uff0810G\/s\uff09<\/p>\n<p>\u9ad8\u5cf0\u671f\u7684\u65f6\u5019 \u6bcf\u79d2\u4f1a\u67095.5\u4e07\u7684\u8bf7\u6c42\u6d8c\u5165\uff0c5.5\/5 = \u5927\u7ea6\u662f\u6bcf\u53f0\u670d\u52a1\u5668\u4f1a\u67091\u4e07\u4e2a\u8bf7\u6c42\u6d8c\u5165\u3002\u6211\u4eec\u4e4b\u524d\u8bf4\u7684\uff0c<code>10000 * 50kb = 488M<\/code> \u4e5f\u5c31\u662f\u6bcf\u6761\u670d\u52a1\u5668\uff0c\u6bcf\u79d2\u8981\u63a5\u53d7488M\u7684\u6570\u636e\u3002\u6570\u636e\u8fd8\u8981\u6709\u526f\u672c\uff0c\u526f\u672c\u4e4b\u95f4\u7684\u540c\u6b65\uff0c\u4e5f\u662f\u8d70\u7684\u7f51\u7edc\u7684\u8bf7\u6c42\u3002<code>488 * 2 = 976m\/s<\/code><\/p>\n<p>\u8bf4\u660e\u4e00\u4e0b\uff1a<\/p>\n<ul>\n<li>\u5f88\u591a\u516c\u53f8\u7684\u6570\u636e\uff0c\u4e00\u4e2a\u8bf7\u6c42\u91cc\u9762\u662f\u6ca1\u670950kb\u8fd9\u4e48\u5927\u7684\uff0c\u6211\u4eec\u516c\u53f8\u662f\u56e0\u4e3a\u4e3b\u673a\u5728\u751f\u4ea7\u7aef\u5c01\u88c5\u4e86\u6570\u636e\uff0c\u7136\u540e\u628a\u591a\u6761\u6570\u636e\u5408\u5e76\u5728\u4e00\u8d77\u4e86\uff0c\u6240\u4ee5\u6211\u4eec\u7684\u4e00\u4e2a\u8bf7\u6c42\u624d\u4f1a\u6709\u8fd9\u4e48\u5927\u3002<\/li>\n<li>\u4e00\u822c\u60c5\u51b5\u4e0b\uff0c\u7f51\u5361\u7684\u5e26\u5bbd\u662f\u8fbe\u4e0d\u5230\u6781\u9650\u7684\uff0c\u5982\u679c\u662f\u5343\u5146\u7684\u7f51\u5361\uff0c\u6211\u4eec\u80fd\u7528\u7684\u4e00\u822c\u5c31\u662f700M\u5de6\u53f3\u3002\u4f46\u662f\u5982\u679c\u6700\u597d\u7684\u60c5\u51b5\uff0c\u6211\u4eec\u8fd8\u662f\u4f7f\u7528\u4e07\u5146\u7684\u7f51\u5361\u3002<\/li>\n<li>\u5982\u679c\u4f7f\u7528\u7684\u662f\u4e07\u5146\u7684\uff0c\u90a3\u5c31\u662f\u5f88\u8f7b\u677e\u3002<\/li>\n<\/ul>\n<h4>11.7 \u96c6\u7fa4\u89c4\u5212<\/h4>\n<ul>\n<li>\u8bf7\u6c42\u91cf<\/li>\n<li>\u89c4\u5212\u7269\u7406\u673a\u7684\u4e2a\u6570<\/li>\n<li>\u5206\u6790\u78c1\u76d8\u7684\u4e2a\u6570\uff0c\u9009\u62e9\u4f7f\u7528\u4ec0\u4e48\u6837\u7684\u78c1\u76d8<\/li>\n<li>\u5185\u5b58<\/li>\n<li>cpu core<\/li>\n<li>\u7f51\u5361<\/li>\n<\/ul>\n<p>\u5c31\u662f\u544a\u8bc9\u5927\u5bb6\uff0c\u4ee5\u540e\u8981\u662f\u516c\u53f8\u91cc\u9762\u6709\u4ec0\u4e48\u9700\u6c42\uff0c\u8fdb\u884c\u8d44\u6e90\u7684\u8bc4\u4f30\uff0c\u670d\u52a1\u5668\u7684\u8bc4\u4f30\uff0c\u5927\u5bb6\u6309\u7167\u6211\u7684\u601d\u8def\u53bb\u8bc4\u4f30\u3002<\/p>\n<p>\u4e00\u6761\u6d88\u606f\u7684\u5927\u5c0f <code>50kb -&gt; 1kb 500byte 1M<\/code><\/p>\n<p>ip \u4e3b\u673a\u540d<\/p>\n<ul>\n<li>192.168.0.100 hadoop1<\/li>\n<li>192.168.0.101 hadoop2<\/li>\n<li>192.168.0.102 hadoop3<\/li>\n<\/ul>\n<p>\u4e3b\u673a\u7684\u89c4\u5212\uff1akafka\u96c6\u7fa4\u67b6\u6784\u7684\u65f6\u5019\uff1a\u4e3b\u4ece\u5f0f\u7684\u67b6\u6784\uff1a<\/p>\n<ul>\n<li>controller -&gt; \u901a\u8fc7zk\u96c6\u7fa4\u6765\u7ba1\u7406\u6574\u4e2a\u96c6\u7fa4\u7684\u5143\u6570\u636e\u3002<\/li>\n<\/ul>\n<p>zookeeper\u96c6\u7fa4<\/p>\n<ul>\n<li>hadoop1<\/li>\n<li>hadoop2<\/li>\n<li>hadoop3<\/li>\n<\/ul>\n<p>kafka\u96c6\u7fa4<\/p>\n<ul>\n<li>\u7406\u8bba\u4e0a\u6765\u8bb2\uff0c\u6211\u4eec\u4e0d\u5e94\u8be5\u628akafka\u7684\u670d\u52a1\u4e8ezk\u7684\u670d\u52a1\u5b89\u88c5\u5728\u4e00\u8d77\u3002<\/li>\n<li>\u4f46\u662f\u6211\u4eec\u8fd9\u513f\u670d\u52a1\u5668\u6709\u9650\u3002\u6240\u4ee5\u6211\u4eeckafka\u96c6\u7fa4\u4e5f\u662f\u5b89\u88c5\u5728hadoop1 haadoop2 hadoop3<\/li>\n<\/ul>\n<h4>11.8 zookeeper\u96c6\u7fa4\u642d\u5efa<\/h4>\n<h4>11.9 \u6838\u5fc3\u53c2\u6570\u8be6\u89e3<\/h4>\n<h4>11.10 \u96c6\u7fa4\u538b\u529b\u6d4b\u8bd5<\/h4>\n<h2><strong>12\u3001kafka\u8fd0\u7ef4<\/strong><\/h2>\n<h4>12.1 \u5e38\u89c1\u8fd0\u7ef4\u5de5\u5177\u4ecb\u7ecd<\/h4>\n<p>KafkaManager \u2014 \u9875\u9762\u7ba1\u7406\u5de5\u5177<\/p>\n<h4>12.2 \u5e38\u89c1\u8fd0\u7ef4\u547d\u4ee4<\/h4>\n<p><strong>\u573a\u666f\u4e00\uff1atopic\u6570\u636e\u91cf\u592a\u5927\uff0c\u8981\u589e\u52a0topic\u6570<\/strong><\/p>\n<p>\u4e00\u5f00\u59cb\u521b\u5efa\u4e3b\u9898\u7684\u65f6\u5019\uff0c\u6570\u636e\u91cf\u4e0d\u5927\uff0c\u7ed9\u7684\u5206\u533a\u6570\u4e0d\u591a\u3002<\/p>\n<pre><code>kafka-topics.sh --create --zookeeper hadoop1:2181,hadoop2:2181,hadoop3:2181 --replication-factor 1 --partitions 1 --topic test6\n\nkafka-topics.sh --alter --zookeeper hadoop1:2181,hadoop2:2181,hadoop3:2181 --partitions 3 --topic test6<\/code><\/pre>\n<p>broker id\uff1a<\/p>\n<ul>\n<li>hadoop1:0<\/li>\n<li>hadoop2:1<\/li>\n<li>hadoop3:2<\/li>\n<\/ul>\n<p>\u5047\u8bbe\u4e00\u4e2apartition\u6709\u4e09\u4e2a\u526f\u672c\uff1apartition0\uff1a<\/p>\n<blockquote><p>a,b,c<\/p><\/blockquote>\n<ul>\n<li>a\uff1aleader partition<\/li>\n<li>b\uff0cc:follower partition<\/li>\n<\/ul>\n<pre><code>ISR:{a,b,c}<\/code><\/pre>\n<p>\u5982\u679c\u4e00\u4e2afollower\u5206\u533a \u8d85\u8fc710\u79d2 \u6ca1\u6709\u5411leader partition\u53bb\u62c9\u53d6\u6570\u636e\uff0c\u90a3\u4e48\u8fd9\u4e2a\u5206\u533a\u5c31\u4eceISR\u5217\u8868\u91cc\u9762\u79fb\u9664\u3002<\/p>\n<p><strong>\u573a\u666f\u4e8c\uff1a\u6838\u5fc3topic\u589e\u52a0\u526f\u672c\u56e0\u5b50<\/strong><\/p>\n<p>\u5982\u679c\u5bf9\u6838\u5fc3\u4e1a\u52a1\u6570\u636e\u9700\u8981\u589e\u52a0\u526f\u672c\u56e0\u5b50<\/p>\n<p>vim test.json\u811a\u672c\uff0c\u5c06\u4e0b\u9762\u4e00\u884cjson\u811a\u672c\u4fdd\u5b58<\/p>\n<pre><code>{\u201cversion\u201d:1,\u201cpartitions\u201d:[{\u201ctopic\u201d:\u201ctest6\u201d,\u201cpartition\u201d:0,\u201creplicas\u201d:[0,1,2]},{\u201ctopic\u201d:\u201ctest6\u201d,\u201cpartition\u201d:1,\u201creplicas\u201d:[0,1,2]},{\u201ctopic\u201d:\u201ctest6\u201d,\u201cpartition\u201d:2,\u201creplicas\u201d:[0,1,2]}]}<\/code><\/pre>\n<p>\u6267\u884c\u4e0a\u9762json\u811a\u672c\uff1a<\/p>\n<pre><code>kafka-reassign-partitions.sh --zookeeper hadoop1:2181,hadoop2:2181,hadoop3:2181 --reassignment-json-file test.json --execute<\/code><\/pre>\n<p><strong>\u573a\u666f\u4e09\uff1a\u8d1f\u8f7d\u4e0d\u5747\u8861\u7684topic\uff0c\u624b\u52a8\u8fc1\u79fb<\/strong><\/p>\n<p>vi topics-to-move.json<\/p>\n<pre><code>{\u201ctopics\u201d: [{\u201ctopic\u201d: \u201ctest01\u201d}, {\u201ctopic\u201d: \u201ctest02\u201d}], \u201cversion\u201d: 1}\n\/\/ \u628a\u4f60\u6240\u6709\u7684topic\u90fd\u5199\u5728\u8fd9\u91cc\n\nkafka-reassgin-partitions.sh --zookeeper hadoop1:2181,hadoop2:2181,hadoop3:2181 --topics-to-move-json-file topics-to-move.json --broker-list \u201c5,6\u201d --generate\n\/\/ \u628a\u4f60\u6240\u6709\u7684\u5305\u62ec\u65b0\u52a0\u5165\u7684broker\u673a\u5668\u90fd\u5199\u5728\u8fd9\u91cc\uff0c\u5c31\u4f1a\u8bf4\u662f\u628a\u6240\u6709\u7684partition\u5747\u5300\u7684\u5206\u6563\u5728\u5404\u4e2abroker\u4e0a\uff0c\u5305\u62ec\u65b0\u8fdb\u6765\u7684broker<\/code><\/pre>\n<p>\u6b64\u65f6\u4f1a\u751f\u6210\u4e00\u4e2a\u8fc1\u79fb\u65b9\u6848\uff0c\u53ef\u4ee5\u4fdd\u5b58\u5230\u4e00\u4e2a\u6587\u4ef6\u91cc\u53bb\uff1aexpand-cluster-reassignment.json<\/p>\n<pre><code>kafka-reassign-partitions.sh --zookeeper hadoop01:2181,hadoop02:2181,hadoop03:2181 --reassignment-json-file expand-cluster-reassignment.json --execute\n\nkafka-reassign-partitions.sh --zookeeper hadoop01:2181,hadoop02:2181,hadoop03:2181 --reassignment-json-file expand-cluster-reassignment.json --verify<\/code><\/pre>\n<p>\u8fd9\u79cd\u6570\u636e\u8fc1\u79fb\u64cd\u4f5c\u4e00\u5b9a\u8981\u5728\u665a\u4e0a\u4f4e\u5cf0\u7684\u65f6\u5019\u6765\u505a\uff0c\u56e0\u4e3a\u4ed6\u4f1a\u5728\u673a\u5668\u4e4b\u95f4\u8fc1\u79fb\u6570\u636e\uff0c\u975e\u5e38\u7684\u5360\u7528\u5e26\u5bbd\u8d44\u6e90<\/p>\n<ul>\n<li>generate: \u6839\u636e\u7ed9\u4e88\u7684Topic\u5217\u8868\u548cBroker\u5217\u8868\u751f\u6210\u8fc1\u79fb\u8ba1\u5212\u3002generate\u5e76\u4e0d\u4f1a\u771f\u6b63\u8fdb\u884c\u6d88\u606f\u8fc1\u79fb\uff0c\u800c\u662f\u5c06\u6d88\u606f\u8fc1\u79fb\u8ba1\u5212\u8ba1\u7b97\u51fa\u6765\uff0c\u4f9bexecute\u547d\u4ee4\u4f7f\u7528\u3002<\/li>\n<li>execute: \u6839\u636e\u7ed9\u4e88\u7684\u6d88\u606f\u8fc1\u79fb\u8ba1\u5212\u8fdb\u884c\u8fc1\u79fb\u3002<\/li>\n<li>verify: \u68c0\u67e5\u6d88\u606f\u662f\u5426\u5df2\u7ecf\u8fc1\u79fb\u5b8c\u6210\u3002<\/li>\n<\/ul>\n<p><strong>\u573a\u666f\u56db\uff1a\u5982\u679c\u67d0\u4e2abroker leader partition\u8fc7\u591a<\/strong><\/p>\n<p>\u6b63\u5e38\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u7684leader partition\u5728\u670d\u52a1\u5668\u4e4b\u95f4\u662f\u8d1f\u8f7d\u5747\u8861\u3002<\/p>\n<ul>\n<li>hadoop1 4<\/li>\n<li>hadoop2 1<\/li>\n<li>hadoop3 1<\/li>\n<\/ul>\n<p>\u73b0\u5728\u5404\u4e2a\u4e1a\u52a1\u65b9\u53ef\u4ee5\u81ea\u884c\u7533\u8bf7\u521b\u5efatopic\uff0c\u5206\u533a\u6570\u91cf\u90fd\u662f\u81ea\u52a8\u5206\u914d\u548c\u540e\u7eed\u52a8\u6001\u8c03\u6574\u7684\uff0ckafka\u672c\u8eab\u4f1a\u81ea\u52a8\u628aleader partition\u5747\u5300\u5206\u6563\u5728\u5404\u4e2a\u673a\u5668\u4e0a\uff0c\u8fd9\u6837\u53ef\u4ee5\u4fdd\u8bc1\u6bcf\u53f0\u673a\u5668\u7684\u8bfb\u5199\u541e\u5410\u91cf\u90fd\u662f\u5747\u5300\u7684\u3002<\/p>\n<p>\u4f46\u662f\u4e5f\u6709\u4f8b\u5916\uff0c\u90a3\u5c31\u662f\u5982\u679c\u67d0\u4e9bbroker\u5b95\u673a\uff0c\u4f1a\u5bfc\u81f4leader partition\u8fc7\u4e8e\u96c6\u4e2d\u5728\u5176\u4ed6\u5c11\u90e8\u5206\u51e0\u53f0broker\u4e0a\uff0c\u8fd9\u4f1a\u5bfc\u81f4\u5c11\u6570\u51e0\u53f0broker\u7684\u8bfb\u5199\u8bf7\u6c42\u538b\u529b\u8fc7\u9ad8\uff0c\u5176\u4ed6\u5b95\u673a\u7684broker\u91cd\u542f\u4e4b\u540e\u90fd\u662ffolloer partition\uff0c\u8bfb\u5199\u8bf7\u6c42\u5f88\u4f4e\u3002<\/p>\n<p>\u9020\u6210\u96c6\u7fa4\u8d1f\u8f7d\u4e0d\u5747\u8861\u6709\u4e00\u4e2a\u53c2\u6570\uff0c<code>auto.leader.rebalance.enable<\/code>\uff0c\u9ed8\u8ba4\u662ftrue\uff0c\u6bcf\u9694300\u79d2\uff08<code>leader.imbalance.check.interval.seconds<\/code>\uff09\u68c0\u67e5leader\u8d1f\u8f7d\u662f\u5426\u5e73\u8861<\/p>\n<p>\u5982\u679c\u4e00\u53f0broker\u4e0a\u7684\u4e0d\u5747\u8861\u7684leader\u8d85\u8fc7\u4e8610%\uff0c<code>leader.imbalance.per.broker.percentage<\/code>\uff0c\u5c31\u4f1a\u5bf9\u8fd9\u4e2abroker\u8fdb\u884c\u9009\u4e3e\u3002<\/p>\n<p>\u914d\u7f6e\u53c2\u6570\uff1a<\/p>\n<ul>\n<li><code>auto.leader.rebalance.enable<\/code> \u9ed8\u8ba4\u662ftrue<\/li>\n<li><code>leader.imbalance.per.broker.percentage<\/code>: \u6bcf\u4e2abroker\u5141\u8bb8\u7684\u4e0d\u5e73\u8861\u7684leader\u7684\u6bd4\u7387\u3002\u5982\u679c\u6bcf\u4e2abroker\u8d85\u8fc7\u4e86\u8fd9\u4e2a\u503c\uff0c\u63a7\u5236\u5668\u4f1a\u89e6\u53d1leader\u7684\u5e73\u8861\u3002\u8fd9\u4e2a\u503c\u8868\u793a\u767e\u5206\u6bd4\u300210%<\/li>\n<li><code>leader.imbalance.check.interval.seconds<\/code>\uff1a\u9ed8\u8ba4\u503c300\u79d2<\/li>\n<\/ul>\n<h2><strong>13\u3001Kafka\u751f\u4ea7\u8005<\/strong><\/h2>\n<h4>13.1 \u6d88\u8d39\u8005\u53d1\u9001\u6d88\u606f\u539f\u7406<\/h4>\n<p><img decoding=\"async\" width=\"723\" height=\"509\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxS\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h4>13.2 \u6d88\u8d39\u8005\u53d1\u9001\u6d88\u606f\u539f\u7406\u2014\u57fa\u7840\u6848\u4f8b\u6f14\u793a<\/h4>\n<p><img decoding=\"async\" width=\"723\" height=\"940\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxT\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h4>13.3 \u5982\u4f55\u63d0\u5347\u541e\u5410\u91cf<\/h4>\n<p>\u5982\u4f55\u63d0\u5347\u541e\u5410\u91cf\uff1a<strong>\u53c2\u6570\u4e00\uff1abuffer.memory\uff1a<\/strong><\/p>\n<p>\u8bbe\u7f6e\u53d1\u9001\u6d88\u606f\u7684\u7f13\u51b2\u533a\uff0c\u9ed8\u8ba4\u503c\u662f33554432\uff0c\u5c31\u662f32MB<\/p>\n<p><strong>\u53c2\u6570\u4e8c\uff1acompression.type\uff1a<\/strong><\/p>\n<p>\u9ed8\u8ba4\u662fnone\uff0c\u4e0d\u538b\u7f29\uff0c\u4f46\u662f\u4e5f\u53ef\u4ee5\u4f7f\u7528lz4\u538b\u7f29\uff0c\u6548\u7387\u8fd8\u662f\u4e0d\u9519\u7684\uff0c\u538b\u7f29\u4e4b\u540e\u53ef\u4ee5\u51cf\u5c0f\u6570\u636e\u91cf\uff0c\u63d0\u5347\u541e\u5410\u91cf\uff0c\u4f46\u662f\u4f1a\u52a0\u5927producer\u7aef\u7684cpu\u5f00\u9500<\/p>\n<p><strong>\u53c2\u6570\u4e09\uff1abatch.size\uff1a<\/strong><\/p>\n<ul>\n<li>\u8bbe\u7f6ebatch\u7684\u5927\u5c0f\uff0c\u5982\u679cbatch\u592a\u5c0f\uff0c\u4f1a\u5bfc\u81f4\u9891\u7e41\u7f51\u7edc\u8bf7\u6c42\uff0c\u541e\u5410\u91cf\u4e0b\u964d\uff1b<\/li>\n<li>\u5982\u679cbatch\u592a\u5927\uff0c\u4f1a\u5bfc\u81f4\u4e00\u6761\u6d88\u606f\u9700\u8981\u7b49\u5f85\u5f88\u4e45\u624d\u80fd\u88ab\u53d1\u9001\u51fa\u53bb\uff0c\u800c\u4e14\u4f1a\u8ba9\u5185\u5b58\u7f13\u51b2\u533a\u6709\u5f88\u5927\u538b\u529b\uff0c\u8fc7\u591a\u6570\u636e\u7f13\u51b2\u5728\u5185\u5b58\u91cc\uff0c\u9ed8\u8ba4\u503c\u662f\uff1a16384\uff0c\u5c31\u662f16kb\uff0c\u4e5f\u5c31\u662f\u4e00\u4e2abatch\u6ee1\u4e8616kb\u5c31\u53d1\u9001\u51fa\u53bb\uff0c\u4e00\u822c\u5728\u5b9e\u9645\u751f\u4ea7\u73af\u5883\uff0c\u8fd9\u4e2abatch\u7684\u503c\u53ef\u4ee5\u589e\u5927\u4e00\u4e9b\u6765\u63d0\u5347\u541e\u5410\u91cf\uff0c\u5982\u679c\u4e00\u4e2a\u6279\u6b21\u8bbe\u7f6e\u5927\u4e86\uff0c\u4f1a\u6709\u5ef6\u8fdf\u3002\u4e00\u822c\u6839\u636e\u4e00\u6761\u6d88\u606f\u5927\u5c0f\u6765\u8bbe\u7f6e\u3002<\/li>\n<li>\u5982\u679c\u6211\u4eec\u6d88\u606f\u6bd4\u8f83\u5c11\u3002\u914d\u5408\u4f7f\u7528\u7684\u53c2\u6570linger.ms\uff0c\u8fd9\u4e2a\u503c\u9ed8\u8ba4\u662f0\uff0c\u610f\u601d\u5c31\u662f\u6d88\u606f\u5fc5\u987b\u7acb\u5373\u88ab\u53d1\u9001\uff0c\u4f46\u662f\u8fd9\u662f\u4e0d\u5bf9\u7684\uff0c\u4e00\u822c\u8bbe\u7f6e\u4e00\u4e2a100\u6beb\u79d2\u4e4b\u7c7b\u7684\uff0c\u8fd9\u6837\u7684\u8bdd\u5c31\u662f\u8bf4\uff0c\u8fd9\u4e2a\u6d88\u606f\u88ab\u53d1\u9001\u51fa\u53bb\u540e\u8fdb\u5165\u4e00\u4e2abatch\uff0c\u5982\u679c100\u6beb\u79d2\u5185\uff0c\u8fd9\u4e2abatch\u6ee1\u4e8616kb\uff0c\u81ea\u7136\u5c31\u4f1a\u53d1\u9001\u51fa\u53bb\u3002<\/li>\n<\/ul>\n<h4>13.4 \u5982\u4f55\u5904\u7406\u5f02\u5e38<\/h4>\n<p><strong>1\u3001LeaderNotAvailableException\uff1a<\/strong><\/p>\n<p>\u8fd9\u4e2a\u5c31\u662f\u5982\u679c\u67d0\u53f0\u673a\u5668\u6302\u4e86\uff0c\u6b64\u65f6leader\u526f\u672c\u4e0d\u53ef\u7528\uff0c\u4f1a\u5bfc\u81f4\u4f60\u5199\u5165\u5931\u8d25\uff0c\u8981\u7b49\u5f85\u5176\u4ed6follower\u526f\u672c\u5207\u6362\u4e3aleader\u526f\u672c\u4e4b\u540e\uff0c\u624d\u80fd\u7ee7\u7eed\u5199\u5165\uff0c\u6b64\u65f6\u53ef\u4ee5\u91cd\u8bd5\u53d1\u9001\u5373\u53ef\uff1b\u5982\u679c\u8bf4\u4f60\u5e73\u65f6\u91cd\u542fkafka\u7684broker\u8fdb\u7a0b\uff0c\u80af\u5b9a\u4f1a\u5bfc\u81f4leader\u5207\u6362\uff0c\u4e00\u5b9a\u4f1a\u5bfc\u81f4\u4f60\u5199\u5165\u62a5\u9519\uff0c\u662f<code>LeaderNotAvailableException<\/code>\u3002<\/p>\n<p><strong>2\u3001NotControllerException\uff1a<\/strong><\/p>\n<p>\u8fd9\u4e2a\u4e5f\u662f\u540c\u7406\uff0c\u5982\u679c\u8bf4Controller\u6240\u5728Broker\u6302\u4e86\uff0c\u90a3\u4e48\u6b64\u65f6\u4f1a\u6709\u95ee\u9898\uff0c\u9700\u8981\u7b49\u5f85Controller\u91cd\u65b0\u9009\u4e3e\uff0c\u6b64\u65f6\u4e5f\u662f\u4e00\u6837\u5c31\u662f\u91cd\u8bd5\u5373\u53ef\u3002<\/p>\n<p><strong>3\u3001NetworkException\uff1a\u7f51\u7edc\u5f02\u5e38 timeout<\/strong><\/p>\n<ul>\n<li>\u914d\u7f6eretries\u53c2\u6570\uff0c\u4ed6\u4f1a\u81ea\u52a8\u91cd\u8bd5\u7684<\/li>\n<li>\u4f46\u662f\u5982\u679c\u91cd\u8bd5\u51e0\u6b21\u4e4b\u540e\u8fd8\u662f\u4e0d\u884c\uff0c\u5c31\u4f1a\u63d0\u4f9bException\u7ed9\u6211\u4eec\u6765\u5904\u7406\u4e86,\u6211\u4eec\u83b7\u53d6\u5230\u5f02\u5e38\u4ee5\u540e\uff0c\u518d\u5bf9\u8fd9\u4e2a\u6d88\u606f\u8fdb\u884c\u5355\u72ec\u5904\u7406\u3002\u6211\u4eec\u4f1a\u6709\u5907\u7528\u7684\u94fe\u8def\u3002\u53d1\u9001\u4e0d\u6210\u529f\u7684\u6d88\u606f\u53d1\u9001\u5230Redis\u6216\u8005\u5199\u5230\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u751a\u81f3\u662f\u4e22\u5f03\u3002<\/li>\n<\/ul>\n<h4>13.5 \u91cd\u8bd5\u673a\u5236<\/h4>\n<p>\u91cd\u8bd5\u4f1a\u5e26\u6765\u4e00\u4e9b\u95ee\u9898\uff1a<\/p>\n<p><strong>\u6d88\u606f\u91cd\u590d<\/strong><\/p>\n<p>\u6709\u7684\u65f6\u5019\u4e00\u4e9bleader\u5207\u6362\u4e4b\u7c7b\u7684\u95ee\u9898\uff0c\u9700\u8981\u8fdb\u884c\u91cd\u8bd5\uff0c\u8bbe\u7f6eretries\u5373\u53ef\uff0c\u4f46\u662f\u6d88\u606f\u91cd\u8bd5\u4f1a\u5bfc\u81f4,\u91cd\u590d\u53d1\u9001\u7684\u95ee\u9898\uff0c\u6bd4\u5982\u8bf4\u7f51\u7edc\u6296\u52a8\u4e00\u4e0b\u5bfc\u81f4\u4ed6\u4ee5\u4e3a\u6ca1\u6210\u529f\uff0c\u5c31\u91cd\u8bd5\u4e86\uff0c\u5176\u5b9e\u4eba\u5bb6\u90fd\u6210\u529f\u4e86.<\/p>\n<p><strong>\u6d88\u606f\u4e71\u5e8f<\/strong>\u6d88\u606f\u91cd\u8bd5\u662f\u53ef\u80fd\u5bfc\u81f4\u6d88\u606f\u7684\u4e71\u5e8f\u7684\uff0c\u56e0\u4e3a\u53ef\u80fd\u6392\u5728\u4f60\u540e\u9762\u7684\u6d88\u606f\u90fd\u53d1\u9001\u51fa\u53bb\u4e86\u3002\u6240\u4ee5\u53ef\u4ee5\u4f7f\u7528&#8221; <code>max.in.flight.requests.per.connection<\/code>&#8220;\u53c2\u6570\u8bbe\u7f6e\u4e3a1\uff0c\u8fd9\u6837\u53ef\u4ee5\u4fdd\u8bc1producer\u540c\u4e00\u65f6\u95f4\u53ea\u80fd\u53d1\u9001\u4e00\u6761\u6d88\u606f\u3002<\/p>\n<p>\u4e24\u6b21\u91cd\u8bd5\u7684\u95f4\u9694\u9ed8\u8ba4\u662f100\u6beb\u79d2\uff0c\u7528&#8221;<code>retry.backoff.ms<\/code>&#8220;\u6765\u8fdb\u884c\u8bbe\u7f6e\uff0c\u57fa\u672c\u4e0a\u5728\u5f00\u53d1\u8fc7\u7a0b\u4e2d\uff0c\u9760\u91cd\u8bd5\u673a\u5236\u57fa\u672c\u5c31\u53ef\u4ee5\u641e\u5b9a95%\u7684\u5f02\u5e38\u95ee\u9898\u3002<\/p>\n<h4>13.6 ACK\u53c2\u6570\u8be6\u89e3<\/h4>\n<p>producer\u7aef<\/p>\n<pre><code>request.required.acks=0\uff1b<\/code><\/pre>\n<ul>\n<li>\u53ea\u8981\u8bf7\u6c42\u5df2\u53d1\u9001\u51fa\u53bb\uff0c\u5c31\u7b97\u662f\u53d1\u9001\u5b8c\u4e86\uff0c\u4e0d\u5173\u5fc3\u6709\u6ca1\u6709\u5199\u6210\u529f\u3002<\/li>\n<li>\u6027\u80fd\u5f88\u597d\uff0c\u5982\u679c\u662f\u5bf9\u4e00\u4e9b\u65e5\u5fd7\u8fdb\u884c\u5206\u6790\uff0c\u53ef\u4ee5\u627f\u53d7\u4e22\u6570\u636e\u7684\u60c5\u51b5\uff0c\u7528\u8fd9\u4e2a\u53c2\u6570\uff0c\u6027\u80fd\u4f1a\u5f88\u597d\u3002<\/li>\n<\/ul>\n<pre><code>request.required.acks=1\uff1b<\/code><\/pre>\n<ul>\n<li>\u53d1\u9001\u4e00\u6761\u6d88\u606f\uff0c\u5f53leader partition\u5199\u5165\u6210\u529f\u4ee5\u540e\uff0c\u624d\u7b97\u5199\u5165\u6210\u529f\u3002<\/li>\n<li>\u4e0d\u8fc7\u8fd9\u79cd\u65b9\u5f0f\u4e5f\u6709\u4e22\u6570\u636e\u7684\u53ef\u80fd\u3002<\/li>\n<\/ul>\n<pre><code>request.required.acks=-1\uff1b<\/code><\/pre>\n<ul>\n<li>\u9700\u8981ISR\u5217\u8868\u91cc\u9762\uff0c\u6240\u6709\u526f\u672c\u90fd\u5199\u5b8c\u4ee5\u540e\uff0c\u8fd9\u6761\u6d88\u606f\u624d\u7b97\u5199\u5165\u6210\u529f\u3002<\/li>\n<li>ISR\uff1a1\u4e2a\u526f\u672c\u30021 leader partition 1 follower partition<\/li>\n<\/ul>\n<p>kafka\u670d\u52a1\u7aef\uff1a<\/p>\n<pre><code>min.insync.replicas\uff1a1<\/code><\/pre>\n<p>\u5982\u679c\u6211\u4eec\u4e0d\u8bbe\u7f6e\u7684\u8bdd\uff0c\u9ed8\u8ba4\u8fd9\u4e2a\u503c\u662f1\uff0c\u4e00\u4e2aleader partition\u4f1a\u7ef4\u62a4\u4e00\u4e2aISR\u5217\u8868\uff0c\u8fd9\u4e2a\u503c\u5c31\u662f\u9650\u5236ISR\u5217\u8868\u91cc\u9762\uff0c\u81f3\u5c11\u5f97\u6709\u51e0\u4e2a\u526f\u672c\uff0c\u6bd4\u5982\u8fd9\u4e2a\u503c\u662f2\uff0c\u90a3\u4e48\u5f53ISR\u5217\u8868\u91cc\u9762\u53ea\u6709\u4e00\u4e2a\u526f\u672c\u7684\u65f6\u5019\u3002\u5f80\u8fd9\u4e2a\u5206\u533a\u63d2\u5165\u6570\u636e\u7684\u65f6\u5019\u4f1a\u62a5\u9519\u3002<\/p>\n<p>\u8bbe\u8ba1\u4e00\u4e2a\u4e0d\u4e22\u6570\u636e\u7684\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u5206\u533a\u526f\u672c &gt;=2<\/li>\n<li>acks = -1<\/li>\n<li>min.insync.replicas &gt;=2<\/li>\n<\/ul>\n<p>\u8fd8\u6709\u53ef\u80fd\u5c31\u662f\u53d1\u9001\u6709\u5f02\u5e38\uff1a\u5bf9\u5f02\u5e38\u8fdb\u884c\u5904\u7406<\/p>\n<h4>13.7 \u81ea\u5b9a\u4e49\u5206\u533a<\/h4>\n<p>\u5206\u533a\uff1a<\/p>\n<ul>\n<li>\u6ca1\u6709\u8bbe\u7f6ekey<\/li>\n<\/ul>\n<p>\u6211\u4eec\u7684\u6d88\u606f\u5c31\u4f1a\u88ab\u8f6e\u8bad\u7684\u53d1\u9001\u5230\u4e0d\u540c\u7684\u5206\u533a\u3002<\/p>\n<ul>\n<li>\u8bbe\u7f6e\u4e86key<\/li>\n<\/ul>\n<p>kafka\u81ea\u5e26\u7684\u5206\u533a\u5668\uff0c\u4f1a\u6839\u636ekey\u8ba1\u7b97\u51fa\u6765\u4e00\u4e2ahash\u503c\uff0c\u8fd9\u4e2ahash\u503c\u4f1a\u5bf9\u5e94\u67d0\u4e00\u4e2a\u5206\u533a\u3002<\/p>\n<p>\u5982\u679ckey\u76f8\u540c\u7684\uff0c\u90a3\u4e48hash\u503c\u5fc5\u7136\u76f8\u540c\uff0ckey\u76f8\u540c\u7684\u503c\uff0c\u5fc5\u7136\u662f\u4f1a\u88ab\u53d1\u9001\u5230\u540c\u4e00\u4e2a\u5206\u533a\u3002<\/p>\n<p>\u4f46\u662f\u6709\u4e9b\u6bd4\u8f83\u7279\u6b8a\u7684\u65f6\u5019\uff0c\u6211\u4eec\u5c31\u9700\u8981\u81ea\u5b9a\u4e49\u5206\u533a<\/p>\n<pre><code>public class HotDataPartitioner implements Partitioner {\nprivate Random random;\n@Override\npublic void configure(Map&lt;String, ?&gt; configs) {\nrandom = new Random();\n}\n@Override\npublic int partition(String topic, Object keyObj, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {\nString key = (String)keyObj;\nList partitionInfoList = cluster.availablePartitionsForTopic(topic);\n\/\/\u83b7\u53d6\u5230\u5206\u533a\u7684\u4e2a\u6570 0,1\uff0c2\nint partitionCount = partitionInfoList.size();\n\/\/\u6700\u540e\u4e00\u4e2a\u5206\u533a\nint hotDataPartition = partitionCount - 1;\nreturn !key.contains(\u201chot_data\u201d) ? random.nextInt(partitionCount - 1) : hotDataPartition;\n}\n}<\/code><\/pre>\n<p>\u5982\u4f55\u4f7f\u7528\uff1a<\/p>\n<p>\u914d\u7f6e\u4e0a\u8fd9\u4e2a\u7c7b\u5373\u53ef\uff1a<code>props.put(\u201dpartitioner.class\u201d, \u201ccom.zhss.HotDataPartitioner\u201d);<\/code><\/p>\n<h4>13.8 \u7efc\u5408\u6848\u4f8b\u6f14\u793a<\/h4>\n<p>\u9700\u6c42\u5206\u6790\uff1a<\/p>\n<blockquote>\n<p>\u7535\u5546\u80cc\u666f -\u300b \u4e8c\u624b\u7684\u7535\u5546\u5e73\u53f0<\/p>\n<p>\u3010\u6b22\u4e50\u9001\u3011\u7684\u9879\u76ee\uff0c\u7528\u6237\u8d2d\u4e70\u4e86\u4e1c\u897f\u4ee5\u540e\u4f1a\u6709\u3010\u661f\u661f\u3011\uff0c\u7528\u661f\u661f\u53bb\u6362\u7269\u54c1\u3002\u4e00\u5757\u94b1\u4e00\u4e2a\u661f\u661f\u3002<\/p>\n<\/blockquote>\n<p>\u8ba2\u5355\u7cfb\u7edf\uff08\u6d88\u606f\u7684\u751f\u4ea7\uff09\uff0c\u53d1\u9001\u4e00\u6761\u6d88\u606f\uff08\u652f\u4ed8\u8ba2\u5355\uff0c\u53d6\u6d88\u8ba2\u5355\uff09 -&gt; Kafka &lt;- \u4f1a\u5458\u7cfb\u7edf\uff0c\u4ecekafak\u91cc\u9762\u53bb\u6d88\u8d39\u6570\u636e\uff0c\u627e\u5230\u5bf9\u5e94\u7528\u6237\u6d88\u8d39\u7684\u91d1\u989d\uff0c\u7136\u540e\u7ed9\u8be5\u7528\u6237\u66f4\u65b0\u661f\u661f\u7684\u6570\u91cf\u3002<\/p>\n<p>\u5206\u6790\u4e00\u4e0b\uff1a<\/p>\n<p>\u53d1\u9001\u6d88\u606f\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u6307\u5b9akey\uff0c\u4e5f\u53ef\u4ee5\u4e0d\u6307\u5b9akey\u3002<\/p>\n<p>1)\u5982\u679c\u4e0d\u6307\u5b9akey<\/p>\n<ul>\n<li>zhangsan -&gt;\u4e0b\u8ba2\u5355 -&gt; 100 -&gt; +100<\/li>\n<li>zhangsan -&gt; \u53d6\u6d88\u8ba2\u5355 -&gt; -100 -&gt; -100<\/li>\n<li>\u4f1a\u5458\u7cfb\u7edf\u6d88\u8d39\u6570\u636e\u7684\u65f6\u5019\uff0c\u6709\u53ef\u80fd\u5148\u6d88\u8d39\u5230\u7684\u662f \u53d6\u6d88\u8ba2\u5355\u7684\u6570\u636e\u3002<\/li>\n<\/ul>\n<p>2)\u5982\u679c\u6307\u5b9akey,key -&gt; hash\uff08\u6570\u5b57\uff09 -&gt; \u5bf9\u5e94\u5206\u533a\u53f7 -&gt; \u53d1\u9001\u5230\u5bf9\u5e94\u7684\u5206\u533a\u91cc\u9762\u3002<\/p>\n<ul>\n<li>\u5982\u679ckey\u76f8\u540c\u7684 -&gt; \u6570\u636e\u80af\u5b9a\u4f1a\u88ab\u53d1\u9001\u5230\u540c\u4e00\u4e2a\u5206\u533a\uff08\u6709\u5e8f\u7684\uff09<\/li>\n<\/ul>\n<p>\u8fd9\u4e2a\u9879\u76ee\u9700\u8981\u6307\u5b9akey\uff0c\u628a\u7528\u6237\u7684id\u6307\u5b9a\u4e3akey.<\/p>\n<h2><strong>14\u3001Kafka\u6d88\u8d39\u8005<\/strong><\/h2>\n<h4>14.1 \u6d88\u8d39\u7ec4\u6982\u5ff5<\/h4>\n<p>groupid\u76f8\u540c\u5c31\u5c5e\u4e8e\u540c\u4e00\u4e2a\u6d88\u8d39\u7ec4<\/p>\n<p>1\uff09\u6bcf\u4e2aconsumer\u90fd\u8981\u5c5e\u4e8e\u4e00\u4e2aconsumer.group\uff0c\u5c31\u662f\u4e00\u4e2a\u6d88\u8d39\u7ec4\uff0ctopic\u7684\u4e00\u4e2a\u5206\u533a\u53ea\u4f1a\u5206\u914d\u7ed9\u4e00\u4e2a\u6d88\u8d39\u7ec4\u4e0b\u7684\u4e00\u4e2aconsumer\u6765\u5904\u7406\uff0c\u6bcf\u4e2aconsumer\u53ef\u80fd\u4f1a\u5206\u914d\u591a\u4e2a\u5206\u533a\uff0c\u4e5f\u6709\u53ef\u80fd\u67d0\u4e2aconsumer\u6ca1\u6709\u5206\u914d\u5230\u4efb\u4f55\u5206\u533a\u3002<\/p>\n<p>2\uff09\u5982\u679c\u60f3\u8981\u5b9e\u73b0\u4e00\u4e2a\u5e7f\u64ad\u7684\u6548\u679c\uff0c\u90a3\u53ea\u9700\u8981\u4f7f\u7528\u4e0d\u540c\u7684group id\u53bb\u6d88\u8d39\u5c31\u53ef\u4ee5\u3002<\/p>\n<p>topicA:<\/p>\n<ul>\n<li>partition0\u3001partition1<\/li>\n<\/ul>\n<p>groupA\uff1a<\/p>\n<ul>\n<li>consumer1:\u6d88\u8d39 partition0<\/li>\n<li>consuemr2:\u6d88\u8d39 partition1<\/li>\n<li>consuemr3:\u6d88\u8d39\u4e0d\u5230\u6570\u636e<\/li>\n<\/ul>\n<p>groupB:<\/p>\n<ul>\n<li>consuemr3:\u6d88\u8d39\u5230partition0\u548cpartition1<\/li>\n<\/ul>\n<p>3\uff09\u5982\u679cconsumer group\u4e2d\u67d0\u4e2a\u6d88\u8d39\u8005\u6302\u4e86\uff0c\u6b64\u65f6\u4f1a\u81ea\u52a8\u628a\u5206\u914d\u7ed9\u4ed6\u7684\u5206\u533a\u4ea4\u7ed9\u5176\u4ed6\u7684\u6d88\u8d39\u8005\uff0c\u5982\u679c\u4ed6\u53c8\u91cd\u542f\u4e86\uff0c\u90a3\u4e48\u53c8\u4f1a\u628a\u4e00\u4e9b\u5206\u533a\u91cd\u65b0\u4ea4\u8fd8\u7ed9\u4ed6<\/p>\n<h4>14.2 \u57fa\u7840\u6848\u4f8b\u6f14\u793a<\/h4>\n<p><img decoding=\"async\" width=\"723\" height=\"639\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxU\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h4>14.3 \u504f\u79fb\u91cf\u7ba1\u7406<\/h4>\n<p>\u6bcf\u4e2aconsumer\u5185\u5b58\u91cc\u6570\u636e\u7ed3\u6784\u4fdd\u5b58\u5bf9\u6bcf\u4e2atopic\u7684\u6bcf\u4e2a\u5206\u533a\u7684\u6d88\u8d39offset\uff0c\u5b9a\u671f\u4f1a\u63d0\u4ea4offset\uff0c\u8001\u7248\u672c\u662f\u5199\u5165zk\uff0c\u4f46\u662f\u90a3\u6837\u9ad8\u5e76\u53d1\u8bf7\u6c42zk\u662f\u4e0d\u5408\u7406\u7684\u67b6\u6784\u8bbe\u8ba1\uff0czk\u662f\u505a\u5206\u5e03\u5f0f\u7cfb\u7edf\u7684\u534f\u8c03\u7684\uff0c\u8f7b\u91cf\u7ea7\u7684\u5143\u6570\u636e\u5b58\u50a8\uff0c\u4e0d\u80fd\u8d1f\u8d23\u9ad8\u5e76\u53d1\u8bfb\u5199\uff0c\u4f5c\u4e3a\u6570\u636e\u5b58\u50a8\u3002<\/p>\n<p>\u73b0\u5728\u65b0\u7684\u7248\u672c\u63d0\u4ea4offset\u53d1\u9001\u7ed9kafka\u5185\u90e8topic\uff1a<code>__consumer_offsets<\/code>\uff0c\u63d0\u4ea4\u8fc7\u53bb\u7684\u65f6\u5019\uff0c key\u662f<code>group.id+topic+\u5206\u533a\u53f7<\/code>\uff0cvalue\u5c31\u662f\u5f53\u524doffset\u7684\u503c\uff0c\u6bcf\u9694\u4e00\u6bb5\u65f6\u95f4\uff0ckafka\u5185\u90e8\u4f1a\u5bf9\u8fd9\u4e2atopic\u8fdb\u884ccompact(\u5408\u5e76)\uff0c\u4e5f\u5c31\u662f\u6bcf\u4e2agroup.id+topic+\u5206\u533a\u53f7\u5c31\u4fdd\u7559\u6700\u65b0\u6570\u636e\u3002<\/p>\n<p><code>__consumer_offsets<\/code>\u53ef\u80fd\u4f1a\u63a5\u6536\u9ad8\u5e76\u53d1\u7684\u8bf7\u6c42\uff0c\u6240\u4ee5\u9ed8\u8ba4\u5206\u533a50\u4e2a(leader partitiron -&gt; 50 kafka)\uff0c\u8fd9\u6837\u5982\u679c\u4f60\u7684kafka\u90e8\u7f72\u4e86\u4e00\u4e2a\u5927\u7684\u96c6\u7fa4\uff0c\u6bd4\u5982\u670950\u53f0\u673a\u5668\uff0c\u5c31\u53ef\u4ee5\u752850\u53f0\u673a\u5668\u6765\u6297offset\u63d0\u4ea4\u7684\u8bf7\u6c42\u538b\u529b\u3002<\/p>\n<ul>\n<li>\u6d88\u8d39\u8005 -&gt; broker\u7aef\u7684\u6570\u636e<\/li>\n<li>message -&gt; \u78c1\u76d8 -&gt; offset \u987a\u5e8f\u9012\u589e<\/li>\n<li>\u4ece\u54ea\u513f\u5f00\u59cb\u6d88\u8d39\uff1f-&gt; offset<\/li>\n<li>\u6d88\u8d39\u8005\uff08offset\uff09<\/li>\n<\/ul>\n<h4>14.4 \u504f\u79fb\u91cf\u76d1\u63a7\u5de5\u5177\u4ecb\u7ecd<\/h4>\n<p>web\u9875\u9762\u7ba1\u7406\u7684\u4e00\u4e2a\u7ba1\u7406\u8f6f\u4ef6(kafka Manager)<\/p>\n<ul>\n<li>\u4fee\u6539bin\/kafka-run-class.sh\u811a\u672c\uff0c\u7b2c\u4e00\u884c\u589e\u52a0<code>JMX_PORT=9988<\/code><\/li>\n<li>\u91cd\u542fkafka\u8fdb\u7a0b<\/li>\n<\/ul>\n<p>\u53e6\u4e00\u4e2a\u8f6f\u4ef6\uff1a\u4e3b\u8981\u76d1\u63a7\u7684consumer\u7684\u504f\u79fb\u91cf\u3002<\/p>\n<p>\u5c31\u662f\u4e00\u4e2ajar\u5305<code>java -cp KafkaOffsetMonitor-assembly-0.3.0-SNAPSHOT.jar<\/code><\/p>\n<pre><code>com.quantifind.kafka.offsetapp.OffsetGetterWeb<\/code><\/pre>\n<ul>\n<li>offsetStorage kafka \/\uff08\u6839\u636e\u7248\u672c\uff1a\u504f\u79fb\u91cf\u5b58\u5728kafka\u5c31\u586bkafka\uff0c\u5b58\u5728zookeeper\u5c31\u586bzookeeper\uff09<\/li>\n<li>zk hadoop1:2181<\/li>\n<li>port 9004<\/li>\n<li>refresh 15.seconds<\/li>\n<li>retain 2.days<\/li>\n<\/ul>\n<p>\u5199\u4e86\u4e00\u6bb5\u7a0b\u5e8f ,\u6d88\u8d39kafka\u91cc\u9762\u7684\u6570\u636e\uff08consumer\uff0c\u5904\u7406\u6570\u636e -&gt; \u4e1a\u52a1\u4ee3\u7801\uff09 -&gt; Kafka \u5982\u4f55\u53bb\u5224\u65ad\u4f60\u7684\u8fd9\u6bb5\u4ee3\u7801\u771f\u7684\u662f\u5b9e\u65f6\u7684\u53bb\u6d88\u8d39\u7684\u5462\uff1f<\/p>\n<p>\u5ef6\u8fdf\u51e0\u4ebf\u6761\u6570\u636e -&gt; \u9608\u503c\uff0820\u4e07\u6761\u7684\u65f6\u5019 \u53d1\u9001\u4e00\u4e2a\u544a\u8b66\u3002\uff09<\/p>\n<h4>14.5 \u6d88\u8d39\u5f02\u5e38\u611f\u77e5<\/h4>\n<p><code>heartbeat.interval.ms<\/code>\uff1a<\/p>\n<ul>\n<li>consumer\u5fc3\u8df3\u65f6\u95f4\u95f4\u9694\uff0c\u5fc5\u987b\u5f97\u4e0ecoordinator\u4fdd\u6301\u5fc3\u8df3\u624d\u80fd\u77e5\u9053consumer\u662f\u5426\u6545\u969c\u4e86\uff0c<\/li>\n<li>\u7136\u540e\u5982\u679c\u6545\u969c\u4e4b\u540e\uff0c\u5c31\u4f1a\u901a\u8fc7\u5fc3\u8df3\u4e0b\u53d1rebalance\u7684\u6307\u4ee4\u7ed9\u5176\u4ed6\u7684consumer\u901a\u77e5\u4ed6\u4eec\u8fdb\u884crebalance\u7684\u64cd\u4f5c<\/li>\n<\/ul>\n<p><code>session.timeout.ms<\/code>\uff1a<\/p>\n<ul>\n<li>kafka\u591a\u957f\u65f6\u95f4\u611f\u77e5\u4e0d\u5230\u4e00\u4e2aconsumer\u5c31\u8ba4\u4e3a\u4ed6\u6545\u969c\u4e86\uff0c\u9ed8\u8ba4\u662f10\u79d2<\/li>\n<\/ul>\n<p><code>max.poll.interval.ms<\/code>\uff1a<\/p>\n<ul>\n<li>\u5982\u679c\u5728\u4e24\u6b21poll\u64cd\u4f5c\u4e4b\u95f4\uff0c\u8d85\u8fc7\u4e86\u8fd9\u4e2a\u65f6\u95f4\uff0c\u90a3\u4e48\u5c31\u4f1a\u8ba4\u4e3a\u8fd9\u4e2aconsume\u5904\u7406\u80fd\u529b\u592a\u5f31\u4e86\uff0c\u4f1a\u88ab\u8e22\u51fa\u6d88\u8d39\u7ec4\uff0c\u5206\u533a\u5206\u914d\u7ed9\u522b\u4eba\u53bb\u6d88\u8d39\uff0c\u4e00\u822c\u6765\u8bf4\u7ed3\u5408\u4e1a\u52a1\u5904\u7406\u7684\u6027\u80fd\u6765\u8bbe\u7f6e\u5c31\u53ef\u4ee5\u4e86\u3002<\/li>\n<\/ul>\n<h4>14.6 \u6838\u5fc3\u53c2\u6570\u89e3\u91ca<\/h4>\n<p><code>fetch.max.bytes<\/code>\uff1a<\/p>\n<p>\u83b7\u53d6\u4e00\u6761\u6d88\u606f\u6700\u5927\u7684\u5b57\u8282\u6570\uff0c\u4e00\u822c\u5efa\u8bae\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u9ed8\u8ba4\u662f1M \u5176\u5b9e\u6211\u4eec\u5728\u4e4b\u524d\u591a\u4e2a\u5730\u65b9\u90fd\u89c1\u5230\u8fc7\u8fd9\u4e2a\u7c7b\u4f3c\u7684\u53c2\u6570\uff0c\u610f\u601d\u5c31\u662f\u8bf4\u4e00\u6761\u4fe1\u606f\u6700\u5927\u80fd\u591a\u5927\uff1f<\/p>\n<ol>\n<li>Producer\uff1a\u53d1\u9001\u7684\u6570\u636e\uff0c\u4e00\u6761\u6d88\u606f\u6700\u5927\u591a\u5927\uff0c -&gt; 10M<\/li>\n<li>Broker\uff1a\u5b58\u50a8\u6570\u636e\uff0c\u4e00\u6761\u6d88\u606f\u6700\u5927\u80fd\u63a5\u53d7\u591a\u5927 -&gt; 10M<\/li>\n<li>Consumer\uff1a<\/li>\n<\/ol>\n<p><code>max.poll.records<\/code>:<\/p>\n<p>\u4e00\u6b21poll\u8fd4\u56de\u6d88\u606f\u7684\u6700\u5927\u6761\u6570\uff0c\u9ed8\u8ba4\u662f500\u6761<\/p>\n<p><code>connection.max.idle.ms<\/code>\uff1a<\/p>\n<p>consumer\u8ddfbroker\u7684socket\u8fde\u63a5\u5982\u679c\u7a7a\u95f2\u8d85\u8fc7\u4e86\u4e00\u5b9a\u7684\u65f6\u95f4\uff0c\u6b64\u65f6\u5c31\u4f1a\u81ea\u52a8\u56de\u6536\u8fde\u63a5\uff0c\u4f46\u662f\u4e0b\u6b21\u6d88\u8d39\u5c31\u8981\u91cd\u65b0\u5efa\u7acbsocket\u8fde\u63a5\uff0c\u8fd9\u4e2a\u5efa\u8bae\u8bbe\u7f6e\u4e3a-1\uff0c\u4e0d\u8981\u53bb\u56de\u6536<\/p>\n<p><code>enable.auto.commit<\/code>:<\/p>\n<p>\u5f00\u542f\u81ea\u52a8\u63d0\u4ea4\u504f\u79fb\u91cf<\/p>\n<p><code>auto.commit.interval.ms<\/code>:<\/p>\n<p>\u6bcf\u9694\u591a\u4e45\u63d0\u4ea4\u4e00\u6b21\u504f\u79fb\u91cf\uff0c\u9ed8\u8ba4\u503c5000\u6beb\u79d2<\/p>\n<p><code>auto.offset.reset<\/code>\uff1a<\/p>\n<ul>\n<li>earliest\uff1a\u5f53\u5404\u5206\u533a\u4e0b\u6709\u5df2\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u63d0\u4ea4\u7684offset\u5f00\u59cb\u6d88\u8d39\uff1b\u65e0\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u5934\u5f00\u59cb\u6d88\u8d39<\/li>\n<li>latest\uff1a\u5f53\u5404\u5206\u533a\u4e0b\u6709\u5df2\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u63d0\u4ea4\u7684offset\u5f00\u59cb\u6d88\u8d39\uff1b\u65e0\u63d0\u4ea4\u7684offset\u65f6\uff0c\u6d88\u8d39\u65b0\u4ea7\u751f\u7684\u8be5\u5206\u533a\u4e0b\u7684\u6570\u636e<\/li>\n<li>none\uff1atopic\u5404\u5206\u533a\u90fd\u5b58\u5728\u5df2\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4eceoffset\u540e\u5f00\u59cb\u6d88\u8d39\uff1b\u53ea\u8981\u6709\u4e00\u4e2a\u5206\u533a\u4e0d\u5b58\u5728\u5df2\u63d0\u4ea4\u7684offset\uff0c\u5219\u629b\u51fa\u5f02\u5e38<\/li>\n<\/ul>\n<h4>14.7 \u7efc\u5408\u6848\u4f8b\u6f14\u793a<\/h4>\n<p>\u5f15\u5165\u6848\u4f8b\uff1a\u4e8c\u624b\u7535\u5546\u5e73\u53f0\uff08\u6b22\u4e50\u9001\uff09\uff0c\u6839\u636e\u7528\u6237\u6d88\u8d39\u7684\u91d1\u989d\uff0c\u5bf9\u7528\u6237\u661f\u661f\u8fdb\u884c\u7d2f\u8ba1\u3002<\/p>\n<ul>\n<li>\u8ba2\u5355\u7cfb\u7edf\uff08\u751f\u4ea7\u8005\uff09 -&gt; Kafka\u96c6\u7fa4\u91cc\u9762\u53d1\u9001\u4e86\u6d88\u606f\u3002<\/li>\n<li>\u4f1a\u5458\u7cfb\u7edf\uff08\u6d88\u8d39\u8005\uff09 -&gt; Kafak\u96c6\u7fa4\u91cc\u9762\u6d88\u8d39\u6d88\u606f\uff0c\u5bf9\u6d88\u606f\u8fdb\u884c\u5904\u7406\u3002<\/li>\n<\/ul>\n<h4>14.8 group coordinator\u539f\u7406<\/h4>\n<p>\u9762\u8bd5\u9898\uff1a\u6d88\u8d39\u8005\u662f\u5982\u4f55\u5b9e\u73b0rebalance\u7684\uff1f\u2014 \u6839\u636ecoordinator\u5b9e\u73b0<\/p>\n<h5>\u4ec0\u4e48\u662fcoordinator<\/h5>\n<p>\u6bcf\u4e2aconsumer group\u90fd\u4f1a\u9009\u62e9\u4e00\u4e2abroker\u4f5c\u4e3a\u81ea\u5df1\u7684coordinator\uff0c\u4ed6\u662f\u8d1f\u8d23\u76d1\u63a7\u8fd9\u4e2a\u6d88\u8d39\u7ec4\u91cc\u7684\u5404\u4e2a\u6d88\u8d39\u8005\u7684\u5fc3\u8df3\uff0c\u4ee5\u53ca\u5224\u65ad\u662f\u5426\u5b95\u673a\uff0c\u7136\u540e\u5f00\u542frebalance\u7684<\/p>\n<h5>\u5982\u4f55\u9009\u62e9coordinator\u673a\u5668<\/h5>\n<p>\u9996\u5148\u5bf9groupId\u8fdb\u884chash\uff08\u6570\u5b57\uff09\uff0c\u63a5\u7740\u5bf9<code>__consumer_offsets<\/code>\u7684\u5206\u533a\u6570\u91cf\u53d6\u6a21\uff0c\u9ed8\u8ba4\u662f50\uff0c<code>_consumer_offsets<\/code>\u7684\u5206\u533a\u6570\u53ef\u4ee5\u901a\u8fc7<code>offsets.topic.num.partitions<\/code>\u6765\u8bbe\u7f6e\uff0c\u627e\u5230\u5206\u533a\u4ee5\u540e\uff0c\u8fd9\u4e2a\u5206\u533a\u6240\u5728\u7684broker\u673a\u5668\u5c31\u662fcoordinator\u673a\u5668\u3002<\/p>\n<p>\u6bd4\u5982\u8bf4\uff1agroupId\uff0c\u201cmyconsumer_group\u201d -&gt; hash\u503c\uff08\u6570\u5b57\uff09-&gt; \u5bf950\u53d6\u6a21 -&gt; 8<code>__consumer_offsets<\/code> \u8fd9\u4e2a\u4e3b\u9898\u76848\u53f7\u5206\u533a\u5728\u54ea\u53f0broker\u4e0a\u9762\uff0c\u90a3\u4e00\u53f0\u5c31\u662fcoordinator \u5c31\u77e5\u9053\u8fd9\u4e2aconsumer group\u4e0b\u7684\u6240\u6709\u7684\u6d88\u8d39\u8005\u63d0\u4ea4offset\u7684\u65f6\u5019\u662f\u5f80\u54ea\u4e2a\u5206\u533a\u53bb\u63d0\u4ea4offset\uff0c<\/p>\n<p>\u8fd0\u884c\u6d41\u7a0b<\/p>\n<ul>\n<li>\u6bcf\u4e2aconsumer\u90fd\u53d1\u9001JoinGroup\u8bf7\u6c42\u5230Coordinator\uff0c<\/li>\n<li>\u7136\u540eCoordinator\u4ece\u4e00\u4e2aconsumer group\u4e2d\u9009\u62e9\u4e00\u4e2aconsumer\u4f5c\u4e3aleader\uff0c<\/li>\n<li>\u628aconsumer group\u60c5\u51b5\u53d1\u9001\u7ed9\u8fd9\u4e2aleader\uff0c<\/li>\n<li>\u63a5\u7740\u8fd9\u4e2aleader\u4f1a\u8d1f\u8d23\u5236\u5b9a\u6d88\u8d39\u65b9\u6848\uff0c<\/li>\n<li>\u901a\u8fc7SyncGroup\u53d1\u7ed9Coordinator<\/li>\n<li>\u63a5\u7740Coordinator\u5c31\u628a\u6d88\u8d39\u65b9\u6848\u4e0b\u53d1\u7ed9\u5404\u4e2aconsumer\uff0c\u4ed6\u4eec\u4f1a\u4ece\u6307\u5b9a\u7684\u5206\u533a\u7684<\/li>\n<\/ul>\n<p>leader broker\u5f00\u59cb\u8fdb\u884csocket\u8fde\u63a5\u4ee5\u53ca\u6d88\u8d39\u6d88\u606f<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"465\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxV\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h4>14.9 rebalance\u7b56\u7565<\/h4>\n<p>consumer group\u9760coordinator\u5b9e\u73b0\u4e86Rebalance<\/p>\n<p>\u8fd9\u91cc\u6709\u4e09\u79cdrebalance\u7684\u7b56\u7565\uff1arange\u3001round-robin\u3001sticky<\/p>\n<p>\u6bd4\u5982\u6211\u4eec\u6d88\u8d39\u7684\u4e00\u4e2a\u4e3b\u9898\u670912\u4e2a\u5206\u533a\uff1a<\/p>\n<pre><code>p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11<\/code><\/pre>\n<p>\u5047\u8bbe\u6211\u4eec\u7684\u6d88\u8d39\u8005\u7ec4\u91cc\u9762\u6709\u4e09\u4e2a\u6d88\u8d39\u8005<\/p>\n<p><strong>range\u7b56\u7565<\/strong><\/p>\n<ul>\n<li>range\u7b56\u7565\u5c31\u662f\u6309\u7167partiton\u7684\u5e8f\u53f7\u8303\u56f4<\/li>\n<li>p0~3 consumer1<\/li>\n<li>p4~7 consumer2<\/li>\n<li>p8~11 consumer3<\/li>\n<li>\u9ed8\u8ba4\u5c31\u662f\u8fd9\u4e2a\u7b56\u7565\uff1b<\/li>\n<\/ul>\n<p><strong>round-robin\u7b56\u7565<\/strong><\/p>\n<ul>\n<li>\u5c31\u662f\u8f6e\u8be2\u5206\u914d<\/li>\n<li>consumer1:0,3,6,9<\/li>\n<li>consumer2:1,4,7,10<\/li>\n<li>consumer3:2,5,8,11<\/li>\n<\/ul>\n<p>\u4f46\u662f\u524d\u9762\u7684\u8fd9\u4e24\u4e2a\u65b9\u6848\u6709\u4e2a\u95ee\u9898\uff1a12 -&gt; 2 \u6bcf\u4e2a\u6d88\u8d39\u8005\u4f1a\u6d88\u8d396\u4e2a\u5206\u533a<\/p>\n<p>\u5047\u8bbeconsuemr1\u6302\u4e86:p0-5\u5206\u914d\u7ed9consumer2,p6-11\u5206\u914d\u7ed9consumer3\uff0c\u8fd9\u6837\u7684\u8bdd\uff0c\u539f\u672c\u5728consumer2\u4e0a\u7684\u7684p6,p7\u5206\u533a\u5c31\u88ab\u5206\u914d\u5230\u4e86 consumer3\u4e0a\u3002<\/p>\n<p><strong>sticky\u7b56\u7565<\/strong><\/p>\n<p>\u6700\u65b0\u7684\u4e00\u4e2asticky\u7b56\u7565\uff0c\u5c31\u662f\u8bf4\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u5728rebalance\u7684\u65f6\u5019\uff0c\u8ba9\u539f\u672c\u5c5e\u4e8e\u8fd9\u4e2aconsumer\u7684\u5206\u533a\u8fd8\u662f\u5c5e\u4e8e\u4ed6\u4eec\uff0c\u7136\u540e\u628a\u591a\u4f59\u7684\u5206\u533a\u518d\u5747\u5300\u5206\u914d\u8fc7\u53bb\uff0c\u8fd9\u6837\u5c3d\u53ef\u80fd\u7ef4\u6301\u539f\u6765\u7684\u5206\u533a\u5206\u914d\u7684\u7b56\u7565<\/p>\n<ul>\n<li>consumer1\uff1a0-3<\/li>\n<li>consumer2: 4-7<\/li>\n<li>consumer3: 8-11<\/li>\n<\/ul>\n<p>\u5047\u8bbeconsumer3\u6302\u4e86<\/p>\n<ul>\n<li>consumer1\uff1a0-3\uff0c+8,9<\/li>\n<li>consumer2: 4-7\uff0c+10,11<\/li>\n<\/ul>\n<h2><strong>15\u3001Broker\u7ba1\u7406<\/strong><\/h2>\n<h4>15.1 Leo\u3001hw\u542b\u4e49<\/h4>\n<ul>\n<li>Kafka\u7684\u6838\u5fc3\u539f\u7406<\/li>\n<li>\u5982\u4f55\u53bb\u8bc4\u4f30\u4e00\u4e2a\u96c6\u7fa4\u8d44\u6e90<\/li>\n<li>\u642d\u5efa\u4e86\u4e00\u5957kafka\u96c6\u7fa4 -\u300b \u4ecb\u7ecd\u4e86\u7b80\u5355\u7684\u4e00\u4e9b\u8fd0\u7ef4\u7ba1\u7406\u7684\u64cd\u4f5c\u3002<\/li>\n<li>\u751f\u4ea7\u8005\uff08\u4f7f\u7528\uff0c\u6838\u5fc3\u7684\u53c2\u6570\uff09<\/li>\n<li>\u6d88\u8d39\u8005\uff08\u539f\u7406\uff0c\u4f7f\u7528\u7684\uff0c\u6838\u5fc3\u53c2\u6570\uff09<\/li>\n<li>broker\u5185\u90e8\u7684\u4e00\u4e9b\u539f\u7406\uff0c\u6838\u5fc3\u7684\u6982\u5ff5\uff1aLEO\uff0cHW<\/li>\n<\/ul>\n<blockquote><p>LEO\uff1a\u662f\u8ddfoffset\u504f\u79fb\u91cf\u6709\u5173\u7cfb\u3002<\/p><\/blockquote>\n<p><strong>LEO\uff1a<\/strong><\/p>\n<p>\u5728kafka\u91cc\u9762\uff0c\u65e0\u8bbaleader partition\u8fd8\u662ffollower partition\u7edf\u4e00\u90fd\u79f0\u4f5c\u526f\u672c\uff08replica\uff09\u3002<\/p>\n<p>\u6bcf\u6b21partition\u63a5\u6536\u5230\u4e00\u6761\u6d88\u606f\uff0c\u90fd\u4f1a\u66f4\u65b0\u81ea\u5df1\u7684LEO\uff0c\u4e5f\u5c31\u662flog end offset\uff0cLEO\u5176\u5b9e\u5c31\u662f\u6700\u65b0\u7684offset + 1<\/p>\n<p><strong>HW\uff1a\u9ad8\u6c34\u4f4d<\/strong><\/p>\n<p>LEO\u6709\u4e00\u4e2a\u5f88\u91cd\u8981\u7684\u529f\u80fd\u5c31\u662f\u66f4\u65b0HW\uff0c\u5982\u679cfollower\u548cleader\u7684LEO\u540c\u6b65\u4e86\uff0c\u6b64\u65f6HW\u5c31\u53ef\u4ee5\u66f4\u65b0<\/p>\n<p>HW\u4e4b\u524d\u7684\u6570\u636e\u5bf9\u6d88\u8d39\u8005\u662f\u53ef\u89c1\uff0c\u6d88\u606f\u5c5e\u4e8ecommit\u72b6\u6001\u3002HW\u4e4b\u540e\u7684\u6d88\u606f\u6d88\u8d39\u8005\u6d88\u8d39\u4e0d\u5230\u3002<\/p>\n<h4>15.2 Leo\u66f4\u65b0<\/h4>\n<p><img decoding=\"async\" width=\"723\" height=\"329\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxW\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<p>15.3 hw\u66f4\u65b0<\/p>\n<p><img decoding=\"async\" width=\"723\" height=\"401\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxX\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h4>15.4 controller\u5982\u4f55\u7ba1\u7406\u6574\u4e2a\u96c6\u7fa4<\/h4>\n<p>1: \u7ade\u4e89controller\u7684<\/p>\n<ul>\n<li><code>\/controller\/id<\/code><\/li>\n<\/ul>\n<p>2\uff1acontroller\u670d\u52a1\u76d1\u542c\u7684\u76ee\u5f55\uff1a<\/p>\n<ul>\n<li><code>\/broker\/ids\/<\/code> \u7528\u6765\u611f\u77e5 broker\u4e0a\u4e0b\u7ebf<\/li>\n<li><code>\/broker\/topics\/<\/code> \u521b\u5efa\u4e3b\u9898\uff0c\u6211\u4eec\u5f53\u65f6\u521b\u5efa\u4e3b\u9898\u547d\u4ee4\uff0c\u63d0\u4f9b\u7684\u53c2\u6570\uff0cZK\u5730\u5740\u3002<\/li>\n<li><code>\/admin\/reassign_partitions<\/code> \u5206\u533a\u91cd\u5206\u914d<\/li>\n<\/ul>\n<p><img decoding=\"async\" width=\"723\" height=\"491\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxY\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<h4>15.5 \u5ef6\u65f6\u4efb\u52a1<\/h4>\n<blockquote><p>kafka\u7684\u5ef6\u8fdf\u8c03\u5ea6\u673a\u5236\uff08\u6269\u5c55\u77e5\u8bc6\uff09<\/p><\/blockquote>\n<p>\u6211\u4eec\u5148\u770b\u4e00\u4e0bkafka\u91cc\u9762\u54ea\u4e9b\u5730\u65b9\u9700\u8981\u6709\u4efb\u52a1\u8981\u8fdb\u884c\u5ef6\u8fdf\u8c03\u5ea6\u3002<\/p>\n<p><strong>\u7b2c\u4e00\u7c7b\u5ef6\u65f6\u7684\u4efb\u52a1\uff1a<\/strong><\/p>\n<p>\u6bd4\u5982\u8bf4producer\u7684acks=-1\uff0c\u5fc5\u987b\u7b49\u5f85leader\u548cfollower\u90fd\u5199\u5b8c\u624d\u80fd\u8fd4\u56de\u54cd\u5e94\u3002<\/p>\n<p>\u6709\u4e00\u4e2a\u8d85\u65f6\u65f6\u95f4\uff0c\u9ed8\u8ba4\u662f30\u79d2\uff08request.timeout.ms\uff09\u3002<\/p>\n<p>\u6240\u4ee5\u9700\u8981\u5728\u5199\u5165\u4e00\u6761\u6570\u636e\u5230leader\u78c1\u76d8\u4e4b\u540e\uff0c\u5c31\u5fc5\u987b\u6709\u4e00\u4e2a\u5ef6\u65f6\u4efb\u52a1\uff0c\u5230\u671f\u65f6\u95f4\u662f30\u79d2\u5ef6\u65f6\u4efb\u52a1 \u653e\u5230DelayedOperationPurgatory\uff08\u5ef6\u65f6\u7ba1\u7406\u5668\uff09\u4e2d\u3002<\/p>\n<p>\u5047\u5982\u572830\u79d2\u4e4b\u524d\u5982\u679c\u6240\u6709follower\u90fd\u5199\u5165\u526f\u672c\u5230\u672c\u5730\u78c1\u76d8\u4e86\uff0c\u90a3\u4e48\u8fd9\u4e2a\u4efb\u52a1\u5c31\u4f1a\u88ab\u81ea\u52a8\u89e6\u53d1\u82cf\u9192\uff0c\u5c31\u53ef\u4ee5\u8fd4\u56de\u54cd\u5e94\u7ed3\u679c\u7ed9\u5ba2\u6237\u7aef\u4e86\uff0c\u5426\u5219\u7684\u8bdd\uff0c\u8fd9\u4e2a\u5ef6\u65f6\u4efb\u52a1\u81ea\u5df1\u6307\u5b9a\u4e86\u6700\u591a\u662f30\u79d2\u5230\u671f\uff0c\u5982\u679c\u5230\u4e86\u8d85\u65f6\u65f6\u95f4\u90fd\u6ca1\u7b49\u5230\uff0c\u5c31\u76f4\u63a5\u8d85\u65f6\u8fd4\u56de\u5f02\u5e38\u3002<\/p>\n<p><strong>\u7b2c\u4e8c\u7c7b\u5ef6\u65f6\u7684\u4efb\u52a1\uff1a<\/strong><\/p>\n<p>follower\u5f80leader\u62c9\u53d6\u6d88\u606f\u7684\u65f6\u5019\uff0c\u5982\u679c\u53d1\u73b0\u662f\u7a7a\u7684\uff0c\u6b64\u65f6\u4f1a\u521b\u5efa\u4e00\u4e2a\u5ef6\u65f6\u62c9\u53d6\u4efb\u52a1<\/p>\n<p>\u5ef6\u65f6\u65f6\u95f4\u5230\u4e86\u4e4b\u540e\uff08\u6bd4\u5982\u5230\u4e86100ms\uff09\uff0c\u5c31\u7ed9follower\u8fd4\u56de\u4e00\u4e2a\u7a7a\u7684\u6570\u636e\uff0c\u7136\u540efollower\u518d\u6b21\u53d1\u9001\u8bf7\u6c42\u8bfb\u53d6\u6d88\u606f\uff0c\u4f46\u662f\u5982\u679c\u5ef6\u65f6\u7684\u8fc7\u7a0b\u4e2d(\u8fd8\u6ca1\u5230100ms)\uff0cleader\u5199\u5165\u4e86\u6d88\u606f\uff0c\u8fd9\u4e2a\u4efb\u52a1\u5c31\u4f1a\u81ea\u52a8\u82cf\u9192\uff0c\u81ea\u52a8\u6267\u884c\u62c9\u53d6\u4efb\u52a1\u3002<\/p>\n<p>\u6d77\u91cf\u7684\u5ef6\u65f6\u4efb\u52a1\uff0c\u9700\u8981\u53bb\u8c03\u5ea6\u3002<\/p>\n<h4>15.6 \u65f6\u95f4\u8f6e\u673a\u5236<\/h4>\n<p><strong>1.\u4ec0\u4e48\u4f1a\u6709\u8981\u8bbe\u8ba1\u65f6\u95f4\u8f6e\uff1f<\/strong><\/p>\n<p>Kafka\u5185\u90e8\u6709\u5f88\u591a\u5ef6\u65f6\u4efb\u52a1\uff0c\u6ca1\u6709\u57fa\u4e8eJDK Timer\u6765\u5b9e\u73b0\uff0c\u90a3\u4e2a\u63d2\u5165\u548c\u5220\u9664\u4efb\u52a1\u7684\u65f6\u95f4\u590d\u6742\u5ea6\u662fO(nlogn)\uff0c\u800c\u662f\u57fa\u4e8e\u4e86\u81ea\u5df1\u5199\u7684\u65f6\u95f4\u8f6e\u6765\u5b9e\u73b0\u7684\uff0c\u65f6\u95f4\u590d\u6742\u5ea6\u662fO(1)\uff0c\u4f9d\u9760\u65f6\u95f4\u8f6e\u673a\u5236\uff0c\u5ef6\u65f6\u4efb\u52a1\u63d2\u5165\u548c\u5220\u9664\uff0cO(1)<\/p>\n<p><strong>2.\u65f6\u95f4\u8f6e\u662f\u4ec0\u4e48\uff1f<\/strong><\/p>\n<p>\u5176\u5b9e\u65f6\u95f4\u8f6e\u8bf4\u767d\u5176\u5b9e\u5c31\u662f\u4e00\u4e2a\u6570\u7ec4\u3002<\/p>\n<ul>\n<li>tickMs:\u65f6\u95f4\u8f6e\u95f4\u9694 1ms<\/li>\n<li>wheelSize\uff1a\u65f6\u95f4\u8f6e\u5927\u5c0f 20<\/li>\n<li>interval\uff1atimckMS * whellSize\uff0c\u4e00\u4e2a\u65f6\u95f4\u8f6e\u7684\u603b\u7684\u65f6\u95f4\u8de8\u5ea6\u300220ms<\/li>\n<li>currentTime\uff1a\u5f53\u65f6\u65f6\u95f4\u7684\u6307\u9488\u3002<\/li>\n<li>\n<ul>\n<li>a:\u56e0\u4e3a\u65f6\u95f4\u8f6e\u662f\u4e00\u4e2a\u6570\u7ec4\uff0c\u6240\u4ee5\u8981\u83b7\u53d6\u91cc\u9762\u6570\u636e\u7684\u65f6\u5019\uff0c\u9760\u7684\u662findex\uff0c\u65f6\u95f4\u590d\u6742\u5ea6\u662fO(1)<\/li>\n<li>b:\u6570\u7ec4\u67d0\u4e2a\u4f4d\u7f6e\u4e0a\u5bf9\u5e94\u7684\u4efb\u52a1\uff0c\u7528\u7684\u662f\u53cc\u5411\u94fe\u8868\u5b58\u50a8\u7684\uff0c\u5f80\u53cc\u5411\u94fe\u8868\u91cc\u9762\u63d2\u5165\uff0c\u5220\u9664\u4efb\u52a1\uff0c\u65f6\u95f4\u590d\u6742\u5ea6\u4e5f\u662fO\uff081\uff09<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p><strong>3.\u591a\u5c42\u7ea7\u7684\u65f6\u95f4\u8f6e<\/strong><\/p>\n<p>\u6bd4\u5982\uff1a\u8981\u63d2\u5165\u4e00\u4e2a110\u6beb\u79d2\u4ee5\u540e\u8fd0\u884c\u7684\u4efb\u52a1\u3002<\/p>\n<ul>\n<li>tickMs:\u65f6\u95f4\u8f6e\u95f4\u9694 20ms<\/li>\n<li>wheelSize\uff1a\u65f6\u95f4\u8f6e\u5927\u5c0f 20<\/li>\n<li>interval\uff1atimckMS * whellSize\uff0c\u4e00\u4e2a\u65f6\u95f4\u8f6e\u7684\u603b\u7684\u65f6\u95f4\u8de8\u5ea6\u300220ms<\/li>\n<li>currentTime\uff1a\u5f53\u65f6\u65f6\u95f4\u7684\u6307\u9488\u3002<\/li>\n<li>\n<ul>\n<li>\u7b2c\u4e00\u5c42\u65f6\u95f4\u8f6e\uff1a1ms * 20<\/li>\n<li>\u7b2c\u4e8c\u5c42\u65f6\u95f4\u8f6e\uff1a20ms * 20<\/li>\n<li>\u7b2c\u4e09\u5c42\u65f6\u95f4\u8f6e\uff1a400ms * 20<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p><img decoding=\"async\" width=\"723\" height=\"208\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/bVc9pxZ\" alt=\"\" title=\"\" loading=\"lazy\"><\/p>\n<p>\u4f5c\u8005\uff1aerainm \/<br \/>\u6765\u6e90\uff1ablog.csdn.net\/eraining\/article\/details\/115860664<\/p>\n<p><strong>\u8fd1\u671f\u70ed\u6587\u63a8\u8350\uff1a<\/strong><\/p>\n<p>1.<a target=\"_blank\" href=\"https:\/\/link.segmentfault.com\/?enc=9Ag3lZuMZ0C1gvzueM2m7A%3D%3D.xE9tdkPmnabET%2Bp18%2B%2Fvpy4AZYaeKxvJmOnsqnm0MQM%3D\" rel=\"noopener noreferrer\">1,000+ \u9053 Java\u9762\u8bd5\u9898\u53ca\u7b54\u6848\u6574\u7406(2022\u6700\u65b0\u7248)<\/a><\/p>\n<p>2.<a target=\"_blank\" href=\"https:\/\/link.segmentfault.com\/?enc=5Mz4J36Ev9fXXtfh5vc2dA%3D%3D.G8In4jjs63CfBIKT4L%2FYZ7enhngcSA3NrZryIa4WGEtvOakzv2WcRcV1Rt%2FjIrUeWHPpGN1sR8499qUk6yhzsw%3D%3D\" rel=\"noopener noreferrer\">\u52b2\u7206\uff01Java \u534f\u7a0b\u8981\u6765\u4e86\u3002\u3002\u3002<\/a><\/p>\n<p>3.<a target=\"_blank\" href=\"https:\/\/link.segmentfault.com\/?enc=khoWCAazlIldCaN1kZK%2Fvg%3D%3D.ealSJHPuAL3vzYctvjvVS%2BhRBGYNhWgb0cA9LalXAYpaD1tXQc8OoW9JRPHNp9OW\" rel=\"noopener noreferrer\">Spring Boot 2.x \u6559\u7a0b\uff0c\u592a\u5168\u4e86\uff01<\/a><\/p>\n<p>4.<a target=\"_blank\" href=\"https:\/\/link.segmentfault.com\/?enc=Iz3qZtEPAlSLUZolBt7tiQ%3D%3D.sVHmijSk8nlN8I7rWZ6P10fcV2chKIkXSOTjIOTuQRCTwudysrLAOVCT9W5zcyHAdhvfeV51LdilcFjPZvdwHA%3D%3D\" rel=\"noopener noreferrer\">\u522b\u518d\u5199\u6ee1\u5c4f\u7684\u7206\u7206\u7206\u70b8\u7c7b\u4e86\uff0c\u8bd5\u8bd5\u88c5\u9970\u5668\u6a21\u5f0f\uff0c\u8fd9\u624d\u662f\u4f18\u96c5\u7684\u65b9\u5f0f\uff01\uff01<\/a><\/p>\n<p>5.<a target=\"_blank\" href=\"https:\/\/link.segmentfault.com\/?enc=i41Adf7anTci4WtG22EuOg%3D%3D.1Z1iLOfQOeflHrOpMlmhPZB9%2Ft8NPQoPUA8z3g3Fvl5FYrpv2A2djH%2F3oS4J2E6vLY%2FzTB6PJ5%2B%2FocQYUjMacfCUNgLsoNuK%2BMxDj1P0AS0%3D\" rel=\"noopener noreferrer\">\u300aJava\u5f00\u53d1\u624b\u518c\uff08\u5d69\u5c71\u7248\uff09\u300b\u6700\u65b0\u53d1\u5e03\uff0c\u901f\u901f\u4e0b\u8f7d\uff01<\/a><\/p>\n<p>\u89c9\u5f97\u4e0d\u9519\uff0c\u522b\u5fd8\u4e86\u968f\u624b\u70b9\u8d5e+\u8f6c\u53d1\u54e6\uff01<\/p>\n<p><a href=\"https:\/\/segmentfault.com\/a\/1190000044149818\">https:\/\/segmentfault.com\/a\/1190000044149818<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>1\u3001\u4e3a\u4ec0\u4e48\u6709\u6d88\u606f\u7cfb\u7edf 1\u3001\u89e3\u8026\u5408 2\u3001\u5f02\u6b65\u5904\u7406 \u4f8b\u5982\u7535\u5546\u5e73\u53f0\uff0c\u79d2\u6740\u6d3b\u52a8\u3002 \u4e00\u822c\u6d41\u7a0b\u4f1a\u5206\u4e3a\uff1a \u98ce\u9669\u63a7\u5236 \u5e93\u5b58\u9501\u5b9a [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[4],"tags":[],"class_list":["post-10008163","post","type-post","status-publish","format-standard","hentry","category-share"],"jetpack_sharing_enabled":true,"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10008163","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=10008163"}],"version-history":[{"count":0,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10008163\/revisions"}],"wp:attachment":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=10008163"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=10008163"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=10008163"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}