{"id":10008000,"date":"2023-06-27T11:58:49","date_gmt":"2023-06-27T03:58:49","guid":{"rendered":"https:\/\/ityet.com\/?p=10008000"},"modified":"2023-06-27T11:58:49","modified_gmt":"2023-06-27T03:58:49","slug":"%e4%b8%80%e7%a7%8d%e8%af%bb%e5%8f%96%e4%ba%bf%e7%ba%a7doris%e6%95%b0%e6%8d%ae%e5%ba%93%e7%9a%84%e6%96%b9%e6%b3%95-%e4%ba%ac%e4%b8%9c%e4%ba%91%e6%8a%80%e6%9c%af%e5%9b%a2%e9%98%9f","status":"publish","type":"post","link":"https:\/\/itmax.cn\/?p=10008000","title":{"rendered":"\u4e00\u79cd\u8bfb\u53d6\u4ebf\u7ea7doris\u6570\u636e\u5e93\u7684\u65b9\u6cd5 | \u4eac\u4e1c\u4e91\u6280\u672f\u56e2\u961f"},"content":{"rendered":"<ol>\n<li>\u5de5\u4f5c\u4e2d\uff0c\u5e38\u5e38\u9700\u8981\u5c06\u7ebf\u4e0adoris\u540c\u6b65\u81f3\u96c6\u5e02\u3002\u8bfb\u53d6doris\u6570\u636e\u540c\u8bfb\u53d6\u5e38\u89c4mysql\u57fa\u672c\u76f8\u540c\u3002\u5982\u679c\u6570\u636e\u884c\u5c0f\u4e8e\u5343\u4e07\uff0c\u6bd4\u8f83\u7b80\u5355\u7684\u65b9\u5f0f\u76f4\u63a5\u5355\u8282\u70b9\u8fde\u63a5\u3001\u8bfb\u53d6\u548c\u5b58\u50a8\u3002Python\u793a\u4f8b\u5982\u4e0b\uff1a<\/li>\n<\/ol>\n<pre><code>def get_data(sql,host='',port=2000,user='',password='',db=''):\n    # \u652f\u6301doris\n    import pymysql\n    connect = pymysql.connect(host=host,port=port,user=user,password=password,db=db,charset='utf8')\n    cursor = connect.cursor()\n    cursor.execute('SET query_timeout = 216000;') #\u5355\u4f4d\u79d2\n    cursor.execute(sql)\n    result = cursor.fetchall()\n    for row in result:\n        pass # \u5b58\u50a8\u683c\u5f0f\u53ef\u4ee5\u81ea\u884c\u63a7\u5236 \n    cursor.close()\n    connect.close()\n    return result<\/code><\/pre>\n<ol start=\"2\">\n<li>\u5982\u679c\u6570\u636e\u91cf\u6bd4\u8f83\u5927\uff0c\u8d85\u8fc7\u5343\u4e07\uff0c\u751a\u81f3\u8fc7\u4ebf\uff0c\u5355\u8282\u70b9\u8bfb\u53d6\u4f1a\u9047\u5230\u8d85\u65f6\u4ee5\u53ca\u65f6\u6548\u8fc7\u4f4e\u7684\u95ee\u9898\u3002\u53ef\u4ee5\u4f7f\u7528spark.read.jdbc\u5206\u5e03\u5f0f\u591a\u8282\u70b9\u5e76\u53d1\u8bfb\u53d6\u3002spark\u8bfb\u53d6\u652f\u6301\u4e24\u79cd\u65b9\u5f0f\u3002<\/li>\n<\/ol>\n<p>\u4e3b\u8981\u53c2\u6570\u4ecb\u7ecd\uff1a<\/p>\n<pre><code>read.jdbc(url=url,table=remote_table,column='item_sku_id',numPartitions=50,lowerBound=lowerBound, upperBound=upperBound,properties=prop)<\/code><\/pre>\n<p>url\uff1a\u683c\u5f0f\u5982&#8217;jdbc:mysql:\/\/**.jd.com:2000\/\u6570\u636e\u5e93\u540d?useUnicode=true&#038;characterEncoding=UTF-8&#038;autoReconnect=true&#038;failOverReadOnly=false&#038;zeroDateTimeBehavior=convertToNull&#038;useSSL=false&#038;serverTimezone=Asia\/Shanghai&#8217;<\/p>\n<p>table\uff1a\u53ef\u4ee5\u662f\u8868\u540d\uff0c\u4e5f\u53ef\u4ee5\u662f\u67e5\u8be2sql(\u4e5f\u5373\u652f\u6301\u6761\u4ef6\u67e5\u8be2)\uff0c\u5982\u679c\u662fsql\uff0c\u683c\u5f0f\u5982&#8221;(SELECT count(*) sku FROM rule_price_result where dt=&#8217;2023-05-10&#8242;) AS tmp&#8221;<\/p>\n<p>numPartitions\uff1a\u63a7\u5236\u5e76\u53d1\u8282\u70b9\u4e2a\u6570<\/p>\n<p>lowerBound+upperBound\u548cproperties\u4e8c\u9009\u4e00\uff0c\u63a7\u5236\u6bcf\u4e2a\u8282\u70b9\u8bfb\u53d6\u7684\u6570\u636e\u8303\u56f4\u3002<\/p>\n<p>lowerBound+upperBound\u65b9\u5f0f\uff1a\u6307\u5b9a\u8bfb\u53d6\u6700\u4f4e\u548c\u6700\u9ad8\u503c\uff0cspark\u4f1a\u7ed3\u5408\u5206\u533a\u4e2a\u6570\u548c\u6700\u4f4e\u6700\u9ad8\u8fb9\u754c\u673a\u68b0\u505a\u5206\u5272\u3002<\/p>\n<p>\u5982\u679c\u6570\u636e\u5206\u5e03\u6709\u503e\u659c\uff0c\u53ef\u4ee5\u901a\u8fc7predicates\u5217\u8868\u81ea\u884c\u63a7\u5236\u8303\u56f4\u3002<\/p>\n<blockquote>\n<p>\u4f5c\u8005\uff1a\u4eac\u4e1c\u96f6\u552e \u8d75\u5947\u731b<\/p>\n<p>\u6765\u6e90\uff1a\u4eac\u4e1c\u4e91\u5f00\u53d1\u8005\u793e\u533a<\/p>\n<\/blockquote>\n<p><a href=\"https:\/\/segmentfault.com\/a\/1190000043940043\">https:\/\/segmentfault.com\/a\/1190000043940043<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u5de5\u4f5c\u4e2d\uff0c\u5e38\u5e38\u9700\u8981\u5c06\u7ebf\u4e0adoris\u540c\u6b65\u81f3\u96c6\u5e02\u3002\u8bfb\u53d6doris\u6570\u636e\u540c\u8bfb\u53d6\u5e38\u89c4mysql\u57fa\u672c\u76f8\u540c\u3002\u5982\u679c\u6570\u636e\u884c\u5c0f\u4e8e\u5343\u4e07 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[4],"tags":[],"class_list":["post-10008000","post","type-post","status-publish","format-standard","hentry","category-share"],"jetpack_sharing_enabled":true,"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10008000","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=10008000"}],"version-history":[{"count":0,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10008000\/revisions"}],"wp:attachment":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=10008000"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=10008000"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=10008000"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}