{"id":10005998,"date":"2022-07-22T19:36:53","date_gmt":"2022-07-22T11:36:53","guid":{"rendered":"https:\/\/ityet.com\/?p=10005998"},"modified":"2022-07-22T19:36:53","modified_gmt":"2022-07-22T11:36:53","slug":"spark%e5%a6%82%e4%bd%95%e5%af%b9%e6%ba%90%e7%ab%af%e6%95%b0%e6%8d%ae%e5%81%9a%e5%88%87%e5%88%86%ef%bc%9f","status":"publish","type":"post","link":"https:\/\/itmax.cn\/?p=10005998","title":{"rendered":"Spark\u5982\u4f55\u5bf9\u6e90\u7aef\u6570\u636e\u505a\u5207\u5206\uff1f"},"content":{"rendered":"<p>\u7b80\u4ecb\uff1a\u00a0\u5178\u578b\u7684Spark\u4f5c\u4e1a\u8bfb\u53d6\u4f4d\u4e8eOSS\u7684Parquet\u5916\u8868\u65f6\uff0c\u6e90\u7aef\u7684\u5e76\u53d1\u5ea6\uff08task\/partition\uff09\u5982\u4f55\u786e\u5b9a\uff1f\u7279\u522b\u662f\u5728\u505aTPCH\u6d4b\u8bd5\u65f6\u6709\u4e00\u4e9b\u7591\u95ee\uff0c\u5982\u6e90\u7aef\u626b\u63cf\u6587\u4ef6\u7684\u5e76\u53d1\u5ea6\u662f\u5982\u4f55\u786e\u5b9a\u7684\uff1f\u662f\u5426\u4e00\u4e2aparquet\u6587\u4ef6\u5bf9\u5e94\u4e00\u4e2apartition\uff1f\u591a\u4e2aparquet\u6587\u4ef6\u5bf9\u5e94\u4e00\u4e2apartition\uff1f\u8fd8\u662f\u4e00\u4e2aparquet\u6587\u4ef6\u5bf9\u5e94\u591a\u4e2apartition\uff1f\u672c\u6587\u5c06\u4ece\u6e90\u7801\u89d2\u5ea6\u8fdb\u884c\u5206\u6790\u8fdb\u800c\u89e3\u7b54\u8fd9\u4e9b\u7591\u95ee\u3002\u5f15\u8a00\u5178\u578b\u7684Spark\u4f5c\u4e1a\u8bfb\u53d6\u4f4d\u4e8eOSS\u7684Parquet\u5916\u8868\u65f6\uff0c\u6e90\u7aef\u7684\u5e76\u53d1\u5ea6\uff08task\/partition\uff09\u5982\u4f55\u786e\u5b9a\uff1f\u7279\u522b\u662f\u5728\u505aTPCH\u6d4b\u8bd5\u65f6\u6709\u4e00\u4e9b\u7591\u95ee\uff0c\u5982\u6e90\u7aef\u626b\u63cf\u6587\u4ef6\u7684\u5e76\u53d1\u5ea6\u662f\u5982\u4f55\u786e\u5b9a\u7684\uff1f\u662f\u5426\u4e00\u4e2aparquet\u6587\u4ef6\u5bf9\u5e94\u4e00\u4e2apartition\uff1f\u591a\u4e2aparquet\u6587\u4ef6\u5bf9\u5e94\u4e00\u4e2apartition\uff1f\u8fd8\u662f\u4e00\u4e2aparquet\u6587\u4ef6\u5bf9\u5e94\u591a\u4e2apartition\uff1f\u672c\u6587\u5c06\u4ece\u6e90\u7801\u89d2\u5ea6\u8fdb\u884c\u5206\u6790\u8fdb\u800c\u89e3\u7b54\u8fd9\u4e9b\u7591\u95ee\u3002\u5206\u6790\u6570\u636e\u6e90\u8bfb\u53d6\u5bf9\u5e94\u7684\u7269\u7406\u6267\u884c\u8282\u70b9\u4e3aFileSourceScanExec\uff0c\u8bfb\u53d6\u6570\u636e\u4ee3\u7801\u5757\u5982\u4e0b\u00a0lazy val inputRDD: RDD[InternalRow] = {<\/p>\n<pre><code>val readFile: (PartitionedFile) =&gt; Iterator[InternalRow] =\n  relation.fileFormat.buildReaderWithPartitionValues(\n    sparkSession = relation.sparkSession,\n    dataSchema = relation.dataSchema,\n    partitionSchema = relation.partitionSchema,\n    requiredSchema = requiredSchema,\n    filters = pushedDownFilters,\n    options = relation.options,\n    hadoopConf = relation.sparkSession.sessionState.newHadoopConfWithOptions(relation.options))\nval readRDD = if (bucketedScan) {\n  createBucketedReadRDD(relation.bucketSpec.get, readFile, dynamicallySelectedPartitions,\n    relation)\n} else {\n  createReadRDD(readFile, dynamicallySelectedPartitions, relation)\n}\nsendDriverMetrics()\nreadRDD<\/code><\/pre>\n<p>}\u4e3b\u8981\u5173\u6ce8\u975ebucket\u7684\u5904\u7406\uff0c\u5bf9\u4e8e\u975ebucket\u7684\u626b\u63cf\u8c03\u7528createReadRDD\u65b9\u6cd5\u5b9a\u4e49\u5982\u4e0b\/**<\/p>\n<ul>\n<li>Create an RDD for non-bucketed reads.<\/li>\n<li>The bucketed variant of this function is [[createBucketedReadRDD]].<br \/>   *<\/li>\n<li>@param readFile a function to read each (part of a) file.<\/li>\n<li>@param selectedPartitions Hive-style partition that are part of the read.<\/li>\n<li>@param fsRelation [[HadoopFsRelation]] associated with the read.<br \/>   *\/<\/li>\n<\/ul>\n<p>private def createReadRDD(<\/p>\n<pre><code>  readFile: (PartitionedFile) =&gt; Iterator[InternalRow],\n  selectedPartitions: Array[PartitionDirectory],\n  fsRelation: HadoopFsRelation): RDD[InternalRow] = {\n\/\/ \u6587\u4ef6\u6253\u5f00\u5f00\u9500\uff0c\u6bcf\u6b21\u6253\u5f00\u6587\u4ef6\u6700\u5c11\u9700\u8981\u8bfb\u53d6\u7684\u5b57\u8282    \nval openCostInBytes = fsRelation.sparkSession.sessionState.conf.filesOpenCostInBytes\n\/\/ \u6700\u5927\u5207\u5206\u5206\u7247\u5927\u5c0f\nval maxSplitBytes =\n  FilePartition.maxSplitBytes(fsRelation.sparkSession, selectedPartitions)\nlogInfo(s&quot;Planning scan with bin packing, max size: $maxSplitBytes bytes, &quot; +\n  s&quot;open cost is considered as scanning $openCostInBytes bytes.&quot;)\n\/\/ Filter files with bucket pruning if possible\nval bucketingEnabled = fsRelation.sparkSession.sessionState.conf.bucketingEnabled\nval shouldProcess: Path =&gt; Boolean = optionalBucketSet match {\n  case Some(bucketSet) if bucketingEnabled =&gt;\n    \/\/ Do not prune the file if bucket file name is invalid\n    filePath =&gt; BucketingUtils.getBucketId(filePath.getName).forall(bucketSet.get)\n  case _ =&gt;\n    _ =&gt; true\n}\n\/\/ \u5bf9\u5206\u533a\u4e0b\u6587\u4ef6\u8fdb\u884c\u5207\u5206\u5e76\u6309\u7167\u4ece\u5927\u5230\u5c0f\u8fdb\u884c\u6392\u5e8f\nval splitFiles = selectedPartitions.flatMap { partition =&gt;\n  partition.files.flatMap { file =&gt;\n    \/\/ getPath() is very expensive so we only want to call it once in this block:\n    val filePath = file.getPath\n    if (shouldProcess(filePath)) {\n      \/\/ \u6587\u4ef6\u662f\u5426\u53efsplit\uff0cparquet\/orc\/avro\u5747\u53ef\u88absplit\n      val isSplitable = relation.fileFormat.isSplitable(\n        relation.sparkSession, relation.options, filePath)\n      \/\/ \u5207\u5206\u6587\u4ef6\n      PartitionedFileUtil.splitFiles(\n        sparkSession = relation.sparkSession,\n        file = file,\n        filePath = filePath,\n        isSplitable = isSplitable,\n        maxSplitBytes = maxSplitBytes,\n        partitionValues = partition.values\n      )\n    } else {\n      Seq.empty\n    }\n  }\n}.sortBy(_.length)(implicitly[Ordering[Long]].reverse)\nval partitions =\n  FilePartition.getFilePartitions(relation.sparkSession, splitFiles, maxSplitBytes)\nnew FileScanRDD(fsRelation.sparkSession, readFile, partitions)<\/code><\/pre>\n<p>}\u00a0\u53ef\u4ee5\u770b\u5230\u786e\u5b9a\u6700\u5927\u5207\u5206\u5206\u7247\u5927\u5c0fmaxSplitBytes\u5bf9\u4e8e\u540e\u7eed\u5207\u5206\u4e3a\u591a\u5c11\u4e2a\u6587\u4ef6\u975e\u5e38\u91cd\u8981\uff0c\u5176\u6838\u5fc3\u903b\u8f91\u5982\u4e0bdef maxSplitBytes(<\/p>\n<pre><code>  sparkSession: SparkSession,\n  selectedPartitions: Seq[PartitionDirectory]): Long = {\n\/\/ \u8bfb\u53d6\u6587\u4ef6\u65f6\u6253\u5305\u6210\u6700\u5927\u7684partition\u5927\u5c0f\uff0c\u9ed8\u8ba4\u4e3a128MB\uff0c\u5bf9\u5e94\u4e00\u4e2ablock\u5927\u5c0f\nval defaultMaxSplitBytes = sparkSession.sessionState.conf.filesMaxPartitionBytes\n\/\/ \u6253\u5f00\u6bcf\u4e2a\u6587\u4ef6\u7684\u5f00\u9500\uff0c\u9ed8\u8ba4\u4e3a4MB\nval openCostInBytes = sparkSession.sessionState.conf.filesOpenCostInBytes\n\/\/ \u5efa\u8bae\u7684\uff08\u4e0d\u4fdd\u8bc1\uff09\u6700\u5c0f\u5206\u5272\u6587\u4ef6\u5206\u533a\u6570\uff0c\u9ed8\u8ba4\u672a\u8bbe\u7f6e\uff0c\u4eceleafNodeDefaultParallelism\u83b7\u53d6\n\/\/ \u4ee3\u7801\u903b\u8f91\u8c03\u7528\u94fe SparkSession#leafNodeDefaultParallelism -&gt; SparkContext#defaultParallelism\n\/\/ -&gt; TaskSchedulerImpl#defaultParallelism -&gt; CoarseGrainedSchedulerBackend#defaultParallelism\n\/\/ -&gt; \u603b\u5171\u591a\u5c11\u6838max(executor core\u603b\u548c, 2)\uff0c\u6700\u5c11\u4e3a2\nval minPartitionNum = sparkSession.sessionState.conf.filesMinPartitionNum\n  .getOrElse(sparkSession.leafNodeDefaultParallelism)\n\/\/ \u603b\u5171\u8bfb\u53d6\u7684\u5927\u5c0f\nval totalBytes = selectedPartitions.flatMap(_.files.map(_.getLen + openCostInBytes)).sum\n\/\/ \u5355core\u8bfb\u53d6\u7684\u5927\u5c0f\nval bytesPerCore = totalBytes \/ minPartitionNum\n\/\/ \u8ba1\u7b97\u5927\u5c0f\uff0c\u4e0d\u4f1a\u8d85\u8fc7\u8bbe\u7f6e\u7684128MB\nMath.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))<\/code><\/pre>\n<p>}\u00a0\u5bf9\u4e8ePartitionedFileUtil#splitFiles\uff0c\u5176\u6838\u5fc3\u903b\u8f91\u5982\u4e0b\uff0c\u8f83\u4e3a\u7b80\u5355\uff0c\u76f4\u63a5\u6309\u7167\u6700\u5927\u5207\u5206\u5927\u5c0f\u5207\u5206\u5927\u6587\u4ef6\u6765\u8fdb\u884c\u5206\u7247def splitFiles(<\/p>\n<pre><code>  sparkSession: SparkSession,\n  file: FileStatus,\n  filePath: Path,\n  isSplitable: Boolean,\n  maxSplitBytes: Long,\n  partitionValues: InternalRow): Seq[PartitionedFile] = {\nif (isSplitable) {\n  \/\/ \u5207\u5206\u4e3a\u591a\u4e2a\u5206\u7247\n  (0L until file.getLen by maxSplitBytes).map { offset =&gt;\n    val remaining = file.getLen - offset\n    val size = if (remaining &gt; maxSplitBytes) maxSplitBytes else remaining\n    val hosts = getBlockHosts(getBlockLocations(file), offset, size)\n    PartitionedFile(partitionValues, filePath.toUri.toString, offset, size, hosts)\n  }\n} else {\n  Seq(getPartitionedFile(file, filePath, partitionValues))\n}<\/code><\/pre>\n<p>}\u5728\u83b7\u53d6\u5230Seq[PartitionedFile]\u5217\u8868\u540e\uff0c\u8fd8\u5e76\u6ca1\u6709\u5b8c\u6210\u5bf9\u6587\u4ef6\u7684\u5207\u5206\uff0c\u8fd8\u9700\u8981\u8c03\u7528FilePartition#getFilePartitions\u505a\u6700\u540e\u7684\u5904\u7406\uff0c\u65b9\u6cd5\u6838\u5fc3\u903b\u8f91\u5982\u4e0bdef getFilePartitions(<\/p>\n<pre><code>  sparkSession: SparkSession,\n  partitionedFiles: Seq[PartitionedFile],\n  maxSplitBytes: Long): Seq[FilePartition] = {\nval partitions = new ArrayBuffer[FilePartition]\nval currentFiles = new ArrayBuffer[PartitionedFile]\nvar currentSize = 0L\n\/** Close the current partition and move to the next. *\/\ndef closePartition(): Unit = {\n  if (currentFiles.nonEmpty) {\n    \/\/ Copy to a new Array.\n    \/\/ \u91cd\u65b0\u751f\u6210\u4e00\u4e2a\u65b0\u7684PartitionFile\n    val newPartition = FilePartition(partitions.size, currentFiles.toArray)\n    partitions += newPartition\n  }\n  currentFiles.clear()\n  currentSize = 0\n}\n\/\/ \u6253\u5f00\u6587\u4ef6\u5f00\u9500\uff0c\u9ed8\u8ba4\u4e3a4MB\nval openCostInBytes = sparkSession.sessionState.conf.filesOpenCostInBytes\n\/\/ Assign files to partitions using &quot;Next Fit Decreasing&quot;\npartitionedFiles.foreach { file =&gt;\n  if (currentSize + file.length &gt; maxSplitBytes) {\n    \/\/ \u5982\u679c\u7d2f\u52a0\u7684\u6587\u4ef6\u5927\u5c0f\u5927\u4e8e\u7684\u6700\u5927\u5207\u5206\u5927\u5c0f\uff0c\u5219\u5173\u95ed\u8be5\u5206\u533a\uff0c\u8868\u793a\u5b8c\u6210\u4e00\u4e2aTask\u8bfb\u53d6\u7684\u6570\u636e\u5207\u5206\n    closePartition()\n  }\n  \/\/ Add the given file to the current partition.\n  currentSize += file.length + openCostInBytes\n  currentFiles += file\n}\n\/\/ \u6700\u540e\u5173\u95ed\u4e00\u6b21\u5206\u533a\uff0c\u6587\u4ef6\u53ef\u80fd\u8f83\u5c0f\nclosePartition()\npartitions.toSeq<\/code><\/pre>\n<p>}\u53ef\u4ee5\u770b\u5230\u7ecf\u8fc7\u8fd9\u4e00\u6b65\u540e\uff0c\u4f1a\u628a\u4e00\u4e9b\u5c0f\u6587\u4ef6\u505a\u5408\u5e76\uff0c\u751f\u6210maxSplitBytes\u5927\u5c0f\u7684PartitionFile\uff0c\u8fd9\u6837\u53ef\u4ee5\u907f\u514d\u62c9\u8d77\u592a\u591atask\u8bfb\u53d6\u592a\u591a\u5c0f\u7684\u6587\u4ef6\u3002\u751f\u6210\u7684FileScanRDD(new FileScanRDD(fsRelation.sparkSession, readFile, partitions))\u7684\u5e76\u53d1\u5ea6\u4e3apartitions\u7684\u957f\u5ea6\uff0c\u4e5f\u5373\u6700\u540eSpark\u751f\u6210\u7684Task\u4e2a\u6570override protected def getPartitions: Array[RDDPartition] = filePartitions.toArray\u6574\u4f53\u6d41\u7a0b\u56fe\u5982\u4e0b\u56fe\u6240\u793a<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209839\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u00a0\u62c6\u5206\u3001\u5408\u5e76\u8fc7\u7a0b\u5982\u4e0b\u56fe\u6240\u793a<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209840\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u5b9e\u6218\u5bf9\u4e8eTPCH 10G\u751f\u6210\u7684customer parquet\u8868<a href=\"https:\/\/link.segmentfault.com\/?enc=5Y02SdqQEBqDk%2FE0AIgiBA%3D%3D.oTAtTOEO%2ByS3Ff9cf6BMy%2BacDiS%2FgsI0%2FYl5wmoqTRSWUO3mK4ooMbyK%2F5gNh3XbowaI9euADZ%2BWz%2FR8lDZZ4cNIiEb0eg3HE%2FteBN6V1qfr4h10MTHq%2FRQZqR8qm1QC3D7z4JTKlabQGJvKRi0L3Z6XdorBQ9bEg2vW8aVjEu8%3D\" rel=\"nofollow\">https:\/\/oss.console.aliyun.co&#8230;<\/a>\u00a0\u51718\u4e2aParquet\u6587\u4ef6\uff0c\u603b\u6587\u4ef6\u5927\u5c0f\u4e3a113.918MB\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209841\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>Spark\u4f5c\u4e1a\u914d\u7f6e\u5982\u4e0b\uff0cexecutor\u53ea\u67091coreconf spark.driver.resourceSpec=small;<br \/>conf spark.executor.instances=1;<br \/>conf spark.executor.resourceSpec=small;<br \/>conf spark.app.name=Spark SQL Test;<br \/>conf spark.adb.connectors=oss;<br \/>use tpcd;<br \/>select * from customer order by C_CUSTKEY desc limit 100;\u6839\u636e\u524d\u9762\u7684\u516c\u5f0f\u8ba1\u7b97defaultMaxSplitBytes = 128MB<br \/>openCostInBytes = 4MB<br \/>minPartitionNum = max(1, 2) = 2<br \/>totalBytes = 113.918 + 8 * 4MB = 145.918MB<br \/>bytesPerCore = 145.918MB \/ 2 = 72.959MB<br \/>maxSplitBytes = 72.959MB = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))\u5f97\u5230maxSplitBytes\u4e3a72.959MB\uff0c\u4ece\u65e5\u5fd7\u4e2d\u4e5f\u53ef\u770b\u5230\u5bf9\u5e94\u5927\u5c0f<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209842\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u7ecf\u8fc7\u6392\u5e8f\u540e\u7684\u6587\u4ef6\u987a\u5e8f\u4e3a(00000, 00001, 00002, 00003, 00004, 00006, 00005, 00007)\uff0c\u518d\u6b21\u7ecf\u8fc7\u5408\u5e76\u540e\u5f97\u52303\u4e2aFilePartitioned\uff0c\u5206\u522b\u5bf9\u5e94FilePartitioned 1: 00000, 00001, 00002FilePartitioned 2: 00003, 00004, 00006FilePartitioned 3: 00005, 00007\u5373\u603b\u5171\u4f1a\u751f\u62103\u4e2aTask\u4eceSpark UI\u67e5\u770b\u786e\u5b9e\u751f\u62103\u4e2aTask<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209843\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u4ece\u65e5\u5fd7\u67e5\u770b\u4e5f\u662f\u751f\u62103\u4e2aTask<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209844\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209845\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><\/p>\n<p><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209846\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u53d8\u66f4Spark\u4f5c\u4e1a\u914d\u7f6e\uff0c5\u4e2aexecutor\u517110coreconf spark.driver.resourceSpec=small;<br \/>conf spark.executor.instances=5;<br \/>conf spark.executor.resourceSpec=medium;<br \/>conf spark.app.name=Spark SQL Test;<br \/>conf spark.adb.connectors=oss;<br \/>use tpcd;<br \/>select * from customer order by C_CUSTKEY desc limit 100;\u6839\u636e\u524d\u9762\u7684\u516c\u5f0f\u8ba1\u7b97defaultMaxSplitBytes = 128MB<br \/>openCostInBytes = 4MB<br \/>minPartitionNum = max(10, 2) = 10<br \/>totalBytes = 113.918 + 8 * 4MB = 145.918MB<br \/>bytesPerCore = 145.918MB \/ 10 = 14.5918MB<br \/>maxSplitBytes = 14.5918MB = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))\u67e5\u770b\u65e5\u5fd7<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209847\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u6b64\u65f6\u53ef\u4ee5\u770b\u523014.5918MB\u4f1a\u5bf9\u6e90\u6587\u4ef6\u8fdb\u884c\u5207\u5206\uff0c\u4f1a\u5bf900001, 00002,00003,00004,00005,00006\u8fdb\u884c\u5207\u5206\uff0c\u5207\u5206\u6210\u4e24\u4efd\uff0c00007\u7531\u4e8e\u5c0f\u4e8e14.5918MB\uff0c\u56e0\u6b64\u4e0d\u4f1a\u8fdb\u884c\u5207\u5206\uff0c\u7ecf\u8fc7PartitionedFileUtil#splitFiles\u540e\uff0c\u603b\u5171\u5b58\u57287 * 2 + 1 = 15\u4e2aPartitionedFile00000(0 -&gt; 14.5918MB), 00000(14.5918MB -&gt; 15.698MB)00001(0 -&gt; 14.5918MB), 00001(14.5918MB -&gt; 15.632MB)00002(0 -&gt; 14.5918MB), 00002(14.5918MB -&gt; 15.629MB)00003(0 -&gt; 14.5918MB), 00003(14.5918MB -&gt; 15.624MB)00004(0 -&gt; 14.5918MB), 00004(14.5918MB -&gt; 15.617MB)00005(0 -&gt; 14.5918MB), 00005(14.5918MB -&gt; 15.536MB)00006(0 -&gt; 14.5918MB), 00006(14.5918MB -&gt; 15.539MB)00007(0 -&gt; 4.634MB)\u7ecf\u8fc7\u6392\u5e8f\u540e\u5f97\u5230\u5982\u4e0b\u4ee5\u53ca\u5408\u5e76\u540e\u5f97\u523010\u4e2aFilePartitioned\uff0c\u5206\u522b\u5bf9\u5e94FilePartitioned 1: 00000(0 -&gt; 14.5918MB)FilePartitioned 2: 00001(0 -&gt; 14.5918MB)FilePartitioned 3: 00002(0 -&gt; 14.5918MB)FilePartitioned 4: 00003(0 -&gt; 14.5918MB)FilePartitioned 5: 00004(0 -&gt; 14.5918MB)FilePartitioned 6: 00005(0 -&gt; 14.5918MB)FilePartitioned 7: 00006(0 -&gt; 14.5918MB)FilePartitioned 8: 00007(0 -&gt; 4.634MB),00000(14.5918MB -&gt; 15.698MB)FilePartitioned 9: 00001(14.5918MB -&gt; 15.632MB),00002(14.5918MB -&gt; 15.629MB),00003(14.5918MB -&gt; 15.624MB)FilePartitioned 10: 00004(14.5918MB -&gt; 15.617MB),00005(14.5918MB -&gt; 15.536MB),00006(14.5918MB -&gt; 15.539MB)\u00a0\u5373\u603b\u5171\u4f1a\u751f\u621010\u4e2aTask\u901a\u8fc7Spark UI\u4e5f\u53ef\u67e5\u770b\u5230\u751f\u6210\u4e8610\u4e2aTask<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209848\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u67e5\u770b\u65e5\u5fd7\uff0c000004(14.5918MB -&gt; 15.617MB),00005(14.5918MB -&gt; 15.536MB),00006(14.5918MB -&gt; 15.539MB)\u5728\u540c\u4e00\u4e2aTask\u4e2d<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209849\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>00007(0 -&gt; 4.634MB),00000(14.5918MB -&gt; 15.698MB)<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209850\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u00a000001(14.5918MB -&gt; 15.632MB),00002(14.5918MB -&gt; 15.629MB),00003(14.5918MB -&gt; 15.624MB)\u5728\u540c\u4e00\u4e2aTask\u4e2d\u00a0<br \/><img decoding=\"async\" referrerpolicy=\"no-referrer\" src=\"https:\/\/segmentfault.com\/img\/remote\/1460000042209851\" alt=\"\u56fe\u7247\" title=\"\u56fe\u7247\" loading=\"lazy\" ><br \/>\u603b\u7ed3\u901a\u8fc7\u6e90\u7801\u53ef\u77e5Spark\u5bf9\u4e8e\u6e90\u7aefPartition\u5207\u5206\uff0c\u4f1a\u8003\u8651\u5230\u5206\u533a\u4e0b\u6240\u6709\u6587\u4ef6\u5927\u5c0f\u4ee5\u53ca\u6253\u5f00\u6bcf\u4e2a\u6587\u4ef6\u7684\u5f00\u9500\uff0c\u540c\u65f6\u4f1a\u6d89\u53ca\u5bf9\u5927\u6587\u4ef6\u7684\u5207\u5206\u4ee5\u53ca\u5c0f\u6587\u4ef6\u7684\u5408\u5e76\uff0c\u6700\u540e\u5f97\u5230\u4e00\u4e2a\u76f8\u5bf9\u5408\u7406\u7684Partition\u3002\u539f\u6587\u94fe\u63a5\uff1a<a href=\"https:\/\/link.segmentfault.com\/?enc=buZ44HH7p9vBEqxleLA4Bw%3D%3D.vLsMl%2Fc8KpyYxeJjWz7v5pQYDcvqld4wbTvEwSvgn%2BGZEAO0RzKNzgeS6I9FD2PT\" rel=\"nofollow\">http:\/\/click.aliyun.com\/m\/100&#8230;<\/a>\u672c\u6587\u4e3a\u963f\u91cc\u4e91\u539f\u521b\u5185\u5bb9\uff0c\u672a\u7ecf\u5141\u8bb8\u4e0d\u5f97\u8f6c\u8f7d\u3002<\/p>\n<p><a href=\"https:\/\/segmentfault.com\/a\/1190000042209837\">https:\/\/segmentfault.com\/a\/1190000042209837<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u7b80\u4ecb\uff1a\u00a0\u5178\u578b\u7684Spark\u4f5c\u4e1a\u8bfb\u53d6\u4f4d\u4e8eOSS\u7684Parquet\u5916\u8868\u65f6\uff0c\u6e90\u7aef\u7684\u5e76\u53d1\u5ea6\uff08task\/partition\uff09 [&hellip;]<\/p>\n","protected":false},"author":0,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[],"tags":[],"class_list":["post-10005998","post","type-post","status-publish","format-standard","hentry"],"jetpack_sharing_enabled":true,"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10005998","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"replies":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=10005998"}],"version-history":[{"count":0,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10005998\/revisions"}],"wp:attachment":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=10005998"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=10005998"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=10005998"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}