工具类--hdfs小文件合并

原创

于 2020-06-22 21:05:22 发布 · 743 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#大数据

package cn.ac.iie

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, Path}
import org.apache.spark.sql.SparkSession

object MergerFile {

  def main(args: Array[String]): Unit = {
    val spark: SparkSession = SparkSession
      .builder()
      .appName("mergeFile")
      .enableHiveSupport()
      .getOrCreate()
    var timestamp = System.currentTimeMillis().toString

    val target = args(0) //待合并的文件夹
    val configuration = new Configuration()
    val output = new Path(target)
    val hdfs = output.getFileSystem(configuration)
    val status = hdfs.listStatus(output);
    var files: List[FileStatus] = List.empty
    var paths: List[String] = List.empty //文件名字符串
    var length = 0l //所有文件的大小
    for (fs <- status) {
      if (fs.getLen == 0) { //大小为0直接删除
        hdfs.delete(fs.getPath, true)
      }
      if (fs.getLen() < 234217728) {