【Hive入门】Hive数据导出完全指南：从HDFS到本地文件系统的专业实践

• 最常用的Hive数据导出方式，语法灵活且支持多种文件格式:

INSERT OVERWRITE DIRECTORY '/output/path'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE
SELECT * FROM source_table;

• Hive支持通过单个查询将数据导出到多个目录：

FROM source_table
INSERT OVERWRITE DIRECTORY '/output/path1'
SELECT col1, col2 WHERE condition1
INSERT OVERWRITE DIRECTORY '/output/path2'
SELECT col3, col4 WHERE condition2;

• 结合动态分区特性实现智能导出

SET hive.exec.dynamic.partition=true;
SET hive.exec.dynamic.partition.mode=nonstrict;

INSERT OVERWRITE DIRECTORY '/output/base_path'
PARTITION(dt, country)
SELECT col1, col2, dt, country 
FROM source_table;

• 常用命令示例：

# 基本导出
hadoop fs -get /user/hive/output/data.csv /home/user/

# 递归导出目录
hadoop fs -getmerge -nl /user/hive/output/ /home/user/merged_data.csv

# 带权限控制导出
hadoop fs -Ddfs.permissions.superusergroup=admin -get /secure/path /local/

• 对于大规模数据迁移，DistCp是更高效的选择：

hadoop distcp \
-Dmapreduce.job.queuename=high_priority \
-update \
-strategy dynamic \
-bandwidth 50 \
hdfs://namenode:8020/source \
file:///local/target/

INSERT OVERWRITE LOCAL DIRECTORY '/tmp/hive_export'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
SELECT * FROM sales_data;

• 示例代码

SET mapreduce.job.reduces=1;

INSERT OVERWRITE LOCAL DIRECTORY '/tmp/single_file'
SELECT * FROM large_table;

mv /tmp/single_file/000000_0 /tmp/single_file/full_data.csv;

• 通过实现Hive的FileFormat接口支持自定义输出：

public class CustomJsonOutputFormat 
    extends FileOutputFormat<NullWritable, Text> {
    // 实现记录写入逻辑
}

• 注册后使用

SET hive.output.fileformat=com.example.CustomJsonOutputFormat;
INSERT OVERWRITE DIRECTORY '/json/output' 
SELECT * FROM table;

• 实现方法

-- 设置并行度
SET hive.exec.parallel=true;
SET hive.exec.parallel.thread.number=8;

-- 分区并行导出
EXPORT TABLE partitioned_table 
PARTITION(dt='2023-01') 
TO '/output/parallel';

• 基于时间戳的增量导出方案：

-- 创建增量记录表
CREATE TABLE export_metadata (
    table_name STRING,
    last_export TIMESTAMP
);

-- 增量导出查询
INSERT OVERWRITE DIRECTORY '/incremental'
SELECT * FROM source_table
WHERE update_time > 
    (SELECT last_export FROM export_metadata 
     WHERE table_name = 'source_table');

• 需求：每日将订单数据导出为CSV供财务系统使用
• 解决方案

#!/bin/bash
# 设置日期变量
EXPORT_DATE=$(date -d "yesterday" +%Y-%m-%d)

# Hive导出到HDFS
hive -e "
SET hive.cli.print.header=true;
INSERT OVERWRITE DIRECTORY '/data/export/orders_${EXPORT_DATE}'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
SELECT * FROM dw_orders 
WHERE dt='${EXPORT_DATE}';"

# 传输到本地系统
hadoop fs -getmerge /data/export/orders_${EXPORT_DATE} \
/home/finance/orders_${EXPORT_DATE}.csv

# 添加标题行
sed -i '1i order_id,user_id,amount,create_time' \
/home/finance/orders_${EXPORT_DATE}.csv

• 场景：将生产集群Hive数据导出到测试集群
• 实现方案：

• 具体命令：

# 生产集群导出
hive -e "EXPORT TABLE prod_db.sales TO '/migration/sales'"

# 跨集群拷贝
hadoop distcp \
hdfs://prod-nn:8020/migration/sales \
hdfs://test-nn:8020/migration/sales

# 测试集群导入
hive -e "IMPORT TABLE test_db.sales FROM '/migration/sales'"

-- 在导出时指定编码
SET hive.exec.default.charset=utf-8;
SET mapreduce.output.fileoutputformat.output.encoding=utf-8;

INSERT OVERWRITE DIRECTORY '/output'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
COLLECTION ITEMS TERMINATED BY '|'
MAP KEYS TERMINATED BY ':'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE
SELECT * FROM chinese_table;

• 典型错误场景：

Permission denied: user=impala, access=WRITE, inode="/export"

• 解决方法：

# 临时方案
hadoop fs -chmod -R 777 /export

# 生产环境推荐
hadoop fs -mkdir /export
hadoop fs -chown impala:supergroup /export
hadoop fs -chmod 750 /export

• 资源分配：适当增加导出任务内存

SET mapreduce.map.memory.mb=4096; 
SET mapreduce.reduce.memory.mb=8192;

• 压缩输出：减少传输数据量

SET hive.exec.compress.output=true;
SET mapreduce.output.fileoutputformat.compress.codec=org.apache.hadoop.io.compress.GzipCodec;

• 文件合并：避免小文件问题

SET hive.merge.mapfiles=true; 
SET hive.merge.size.per.task=256000000;

• 错误处理：自动重试机制

SET mapreduce.map.maxattempts=4; 
SET mapreduce.reduce.maxattempts=4;