【Hive入门】Hive增量数据导入：基于Sqoop的关系型数据库同步方案深度解析

-- 创建目标Hive表
CREATE TABLE orders (
    order_id INT,
    customer_id INT,
    order_date TIMESTAMP,
    amount DECIMAL(10,2)
) STORED AS ORC;

• Sqoop命令示例：

sqoop job --create inc_order_import \
-- import \
--connect jdbc:mysql://mysql-server:3306/sales \
--username etl_user \
--password-file /user/password.txt \
--table orders \
--hive-import \
--hive-table orders \
--incremental append \
--check-column order_id \
--last-value 0 \
--split-by order_id

• 关键参数：

--incremental lastmodified \
--check-column update_time \
--last-value "2025-05-03 00:00:00" \
--append

-- 目标表需支持ACID
CREATE TABLE customer_merge (
    id INT,
    name STRING,
    email STRING,
    last_update TIMESTAMP
) STORED AS ORC TBLPROPERTIES ('transactional'='true');

• Sqoop命令示例：

sqoop import \
--connect jdbc:oracle:thin:@//oracle-host:1521/ORCL \
--username scott \
--password tiger \
--table customers \
--hive-import \
--hive-table customer_merge \
--incremental lastmodified \
--check-column last_update \
--last-value "2023-01-01" \
--merge-key id

• 按日分区表示例：

CREATE TABLE sales_partitioned (
    id INT,
    product STRING,
    quantity INT,
    update_time TIMESTAMP
) PARTITIONED BY (dt STRING)
STORED AS PARQUET;

• 增量同步脚本：

#!/bin/bash
LAST_DATE=$(hive -e "SELECT MAX(dt) FROM sales_partitioned")
CURRENT_DATE=$(date +%Y-%m-%d)
sqoop import \
--connect jdbc:postgresql://pg-server/db \
--table sales \
--where "update_time BETWEEN '$LAST_DATE' AND '$CURRENT_DATE'" \
--hive-import \
--hive-table sales_partitioned \
--hive-partition-key dt \
--hive-partition-value $CURRENT_DATE \
--incremental lastmodified \
--check-column update_time \
--last-value "$LAST_DATE"

# 控制事务大小
--batch
--fetch-size 1000

# 内存优化
-Dmapreduce.map.memory.mb=4096
-Dmapreduce.reduce.memory.mb=8192

# 连接池配置
-Dsqoop.connection.pool.size=10
-Dsqoop.connection.pool.timeout=300

-- 创建支持压缩的Hive表
CREATE TABLE compressed_orders (
    id INT,
    -- 其他列...
) STORED AS ORC
TBLPROPERTIES ("orc.compress"="SNAPPY");

• Sqoop压缩参数：

--compress 
--compression-codec org.apache.hadoop.io.compress.SnappyCodec

• 校验脚本示例：

-- 记录数比对
SELECT 
    (SELECT COUNT(*) FROM rdb_table) AS source_count,
    (SELECT COUNT(*) FROM hive_temp_table) AS target_count,
    (SELECT COUNT(*) FROM hive_temp_table t JOIN rdb_table r ON t.id=r.id) AS match_count;

# 显式指定时区
-Duser.timezone=UTC
--map-column-java update_time=java.sql.Timestamp
--hive-overwrite
--hive-import

• 分片导入技术：

# 按ID范围分批导入
for i in {0..9}; do
    sqoop import \
    --query "SELECT * FROM big_table WHERE MOD(id,10)=$i AND \$CONDITIONS" \
    --split-by id \
    --target-dir /data/big_table/part=$i
done