本文共 1245 字,大约阅读时间需要 4 分钟。
开启动态分区
set hive.exec.dynamic.partition.mode=nonstrict;set hive.exec.dynamic.partition=true;
开启压缩
set hive.exec.compress.output=true;set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;
开启文件合并默认大小为128M
set hive.merge.tezfiles=true;set hive.merge.smallfiles.avgsize=128000000;set hive.merge.size.per.task=128000000;
创建表
CREATE EXTERNAL TABLE `namespace.tableName`(`字段名称` string COMMENT "描述")PARTITIONED BY ( `ymd` string)ROW FORMAT DELIMITED FIELDS TERMINATED BY '\u0005' STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'LOCATION 'hdfs路径';
查询分区数据写入对应分区
insert into table namespace.tableName partition(分区字段)select 'sms',mktactivitynbr,phonenum as phonecase sms_result when '0' then '1' else '0' end as code,ymdfrom namespace.tableName where ymd>=20210104 and ymd<=20210110 and 字段 in('0','1','23') ;
set hive.exec.dynamic.partition.mode=nonstrict;set hive.exec.dynamic.partition=true;set hive.exec.compress.output=true;set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;insert overwrite table dmp_mcc_contact_view partition(ymd) select * from dmp_mcc_contact_view where ymd>=20201127 and ymd<=20201129;
转载地址:http://ximzi.baihongyu.com/