hadoop - 配置单元查询错误java.lang.RuntimeException:org.apache.hadoop.hive.ql.metadata.HiveException

Question

您好，我正在执行一个Hive查询:

CREATE TABLE temp_session_orgid as
SELECT
    sorgid.property_num, sorgid.visitid, sorgid.fullvisitorid, sorgid.adate, sorgid.hits_customvariables_customvarvalue as orgid
FROM
    (
        SELECT
            *,
            row_number() over (partition by property_num, visitid, fullvisitorid, adate order by hitsid) as rn
        FROM bt_hits_custom_vars
        WHERE hits_customvariables_customvarname = 'orgId'
    ) sorgid
WHERE
    sorgid.rn = 1
;

Hive:2.1.1
EMR:5.3.1  

我收到以下错误:

Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.nio.channels.ClosedChannelException at org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:785) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:879) at org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:879) at org.apache.hadoop.hive.ql.exec.FilterOperator.process(FilterOperator.java:126) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:879) at org.apache.hadoop.hive.ql.exec.PTFOperator$PTFInvocation.finishPartition(PTFOperator.java:373) at org.apache.hadoop.hive.ql.exec.PTFOperator.process(PTFOperator.java:122) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:879) at org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95) at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:348) ... 17 more Caused by: java.nio.channels.ClosedChannelException at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1546) at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:104) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:60) at java.io.DataOutputStream.write(DataOutputStream.java:107) at org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat$1.write(HiveIgnoreKeyTextOutputFormat.java:87) at org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:751) ... 27 more ], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) : attempt_1501196537592_0020_2_01_000000_3:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row (tag=0) {"key":{"reducesinkkey0":"89634781","reducesinkkey1":"1442844353","reducesinkkey2":"5186210141339993001","reducesinkkey3":"20150921","reducesinkkey4":"1"},"value":{"_col1":"CUSTOMER"}} at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)

此错误的原因是什么？解决此错误的解决方案是什么？

Answer 1

在 hive 中使用以下设置。

set hive.auto.convert.join=false;
set hive.vectorized.execution.enabled=false;
set hive.vectorized.execution.reduce.enabled=false;
set hive.merge.cardinality.check=false;
set hive.exec.dynamic.partition=true;
set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.max.dynamic.partitions.pernode=500;
set hive.auto.convert.join.noconditionaltask = true;
set hive.auto.convert.join.noconditionaltask.size = 10000000;
set hive.auto.convert.sortmerge.join=true;