[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO - Exception in thread "main" org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO - Exchange hashpartitioning(app_id#208, user_id#210, 200)
[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO - +- *HashAggregate(keys=[app_id#208, user_id#210], functions=[partial_count(distinct substring(visited_at#213, 1, 10)#243)], output=[app_id#208, user_id#210, count#246L])
[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO -    +- *HashAggregate(keys=[app_id#208, user_id#210, substring(visited_at#213, 1, 10)#243], functions=[], output=[app_id#208, user_id#210, substring(visited_at#213, 1, 10)#243])
[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO -       +- Exchange hashpartitioning(app_id#208, user_id#210, substring(visited_at#213, 1, 10)#243, 200)
[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO -          +- *HashAggregate(keys=[app_id#208, user_id#210, substring(visited_at#213, 1, 10) AS substring(visited_at#213, 1, 10)#243], functions=[], output=[app_id#208, user_id#210, substring(visited_at#213, 1, 10)#243])
[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO -             +- HiveTableScan [app_id#208, user_id#210, visited_at#213], MetastoreRelation db_mid, m_visit_log_h, [isnotnull(statdate#207), (statdate#207 >= 2019062000), (statdate#207 < 2019062700)]
[2019-06-27 11:18:54,006] {bash_operator.py:110} INFO - 
[2019-06-27 11:18:54,007] {bash_operator.py:110} INFO -     at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
[2019-06-27 11:18:54,007] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.exchange.ShuffleExchange.doExecute(ShuffleExchange.scala:114)
[2019-06-27 11:18:54,007] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,007] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,007] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:136)
[2019-06-27 11:18:54,007] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
[2019-06-27 11:18:54,007] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:133)
[2019-06-27 11:18:54,008] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:114)
[2019-06-27 11:18:54,008] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:233)
[2019-06-27 11:18:54,008] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:138)
[2019-06-27 11:18:54,008] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.FilterExec.inputRDDs(basicPhysicalOperators.scala:115)
[2019-06-27 11:18:54,008] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:36)
[2019-06-27 11:18:54,008] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:361)
[2019-06-27 11:18:54,008] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,009] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,009] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:136)
[2019-06-27 11:18:54,009] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
[2019-06-27 11:18:54,009] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:133)
[2019-06-27 11:18:54,009] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:114)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:240)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:323)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:39)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset$$anonfun$org$apache$spark$sql$Dataset$$execute$1$1.apply(Dataset.scala:2193)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2546)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$execute$1(Dataset.scala:2192)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collect(Dataset.scala:2199)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:1935)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:1934)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2576)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.head(Dataset.scala:1934)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.take(Dataset.scala:2149)
[2019-06-27 11:18:54,010] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.showString(Dataset.scala:239)
[2019-06-27 11:18:54,011] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.show(Dataset.scala:526)
[2019-06-27 11:18:54,011] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.show(Dataset.scala:486)
[2019-06-27 11:18:54,011] {bash_operator.py:110} INFO -     at org.apache.spark.sql.Dataset.show(Dataset.scala:495)
[2019-06-27 11:18:54,012] {bash_operator.py:110} INFO -     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[2019-06-27 11:18:54,012] {bash_operator.py:110} INFO -     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[2019-06-27 11:18:54,012] {bash_operator.py:110} INFO -     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[2019-06-27 11:18:54,012] {bash_operator.py:110} INFO -     at java.lang.reflect.Method.invoke(Method.java:498)
[2019-06-27 11:18:54,012] {bash_operator.py:110} INFO -     at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:736)
[2019-06-27 11:18:54,012] {bash_operator.py:110} INFO -     at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)
[2019-06-27 11:18:54,012] {bash_operator.py:110} INFO -     at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO - Caused by: org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO - Exchange hashpartitioning(app_id#208, user_id#210, substring(visited_at#213, 1, 10)#243, 200)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO - +- *HashAggregate(keys=[app_id#208, user_id#210, substring(visited_at#213, 1, 10) AS substring(visited_at#213, 1, 10)#243], functions=[], output=[app_id#208, user_id#210, substring(visited_at#213, 1, 10)#243])
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -    +- HiveTableScan [app_id#208, user_id#210, visited_at#213], MetastoreRelation db_mid, m_visit_log_h, [isnotnull(statdate#207), (statdate#207 >= 2019062000), (statdate#207 < 2019062700)]
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO - 
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.exchange.ShuffleExchange.doExecute(ShuffleExchange.scala:114)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:136)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:133)
[2019-06-27 11:18:54,013] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:114)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:233)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:138)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:138)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:361)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:136)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:133)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:114)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.exchange.ShuffleExchange.prepareShuffleDependency(ShuffleExchange.scala:87)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:123)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:114)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     ... 46 more
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO - Caused by: java.lang.RuntimeException: serious problem
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.generateSplitsInfo(OrcInputFormat.java:1021)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getSplits(OrcInputFormat.java:1048)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:199)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
[2019-06-27 11:18:54,014] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.Option.getOrElse(Option.scala:121)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.Option.getOrElse(Option.scala:121)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.Option.getOrElse(Option.scala:121)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.UnionRDD$$anonfun$1.apply(UnionRDD.scala:84)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at org.apache.spark.rdd.UnionRDD$$anonfun$1.apply(UnionRDD.scala:84)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.collection.parallel.AugmentedIterableIterator$class.map2combiner(RemainsIterator.scala:115)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:62)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1054)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply$mcV$sp(Tasks.scala:49)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:48)
[2019-06-27 11:18:54,015] {bash_operator.py:110} INFO -     at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:48)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.collection.parallel.Task$class.tryLeaf(Tasks.scala:51)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1051)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask$class.internal(Tasks.scala:169)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.internal(Tasks.scala:443)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask$class.compute(Tasks.scala:149)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:443)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.concurrent.forkjoin.RecursiveAction.exec(RecursiveAction.java:160)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO - Caused by: java.lang.NullPointerException
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$BISplitStrategy.getSplits(OrcInputFormat.java:560)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.generateSplitsInfo(OrcInputFormat.java:1010)
[2019-06-27 11:18:54,016] {bash_operator.py:110} INFO -     ... 35 more

原因：

    sparksql 语句查询的分区，分区中存在空文件，删除空文件即可解决问题