· 9 years ago · Jan 27, 2017, 11:00 AM
1object Main {
2def main(args: Array[String]): Unit = {
3val conf= new SparkConf().setAppName("testSqlCreateHiveStatement").setMaster("local")
4var sparkContext =new SparkContext(conf)
5val sqlContext = new org.apache.spark.sql.hive.HiveContext(sparkContext)
6// I have hardcoded 2 ddl's to get better understanding.
7val lines = """
8CREATE EXTERNAL TABLE IF NOT EXISTS test_sh.abc (
9x String,
10y String,
11z String
12)
13ROW FORMAT DELIMITED
14FIELDS TERMINATED BY "|"
15LOCATION 'hdfslocation';
16
17CREATE EXTERNAL TABLE IF NOT EXISTS test_sh2.abc (
18 x String,
19 y String,
20 z String
21)
22ROW FORMAT DELIMITED
23FIELDS TERMINATED BY "|"
24STORED AS PARQUET
25LOCATION 'hdfslocation2'
26TBLPROPERTIES("hive.exec.compress.output"="true","mapreduce.output.fileoutputformat.compress"="true","parquet.compress"="SNAPPY");
27
28"""
29
30def generateHiveCreateTableStatement(lines:String): Array[String] ={
31 val statement_1 = lines.split(";")(0)
32 val statement_2 = lines.split(";")(1)
33 Array(statement_1,statement_2)}
34
35val statements = generateHiveCreateTableStatement(lines)
36sqlContext.sql(statements(0))
37
38}
39}
40
41java.lang.LinkageError: ClassCastException: attempting to castjar:file:/usr/local/spark/lib/spark-assembly-1.6.1-hadoop2.6.0.jar!/javax/ws/rs/ext/RuntimeDelegate.classtojar:file:/usr/local/spark/lib/spark-assembly-1.6.1-hadoop2.6.0.jar!/javax/ws/rs/ext/RuntimeDelegate.class
42 at javax.ws.rs.ext.RuntimeDelegate.findDelegate(RuntimeDelegate.java:116)
43 at javax.ws.rs.ext.RuntimeDelegate.getInstance(RuntimeDelegate.java:91)
44 at javax.ws.rs.core.MediaType.<clinit>(MediaType.java:44)
45 at com.sun.jersey.core.header.MediaTypes.<clinit>(MediaTypes.java:64)
46 at com.sun.jersey.core.spi.factory.MessageBodyFactory.initReaders(MessageBodyFactory.java:182)
47 at com.sun.jersey.core.spi.factory.MessageBodyFactory.initReaders(MessageBodyFactory.java:175)
48 at com.sun.jersey.core.spi.factory.MessageBodyFactory.init(MessageBodyFactory.java:162)
49 at com.sun.jersey.api.client.Client.init(Client.java:342)
50 at com.sun.jersey.api.client.Client.access$000(Client.java:118)
51 at com.sun.jersey.api.client.Client$1.f(Client.java:191)
52 at com.sun.jersey.api.client.Client$1.f(Client.java:187)
53 at com.sun.jersey.spi.inject.Errors.processWithErrors(Errors.java:193)
54 at com.sun.jersey.api.client.Client.<init>(Client.java:187)
55 at com.sun.jersey.api.client.Client.<init>(Client.java:170)
56 at org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.serviceInit(TimelineClientImpl.java:268)
57 at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
58 at org.apache.hadoop.hive.ql.hooks.ATSHook.<init>(ATSHook.java:67)
59 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
60 at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
61 at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
62 at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
63 at java.lang.Class.newInstance(Class.java:442)
64 at org.apache.hadoop.hive.ql.hooks.HookUtils.getHooks(HookUtils.java:60)
65 at org.apache.hadoop.hive.ql.Driver.getHooks(Driver.java:1309)
66 at org.apache.hadoop.hive.ql.Driver.getHooks(Driver.java:1293)
67 at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1347)
68 at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1195)
69 at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1059)
70 at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
71 at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$runHive$1.apply(ClientWrapper.scala:495)
72 at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$runHive$1.apply(ClientWrapper.scala:484)
73 at org.apache.spark.sql.hive.client.ClientWrapper$$anonfun$withHiveState$1.apply(ClientWrapper.scala:290)
74 at org.apache.spark.sql.hive.client.ClientWrapper.liftedTree1$1(ClientWrapper.scala:237)
75 at org.apache.spark.sql.hive.client.ClientWrapper.retryLocked(ClientWrapper.scala:236)
76 at org.apache.spark.sql.hive.client.ClientWrapper.withHiveState(ClientWrapper.scala:279)
77 at org.apache.spark.sql.hive.client.ClientWrapper.runHive(ClientWrapper.scala:484)
78 at org.apache.spark.sql.hive.client.ClientWrapper.runSqlHive(ClientWrapper.scala:474)
79 at org.apache.spark.sql.hive.HiveContext.runSqlHive(HiveContext.scala:605)
80 at org.apache.spark.sql.hive.execution.HiveNativeCommand.run(HiveNativeCommand.scala:33)
81 at org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult$lzycompute(commands.scala:58)
82 at org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult(commands.scala:56)
83 at org.apache.spark.sql.execution.ExecutedCommand.doExecute(commands.scala:70)
84 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:132)
85 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:130)
86 at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
87 at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:130)
88 at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:55)
89 at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55)
90 at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:145)
91 at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:130)
92 at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:52)
93 at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:817)
94 at Main$.main(Main.scala:87)
95 at Main.main(Main.scala)
96 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
97 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
98 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
99 at java.lang.reflect.Method.invoke(Method.java:497)
100 at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)
101 at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
102 at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
103 at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
104 at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
105
106file:/usr/local/spark/lib/spark-assembly-1.6.1-hadoop2.6.0.jar!/javax/ws/rs/ext/RuntimeDelegate.class
107 tojar:
108 file:/usr/local/spark/lib/spark-assembly-1.6.1-hadoop2.6.0.jar!/javax/ws/rs/ext/RuntimeDelegate.class
109
110spark-submit testyarnmasteroncluster_2.10-1.0.jar