· 8 years ago · Nov 07, 2017, 05:10 AM
1create table customerspart1(id int, fname varchar(64), lname varchar(64), email varchar(64), street varchar(64), city varchar(10), zipcode int) partitioned by(state string, cob_dt string) row format delimited fields terminated by '|' location '/user/hduser/sqoop_imports/customerpart1';
2
3sqoop job --create customerspart2 -- import --bindir $SQOOP_HOME/lib --connect jdbc:mysql://localhost/mydb --username root ----query "select id,fname,lname,email,street,city,zipcode,state, cob_dt from customer where $CONDITIONS" --check-column cob_dt --incremental lastmodified --last-value 0 --split-by state --merge-key id -m 2 --hive-import --hive-database mydbbackup --hive-table customerspart1 --fields-terminated-by '|' --target-dir '/user/hduser/sqoop_imports/customerpart1/';
4
517/11/07 09:32:06 INFO parse.CalcitePlanner: Creating table mydbbackup.customerspart1 position=27
617/11/07 09:32:06 INFO metastore.HiveMetaStore: 0: get_table : db=mydbbackup tbl=customerspart1
717/11/07 09:32:06 INFO HiveMetaStore.audit: ugi=hduser ip=unknown-ip-addr cmd=get_table : db=mydbbackup tbl=customerspart1
817/11/07 09:32:07 INFO ql.Driver: Semantic Analysis Completed
917/11/07 09:32:07 INFO ql.Driver: Returning Hive schema: Schema(fieldSchemas:null, properties:null)
1017/11/07 09:32:07 INFO ql.Driver: Completed compiling command(queryId=hduser_20171107093158_19c8f388-1c06-4543-8a26-08dd0438ab5e); Time taken: 9.237 seconds
1117/11/07 09:32:07 INFO ql.Driver: Concurrency mode is disabled, not creating a lock manager
1217/11/07 09:32:07 INFO ql.Driver: Executing command(queryId=hduser_20171107093158_19c8f388-1c06-4543-8a26-08dd0438ab5e): CREATE TABLE IF NOT EXISTS `mydbbackup`.`customerspart1` ( `id` INT, `fname` STRING, `lname` STRING, `email` STRING, `street` STRING, `city` STRING, `zipcode` INT, `state` STRING, `cob_dt` STRING) COMMENT 'Imported by sqoop on 2017/11/07 09:31:55' ROW FORMAT DELIMITED FIELDS TERMINATED BY '174' LINES TERMINATED BY ' 12' STORED AS TEXTFILE
1317/11/07 09:32:07 INFO sqlstd.SQLStdHiveAccessController: Created SQLStdHiveAccessController for session context : HiveAuthzSessionContext [sessionString=6920d0d0-8179-44b7-86bb-c397bb26b8c1, clientType=HIVECLI]
1417/11/07 09:32:07 WARN session.SessionState: METASTORE_FILTER_HOOK will be ignored, since hive.security.authorization.manager is set to instance of HiveAuthorizerFactory.
1517/11/07 09:32:07 INFO hive.metastore: Mestastore configuration hive.metastore.filter.hook changed from org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl to org.apache.hadoop.hive.ql.security.authorization.plugin.AuthorizationMetaStoreFilterHook
1617/11/07 09:32:07 INFO metastore.HiveMetaStore: 0: Cleaning up thread local RawStore...
1717/11/07 09:32:07 INFO HiveMetaStore.audit: ugi=hduser ip=unknown-ip-addr cmd=Cleaning up thread local RawStore...
1817/11/07 09:32:07 INFO metastore.HiveMetaStore: 0: Done cleaning up thread local RawStore
1917/11/07 09:32:07 INFO HiveMetaStore.audit: ugi=hduser ip=unknown-ip-addr cmd=Done cleaning up thread local RawStore
2017/11/07 09:32:07 INFO ql.Driver: Completed executing command(queryId=hduser_20171107093158_19c8f388-1c06-4543-8a26-08dd0438ab5e); Time taken: 0.06 seconds
21OK
2217/11/07 09:32:07 INFO ql.Driver: OK
23Time taken: 9.31 seconds
2417/11/07 09:32:07 INFO CliDriver: Time taken: 9.31 seconds
2517/11/07 09:32:07 INFO conf.HiveConf: Using the default value passed in for log id: 6920d0d0-8179-44b7-86bb-c397bb26b8c1
2617/11/07 09:32:07 INFO session.SessionState: Resetting thread name to main
2717/11/07 09:32:07 INFO conf.HiveConf: Using the default value passed in for log id: 6920d0d0-8179-44b7-86bb-c397bb26b8c1
2817/11/07 09:32:07 INFO session.SessionState: Updating thread name to 6920d0d0-8179-44b7-86bb-c397bb26b8c1 main
2917/11/07 09:32:07 INFO ql.Driver: Compiling command(queryId=hduser_20171107093207_0142e6e7-c66f-424f-b6b9-673200a81589):
30LOAD DATA INPATH 'hdfs://localhost:54310/user/hduser/_sqoop/de554e0506484231837b47e42c796c68_16a1bffb' INTO TABLE `mydbbackup`.`customerspart1`
3117/11/07 09:32:07 INFO metastore.HiveMetaStore: 0: Opening raw store with implementation class:org.apache.hadoop.hive.metastore.ObjectStore
3217/11/07 09:32:07 INFO metastore.ObjectStore: ObjectStore, initialize called
3317/11/07 09:32:07 INFO metastore.MetaStoreDirectSql: Using direct SQL, underlying DB is DERBY
3417/11/07 09:32:07 INFO metastore.ObjectStore: Initialized ObjectStore
3517/11/07 09:32:07 INFO metastore.HiveMetaStore: 0: get_table : db=mydbbackup tbl=customerspart1
3617/11/07 09:32:07 INFO HiveMetaStore.audit: ugi=hduser ip=unknown-ip-addr cmd=get_table : db=mydbbackup tbl=customerspart1
37FAILED: SemanticException [Error 10062]: Need to specify partition columns because the destination table is partitioned
3817/11/07 09:32:07 ERROR ql.Driver: FAILED: SemanticException [Error 10062]: Need to specify partition columns because the destination table is partitioned
39org.apache.hadoop.hive.ql.parse.SemanticException: Need to specify partition columns because the destination table is partitioned
40 at org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer.analyzeInternal(LoadSemanticAnalyzer.java:220)
41 at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:258)
42 at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:511)
43 at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1316)
44 at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1456)
45 at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1236)
46 at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1226)
47 at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:233)
48 at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:184)
49 at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:403)
50 at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:336)
51 at org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:474)
52 at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:490)
53 at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:793)
54 at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:759)
55 at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:686)
56 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
57 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
58 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
59 at java.lang.reflect.Method.invoke(Method.java:498)
60 at org.apache.sqoop.hive.HiveImport.executeScript(HiveImport.java:333)
61 at org.apache.sqoop.hive.HiveImport.importTable(HiveImport.java:240)
62 at org.apache.sqoop.tool.ImportTool.importTable(ImportTool.java:514)
63 at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:605)
64 at org.apache.sqoop.tool.JobTool.execJob(JobTool.java:228)
65 at org.apache.sqoop.tool.JobTool.run(JobTool.java:283)
66 at org.apache.sqoop.Sqoop.run(Sqoop.java:143)
67 at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
68 at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:179)
69 at org.apache.sqoop.Sqoop.runTool(Sqoop.java:218)
70 at org.apache.sqoop.Sqoop.runTool(Sqoop.java:227)
71 at org.apache.sqoop.Sqoop.main(Sqoop.java:236)
72
7317/11/07 09:32:07 INFO ql.Driver: Completed compiling command(queryId=hduser_20171107093207_0142e6e7-c66f-424f-b6b9-673200a81589); Time taken: 0.24 seconds
7417/11/07 09:32:07 INFO conf.HiveConf: Using the default value passed in for log id: 6920d0d0-8179-44b7-86bb-c397bb26b8c1
7517/11/07 09:32:07 INFO session.SessionState: Resetting thread name to main
7617/11/07 09:32:07 INFO conf.HiveConf: Using the default value passed in for log id: 6920d0d0-8179-44b7-86bb-c397bb26b8c1
7717/11/07 09:32:07 INFO session.SessionState: Deleted directory: /tmp/hive/hduser/6920d0d0-8179-44b7-86bb-c397bb26b8c1 on fs with scheme hdfs
7817/11/07 09:32:07 INFO session.SessionState: Deleted directory: /tmp/hduser/6920d0d0-8179-44b7-86bb-c397bb26b8c1 on fs with scheme file
7917/11/07 09:32:07 INFO metastore.HiveMetaStore: 0: Cleaning up thread local RawStore...
8017/11/07 09:32:07 INFO HiveMetaStore.audit: ugi=hduser ip=unknown-ip-addr cmd=Cleaning up thread local RawStore...
8117/11/07 09:32:07 INFO metastore.HiveMetaStore: 0: Done cleaning up thread local RawStore
8217/11/07 09:32:07 INFO HiveMetaStore.audit: ugi=hduser ip=unknown-ip-addr cmd=Done cleaning up thread local RawStore
8317/11/07 09:32:07 ERROR tool.ImportTool: Encountered IOException running import job: java.io.IOException: Hive CliDriver exited with status=10062
84 at org.apache.sqoop.hive.HiveImport.executeScript(HiveImport.java:357)
85 at org.apache.sqoop.hive.HiveImport.importTable(HiveImport.java:240)
86 at org.apache.sqoop.tool.ImportTool.importTable(ImportTool.java:514)
87 at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:605)
88 at org.apache.sqoop.tool.JobTool.execJob(JobTool.java:228)
89 at org.apache.sqoop.tool.JobTool.run(JobTool.java:283)
90 at org.apache.sqoop.Sqoop.run(Sqoop.java:143)
91 at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
92 at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:179)
93 at org.apache.sqoop.Sqoop.runTool(Sqoop.java:218)
94 at org.apache.sqoop.Sqoop.runTool(Sqoop.java:227)
95 at org.apache.sqoop.Sqoop.main(Sqoop.java:236)