· 10 years ago · Aug 23, 2016, 09:04 PM
1(PY27)~/src/ruff/common/report 662 $ spark-submit --driver-memory 10G --executor-memory 10G --packages com.databricks:spark-csv_2.10:1.2.0 parquet_to_csv.py
2WARNING: User-defined SPARK_HOME (/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/lib/spark) overrides detected (/opt/cloudera/parcels/CDH/lib/spark).
3WARNING: Running spark-class from user-defined location.
4Ivy Default Cache set to: /home/anave/.ivy2/cache
5The jars for the packages stored in: /home/anave/.ivy2/jars
6:: loading settings :: url = jar:file:/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/jars/spark-assembly-1.6.0-cdh5.7.1-hadoop2.6.0-cdh5.7.1.jar!/org/apache/ivy/core/settings/ivysettings.xml
7com.databricks#spark-csv_2.10 added as a dependency
8:: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0
9 confs: [default]
10 found com.databricks#spark-csv_2.10;1.2.0 in central
11 found org.apache.commons#commons-csv;1.1 in central
12 found com.univocity#univocity-parsers;1.5.1 in central
13:: resolution report :: resolve 728ms :: artifacts dl 22ms
14 :: modules in use:
15 com.databricks#spark-csv_2.10;1.2.0 from central in [default]
16 com.univocity#univocity-parsers;1.5.1 from central in [default]
17 org.apache.commons#commons-csv;1.1 from central in [default]
18 ---------------------------------------------------------------------
19 | | modules || artifacts |
20 | conf | number| search|dwnlded|evicted|| number|dwnlded|
21 ---------------------------------------------------------------------
22 | default | 3 | 0 | 0 | 0 || 3 | 0 |
23 ---------------------------------------------------------------------
24:: retrieving :: org.apache.spark#spark-submit-parent
25 confs: [default]
26 0 artifacts copied, 3 already retrieved (0kB/30ms)
27hadoop fs -mkdir /data/res/warehouse/ruff/user/anave/ruff_daily_summary
28mkdir: `/data/res/warehouse/ruff/user/anave/ruff_daily_summary': File exists
29hadoop fs -rm -R /data/res/warehouse/ruff/user/anave/ruff_daily_summary/parquet_states
30rm: `/data/res/warehouse/ruff/user/anave/ruff_daily_summary/parquet_states': No such file or directory
31hadoop fs -mkdir /data/res/warehouse/ruff/user/anave/ruff_daily_summary/parquet_states
32Traceback (most recent call last):
33 File "/home/anave/src/ruff/common/report/parquet_to_csv.py", line 218, in <module>
34 impala_query()
35 File "/home/anave/src/ruff/common/report/parquet_to_csv.py", line 211, in impala_query
36 term_cnt=term_cnt_col, IMPALA_QUERY_PARQUET=IMPALA_QUERY_PARQUET)
37KeyError: 'REPORT_NAME'
38(PY27)~/src/ruff/common/report 663 $ spark-submit --driver-memory 10G --executor-memory 10G --packages com.databricks:spark-csv_2.10:1.2.0 parquet_to_csv.py
39WARNING: User-defined SPARK_HOME (/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/lib/spark) overrides detected (/opt/cloudera/parcels/CDH/lib/spark).
40WARNING: Running spark-class from user-defined location.
41Ivy Default Cache set to: /home/anave/.ivy2/cache
42The jars for the packages stored in: /home/anave/.ivy2/jars
43:: loading settings :: url = jar:file:/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/jars/spark-assembly-1.6.0-cdh5.7.1-hadoop2.6.0-cdh5.7.1.jar!/org/apache/ivy/core/settings/ivysettings.xml
44com.databricks#spark-csv_2.10 added as a dependency
45:: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0
46 confs: [default]
47 found com.databricks#spark-csv_2.10;1.2.0 in central
48 found org.apache.commons#commons-csv;1.1 in central
49 found com.univocity#univocity-parsers;1.5.1 in central
50:: resolution report :: resolve 590ms :: artifacts dl 32ms
51 :: modules in use:
52 com.databricks#spark-csv_2.10;1.2.0 from central in [default]
53 com.univocity#univocity-parsers;1.5.1 from central in [default]
54 org.apache.commons#commons-csv;1.1 from central in [default]
55 ---------------------------------------------------------------------
56 | | modules || artifacts |
57 | conf | number| search|dwnlded|evicted|| number|dwnlded|
58 ---------------------------------------------------------------------
59 | default | 3 | 0 | 0 | 0 || 3 | 0 |
60 ---------------------------------------------------------------------
61:: retrieving :: org.apache.spark#spark-submit-parent
62 confs: [default]
63 0 artifacts copied, 3 already retrieved (0kB/16ms)
64hadoop fs -mkdir /data/res/warehouse/ruff/user/anave/ruff_daily_summary
65mkdir: `/data/res/warehouse/ruff/user/anave/ruff_daily_summary': File exists
66hadoop fs -rm -R /data/res/warehouse/ruff/user/anave/ruff_daily_summary/parquet_states
67hadoop fs -mkdir /data/res/warehouse/ruff/user/anave/ruff_daily_summary/parquet_states
68Starting Impala Shell without Kerberos authentication
69Error connecting: TTransportException, TSocket read 0 bytes
70Kerberos ticket found in the credentials cache, retrying the connection with a secure transport.
71Connected to hrtimpslb.allstate.com:21000
72Server version: impalad version 2.5.0-cdh5.7.1 RELEASE (build 27a4325c18c2a01c7a8097681a0eccf6d4335ea1)
73Query: drop table if exists dru_anave.ruff_loss_report
74Query: create external table dru_anave.ruff_loss_report
75 stored as parquet location '/data/res/warehouse/ruff/user/anave/ruff_daily_summary/ruff_loss_report' as
76 select
77 drv_endorse_dt,
78drv_endorse_end_dt,
79ply_pt_state_cd,
80ply_line_cd,
81pls_company_cd,
82ply_ifs_group_cd,
83ply_opt_pkg_cd,
84count(*) as iif,
85count (distinct ply_policy_id) as pif,
86 sum(case when drv_aa_exposure > 0 then 0.002739726 else 0 end) as drv_aa_exposure,
87sum(case when drv_ac_exposure > 0 then 0.002739726 else 0 end) as drv_ac_exposure,
88sum(case when drv_bb_exposure > 0 then 0.002739726 else 0 end) as drv_bb_exposure,
89sum(case when drv_bt_exposure > 0 then 0.002739726 else 0 end) as drv_bt_exposure,
90sum(case when drv_bv_exposure > 0 then 0.002739726 else 0 end) as drv_bv_exposure,
91sum(case when drv_cc_exposure > 0 then 0.002739726 else 0 end) as drv_cc_exposure,
92sum(case when drv_dd_exposure > 0 then 0.002739726 else 0 end) as drv_dd_exposure,
93sum(case when drv_da_exposure > 0 then 0.002739726 else 0 end) as drv_da_exposure,
94sum(case when drv_db_exposure > 0 then 0.002739726 else 0 end) as drv_db_exposure,
95sum(case when drv_hh_exposure > 0 then 0.002739726 else 0 end) as drv_hh_exposure,
96sum(case when drv_jj_exposure > 0 then 0.002739726 else 0 end) as drv_jj_exposure,
97sum(case when drv_uu_exposure > 0 then 0.002739726 else 0 end) as drv_uu_exposure,
98sum(case when drv_va_exposure > 0 then 0.002739726 else 0 end) as drv_va_exposure,
99sum(case when drv_vb_exposure > 0 then 0.002739726 else 0 end) as drv_vb_exposure,
100sum(case when drv_vo_exposure > 0 then 0.002739726 else 0 end) as drv_vo_exposure,
101 sum (cdf_cmloss_total_aa) as cdf_cmloss_total_aa,
102sum (cdf_cmexp_total_aa) as cdf_cmexp_total_aa,
103sum (cdf_caser_total_aa) as cdf_caser_total_aa,
104sum (cdf_notct_total_aa) as cdf_notct_total_aa,
105sum (cdf_casupl_total_aa) as cdf_casupl_total_aa,
106sum (cdf_clcwp_total_aa) as cdf_clcwp_total_aa,
107sum (cdf_clect_total_aa) as cdf_clect_total_aa,
108sum (cdf_cllct_total_aa) as cdf_cllct_total_aa,
109sum (cdf_cllect_total_aa) as cdf_cllect_total_aa,
110sum (cdf_cmloss_total_ac) as cdf_cmloss_total_ac,
111sum (cdf_cmexp_total_ac) as cdf_cmexp_total_ac,
112sum (cdf_caser_total_ac) as cdf_caser_total_ac,
113sum (cdf_notct_total_ac) as cdf_notct_total_ac,
114sum (cdf_casupl_total_ac) as cdf_casupl_total_ac,
115sum (cdf_clcwp_total_ac) as cdf_clcwp_total_ac,
116sum (cdf_clect_total_ac) as cdf_clect_total_ac,
117sum (cdf_cllct_total_ac) as cdf_cllct_total_ac,
118sum (cdf_cllect_total_ac) as cdf_cllect_total_ac,
119sum (cdf_cmloss_total_bb) as cdf_cmloss_total_bb,
120sum (cdf_cmexp_total_bb) as cdf_cmexp_total_bb,
121sum (cdf_caser_total_bb) as cdf_caser_total_bb,
122sum (cdf_notct_total_bb) as cdf_notct_total_bb,
123sum (cdf_casupl_total_bb) as cdf_casupl_total_bb,
124sum (cdf_clcwp_total_bb) as cdf_clcwp_total_bb,
125sum (cdf_clect_total_bb) as cdf_clect_total_bb,
126sum (cdf_cllct_total_bb) as cdf_cllct_total_bb,
127sum (cdf_cllect_total_bb) as cdf_cllect_total_bb,
128sum (cdf_cmloss_total_bt) as cdf_cmloss_total_bt,
129sum (cdf_cmexp_total_bt) as cdf_cmexp_total_bt,
130sum (cdf_caser_total_bt) as cdf_caser_total_bt,
131sum (cdf_notct_total_bt) as cdf_notct_total_bt,
132sum (cdf_casupl_total_bt) as cdf_casupl_total_bt,
133sum (cdf_clcwp_total_bt) as cdf_clcwp_total_bt,
134sum (cdf_clect_total_bt) as cdf_clect_total_bt,
135sum (cdf_cllct_total_bt) as cdf_cllct_total_bt,
136sum (cdf_cllect_total_bt) as cdf_cllect_total_bt,
137sum (cdf_cmloss_total_bv) as cdf_cmloss_total_bv,
138sum (cdf_cmexp_total_bv) as cdf_cmexp_total_bv,
139sum (cdf_caser_total_bv) as cdf_caser_total_bv,
140sum (cdf_notct_total_bv) as cdf_notct_total_bv,
141sum (cdf_casupl_total_bv) as cdf_casupl_total_bv,
142sum (cdf_clcwp_total_bv) as cdf_clcwp_total_bv,
143sum (cdf_clect_total_bv) as cdf_clect_total_bv,
144sum (cdf_cllct_total_bv) as cdf_cllct_total_bv,
145sum (cdf_cllect_total_bv) as cdf_cllect_total_bv,
146sum (cdf_cmloss_total_cc) as cdf_cmloss_total_cc,
147sum (cdf_cmexp_total_cc) as cdf_cmexp_total_cc,
148sum (cdf_caser_total_cc) as cdf_caser_total_cc,
149sum (cdf_notct_total_cc) as cdf_notct_total_cc,
150sum (cdf_casupl_total_cc) as cdf_casupl_total_cc,
151sum (cdf_clcwp_total_cc) as cdf_clcwp_total_cc,
152sum (cdf_clect_total_cc) as cdf_clect_total_cc,
153sum (cdf_cllct_total_cc) as cdf_cllct_total_cc,
154sum (cdf_cllect_total_cc) as cdf_cllect_total_cc,
155sum (cdf_cmloss_total_dd) as cdf_cmloss_total_dd,
156sum (cdf_cmexp_total_dd) as cdf_cmexp_total_dd,
157sum (cdf_caser_total_dd) as cdf_caser_total_dd,
158sum (cdf_notct_total_dd) as cdf_notct_total_dd,
159sum (cdf_casupl_total_dd) as cdf_casupl_total_dd,
160sum (cdf_clcwp_total_dd) as cdf_clcwp_total_dd,
161sum (cdf_clect_total_dd) as cdf_clect_total_dd,
162sum (cdf_cllct_total_dd) as cdf_cllct_total_dd,
163sum (cdf_cllect_total_dd) as cdf_cllect_total_dd,
164sum (cdf_cmloss_total_da) as cdf_cmloss_total_da,
165sum (cdf_cmexp_total_da) as cdf_cmexp_total_da,
166sum (cdf_caser_total_da) as cdf_caser_total_da,
167sum (cdf_notct_total_da) as cdf_notct_total_da,
168sum (cdf_casupl_total_da) as cdf_casupl_total_da,
169sum (cdf_clcwp_total_da) as cdf_clcwp_total_da,
170sum (cdf_clect_total_da) as cdf_clect_total_da,
171sum (cdf_cllct_total_da) as cdf_cllct_total_da,
172sum (cdf_cllect_total_da) as cdf_cllect_total_da,
173sum (cdf_cmloss_total_db) as cdf_cmloss_total_db,
174sum (cdf_cmexp_total_db) as cdf_cmexp_total_db,
175sum (cdf_caser_total_db) as cdf_caser_total_db,
176sum (cdf_notct_total_db) as cdf_notct_total_db,
177sum (cdf_casupl_total_db) as cdf_casupl_total_db,
178sum (cdf_clcwp_total_db) as cdf_clcwp_total_db,
179sum (cdf_clect_total_db) as cdf_clect_total_db,
180sum (cdf_cllct_total_db) as cdf_cllct_total_db,
181sum (cdf_cllect_total_db) as cdf_cllect_total_db,
182sum (cdf_cmloss_total_hh) as cdf_cmloss_total_hh,
183sum (cdf_cmexp_total_hh) as cdf_cmexp_total_hh,
184sum (cdf_caser_total_hh) as cdf_caser_total_hh,
185sum (cdf_notct_total_hh) as cdf_notct_total_hh,
186sum (cdf_casupl_total_hh) as cdf_casupl_total_hh,
187sum (cdf_clcwp_total_hh) as cdf_clcwp_total_hh,
188sum (cdf_clect_total_hh) as cdf_clect_total_hh,
189sum (cdf_cllct_total_hh) as cdf_cllct_total_hh,
190sum (cdf_cllect_total_hh) as cdf_cllect_total_hh,
191sum (cdf_cmloss_total_jj) as cdf_cmloss_total_jj,
192sum (cdf_cmexp_total_jj) as cdf_cmexp_total_jj,
193sum (cdf_caser_total_jj) as cdf_caser_total_jj,
194sum (cdf_notct_total_jj) as cdf_notct_total_jj,
195sum (cdf_casupl_total_jj) as cdf_casupl_total_jj,
196sum (cdf_clcwp_total_jj) as cdf_clcwp_total_jj,
197sum (cdf_clect_total_jj) as cdf_clect_total_jj,
198sum (cdf_cllct_total_jj) as cdf_cllct_total_jj,
199sum (cdf_cllect_total_jj) as cdf_cllect_total_jj,
200sum (cdf_cmloss_total_uu) as cdf_cmloss_total_uu,
201sum (cdf_cmexp_total_uu) as cdf_cmexp_total_uu,
202sum (cdf_caser_total_uu) as cdf_caser_total_uu,
203sum (cdf_notct_total_uu) as cdf_notct_total_uu,
204sum (cdf_casupl_total_uu) as cdf_casupl_total_uu,
205sum (cdf_clcwp_total_uu) as cdf_clcwp_total_uu,
206sum (cdf_clect_total_uu) as cdf_clect_total_uu,
207sum (cdf_cllct_total_uu) as cdf_cllct_total_uu,
208sum (cdf_cllect_total_uu) as cdf_cllect_total_uu,
209sum (cdf_cmloss_total_va) as cdf_cmloss_total_va,
210sum (cdf_cmexp_total_va) as cdf_cmexp_total_va,
211sum (cdf_caser_total_va) as cdf_caser_total_va,
212sum (cdf_notct_total_va) as cdf_notct_total_va,
213sum (cdf_casupl_total_va) as cdf_casupl_total_va,
214sum (cdf_clcwp_total_va) as cdf_clcwp_total_va,
215sum (cdf_clect_total_va) as cdf_clect_total_va,
216sum (cdf_cllct_total_va) as cdf_cllct_total_va,
217sum (cdf_cllect_total_va) as cdf_cllect_total_va,
218sum (cdf_cmloss_total_vb) as cdf_cmloss_total_vb,
219sum (cdf_cmexp_total_vb) as cdf_cmexp_total_vb,
220sum (cdf_caser_total_vb) as cdf_caser_total_vb,
221sum (cdf_notct_total_vb) as cdf_notct_total_vb,
222sum (cdf_casupl_total_vb) as cdf_casupl_total_vb,
223sum (cdf_clcwp_total_vb) as cdf_clcwp_total_vb,
224sum (cdf_clect_total_vb) as cdf_clect_total_vb,
225sum (cdf_cllct_total_vb) as cdf_cllct_total_vb,
226sum (cdf_cllect_total_vb) as cdf_cllect_total_vb,
227sum (cdf_cmloss_total_vo) as cdf_cmloss_total_vo,
228sum (cdf_cmexp_total_vo) as cdf_cmexp_total_vo,
229sum (cdf_caser_total_vo) as cdf_caser_total_vo,
230sum (cdf_notct_total_vo) as cdf_notct_total_vo,
231sum (cdf_casupl_total_vo) as cdf_casupl_total_vo,
232sum (cdf_clcwp_total_vo) as cdf_clcwp_total_vo,
233sum (cdf_clect_total_vo) as cdf_clect_total_vo,
234sum (cdf_cllct_total_vo) as cdf_cllct_total_vo,
235sum (cdf_cllect_total_vo) as cdf_cllect_total_vo,
236 group_concat(case when drv_termination_dt != '0' then drv_termination_dt end , ' ') as term_cnt
237 from
238 dra_ruffprod.full_loss_rollup_parquet where ply_pt_state_cd = 'PR' or ply_pt_state_cd = 'NH'
239 group by
240 drv_endorse_dt,
241drv_endorse_end_dt,
242ply_pt_state_cd,
243ply_line_cd,
244pls_company_cd,
245ply_ifs_group_cd,
246ply_opt_pkg_cd
247ERROR: AuthorizationException: User 'anave@AD.ALLSTATE.COM' does not have privileges to access: hdfs://nameservice1/data/res/warehouse/ruff/user/anave/ruff_daily_summary/ruff_loss_report
248
249Could not execute command: create external table dru_anave.ruff_loss_report
250 stored as parquet location '/data/res/warehouse/ruff/user/anave/ruff_daily_summary/ruff_loss_report' as
251 select
252 drv_endorse_dt,
253drv_endorse_end_dt,
254ply_pt_state_cd,
255ply_line_cd,
256pls_company_cd,
257ply_ifs_group_cd,
258ply_opt_pkg_cd,
259count(*) as iif,
260count (distinct ply_policy_id) as pif,
261 sum(case when drv_aa_exposure > 0 then 0.002739726 else 0 end) as drv_aa_exposure,
262sum(case when drv_ac_exposure > 0 then 0.002739726 else 0 end) as drv_ac_exposure,
263sum(case when drv_bb_exposure > 0 then 0.002739726 else 0 end) as drv_bb_exposure,
264sum(case when drv_bt_exposure > 0 then 0.002739726 else 0 end) as drv_bt_exposure,
265sum(case when drv_bv_exposure > 0 then 0.002739726 else 0 end) as drv_bv_exposure,
266sum(case when drv_cc_exposure > 0 then 0.002739726 else 0 end) as drv_cc_exposure,
267sum(case when drv_dd_exposure > 0 then 0.002739726 else 0 end) as drv_dd_exposure,
268sum(case when drv_da_exposure > 0 then 0.002739726 else 0 end) as drv_da_exposure,
269sum(case when drv_db_exposure > 0 then 0.002739726 else 0 end) as drv_db_exposure,
270sum(case when drv_hh_exposure > 0 then 0.002739726 else 0 end) as drv_hh_exposure,
271sum(case when drv_jj_exposure > 0 then 0.002739726 else 0 end) as drv_jj_exposure,
272sum(case when drv_uu_exposure > 0 then 0.002739726 else 0 end) as drv_uu_exposure,
273sum(case when drv_va_exposure > 0 then 0.002739726 else 0 end) as drv_va_exposure,
274sum(case when drv_vb_exposure > 0 then 0.002739726 else 0 end) as drv_vb_exposure,
275sum(case when drv_vo_exposure > 0 then 0.002739726 else 0 end) as drv_vo_exposure,
276 sum (cdf_cmloss_total_aa) as cdf_cmloss_total_aa,
277sum (cdf_cmexp_total_aa) as cdf_cmexp_total_aa,
278sum (cdf_caser_total_aa) as cdf_caser_total_aa,
279sum (cdf_notct_total_aa) as cdf_notct_total_aa,
280sum (cdf_casupl_total_aa) as cdf_casupl_total_aa,
281sum (cdf_clcwp_total_aa) as cdf_clcwp_total_aa,
282sum (cdf_clect_total_aa) as cdf_clect_total_aa,
283sum (cdf_cllct_total_aa) as cdf_cllct_total_aa,
284sum (cdf_cllect_total_aa) as cdf_cllect_total_aa,
285sum (cdf_cmloss_total_ac) as cdf_cmloss_total_ac,
286sum (cdf_cmexp_total_ac) as cdf_cmexp_total_ac,
287sum (cdf_caser_total_ac) as cdf_caser_total_ac,
288sum (cdf_notct_total_ac) as cdf_notct_total_ac,
289sum (cdf_casupl_total_ac) as cdf_casupl_total_ac,
290sum (cdf_clcwp_total_ac) as cdf_clcwp_total_ac,
291sum (cdf_clect_total_ac) as cdf_clect_total_ac,
292sum (cdf_cllct_total_ac) as cdf_cllct_total_ac,
293sum (cdf_cllect_total_ac) as cdf_cllect_total_ac,
294sum (cdf_cmloss_total_bb) as cdf_cmloss_total_bb,
295sum (cdf_cmexp_total_bb) as cdf_cmexp_total_bb,
296sum (cdf_caser_total_bb) as cdf_caser_total_bb,
297sum (cdf_notct_total_bb) as cdf_notct_total_bb,
298sum (cdf_casupl_total_bb) as cdf_casupl_total_bb,
299sum (cdf_clcwp_total_bb) as cdf_clcwp_total_bb,
300sum (cdf_clect_total_bb) as cdf_clect_total_bb,
301sum (cdf_cllct_total_bb) as cdf_cllct_total_bb,
302sum (cdf_cllect_total_bb) as cdf_cllect_total_bb,
303sum (cdf_cmloss_total_bt) as cdf_cmloss_total_bt,
304sum (cdf_cmexp_total_bt) as cdf_cmexp_total_bt,
305sum (cdf_caser_total_bt) as cdf_caser_total_bt,
306sum (cdf_notct_total_bt) as cdf_notct_total_bt,
307sum (cdf_casupl_total_bt) as cdf_casupl_total_bt,
308sum (cdf_clcwp_total_bt) as cdf_clcwp_total_bt,
309sum (cdf_clect_total_bt) as cdf_clect_total_bt,
310sum (cdf_cllct_total_bt) as cdf_cllct_total_bt,
311sum (cdf_cllect_total_bt) as cdf_cllect_total_bt,
312sum (cdf_cmloss_total_bv) as cdf_cmloss_total_bv,
313sum (cdf_cmexp_total_bv) as cdf_cmexp_total_bv,
314sum (cdf_caser_total_bv) as cdf_caser_total_bv,
315sum (cdf_notct_total_bv) as cdf_notct_total_bv,
316sum (cdf_casupl_total_bv) as cdf_casupl_total_bv,
317sum (cdf_clcwp_total_bv) as cdf_clcwp_total_bv,
318sum (cdf_clect_total_bv) as cdf_clect_total_bv,
319sum (cdf_cllct_total_bv) as cdf_cllct_total_bv,
320sum (cdf_cllect_total_bv) as cdf_cllect_total_bv,
321sum (cdf_cmloss_total_cc) as cdf_cmloss_total_cc,
322sum (cdf_cmexp_total_cc) as cdf_cmexp_total_cc,
323sum (cdf_caser_total_cc) as cdf_caser_total_cc,
324sum (cdf_notct_total_cc) as cdf_notct_total_cc,
325sum (cdf_casupl_total_cc) as cdf_casupl_total_cc,
326sum (cdf_clcwp_total_cc) as cdf_clcwp_total_cc,
327sum (cdf_clect_total_cc) as cdf_clect_total_cc,
328sum (cdf_cllct_total_cc) as cdf_cllct_total_cc,
329sum (cdf_cllect_total_cc) as cdf_cllect_total_cc,
330sum (cdf_cmloss_total_dd) as cdf_cmloss_total_dd,
331sum (cdf_cmexp_total_dd) as cdf_cmexp_total_dd,
332sum (cdf_caser_total_dd) as cdf_caser_total_dd,
333sum (cdf_notct_total_dd) as cdf_notct_total_dd,
334sum (cdf_casupl_total_dd) as cdf_casupl_total_dd,
335sum (cdf_clcwp_total_dd) as cdf_clcwp_total_dd,
336sum (cdf_clect_total_dd) as cdf_clect_total_dd,
337sum (cdf_cllct_total_dd) as cdf_cllct_total_dd,
338sum (cdf_cllect_total_dd) as cdf_cllect_total_dd,
339sum (cdf_cmloss_total_da) as cdf_cmloss_total_da,
340sum (cdf_cmexp_total_da) as cdf_cmexp_total_da,
341sum (cdf_caser_total_da) as cdf_caser_total_da,
342sum (cdf_notct_total_da) as cdf_notct_total_da,
343sum (cdf_casupl_total_da) as cdf_casupl_total_da,
344sum (cdf_clcwp_total_da) as cdf_clcwp_total_da,
345sum (cdf_clect_total_da) as cdf_clect_total_da,
346sum (cdf_cllct_total_da) as cdf_cllct_total_da,
347sum (cdf_cllect_total_da) as cdf_cllect_total_da,
348sum (cdf_cmloss_total_db) as cdf_cmloss_total_db,
349sum (cdf_cmexp_total_db) as cdf_cmexp_total_db,
350sum (cdf_caser_total_db) as cdf_caser_total_db,
351sum (cdf_notct_total_db) as cdf_notct_total_db,
352sum (cdf_casupl_total_db) as cdf_casupl_total_db,
353sum (cdf_clcwp_total_db) as cdf_clcwp_total_db,
354sum (cdf_clect_total_db) as cdf_clect_total_db,
355sum (cdf_cllct_total_db) as cdf_cllct_total_db,
356sum (cdf_cllect_total_db) as cdf_cllect_total_db,
357sum (cdf_cmloss_total_hh) as cdf_cmloss_total_hh,
358sum (cdf_cmexp_total_hh) as cdf_cmexp_total_hh,
359sum (cdf_caser_total_hh) as cdf_caser_total_hh,
360sum (cdf_notct_total_hh) as cdf_notct_total_hh,
361sum (cdf_casupl_total_hh) as cdf_casupl_total_hh,
362sum (cdf_clcwp_total_hh) as cdf_clcwp_total_hh,
363sum (cdf_clect_total_hh) as cdf_clect_total_hh,
364sum (cdf_cllct_total_hh) as cdf_cllct_total_hh,
365sum (cdf_cllect_total_hh) as cdf_cllect_total_hh,
366sum (cdf_cmloss_total_jj) as cdf_cmloss_total_jj,
367sum (cdf_cmexp_total_jj) as cdf_cmexp_total_jj,
368sum (cdf_caser_total_jj) as cdf_caser_total_jj,
369sum (cdf_notct_total_jj) as cdf_notct_total_jj,
370sum (cdf_casupl_total_jj) as cdf_casupl_total_jj,
371sum (cdf_clcwp_total_jj) as cdf_clcwp_total_jj,
372sum (cdf_clect_total_jj) as cdf_clect_total_jj,
373sum (cdf_cllct_total_jj) as cdf_cllct_total_jj,
374sum (cdf_cllect_total_jj) as cdf_cllect_total_jj,
375sum (cdf_cmloss_total_uu) as cdf_cmloss_total_uu,
376sum (cdf_cmexp_total_uu) as cdf_cmexp_total_uu,
377sum (cdf_caser_total_uu) as cdf_caser_total_uu,
378sum (cdf_notct_total_uu) as cdf_notct_total_uu,
379sum (cdf_casupl_total_uu) as cdf_casupl_total_uu,
380sum (cdf_clcwp_total_uu) as cdf_clcwp_total_uu,
381sum (cdf_clect_total_uu) as cdf_clect_total_uu,
382sum (cdf_cllct_total_uu) as cdf_cllct_total_uu,
383sum (cdf_cllect_total_uu) as cdf_cllect_total_uu,
384sum (cdf_cmloss_total_va) as cdf_cmloss_total_va,
385sum (cdf_cmexp_total_va) as cdf_cmexp_total_va,
386sum (cdf_caser_total_va) as cdf_caser_total_va,
387sum (cdf_notct_total_va) as cdf_notct_total_va,
388sum (cdf_casupl_total_va) as cdf_casupl_total_va,
389sum (cdf_clcwp_total_va) as cdf_clcwp_total_va,
390sum (cdf_clect_total_va) as cdf_clect_total_va,
391sum (cdf_cllct_total_va) as cdf_cllct_total_va,
392sum (cdf_cllect_total_va) as cdf_cllect_total_va,
393sum (cdf_cmloss_total_vb) as cdf_cmloss_total_vb,
394sum (cdf_cmexp_total_vb) as cdf_cmexp_total_vb,
395sum (cdf_caser_total_vb) as cdf_caser_total_vb,
396sum (cdf_notct_total_vb) as cdf_notct_total_vb,
397sum (cdf_casupl_total_vb) as cdf_casupl_total_vb,
398sum (cdf_clcwp_total_vb) as cdf_clcwp_total_vb,
399sum (cdf_clect_total_vb) as cdf_clect_total_vb,
400sum (cdf_cllct_total_vb) as cdf_cllct_total_vb,
401sum (cdf_cllect_total_vb) as cdf_cllect_total_vb,
402sum (cdf_cmloss_total_vo) as cdf_cmloss_total_vo,
403sum (cdf_cmexp_total_vo) as cdf_cmexp_total_vo,
404sum (cdf_caser_total_vo) as cdf_caser_total_vo,
405sum (cdf_notct_total_vo) as cdf_notct_total_vo,
406sum (cdf_casupl_total_vo) as cdf_casupl_total_vo,
407sum (cdf_clcwp_total_vo) as cdf_clcwp_total_vo,
408sum (cdf_clect_total_vo) as cdf_clect_total_vo,
409sum (cdf_cllct_total_vo) as cdf_cllct_total_vo,
410sum (cdf_cllect_total_vo) as cdf_cllect_total_vo,
411 group_concat(case when drv_termination_dt != '0' then drv_termination_dt end , ' ') as term_cnt
412 from
413 dra_ruffprod.full_loss_rollup_parquet where ply_pt_state_cd = 'PR' or ply_pt_state_cd = 'NH'
414 group by
415 drv_endorse_dt,
416drv_endorse_end_dt,
417ply_pt_state_cd,
418ply_line_cd,
419pls_company_cd,
420ply_ifs_group_cd,
421ply_opt_pkg_cd
42216/08/23 15:52:49 WARN spark.SparkConf: Detected deprecated memory fraction settings: [spark.storage.memoryFraction]. As of Spark 1.6, execution and storage memory management are unified. All memory fractions used in the old model are now deprecated and no longer read. If you wish to use the old memory management, you may explicitly enable `spark.memory.useLegacyMode` (not recommended).
42316/08/23 15:52:51 WARN util.Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.
42416/08/23 15:52:51 WARN util.Utils: Service 'SparkUI' could not bind on port 4041. Attempting port 4042.
42516/08/23 15:52:51 WARN util.Utils: Service 'SparkUI' could not bind on port 4042. Attempting port 4043.
42616/08/23 15:52:51 WARN util.Utils: Service 'SparkUI' could not bind on port 4043. Attempting port 4044.
427Traceback (most recent call last):
428 File "/home/anave/src/ruff/common/report/parquet_to_csv.py", line 232, in <module>
429 df = sqlContext.read.parquet(IMPALA_QUERY_PARQUET)
430 File "/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/lib/spark/python/lib/pyspark.zip/pyspark/sql/readwriter.py", line 215, in parquet
431 File "/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/lib/spark/python/lib/py4j-0.9-src.zip/py4j/java_gateway.py", line 813, in __call__
432 File "/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/lib/spark/python/lib/pyspark.zip/pyspark/sql/utils.py", line 45, in deco
433 File "/opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p1657.1607/lib/spark/python/lib/py4j-0.9-src.zip/py4j/protocol.py", line 308, in get_return_value
434py4j.protocol.Py4JJavaError: An error occurred while calling o48.parquet.
435: java.lang.AssertionError: assertion failed: No predefined schema found, and no Parquet data files or summary files found under hdfs://nameservice1/data/res/warehouse/ruff/user/anave/ruff_daily_summary/ruff_loss_report.
436 at scala.Predef$.assert(Predef.scala:179)
437 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation$MetadataCache.org$apache$spark$sql$execution$datasources$parquet$ParquetRelation$MetadataCache$$readSchema(ParquetRelation.scala:512)
438 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation$MetadataCache$$anonfun$12.apply(ParquetRelation.scala:421)
439 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation$MetadataCache$$anonfun$12.apply(ParquetRelation.scala:421)
440 at scala.Option.orElse(Option.scala:257)
441 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation$MetadataCache.refresh(ParquetRelation.scala:421)
442 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation.org$apache$spark$sql$execution$datasources$parquet$ParquetRelation$$metadataCache$lzycompute(ParquetRelation.scala:145)
443 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation.org$apache$spark$sql$execution$datasources$parquet$ParquetRelation$$metadataCache(ParquetRelation.scala:143)
444 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation$$anonfun$6.apply(ParquetRelation.scala:202)
445 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation$$anonfun$6.apply(ParquetRelation.scala:202)
446 at scala.Option.getOrElse(Option.scala:120)
447 at org.apache.spark.sql.execution.datasources.parquet.ParquetRelation.dataSchema(ParquetRelation.scala:202)
448 at org.apache.spark.sql.sources.HadoopFsRelation.schema$lzycompute(interfaces.scala:636)
449 at org.apache.spark.sql.sources.HadoopFsRelation.schema(interfaces.scala:635)
450 at org.apache.spark.sql.execution.datasources.LogicalRelation.<init>(LogicalRelation.scala:37)
451 at org.apache.spark.sql.SQLContext.baseRelationToDataFrame(SQLContext.scala:442)
452 at org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:316)
453 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
454 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
455 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
456 at java.lang.reflect.Method.invoke(Method.java:606)
457 at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
458 at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
459 at py4j.Gateway.invoke(Gateway.java:259)
460 at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
461 at py4j.commands.CallCommand.execute(CallCommand.java:79)
462 at py4j.GatewayConnection.run(GatewayConnection.java:209)
463 at java.lang.Thread.run(Thread.java:745)
464
465(PY27)~/src/ruff/common/report 663 $