QSt2Ey0Q

1#Table is a List of Rows from small Hive table I loaded using
2#query = "SELECT * FROM Table"
3#Table = sqlContext.sql(query).collect()
4
5for i in range(len(Table)):
6
7    rows = sqlContext.sql(qry)
8    val1 = Table[i][0]
9    val2 = Table[i][1]
10    count = Table[i][2]
11    x = 100 - count
12
13#hivetemp is a table that I copied from Hive to my hfs using:
14#create external table IF NOT EXISTS hive temp LIKE hivetableIwant2copy LOCATION "/user/name/hiveBackup";
15#INSERT OVERWRITE TABLE hivetemp SELECT * FROM hivetableIwant2copy;
16
17    query = "SELECT * FROM hivetemp WHERE col1<>""+val1+"" AND col2 ==""+val2+"" ORDER BY RAND() LIMIT "+str(x)
18
19    rows = sqlContext.sql(query)
20    rows = rows.withColumn("col4", lit(10))
21    rows = rows.withColumn("col5", lit(some_string))
22#writing to parquet is heck slow AND I can't work with pandas due to the library not installed on the server
23    rows.saveAsParquetFile("rows"+str(i)+".parquet")
24#tried this before and heck slow also
25    #rows_list = rows.collect()
26    #shuffle(rows_list)
27	
28rdd = rdd.map(lambda x: pyspark.sql.Row(**x))
29# ** is to unpack the dictionary since the Row constructor
30# only takes keyword arguments