· 9 years ago · Dec 15, 2016, 07:09 PM
1# -*- coding: utf-8 -*-
2
3'''
4 Phoenix Add-on
5 Code ported from Shani's LiveStreamsPro Add-on
6
7 This program is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>.
19'''
20
21import re
22import os
23import sys
24import urllib
25import urllib2
26import xbmc
27import xbmcaddon
28import traceback
29import cookielib
30import base64
31
32profile = functions_dir = xbmc.translatePath(xbmcaddon.Addon().getAddonInfo('profile').decode('utf-8'))
33
34from resources.lib.modules import client
35from resources.lib.modules import control
36
37
38def fetch(regex):
39 try:
40 cacheFile = os.path.join(control.dataPath, 'regex.db')
41 dbcon = database.connect(cacheFile)
42 dbcur = dbcon.cursor()
43 dbcur.execute("SELECT * FROM regex WHERE regex = '%s'" % regex)
44 regex = dbcur.fetchone()[1]
45 return regex
46 except:
47 return
48
49
50def insert(data):
51 try:
52 control.makeFile(control.dataPath)
53 cacheFile = os.path.join(control.dataPath, 'regex.db')
54 dbcon = database.connect(cacheFile)
55 dbcur = dbcon.cursor()
56 dbcur.execute("CREATE TABLE IF NOT EXISTS regex (""regex TEXT, ""response TEXT, ""UNIQUE(regex)"");")
57 for i in data:
58 try: dbcur.execute("INSERT INTO regex Values (?, ?)", (i['regex'], i['response']))
59 except: pass
60 dbcon.commit()
61 except:
62 return
63
64
65def clear():
66 try:
67 cacheFile = os.path.join(control.dataPath, 'regex.db')
68 dbcon = database.connect(cacheFile)
69 dbcur = dbcon.cursor()
70 dbcur.execute("DROP TABLE IF EXISTS regex")
71 dbcur.execute("VACUUM")
72 dbcon.commit()
73 except:
74 pass
75
76
77def resolve(regex):
78 try:
79 vanilla = re.compile('(<regex>.+)', re.MULTILINE|re.DOTALL).findall(regex)[0]
80 cddata = re.compile('<\!\[CDATA\[(.+?)\]\]>', re.MULTILINE|re.DOTALL).findall(regex)
81 for i in cddata:
82 regex = regex.replace('<![CDATA['+i+']]>', urllib.quote_plus(i))
83
84 regexs = re.compile('(<regex>.+)', re.MULTILINE|re.DOTALL).findall(regex)[0]
85 regexs = re.compile('<regex>(.+?)</regex>', re.MULTILINE|re.DOTALL).findall(regexs)
86 regexs = [re.compile('<(.+?)>(.*?)</.+?>', re.MULTILINE|re.DOTALL).findall(i) for i in regexs]
87
88 regexs = [dict([(client.replaceHTMLCodes(x[0]), client.replaceHTMLCodes(urllib.unquote_plus(x[1]))) for x in i]) for i in regexs]
89 regexs = [(i['name'], i) for i in regexs]
90 regexs = dict(regexs)
91
92 url = regex.split('<regex>', 1)[0].strip()
93 url = client.replaceHTMLCodes(url)
94 url = url.encode('utf-8')
95
96 r = getRegexParsed(regexs, url)
97
98 try:
99 ln = ''
100 ret = r[1]
101 listrepeat = r[2]['listrepeat']
102 regexname = r[2]['name']
103
104 for obj in ret:
105 try:
106 item = listrepeat
107 for i in range(len(obj)+1):
108 item = item.replace('[%s.param%s]' % (regexname, str(i)), obj[i-1])
109
110 item2 = vanilla
111 for i in range(len(obj)+1):
112 item2 = item2.replace('[%s.param%s]' % (regexname, str(i)), obj[i-1])
113
114 item2 = re.compile('(<regex>.+?</regex>)', re.MULTILINE|re.DOTALL).findall(item2)
115 item2 = [x for x in item2 if not '<name>%s</name>' % regexname in x]
116 item2 = ''.join(item2)
117
118 ln += '\n<item>%s\n%s</item>\n' % (item, item2)
119 except:
120 pass
121
122 return ln
123 except:
124 pass
125
126 if r[1] == True:
127 return r[0]
128 except:
129 return
130
131
132class NoRedirection(urllib2.HTTPErrorProcessor):
133 def http_response(self, request, response):
134 return response
135 https_response = http_response
136
137
138def getRegexParsed(regexs, url,cookieJar=None,forCookieJarOnly=False,recursiveCall=False,cachedPages={}, rawPost=False, cookie_jar_file=None):#0,1,2 = URL, regexOnly, CookieJarOnly
139 #cachedPages = {}
140 #print 'url',url
141 doRegexs = re.compile('\$doregex\[([^\]]*)\]').findall(url)
142# print 'doRegexs',doRegexs,regexs
143 setresolved=True
144 for k in doRegexs:
145 if k in regexs:
146 #print 'processing ' ,k
147 m = regexs[k]
148 #print m
149 cookieJarParam=False
150 if 'cookiejar' in m: # so either create or reuse existing jar
151 #print 'cookiejar exists',m['cookiejar']
152 cookieJarParam=m['cookiejar']
153 if '$doregex' in cookieJarParam:
154 cookieJar=getRegexParsed(regexs, m['cookiejar'],cookieJar,True, True,cachedPages)
155 cookieJarParam=True
156 else:
157 cookieJarParam=True
158 #print 'm[cookiejar]',m['cookiejar'],cookieJar
159 if cookieJarParam:
160 if cookieJar==None:
161 #print 'create cookie jar'
162 cookie_jar_file=None
163 if 'open[' in m['cookiejar']:
164 cookie_jar_file=m['cookiejar'].split('open[')[1].split(']')[0]
165# print 'cookieJar from file name',cookie_jar_file
166
167 cookieJar=getCookieJar(cookie_jar_file)
168# print 'cookieJar from file',cookieJar
169 if cookie_jar_file:
170 saveCookieJar(cookieJar,cookie_jar_file)
171 #import cookielib
172 #cookieJar = cookielib.LWPCookieJar()
173 #print 'cookieJar new',cookieJar
174 elif 'save[' in m['cookiejar']:
175 cookie_jar_file=m['cookiejar'].split('save[')[1].split(']')[0]
176 complete_path=os.path.join(profile,cookie_jar_file)
177# print 'complete_path',complete_path
178 saveCookieJar(cookieJar,cookie_jar_file)
179
180 if m['page'] and '$doregex' in m['page']:
181 pg=getRegexParsed(regexs, m['page'],cookieJar,recursiveCall=True,cachedPages=cachedPages)
182 if len(pg)==0:
183 pg='http://regexfailed'
184 m['page']=pg
185
186 if 'setcookie' in m and m['setcookie'] and '$doregex' in m['setcookie']:
187 m['setcookie']=getRegexParsed(regexs, m['setcookie'],cookieJar,recursiveCall=True,cachedPages=cachedPages)
188 if 'appendcookie' in m and m['appendcookie'] and '$doregex' in m['appendcookie']:
189 m['appendcookie']=getRegexParsed(regexs, m['appendcookie'],cookieJar,recursiveCall=True,cachedPages=cachedPages)
190
191
192 if 'post' in m and '$doregex' in m['post']:
193 m['post']=getRegexParsed(regexs, m['post'],cookieJar,recursiveCall=True,cachedPages=cachedPages)
194# print 'post is now',m['post']
195
196 if 'rawpost' in m and '$doregex' in m['rawpost']:
197 m['rawpost']=getRegexParsed(regexs, m['rawpost'],cookieJar,recursiveCall=True,cachedPages=cachedPages,rawPost=True)
198 #print 'rawpost is now',m['rawpost']
199
200 if 'rawpost' in m and '$epoctime$' in m['rawpost']:
201 m['rawpost']=m['rawpost'].replace('$epoctime$',getEpocTime())
202
203 if 'rawpost' in m and '$epoctime2$' in m['rawpost']:
204 m['rawpost']=m['rawpost'].replace('$epoctime2$',getEpocTime2())
205
206
207 link=''
208 if m['page'] and m['page'] in cachedPages and not 'ignorecache' in m and forCookieJarOnly==False :
209 #print 'using cache page',m['page']
210 link = cachedPages[m['page']]
211 else:
212 if m['page'] and not m['page']=='' and m['page'].startswith('http'):
213 if '$epoctime$' in m['page']:
214 m['page']=m['page'].replace('$epoctime$',getEpocTime())
215 if '$epoctime2$' in m['page']:
216 m['page']=m['page'].replace('$epoctime2$',getEpocTime2())
217
218 #print 'Ingoring Cache',m['page']
219 page_split=m['page'].split('|')
220 pageUrl=page_split[0]
221 header_in_page=None
222 if len(page_split)>1:
223 header_in_page=page_split[1]
224
225# if
226# proxy = urllib2.ProxyHandler({ ('https' ? proxytouse[:5]=="https":"http") : proxytouse})
227# opener = urllib2.build_opener(proxy)
228# urllib2.install_opener(opener)
229
230
231
232# import urllib2
233# print 'urllib2.getproxies',urllib2.getproxies()
234 current_proxies=urllib2.ProxyHandler(urllib2.getproxies())
235
236
237 #print 'getting pageUrl',pageUrl
238 req = urllib2.Request(pageUrl)
239 if 'proxy' in m:
240 proxytouse= m['proxy']
241# print 'proxytouse',proxytouse
242# urllib2.getproxies= lambda: {}
243 if pageUrl[:5]=="https":
244 proxy = urllib2.ProxyHandler({ 'https' : proxytouse})
245 #req.set_proxy(proxytouse, 'https')
246 else:
247 proxy = urllib2.ProxyHandler({ 'http' : proxytouse})
248 #req.set_proxy(proxytouse, 'http')
249 opener = urllib2.build_opener(proxy)
250 urllib2.install_opener(opener)
251
252
253 req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/14.0.1')
254 proxytouse=None
255
256 if 'referer' in m:
257 req.add_header('Referer', m['referer'])
258 if 'accept' in m:
259 req.add_header('Accept', m['accept'])
260 if 'agent' in m:
261 req.add_header('User-agent', m['agent'])
262 if 'x-req' in m:
263 req.add_header('X-Requested-With', m['x-req'])
264 if 'x-addr' in m:
265 req.add_header('x-addr', m['x-addr'])
266 if 'x-forward' in m:
267 req.add_header('X-Forwarded-For', m['x-forward'])
268 if 'setcookie' in m:
269# print 'adding cookie',m['setcookie']
270 req.add_header('Cookie', m['setcookie'])
271 if 'appendcookie' in m:
272# print 'appending cookie to cookiejar',m['appendcookie']
273 cookiestoApend=m['appendcookie']
274 cookiestoApend=cookiestoApend.split(';')
275 for h in cookiestoApend:
276 n,v=h.split('=')
277 w,n= n.split(':')
278 ck = cookielib.Cookie(version=0, name=n, value=v, port=None, port_specified=False, domain=w, domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
279 cookieJar.set_cookie(ck)
280 if 'origin' in m:
281 req.add_header('Origin', m['origin'])
282 if header_in_page:
283 header_in_page=header_in_page.split('&')
284 for h in header_in_page:
285 n,v=h.split('=')
286 req.add_header(n,v)
287
288 if not cookieJar==None:
289# print 'cookieJarVal',cookieJar
290 cookie_handler = urllib2.HTTPCookieProcessor(cookieJar)
291 opener = urllib2.build_opener(cookie_handler, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
292 opener = urllib2.install_opener(opener)
293# print 'noredirect','noredirect' in m
294
295 if 'noredirect' in m:
296 opener = urllib2.build_opener(cookie_handler,NoRedirection, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
297 opener = urllib2.install_opener(opener)
298 elif 'noredirect' in m:
299 opener = urllib2.build_opener(NoRedirection, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
300 opener = urllib2.install_opener(opener)
301
302
303 if 'connection' in m:
304# print '..........................connection//////.',m['connection']
305 from keepalive import HTTPHandler
306 keepalive_handler = HTTPHandler()
307 opener = urllib2.build_opener(keepalive_handler)
308 urllib2.install_opener(opener)
309
310
311 #print 'after cookie jar'
312 post=None
313
314 if 'post' in m:
315 postData=m['post']
316 #if '$LiveStreamRecaptcha' in postData:
317 # (captcha_challenge,catpcha_word,idfield)=processRecaptcha(m['page'],cookieJar)
318 # if captcha_challenge:
319 # postData=postData.replace('$LiveStreamRecaptcha','manual_recaptcha_challenge_field:'+captcha_challenge+',recaptcha_response_field:'+catpcha_word+',id:'+idfield)
320 splitpost=postData.split(',');
321 post={}
322 for p in splitpost:
323 n=p.split(':')[0];
324 v=p.split(':')[1];
325 post[n]=v
326 post = urllib.urlencode(post)
327
328 if 'rawpost' in m:
329 post=m['rawpost']
330 #if '$LiveStreamRecaptcha' in post:
331 # (captcha_challenge,catpcha_word,idfield)=processRecaptcha(m['page'],cookieJar)
332 # if captcha_challenge:
333 # post=post.replace('$LiveStreamRecaptcha','&manual_recaptcha_challenge_field='+captcha_challenge+'&recaptcha_response_field='+catpcha_word+'&id='+idfield)
334 link=''
335 try:
336
337 if post:
338 response = urllib2.urlopen(req,post)
339 else:
340 response = urllib2.urlopen(req)
341 if response.info().get('Content-Encoding') == 'gzip':
342 from StringIO import StringIO
343 import gzip
344 buf = StringIO( response.read())
345 f = gzip.GzipFile(fileobj=buf)
346 link = f.read()
347 else:
348 link=response.read()
349
350
351
352 if 'proxy' in m and not current_proxies is None:
353 urllib2.install_opener(urllib2.build_opener(current_proxies))
354
355 link=javascriptUnEscape(link)
356 #print repr(link)
357 #print link This just print whole webpage in LOG
358 if 'includeheaders' in m:
359 #link+=str(response.headers.get('Set-Cookie'))
360 link+='$$HEADERS_START$$:'
361 for b in response.headers:
362 link+= b+':'+response.headers.get(b)+'\n'
363 link+='$$HEADERS_END$$:'
364 # print link
365
366 response.close()
367 except:
368 pass
369 cachedPages[m['page']] = link
370 #print link
371 #print 'store link for',m['page'],forCookieJarOnly
372
373 if forCookieJarOnly:
374 return cookieJar# do nothing
375 elif m['page'] and not m['page'].startswith('http'):
376 if m['page'].startswith('$pyFunction:'):
377 val=doEval(m['page'].split('$pyFunction:')[1],'',cookieJar,m )
378 if forCookieJarOnly:
379 return cookieJar# do nothing
380 link=val
381 link=javascriptUnEscape(link)
382 else:
383 link=m['page']
384
385 if '$doregex' in m['expres']:
386 m['expres']=getRegexParsed(regexs, m['expres'],cookieJar,recursiveCall=True,cachedPages=cachedPages)
387
388 if not m['expres']=='':
389 #print 'doing it ',m['expres']
390 if '$LiveStreamCaptcha' in m['expres']:
391 val=askCaptcha(m,link,cookieJar)
392 #print 'url and val',url,val
393 url = url.replace("$doregex[" + k + "]", val)
394
395 elif m['expres'].startswith('$pyFunction:') or '#$pyFunction' in m['expres']:
396 #print 'expeeeeeeeeeeeeeeeeeee',m['expres']
397 val=''
398 if m['expres'].startswith('$pyFunction:'):
399 val=doEval(m['expres'].split('$pyFunction:')[1],link,cookieJar,m)
400 else:
401 val=doEvalFunction(m['expres'],link,cookieJar,m)
402 if 'ActivateWindow' in m['expres']: return
403 if forCookieJarOnly:
404 return cookieJar# do nothing
405 if 'listrepeat' in m:
406 listrepeat=m['listrepeat']
407 return listrepeat,eval(val), m,regexs,cookieJar
408
409 try:
410 url = url.replace(u"$doregex[" + k + "]", val)
411 except: url = url.replace("$doregex[" + k + "]", val.decode("utf-8"))
412 else:
413 if 'listrepeat' in m:
414 listrepeat=m['listrepeat']
415 ret=re.findall(m['expres'],link)
416 return listrepeat,ret, m,regexs
417
418 val=''
419 if not link=='':
420 #print 'link',link
421 reg = re.compile(m['expres']).search(link)
422 try:
423 val=reg.group(1).strip()
424 except: traceback.print_exc()
425 elif m['page']=='' or m['page']==None:
426 val=m['expres']
427
428 if rawPost:
429# print 'rawpost'
430 val=urllib.quote_plus(val)
431 if 'htmlunescape' in m:
432 #val=urllib.unquote_plus(val)
433 import HTMLParser
434 val=HTMLParser.HTMLParser().unescape(val)
435 try:
436 url = url.replace("$doregex[" + k + "]", val)
437 except: url = url.replace("$doregex[" + k + "]", val.decode("utf-8"))
438 #print 'ur',url
439 #return val
440 else:
441 url = url.replace("$doregex[" + k + "]",'')
442 if '$epoctime$' in url:
443 url=url.replace('$epoctime$',getEpocTime())
444 if '$epoctime2$' in url:
445 url=url.replace('$epoctime2$',getEpocTime2())
446
447 if '$GUID$' in url:
448 import uuid
449 url=url.replace('$GUID$',str(uuid.uuid1()).upper())
450 if '$get_cookies$' in url:
451 url=url.replace('$get_cookies$',getCookiesString(cookieJar))
452
453 if recursiveCall: return url
454 #print 'final url',repr(url)
455 if url=="":
456 return
457 else:
458 return url,setresolved
459
460
461def get_unwise( str_eval):
462 page_value=""
463 try:
464 ss="w,i,s,e=("+str_eval+')'
465 exec (ss)
466 page_value=unwise_func(w,i,s,e)
467 except: traceback.print_exc(file=sys.stdout)
468 #print 'unpacked',page_value
469 return page_value
470
471
472def unwise_func( w, i, s, e):
473 lIll = 0;
474 ll1I = 0;
475 Il1l = 0;
476 ll1l = [];
477 l1lI = [];
478 while True:
479 if (lIll < 5):
480 l1lI.append(w[lIll])
481 elif (lIll < len(w)):
482 ll1l.append(w[lIll]);
483 lIll+=1;
484 if (ll1I < 5):
485 l1lI.append(i[ll1I])
486 elif (ll1I < len(i)):
487 ll1l.append(i[ll1I])
488 ll1I+=1;
489 if (Il1l < 5):
490 l1lI.append(s[Il1l])
491 elif (Il1l < len(s)):
492 ll1l.append(s[Il1l]);
493 Il1l+=1;
494 if (len(w) + len(i) + len(s) + len(e) == len(ll1l) + len(l1lI) + len(e)):
495 break;
496
497 lI1l = ''.join(ll1l)#.join('');
498 I1lI = ''.join(l1lI)#.join('');
499 ll1I = 0;
500 l1ll = [];
501 for lIll in range(0,len(ll1l),2):
502 #print 'array i',lIll,len(ll1l)
503 ll11 = -1;
504 if ( ord(I1lI[ll1I]) % 2):
505 ll11 = 1;
506 #print 'val is ', lI1l[lIll: lIll+2]
507 l1ll.append(chr( int(lI1l[lIll: lIll+2], 36) - ll11));
508 ll1I+=1;
509 if (ll1I >= len(l1lI)):
510 ll1I = 0;
511 ret=''.join(l1ll)
512 if 'eval(function(w,i,s,e)' in ret:
513# print 'STILL GOing'
514 ret=re.compile('eval\(function\(w,i,s,e\).*}\((.*?)\)').findall(ret)[0]
515 return get_unwise(ret)
516 else:
517# print 'FINISHED'
518 return ret
519
520
521def get_unpacked( page_value, regex_for_text='', iterations=1, total_iteration=1):
522 try:
523 reg_data=None
524 if page_value.startswith("http"):
525 page_value= getUrl(page_value)
526# print 'page_value',page_value
527 if regex_for_text and len(regex_for_text)>0:
528 try:
529 page_value=re.compile(regex_for_text).findall(page_value)[0] #get the js variable
530 except: return 'NOTPACKED'
531
532 page_value=unpack(page_value,iterations,total_iteration)
533 except:
534 page_value='UNPACKEDFAILED'
535 traceback.print_exc(file=sys.stdout)
536# print 'unpacked',page_value
537 if 'sav1live.tv' in page_value:
538 page_value=page_value.replace('sav1live.tv','sawlive.tv') #quick fix some bug somewhere
539# print 'sav1 unpacked',page_value
540 return page_value
541
542
543def unpack(sJavascript,iteration=1, totaliterations=2 ):
544# print 'iteration',iteration
545 if sJavascript.startswith('var _0xcb8a='):
546 aSplit=sJavascript.split('var _0xcb8a=')
547 ss="myarray="+aSplit[1].split("eval(")[0]
548 exec(ss)
549 a1=62
550 c1=int(aSplit[1].split(",62,")[1].split(',')[0])
551 p1=myarray[0]
552 k1=myarray[3]
553 with open('temp file'+str(iteration)+'.js', "wb") as filewriter:
554 filewriter.write(str(k1))
555 #aa=1/0
556 else:
557
558 if "rn p}('" in sJavascript:
559 aSplit = sJavascript.split("rn p}('")
560 else:
561 aSplit = sJavascript.split("rn A}('")
562# print aSplit
563
564 p1,a1,c1,k1=('','0','0','')
565
566 ss="p1,a1,c1,k1=('"+aSplit[1].split(".spli")[0]+')'
567 exec(ss)
568 k1=k1.split('|')
569 aSplit = aSplit[1].split("))'")
570
571 e = ''
572 d = ''#32823
573
574 #sUnpacked = str(__unpack(p, a, c, k, e, d))
575 sUnpacked1 = str(__unpack(p1, a1, c1, k1, e, d,iteration))
576
577 #print sUnpacked[:200]+'....'+sUnpacked[-100:], len(sUnpacked)
578# print sUnpacked1[:200]+'....'+sUnpacked1[-100:], len(sUnpacked1)
579
580 #exec('sUnpacked1="'+sUnpacked1+'"')
581 if iteration>=totaliterations:
582# print 'final res',sUnpacked1[:200]+'....'+sUnpacked1[-100:], len(sUnpacked1)
583 return sUnpacked1#.replace('\\\\', '\\')
584 else:
585# print 'final res for this iteration is',iteration
586 return unpack(sUnpacked1,iteration+1)#.replace('\\', ''),iteration)#.replace('\\', '');#unpack(sUnpacked.replace('\\', ''))
587
588
589def __unpack(p, a, c, k, e, d, iteration,v=1):
590
591 #with open('before file'+str(iteration)+'.js', "wb") as filewriter:
592 # filewriter.write(str(p))
593 while (c >= 1):
594 c = c -1
595 if (k[c]):
596 aa=str(__itoaNew(c, a))
597 if v==1:
598 p=re.sub('\\b' + aa +'\\b', k[c], p)# THIS IS Bloody slow!
599 else:
600 p=findAndReplaceWord(p,aa,k[c])
601
602 #p=findAndReplaceWord(p,aa,k[c])
603
604
605 #with open('after file'+str(iteration)+'.js', "wb") as filewriter:
606 # filewriter.write(str(p))
607 return p
608
609
610def __itoa(num, radix):
611# print 'num red',num, radix
612 result = ""
613 if num==0: return '0'
614 while num > 0:
615 result = "0123456789abcdefghijklmnopqrstuvwxyz"[num % radix] + result
616 num /= radix
617 return result
618
619
620def __itoaNew(cc, a):
621 aa="" if cc < a else __itoaNew(int(cc / a),a)
622 cc = (cc % a)
623 bb=chr(cc + 29) if cc> 35 else str(__itoa(cc,36))
624 return aa+bb
625
626
627def findAndReplaceWord(source_str, word_to_find,replace_with):
628 splits=None
629 splits=source_str.split(word_to_find)
630 if len(splits)>1:
631 new_string=[]
632 current_index=0
633 for current_split in splits:
634 #print 'here',i
635 new_string.append(current_split)
636 val=word_to_find#by default assume it was wrong to split
637
638 #if its first one and item is blank then check next item is valid or not
639 if current_index==len(splits)-1:
640 val='' # last one nothing to append normally
641 else:
642 if len(current_split)==0: #if blank check next one with current split value
643 if ( len(splits[current_index+1])==0 and word_to_find[0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_') or (len(splits[current_index+1])>0 and splits[current_index+1][0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_'):# first just just check next
644 val=replace_with
645 #not blank, then check current endvalue and next first value
646 else:
647 if (splits[current_index][-1].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_') and (( len(splits[current_index+1])==0 and word_to_find[0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_') or (len(splits[current_index+1])>0 and splits[current_index+1][0].lower() not in 'abcdefghijklmnopqrstuvwxyz1234567890_')):# first just just check next
648 val=replace_with
649
650 new_string.append(val)
651 current_index+=1
652 #aaaa=1/0
653 source_str=''.join(new_string)
654 return source_str
655
656
657def re_me(data, re_patten):
658 match = ''
659 m = re.search(re_patten, data)
660 if m != None:
661 match = m.group(1)
662 else:
663 match = ''
664 return match
665
666
667def getCookiesString(cookieJar):
668 try:
669 cookieString=""
670 for index, cookie in enumerate(cookieJar):
671 cookieString+=cookie.name + "=" + cookie.value +";"
672 except: pass
673 #print 'cookieString',cookieString
674 return cookieString
675
676
677def saveCookieJar(cookieJar,COOKIEFILE):
678 try:
679 complete_path=os.path.join(profile,COOKIEFILE)
680 cookieJar.save(complete_path,ignore_discard=True)
681 except: pass
682
683
684def getCookieJar(COOKIEFILE):
685 cookieJar=None
686 if COOKIEFILE:
687 try:
688 complete_path=os.path.join(profile,COOKIEFILE)
689 cookieJar = cookielib.LWPCookieJar()
690 cookieJar.load(complete_path,ignore_discard=True)
691 except:
692 cookieJar=None
693
694 if not cookieJar:
695 cookieJar = cookielib.LWPCookieJar()
696
697 return cookieJar
698
699
700def doEval(fun_call,page_data,Cookie_Jar,m):
701 ret_val=''
702 #print fun_call
703 if functions_dir not in sys.path:
704 sys.path.append(functions_dir)
705
706# print fun_call
707 try:
708 py_file='import '+fun_call.split('.')[0]
709# print py_file,sys.path
710 exec( py_file)
711# print 'done'
712 except:
713 #print 'error in import'
714 traceback.print_exc(file=sys.stdout)
715# print 'ret_val='+fun_call
716 exec ('ret_val='+fun_call)
717# print ret_val
718 #exec('ret_val=1+1')
719 try:
720 return str(ret_val)
721 except: return ret_val
722
723def doEvalFunction(fun_call,page_data,Cookie_Jar,m):
724# print 'doEvalFunction'
725 ret_val=''
726 if functions_dir not in sys.path:
727 sys.path.append(functions_dir)
728
729 f=open(functions_dir+"/LSProdynamicCode.py","w")
730 f.write(fun_call);
731 f.close()
732 import LSProdynamicCode
733 ret_val=LSProdynamicCode.GetLSProData(page_data,Cookie_Jar,m)
734 try:
735 return str(ret_val)
736 except: return ret_val
737
738def getUrl(url, cookieJar=None,post=None, timeout=20, headers=None, noredir=False):
739 cookie_handler = urllib2.HTTPCookieProcessor(cookieJar)
740
741 if noredir:
742 opener = urllib2.build_opener(NoRedirection,cookie_handler, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
743 else:
744 opener = urllib2.build_opener(cookie_handler, urllib2.HTTPBasicAuthHandler(), urllib2.HTTPHandler())
745 #opener = urllib2.install_opener(opener)
746 req = urllib2.Request(url)
747 req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36')
748 if headers:
749 for h,hv in headers:
750 req.add_header(h,hv)
751
752 response = opener.open(req,post,timeout=timeout)
753 link=response.read()
754 response.close()
755 return link
756
757def get_decode(str,reg=None):
758 if reg:
759 str=re.findall(reg, str)[0]
760 s1 = urllib.unquote(str[0: len(str)-1]);
761 t = '';
762 for i in range( len(s1)):
763 t += chr(ord(s1[i]) - s1[len(s1)-1]);
764 t=urllib.unquote(t)
765# print t
766 return t
767
768
769def javascriptUnEscape(str):
770 js=re.findall('unescape\(\'(.*?)\'',str)
771# print 'js',js
772 if (not js==None) and len(js)>0:
773 for j in js:
774 #print urllib.unquote(j)
775 str=str.replace(j ,urllib.unquote(j))
776 return str
777
778
779def getEpocTime():
780 import time
781 return str(int(time.time()*1000))
782
783
784def getEpocTime2():
785 import time
786 return str(int(time.time()))