· 8 years ago · Jun 03, 2018, 09:46 AM
1# Copyright 2014 SolidBuilds.com. All rights reserved
2#
3# Authors: Ling Thio <ling.thio@gmail.com>
4
5
6from flask import redirect, render_template, render_template_string, Blueprint, request
7from flask import url_for
8from flask_user import current_user, login_required, roles_accepted
9from app.init_app import app, db
10from app.models import UserProfileForm
11import tweepy
12import indicoio
13import re
14import feedparser
15import sqlite3
16import HTMLParser
17import json
18
19@app.errorhandler(404)
20def page_not_found(e):
21 return render_template('pages/404.html'), 404
22
23@app.errorhandler(500)
24def page_not_found(e):
25 return render_template('pages/500.html'), 500
26
27# The Home page is accessible to anyone
28@app.route('/')
29def home_page():
30 return render_template('pages/home.html')
31
32@app.route('/chart')
33def chart():
34
35 conn = sqlite3.connect("databases/trial.db") # or use :memory: to put it in RAM
36 cursor = conn.cursor()
37
38 query = 'New York City'
39 average_history = []
40
41 for row in cursor.execute("SELECT average, sqltime FROM average WHERE +keyword=? ORDER BY sqltime DESC LIMIT 10", (query,)):
42 average_history.append(row)
43
44 return render_template('pages/chart.html', average_history=average_history)
45
46
47# The User page is accessible to authenticated users (users that have logged in)
48@app.route('/dashboard')
49@login_required # Limits access to authenticated users
50def user_page():
51
52 user = str(current_user.email)
53 user_name = str(current_user.first_name)
54
55 conn = sqlite3.connect("databases/trial.db") # or use :memory: to put it in RAM
56 cursor = conn.cursor()
57
58 keylist = []
59
60 for row in cursor.execute("SELECT DISTINCT keyword FROM panel_01 WHERE +user=? ORDER BY sqltime ASC", (user,)):
61 keylist.append(row[0])
62
63 for row in cursor.execute("SELECT DISTINCT keyword FROM panel_02 WHERE +user=? ORDER BY sqltime ASC", (user,)):
64 keylist.append(row[0])
65
66 for row in cursor.execute("SELECT DISTINCT keyword FROM panel_03 WHERE +user=? ORDER BY sqltime ASC", (user,)):
67 keylist.append(row[0])
68
69 return render_template('pages/dashboard.html', user_name=user_name, keylist=keylist)
70
71
72# The Admin page is accessible to users with the 'admin' role
73@app.route('/admin')
74@roles_accepted('admin')
75def admin_page():
76
77
78 # allhead = []
79 last_keys = []
80 allusers = []
81
82 # conn = sqlite3.connect("newsdata.db")
83 # cursor = conn.cursor()
84
85 # for row in cursor.execute("SELECT DISTINCT tweet, sentiment FROM data_01 LIMIT 25"):
86 # allhead.append(row)
87
88 # conn = sqlite3.connect("panel_01.db")
89 # cursor = conn.cursor()
90
91 # for row in cursor.execute("SELECT DISTINCT keyword FROM data_01"):
92 # last_keys.append(row[0])
93
94 # import os.path
95 # BASE_DIR = os.path.dirname(os.path.abspath(__file__))
96 # db_path = os.path.join(BASE_DIR, "app.db")
97 # with sqlite3.connect(db_path) as db:
98
99 # conn = sqlite3.connect("app.db")
100 # cursor = conn.cursor()
101
102 # for row in cursor.execute("SELECT email, first_name FROM users"):
103 # allusers.append(row)
104
105 return render_template('pages/admin_page.html')
106
107
108@app.route('/pages/profile', methods=['GET', 'POST'])
109@login_required
110def user_profile_page():
111 # Initialize form
112 form = UserProfileForm(request.form, current_user)
113
114 # Process valid POST
115 if request.method == 'POST' and form.validate():
116 # Copy form fields to user_profile fields
117 form.populate_obj(current_user)
118
119 # Save user_profile
120 db.session.commit()
121
122 # Redirect to home page
123 return redirect(url_for('home_page'))
124
125 # Process GET or invalid POST
126 return render_template('pages/user_profile_page.html',
127 form=form)
128
129@app.route('/pricing')
130def pricing():
131 return render_template('pages/pricing.html')
132
133
134##############################################################
135
136
137
138
139
140indicoio.config.api_key = '6dc10f3befc35a07132811f7c17021eb'
141
142consumer_key = 'n1SOuktQN8CDxB1qE4T6t6k2R'
143consumer_secret = 'xrIpbAj6AyrnhQbv6OORgibkGExXcBqRJFpSdVUXyhsAXJ0NwY'
144access_token = '822327888498176003-8IlqBYsFajJYbGocmtA1qufY4MNM8MU'
145access_token_secret = 'fJMfqTIYIC9crsn3nUmVJyLYZ2znLq3OwVoyclxBCjtTF'
146
147auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
148auth.set_access_token(access_token, access_token_secret)
149api = tweepy.API(auth)
150
151
152
153
154
155
156##############################################################
157
158
159
160
161
162
163
164@app.route('/realtime', methods=['GET','POST'])
165def results():
166
167 conn = sqlite3.connect("databases/trial.db") # or use :memory: to put it in RAM
168 cursor = conn.cursor()
169
170 cursor.execute("""
171 CREATE TABLE IF NOT EXISTS panel_01(
172 User TEXT,
173 Keyword TEXT,
174 Tweet TEXT UNIQUE,
175 Score INTEGER,
176 -- Timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
177 sqltime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
178 );
179 """)
180
181 cursor.execute("""
182 CREATE TABLE IF NOT EXISTS average(
183 User TEXT,
184 Keyword TEXT,
185 Average INTEGER,
186 -- Timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
187 sqltime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
188 );
189 """)
190
191 num_tweets = 99
192 keyword1 = str(request.args.get( "keyword1" , None ))
193 results = []
194 sent_list = []
195 stored = []
196 exact = len(stored)
197
198 if current_user.is_authenticated:
199 user = str(current_user.email)
200 else:
201 user = str('Guest')
202
203
204 for tweet in tweepy.Cursor(api.search,q=str(keyword1),
205 result_type='recent',
206 lang="en").items(num_tweets):
207
208 clean = HTMLParser.HTMLParser().unescape(tweet.text)
209
210 final = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', clean)
211
212 words = len(final.split())
213
214 if (words > 10):
215 rows_affected = cursor.rowcount
216 result = indicoio.sentiment_hq(final)
217
218 calc = (result*100)
219 result2 = ("%.2f" % calc)
220 results.append((result2, final))
221 sent_list.append(result*100)
222
223 total = (sum(sent_list))
224 lenght = (len(sent_list))
225 before = (total/lenght)
226 average = ("%.2f" % before)
227
228 nline = user, keyword1, final, result2
229 cursor.execute('INSERT OR IGNORE INTO panel_01 (user, keyword, tweet, score) VALUES (?, ?, ?, ?)', (nline))
230 conn.commit()
231
232 if (rows_affected == 1):
233 stored.append(final)
234 else:
235 pass
236
237 if (len(stored) < 5):
238 continue
239 else:
240 (len(stored) == 5)
241 break
242
243 nline = user, keyword1, average
244 cursor.execute('INSERT OR IGNORE INTO average (user, keyword, average) VALUES (?, ?, ?)', (nline))
245 conn.commit()
246
247 bestlist = []
248 worstlist = []
249 neutral = []
250 average_history = []
251
252 query = keyword1
253
254 for row in cursor.execute("SELECT tweet, score FROM panel_01 WHERE score > 50 AND +keyword=? ORDER BY score DESC LIMIT 5", (query,)):
255 bestlist.append(row)
256
257 for row in cursor.execute("SELECT tweet, score FROM panel_01 WHERE score < 50 AND +keyword=? ORDER BY score ASC LIMIT 5", (query,)):
258 worstlist.append(row)
259
260 for row in cursor.execute("SELECT tweet, score FROM panel_01 WHERE score BETWEEN 40 AND 70 AND +keyword=? ORDER BY score DESC LIMIT 5", (query,)):
261 neutral.append(row)
262
263 for row in cursor.execute("SELECT keyword, average, sqltime FROM average WHERE +keyword=? ORDER BY sqltime DESC LIMIT 10", (query,)):
264 average_history.append(row)
265
266 return render_template('pages/realtime.html', average_history=average_history, results=results, average=average, keyword1=keyword1, bestlist=bestlist, worstlist=worstlist)
267
268
269
270
271
272@app.route('/nlp', methods=['GET','POST'])
273def nlp():
274
275 conn = sqlite3.connect("databases/trial.db") # or use :memory: to put it in RAM
276 cursor = conn.cursor()
277
278 cursor.execute("""
279 CREATE TABLE IF NOT EXISTS panel_02(
280 User Text,
281 Keyword TEXT,
282 Tweet TEXT UNIQUE,
283 Op INTEGER,
284 Ex INTEGER,
285 Ag INTEGER,
286 Co INTEGER,
287 -- Timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
288 sqltime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
289 );
290 """)
291
292 user = str(current_user.email)
293
294 # Parameters
295
296 num_tweets = 20
297 keyword2 = str(request.args.get( "keyword2" , None ))
298
299 results2 = []
300
301 op_list = []
302 ex_list = []
303 ag_list = []
304 co_list = []
305 data_list = []
306
307 stored = []
308
309 # Twitter For Loop
310
311 for tweet in tweepy.Cursor(api.search,q=str(keyword2)+
312 " -filter:retweets",
313 result_type='recent',
314 lang="en").items(num_tweets):
315
316 # Text Processing
317
318 clean = HTMLParser.HTMLParser().unescape(tweet.text)
319
320 final = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', clean)
321
322 words = len(final.split())
323
324 # Personality Analysis API
325
326 if (words > 10):
327 rows_affected = cursor.rowcount
328 result = indicoio.personality(final)
329
330 # Math Calculation
331
332 op = float("%.1f" % (result['openness']*100))
333 ex = float("%.1f" % (result['extraversion']*100))
334 ag = float("%.1f" % (result['agreeableness']*100))
335 co = float("%.1f" % (result['conscientiousness']*100))
336
337 nline = user, keyword2, final, op, ex, ag, co
338 cursor.execute('INSERT OR IGNORE INTO panel_02 (user, keyword, tweet, op, ex, ag, co) VALUES (?, ?, ?, ?, ?, ?, ?)', (nline))
339 conn.commit()
340
341 results2.append([final, op, ex, ag, co])
342
343 op_list.append(float(op))
344 ex_list.append(float(ex))
345 ag_list.append(float(ag))
346 co_list.append(float(co))
347
348 if (rows_affected == 1):
349 stored.append(final)
350 else:
351 pass
352
353 if (len(stored) < 10):
354 continue
355 else:
356 (len(stored) == 10)
357 break
358
359 # For Open
360
361 total_o = (sum(op_list))
362 lenght_o = (len(op_list))
363 before_o = (total_o/lenght_o)
364 average_o = ("%.0f" % before_o)
365
366 # For Extra
367
368 total_e = (sum(ex_list))
369 lenght_e = (len(ex_list))
370 before_e = (total_e/lenght_e)
371 average_e = ("%.0f" % before_e)
372
373 # For Agree
374
375 total_a = (sum(ag_list))
376 lenght_a = (len(ag_list))
377 before_a = (total_a/lenght_a)
378 average_a = ("%.0f" % before_a)
379
380 # For Consi
381
382 total_c = (sum(co_list))
383 lenght_c = (len(co_list))
384 before_c = (total_c/lenght_c)
385 average_c = ("%.0f" % before_c)
386
387 query = keyword2
388
389 for row in cursor.execute("SELECT tweet, op, ex, ag, co FROM panel_02 WHERE +keyword=? ORDER BY sqltime DESC", (query,)):
390 data_list.append(row)
391
392 return render_template('pages/nlp.html', data_list=data_list, results2=results2, final=final, keyword2=keyword2, op=op, ex=ex, ag=ag, co=co, average_o=average_o, average_e=average_e, average_a=average_a, average_c=average_c)
393
394
395
396
397
398@app.route('/emotion')
399def emotion():
400
401 conn = sqlite3.connect("databases/trial.db")
402 cursor = conn.cursor()
403
404 cursor.execute("""
405 CREATE TABLE IF NOT EXISTS panel_03(
406 User TEXT,
407 Keyword TEXT,
408 Tweet TEXT UNIQUE,
409 An INTEGER,
410 Su INTEGER,
411 Sa INTEGER,
412 Fe INTEGER,
413 Jo INTEGER,
414 -- Timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
415 sqltime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
416 );
417 """)
418
419 user = str(current_user.email)
420
421 num_tweets = 20
422 keyword3 = str(request.args.get( "keyword3" , None ))
423
424 results3 = []
425
426 an_list = []
427 su_list = []
428 sa_list = []
429 fe_list = []
430 jo_list = []
431
432 emo_data = []
433
434 stored = []
435
436 # Twitter For Loop
437
438 for tweet in tweepy.Cursor(api.search,q=str(keyword3)+
439 " -filter:retweets",
440 result_type='recent',
441 lang="en").items(num_tweets):
442
443 clean = HTMLParser.HTMLParser().unescape(tweet.text)
444
445 final = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', clean)
446
447 words = len(final.split())
448
449 # Personality Analysis API
450
451 if (words > 10):
452 rows_affected = cursor.rowcount
453 result = indicoio.emotion(final)
454
455 # Math Calculation
456
457 an = float("%.1f" % (result['anger']*100))
458 su = float("%.1f" % (result['surprise']*100))
459 sa = float("%.1f" % (result['sadness']*100))
460 fe = float("%.1f" % (result['fear']*100))
461 jo = float("%.1f" % (result['joy']*100))
462
463 results3.append([final, an, su, sa, fe, jo])
464
465 nline = user, keyword3, final, an, su, sa, fe, jo
466 cursor.execute('INSERT OR IGNORE INTO panel_03 (user, keyword, tweet, an, su, sa, fe, jo) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (nline))
467 conn.commit()
468
469 an_list.append(float(an))
470 su_list.append(float(su))
471 sa_list.append(float(sa))
472 fe_list.append(float(fe))
473 jo_list.append(float(jo))
474
475 if (rows_affected == 1):
476 stored.append(final)
477 else:
478 pass
479
480 if (len(stored) < 10):
481 continue
482 else:
483 (len(stored) == 10)
484 break
485
486 # For Anger
487
488 total_an = (sum(an_list))
489 lenght_an = (len(an_list))
490 before_an = (total_an/lenght_an)
491 average_an = ("%.0f" % before_an)
492
493 # For Surprise
494
495 total_su = (sum(su_list))
496 lenght_su = (len(su_list))
497 before_su = (total_su/lenght_su)
498 average_su = ("%.0f" % before_su)
499
500 # For Sadness
501
502 total_sa = (sum(sa_list))
503 lenght_sa = (len(sa_list))
504 before_sa = (total_sa/lenght_sa)
505 average_sa = ("%.0f" % before_sa)
506
507 # For Fear
508
509 total_fe = (sum(fe_list))
510 lenght_fe = (len(fe_list))
511 before_fe = (total_fe/lenght_fe)
512 average_fe = ("%.0f" % before_fe)
513
514 # For Joy
515
516 total_jo = (sum(jo_list))
517 lenght_jo = (len(jo_list))
518 before_jo = (total_jo/lenght_jo)
519 average_jo = ("%.0f" % before_jo)
520
521 query = keyword3
522 for row in cursor.execute("SELECT tweet, an, su, sa, fe, jo FROM panel_03 WHERE +keyword=? ORDER BY sqltime DESC", (query,)):
523 emo_data.append(row)
524
525 return render_template('pages/emotion.html', emo_data=emo_data, results3=results3, final=final, keyword3=keyword3, an=an, su=su, sa=sa, fe=fe, jo=jo, average_an=average_an, average_su=average_su, average_sa=average_sa, average_fe=average_fe, average_jo=average_jo)
526
527
528
529
530@app.route('/trends')
531def trends():
532
533 conn = sqlite3.connect("databases/news.db") # or use :memory: to put it in RAM
534 cursor = conn.cursor()
535
536 cursor.execute("""
537 CREATE TABLE IF NOT EXISTS newstable(
538 headline TEXT UNIQUE,
539 sentiment REAL,
540 -- Timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
541 sqltime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
542 );
543 """)
544
545
546 def parseRSS( rss_url ):
547 return feedparser.parse( rss_url )
548
549 def getHeadlines( rss_url ):
550 headlines = []
551
552 feed = parseRSS( rss_url )
553 for newsitem in feed['items']:
554 headlines.append(newsitem['title'])
555
556 return headlines
557
558 allheadlines = []
559
560 newsurls = {
561 'Reuters Company': 'http://feeds.reuters.com/reuters/companyNews',
562 # 'Reuters Money': 'http://feeds.reuters.com/news/wealth',
563 # 'MW Stories': 'http://www.marketwatch.com/rss/pf',
564 'MW Top': 'http://www.marketwatch.com/rss/topstories',
565 # 'CNN Top': 'http://rss.cnn.com/rss/money_topstories.rss',
566 'CNN Company': 'http://rss.cnn.com/rss/money_news_companies.rss'
567 }
568
569 for key,url in newsurls.items():
570 allheadlines.extend( getHeadlines( url ) )
571
572
573 for hl in allheadlines:
574 bresult = (indicoio.sentiment_hq(hl)*100)
575 result = ("%.2f" % bresult)
576
577 nline = hl,result
578 cursor.execute('INSERT INTO newstable (headline, sentiment) VALUES (?, ?)', (nline))
579 conn.commit()
580
581 worstlist = []
582 bestlist = []
583
584 for row in cursor.execute("SELECT DISTINCT headline, sentiment FROM newstable WHERE sentiment > 50 AND sqltime >= strftime('%s', 'now', '-1 day') ORDER BY sentiment DESC LIMIT 10"):
585 bestlist.append(row)
586
587
588 for row in cursor.execute("SELECT DISTINCT headline, sentiment FROM newstable WHERE sentiment < 50 AND sqltime >= strftime('%s', 'now', '-1 day') ORDER BY sentiment DESC LIMIT 10"):
589 worstlist.append(row)
590
591 return render_template('pages/trends.html', bestlist=bestlist, worstlist=worstlist)
592
593
594
595
596# @app.route('/app', methods=['GET','POST'])
597# def appp():
598
599# num_tweets = 15
600# keyword = str(request.args.get( "keyword" , None ))
601
602# results = []
603# sent_list = []
604
605# results2 = []
606
607# op_list = []
608# ex_list = []
609# ag_list = []
610# co_list = []
611
612# results3 = []
613
614# an_list = []
615# su_list = []
616# sa_list = []
617# fe_list = []
618# jo_list = []
619
620# for tweet in tweepy.Cursor(api.search,q=str(keyword)+
621# " -filter:retweets",
622# result_type='recent',
623# lang="en").items(num_tweets):
624
625# clean = HTMLParser.HTMLParser().unescape(tweet.text)
626# final = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', clean)
627
628# senthq = indicoio.sentiment_hq(final)
629# perso = indicoio.personality(final)
630# result = indicoio.emotion(final)
631
632# calc = (senthq*100)
633# result2 = ("%.2f" % calc)
634# results.append((result2, final))
635# sent_list.append(senthq*100)
636
637# total = (sum(sent_list))
638# lenght = (len(sent_list))
639# before = (total/lenght)
640# average = ("%.1f" % before)
641
642# op = float("%.1f" % (perso['openness']*100))
643# ex = float("%.1f" % (perso['extraversion']*100))
644# ag = float("%.1f" % (perso['agreeableness']*100))
645# co = float("%.1f" % (perso['conscientiousness']*100))
646
647# results2.append([final, op, ex, ag, co])
648
649# op_list.append(float(op))
650# ex_list.append(float(ex))
651# ag_list.append(float(ag))
652# co_list.append(float(co))
653
654# # For Open
655
656# total_o = (sum(op_list))
657# lenght_o = (len(op_list))
658# before_o = (total_o/lenght_o)
659# average_o = ("%.0f" % before_o)
660
661# # For Extra
662
663# total_e = (sum(ex_list))
664# lenght_e = (len(ex_list))
665# before_e = (total_e/lenght_e)
666# average_e = ("%.0f" % before_e)
667
668# # For Agree
669
670# total_a = (sum(ag_list))
671# lenght_a = (len(ag_list))
672# before_a = (total_a/lenght_a)
673# average_a = ("%.0f" % before_a)
674
675# # For Consi
676
677# total_c = (sum(co_list))
678# lenght_c = (len(co_list))
679# before_c = (total_c/lenght_c)
680# average_c = ("%.0f" % before_c)
681
682# an = float("%.1f" % (result['anger']*100))
683# su = float("%.1f" % (result['surprise']*100))
684# sa = float("%.1f" % (result['sadness']*100))
685# fe = float("%.1f" % (result['fear']*100))
686# jo = float("%.1f" % (result['joy']*100))
687
688# results3.append([final, an, su, sa, fe, jo])
689
690# an_list.append(float(an))
691# su_list.append(float(su))
692# sa_list.append(float(sa))
693# fe_list.append(float(fe))
694# jo_list.append(float(jo))
695
696# # For Anger
697
698# total_an = (sum(an_list))
699# lenght_an = (len(an_list))
700# before_an = (total_an/lenght_an)
701# average_an = ("%.0f" % before_an)
702
703# # For Surprise
704
705# total_su = (sum(su_list))
706# lenght_su = (len(su_list))
707# before_su = (total_su/lenght_su)
708# average_su = ("%.0f" % before_su)
709
710# # For Sadness
711
712# total_sa = (sum(sa_list))
713# lenght_sa = (len(sa_list))
714# before_sa = (total_sa/lenght_sa)
715# average_sa = ("%.0f" % before_sa)
716
717# # For Fear
718
719# total_fe = (sum(fe_list))
720# lenght_fe = (len(fe_list))
721# before_fe = (total_fe/lenght_fe)
722# average_fe = ("%.0f" % before_fe)
723
724# # For Joy
725
726# total_jo = (sum(jo_list))
727# lenght_jo = (len(jo_list))
728# before_jo = (total_jo/lenght_jo)
729# average_jo = ("%.0f" % before_jo)
730
731# return render_template('pages/app.html',keyword=keyword, results=results, results2=results2, results3=results3, average=average, an=an, su=su, sa=sa, fe=fe, jo=jo, average_an=average_an, average_su=average_su, average_sa=average_sa, average_fe=average_fe, average_jo=average_jo, final=final, op=op, ex=ex, ag=ag, co=co, average_o=average_o, average_e=average_e, average_a=average_a, average_c=average_c)