dSjbKJFS

1import os
2import csv
3import sys
4import logging
5import Algorithmia
6
7# Logging
8logger = logging.getLogger(__name__)
9logger.setLevel(logging.INFO)
10
11logFile = logging.FileHandler(
12    'logs/twitter_pull_data.log')
13logFile.setLevel(logging.INFO)
14
15# Creating a custom log format for each line in the log file
16formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(message)s')
17logFile.setFormatter(formatter)
18logger.addHandler(logFile)
19
20# Pass in string query as sys.argv
21q_input = sys.argv[1]
22
23def pull_tweets():
24    input = {
25        "query": q_input,
26        "numTweets": "700",
27        "auth": {
28            "app_key": 'your_consumer_key',
29            "app_secret": 'your_consumer_secret_key',
30            "oauth_token": 'your_access_token',
31            "oauth_token_secret": 'your_access_token_secret'
32        }
33    }
34    client = Algorithmia.client('your_algorithmia_api_key')
35    algo = client.algo('twitter/RetrieveTweetsWithKeyword/0.1.3')
36
37    tweet_list = [{'user_id': record['user']['id'], 'retweet_count': record['retweet_count'],
38                   'text': record['text']} for record in algo.pipe(input).result]
39    return tweet_list
40
41def write_data():
42    # Write tweet records to csv for later data processing
43    data = pull_tweets()
44    filename = os.path.join(q_input.replace(' ', '-'))
45    try:
46        with open('data/{0}.csv'.format(filename), 'w') as f:
47            fieldnames = ['user_id', 'retweet_count', 'text']
48            writer = csv.DictWriter(f, fieldnames=fieldnames)
49            writer.writeheader()
50            for record in data:
51                writer.writerow(record)
52
53    except Exception as e:
54        logger.info(e)
55
56if __name__ == '__main__':
57    write_data()