ConferenceTweetMapper/archive.py at master · identitymonk/ConferenceTweetMapper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import tweepy
import json
import time
import configparser
import argparse

#Configuration initialization
config = configparser.ConfigParser()

parser = argparse.ArgumentParser(description='Export tweets that match the search query')
subparsers = parser.add_subparsers(help='Add configuration from Ini file or through arguments')

parser_file = subparsers.add_parser('file', help='Adding configuration from a file (Default: Ini/Default.ini)')
parser_file.add_argument('cmd', const='file', action='store_const')
parser_file.add_argument('-i', '--ini_file', default='Ini/Default.ini', help='Path to the Ini file  (Default: Ini/Default.ini)')

parser_line = subparsers.add_parser('line', help='Adding configuration from a arguments in the command line')
parser_line.add_argument('cmd', const='line', action='store_const')
parser_line.add_argument('-s', '--search', required=True, help='Twitter search filter')
parser_line.add_argument('-ck', '--consumer_key', required=True, help='Twitter consumer key obtained from your Twitter account')
parser_line.add_argument('-cs', '--consumer_secret', required=True, help='Twitter consumer secret obtained from your Twitter account')
parser_line.add_argument('-ak', '--access_key', required=True, help='Twitter access key obtained from your Twitter account')
parser_line.add_argument('-as', '--access_secret', required=True, help='Twitter access_secret obtained from your Twitter account')
parser_line.add_argument('-o', '--output_filename', required=True, help='Name of the results output file')
parser_line.add_argument('-b', '--backup_ini_file_name', help='Name of the Ini file to backup from this request parameters')

args = parser.parse_args()
elements = vars(args)

#Configuration

if elements['cmd'] == 'file':
    #mettre une condition de verification de i, si i n'existe pas, i est forcement dans Ini/
    config.read(elements['ini_file'])

    output_filenane = config['DEFAULT']['output_filename']
    search_query = config['DEFAULT']['search']

    consumer_key = config['Twitter']['consumer_key']
    consumer_secret = config['Twitter']['consumer_secret']
    access_key = config['Twitter']['access_key']
    access_secret = config['Twitter']['access_secret']

elif elements['cmd'] == 'line':
    output_filenane = elements['output_filename']
    search_query = elements['search']

    consumer_key = elements['consumer_key']
    consumer_secret = elements['consumer_secret']
    access_key = elements['access_key']
    access_secret = elements['access_secret']

    if elements['backup_ini_file_name'] != None:
        if elements['backup_ini_file_name'] !=  'Ini/Default.ini':
            config['DEFAULT'] = {'output_filename': output_filenane, 'search': search_query}
            config['Twitter'] =  {'consumer_key': consumer_key, 'consumer_secret': consumer_secret, 'access_key': access_key, 'access_secret': access_secret }
            with open('Ini/' + elements['backup_ini_file_name'], 'w') as configFile:
                config.write(configFile)
        else:
            print('You cannot overwrite the Default Ini file')
            exit
else:
    print('Sorry this command is invalid')
    exit

print(search_query)

#Twitter connection
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
#refer http://docs.tweepy.org/en/v3.2.0/api.html#API
#tells tweepy.API to automatically wait for rate limits to replenish

#Twitter search terms
users =tweepy.Cursor(api.search,q=search_query, tweet_mode='extended').items()
count = 0
errorCount=0

#Result file
file = open(output_filenane, 'w')

#Processing
data = []
while True:
    try:
        user = next(users)
        count += 1
        #use count-break during dev to avoid twitter restrictions
        #if (count>10):
        #    break
    except tweepy.TweepError:
        #catches TweepError when rate limiting occurs, sleeps, then restarts.
        #nominally 15 minnutes, make a bit longer to avoid attention.
        print("sleeping....")
        time.sleep(60*16)
        user = next(users)
    except StopIteration:
        break
    try:
        print("Writing to JSON tweet number:"+str(count))
        #json.dump(user._json,file,sort_keys = True,indent = 4)
        data.append(user._json)
    except UnicodeEncodeError:
        errorCount += 1
        print("UnicodeEncodeError,errorCount ="+str(errorCount))

json.dump(data,file,sort_keys = True,indent = 4)
print("completed, errorCount ="+str(errorCount)+" total tweets="+str(count))

    #todo: write users to file, search users for interests, locations etc.

"""
http://docs.tweepy.org/en/v3.5.0/api.html?highlight=tweeperror#TweepError
NB: RateLimitError inherits TweepError.
http://docs.tweepy.org/en/v3.2.0/api.html#API  wait_on_rate_limit & wait_on_rate_limit_notify
NB: possibly makes the sleep redundant but leave until verified.

"""