text-mining/testdata.py at master · superleggera-21/text-mining · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# The actual code begins here
# This file is intended to load everything downloaded from loaddata.py, preventing user getting banned from IMDB
# The code is written to see what are some key words of the reviews from critics and normal viewers
# And to see what are some of the differences
# The second task is to asses the people's emotion vs. actual score given

# First, we need to load back everything we dumped to folder via pickle.

import pickle
print('loading data...')

with open('movienumbers.pickle','rb') as input_file:
    movienumbers = pickle.load(input_file)

with open('ratings.pickle','rb') as input_file:
    ratings = pickle.load(input_file)

with open('userratings.pickle','rb') as input_file:
    userratings = pickle.load(input_file)

with open('metaratings.pickle','rb') as input_file:
    metaratings = pickle.load(input_file)

print('Pickled data successfully loaded.')

# then, it's time to use nltp to see the score of the critics vs. viewers on movies

from nltk.sentiment.vader import SentimentIntensityAnalyzer

# print(movienumbers)
# print(ratings)
# print(userratings)
# print(metaratings)

# Userratings is a dictionary in ways like this "ttxxxxxx : [reviews1, reviews2,...]"

# print(userratings['tt0111161'])
#
# print(metaratings['tt0111161'])
# print(ratings['tt0111161'])

userscore = {}
for movieid, reviews in userratings.items():
    score = 0
    for eachreviews in reviews:
        score += SentimentIntensityAnalyzer().polarity_scores(eachreviews)['compound']
    average = score / len(reviews)
    userscore[movieid] = average

print(userscore)

# Meta ratings is a dictionary in ways like this "ttxxxxxx : [reviews1, reviews2,...]"


criticsscore = {}
for movieid, reviews in metaratings.items():
    score_1 = 0
    for eachreviews in reviews:
        score_1 += SentimentIntensityAnalyzer().polarity_scores(eachreviews)['compound']
    average = score_1 / len(reviews)
    criticsscore[movieid] = average

print(criticsscore)


# Question 1: Are critics always more positive than the audience?

counter = 0
for movieid, score in userscore.items():
    if movieid in criticsscore and criticsscore[movieid] > score:
        counter += 1
    else:
        counter += 0

# Displaying results to question 1
print("Critics overpraise these movies " + str(counter) + " times more than normal viewers out of "
      + str(len(criticsscore)) + " movies in total.")
if counter < (len(criticsscore) - counter):
    print("Because the critics overpraise less than half of the movies sampled here, the critics are more refrained "
          "than the users on IMDb.")
else:
    print("Because the critics overpraise no less than half of the movies sampled here, the critics are less refrained "
          "than the users on IMDb.")

# Question 2: Is the IMDB score closer to the users' sentiment? Or the critics.

useriscloser = 0
criticiscloser = 0
for movieid, score in criticsscore.items():
    if abs(userscore[movieid] - (ratings[movieid])/10) > abs(score - (ratings[movieid]/10)):
        useriscloser += 1
    else:
        criticiscloser += 1

# Displaying results to question 2
print("Critics are more closer to the ratings for " + str(criticiscloser) +
      " times, while normal viewers are closer " + str(useriscloser) + " times out of " +
      str(len(criticsscore)) + " movies in total.")

if useriscloser > criticiscloser:
    print("Because the more movies have users resembling closer to the rating, the critics are less accurate "
          "than the users on IMDb.")
else:
    print("Because the more movies have critics resembling closer to the rating, the users are less accurate "
          "than the users on IMDb.")