youtube-comment-collector/youtube.py at master · UUDigitalHumanitieslab/youtube-comment-collector · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
import feedparser
import requests
import csv
import time
import ConfigParser


def main():
	config = ConfigParser.RawConfigParser()
	config.read('example.cfg')

	with open('comments.csv', 'wb') as csv_file:
		# write heading
		csv_file.write(u'\uFEFF'.encode('utf-8')) # the UTF-8 BOM to hint Excel we are using that...
		csv_writer = csv.writer(csv_file, dialect='excel', delimiter=';', quoting=csv.QUOTE_MINIMAL)
		heading = ['username', 'firstname', 'lastname', 'location', \
			'birthday', 'gender', 'aboutme', 'organizations', 'tagline', \
			'occupation', 'skills', 'statistics', 'replycount', 'activity_id', 'published', 'comment']
		csv_writer.writerow(heading)

		# get first feed
		url = 'https://gdata.youtube.com/feeds/api/videos/' + config.get('youtube', 'video_id') + '/comments'
		r = requests.get(url, params={'v': '2.1', 'orderby': 'published', 'max-results': '50'})

		# parse first feed
		i = 1
		print 'Now parsing feed ' + str(i) + '...'
		feed = feedparser.parse(r.url)
		csv_writer.writerows(dump_comments(feed, config))

		# get new comments while there's a next url
		next_url = get_next_url(feed)
		while next_url:
			i += 1
			print 'Now parsing feed ' + str(i) + '...'
			feed = get_next_feed(next_url)
			csv_writer.writerows(dump_comments(feed, config))
			next_url = get_next_url(feed)


def get_next_feed(next_url):
	"""Try a few times to retrieve the feed, wait a few seconds on error"""
	for attempt in range(5):
		feed = feedparser.parse(next_url)
		if feed.status == 403:
			sleep = 30 * (attempt + 1)
			print 'Waiting ' + str(sleep) + ' seconds before retrying'
			time.sleep(sleep)
		else:
			return feed
	else:
		raise ValueError("Feed unreachable")


def get_next_url(feed):
	"""Retrieve the next URL"""
	next_url = ''
	for l in feed.feed.links:
		if l.rel == 'next':
			next_url = l.href
	return next_url


def dump_comments(feed, config):
	comments = list()
	for entry in feed.entries:
		body = entry.content[0].value.replace('\n', ' ')

		author_feed = feedparser.parse(entry.author_detail.href)
		google_id = get_author_attr(author_feed, 'yt_googleplususerid')
		r = requests.get(config.get('googleplus', 'url') + google_id,
			params={'key' : config.get('googleplus', 'api_key')})
		j = r.json()

		firstname = j.get('name', {}).get('givenName', '')
		lastname = j.get('name', {}).get('familyName', '')
		location = get_author_attr(author_feed, 'yt_location')
		birthday = j.get('birthday', '')
		gender = j.get('gender', '')
		aboutme = j.get('aboutMe', '')
		organizations = str(j.get('organizations', ''))
		tagline = j.get('tagline', '')
		occupation = j.get('occupation', '')
		skills = j.get('skills', '')
		statistics = get_author_stats(author_feed)

		reply_count = entry.yt_replycount
		activity_id = entry.id.split(':')[-1]

		comment = [entry.author, firstname, lastname, location, \
			birthday, gender, aboutme, organizations, tagline, \
			occupation, skills, statistics, reply_count, activity_id, entry.published, body]
		comment = [i.encode('utf-8') for i in comment]
		comments.append(comment)
	return comments


def get_author_attr(author_feed, attribute):
	try:
		value = getattr(author_feed.entries[0], attribute)
	except (AttributeError, IndexError) as e:
		value = ''
	return value


def get_author_stats(author_feed):
	try:
		value = author_feed.entries[0].yt_statistics
	except (AttributeError, IndexError) as e:
		value = ''
	return str(value)


if __name__ == "__main__":
	main()