fetch_medium_responses.py

import json
import requests
import traceback

def fetch_post(post_url):
	try:
		res = requests.get(post_url + '?format=json').text
		res = res[res.find('{'):]
		data = json.loads(res)
		return data
	except:
		traceback.print_exc()


def fetch_post_by_id(post_id):
	try:
		post_url = 'http://medium.com/post/{}?format=json'.format(post_id)
		res = requests.get(post_url).text
		res = res[res.find('{'):]
		data = json.loads(res)
		return data
	except:
		traceback.print_exc()


def fetch_comment_list(post_id):
	req_url = 'https://medium.com/_/api/posts/{}/responsesStream'.format(post_id)
	res = requests.get(req_url).text
	res = res[res.find('{'):]

	data = json.loads(res)

	comment_ids = []
	for it in data['payload']['streamItems']:
		if 'postPreview' in it:
			comment_ids.append(it['postPreview']['postId'])
	
	return comment_ids


def fetch_comment_content(comment_id):
	return fetch_post_by_id(comment_id)


def fetch_comments(post_id):
	comment_ids = fetch_comment_list(post_id)
	for cid in comment_ids:
		comment = fetch_post_by_id(cid)
		yield(comment)


if __name__ == '__main__':
	post_url = 'https://medium.learningbyshipping.com/nikon-versus-canon-a-story-of-technology-change-45777098038c'

	post_data = fetch_post(post_url)
	post_id = post_data['payload']['value']['id']
	print('Post id:', post_id)
	
	comments = []
	for comm in fetch_comments(post_id):
		comm_text = []
		for p in comm['payload']['value']['content']['bodyModel']['paragraphs']:
			comm_text.append(p['text'])
		comments.append((comm['payload']['value']['id'], comm_text))
		print(comments[-1])
		print('-----')