【Python】如何获取知乎最有价值的内容

一 前言 
   相信大部分能看到这篇blog的人都听说过知乎吧？如果你没有听说过，那么链接在这里   知乎  作为一个知乎er，为了更加深入的理解“xxx 是一种什么体验”（的图片），为了践行 “技术改变生活”（实则有些wuliao) ，使用requsets 爬取知乎中最优价值的内容，本文本着探索的精神，写一段获取内容的python程序。

二 践行

1. #!/usr/bin/env python
2. #-*- coding:utf-8 -*-
3. import re
4. import requests
5. import os
6. from urlparse import urlsplit
7. from os.path import basename
8. def getHtml(url):
9.     session = requests.Session()
10.     # 模拟浏览器访问
11.     header = {
12.         'User-Agent': "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
13.         'Accept-Encoding': 'gzip, deflate'}
14.     res = session.get(url, headers=header)
15.     if res.status_code == 200:
16.         content = res.content
17.     else:
18.         content = ''
19.     return content
20. def mkdir(path):
21.     if not os.path.exists(path):
22.         print '新建文件夹:', path
23.         os.makedirs(path)
24.         return True
25.     else:
26.         print u"图片存放于:", os.getcwd() + os.sep + path
27.         return False
28. def download_pic(img_lists, dir_name):
29.     print "一共有 {num} 张照片".format(num=len(img_lists))
30.     for image_url in img_lists:
31.         response = requests.get(image_url, stream=True)
32.         if response.status_code == 200:
33.             image = response.content
34.         else:
35.             continue
36.         file_name = dir_name + os.sep + basename(urlsplit(image_url)[2])
37.         try:
38.             with open(file_name, "wb") as picture:
39.                 picture.write(image)
40.         except IOError:
41.             print("IO Error\n")
42.             return
43.         finally:
44.             picture.close
45.             print "下载 {pic_name} 完成!".format(pic_name=file_name)
46. def getAllImg(html):
47.     # 利用正则表达式把源代码中的图片地址过滤出来
48.     #reg = r'data-actualsrc="(.*?)">'
49.     reg = r'https://pic\d.zhimg.com/[a-fA-F0-9]{5,32}_\w+.jpg'
50.     imgre = re.compile(reg, re.S)
51.     tmp_list = imgre.findall(html) # 表示在整个网页中过滤出所有图片的地址，放在imglist中
52.     # 清理掉头像和去重 获取data-original的内容
53.     tmp_list = list(set(tmp_list)) # 去重
54.     imglist = []
55.     for item in tmp_list:
56.         if item.endswith('r.jpg'):
57.             img_list.append(item)
58.     print 'num : %d' % (len(imglist))
59.     return imglist
60. if __name__ == '__main__':
61.     question_id = 35990613
62.     zhihu_url = "https://www.zhihu.com/question/{qid}".format(qid=question_id)
63.     html_content = getHtml(zhihu_url)
64.     path = 'zhihu_pic'
65.     mkdir(path) # 创建本地文件夹
66.     img_list = getAllImg(html_content) # 获取图片的地址列表
67.     download_pic(img_list, path)       # 保存图片

本代码还存在一些不足的地方，无法完全获取全部的图片，需要在兼容 自动点击 ”更多“ 加载更多答案。
代码第二版解决了第一版代码中不能自动加载的问题。

1. #!/usr/bin/env python
2. #-*- coding:utf-8 -*-
3. import re
4. import requests
5. import os
6. from urlparse import urlsplit
7. from os.path import basename
8. headers = {
9.     'User-Agent': "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
10.     'Accept-Encoding': 'gzip, deflate'}
11. def mkdir(path):
12.     if not os.path.exists(path):
13.         print '新建文件夹:', path
14.         os.makedirs(path)
15.         return True
16.     else:
17.         print u"图片存放于:", os.getcwd() + os.sep + path
18.         return False
19. def download_pic(img_lists, dir_name):
20.     print "一共有 {num} 张照片".format(num=len(img_lists))
21.     for image_url in img_lists:
22.         response = requests.get(image_url, stream=True)
23.         if response.status_code == 200:
24.             image = response.content
25.         else:
26.             continue
27.         file_name = dir_name + os.sep + basename(urlsplit(image_url)[2])
28.         try:
29.             with open(file_name, "wb") as picture:
30.                 picture.write(image)
31.         except IOError:
32.             print("IO Error\n")
33.             continue
34.         finally:
35.             picture.close
36.             print "下载 {pic_name} 完成!".format(pic_name=file_name)
37. def get_image_url(qid, headers):
38.     # 利用正则表达式把源代码中的图片地址过滤出来
39.     #reg = r'data-actualsrc="(.*?)">'
40.     tmp_url = "https://www.zhihu.com/node/QuestionAnswerListV2"
41.     size = 10
42.     image_urls = []
43.     session = requests.Session()
44.     # 利用循环自动完成需要点击 “更多” 获取所有答案，每个分页作为一个answer集合。
45.     while True:
46.         postdata = {'method': 'next', 'params': '{"url_token":' +
47.                     str(qid) + ',"pagesize": "10",' + '"offset":' + str(size) + "}"}
48.         page = session.post(tmp_url, headers=headers, data=postdata)
49.         ret = eval(page.text)
50.         answers = ret['msg']
51.         size += 10
52.         if not answers:
53.             print "图片URL获取完毕, 页数: ", (size - 10) / 10
54.             return image_urls
55.         #reg = r'https://pic\d.zhimg.com/[a-fA-F0-9]{5,32}_\w+.jpg'
56.         imgreg = re.compile('data-original="(.*?)"', re.S)
57.         for answer in answers:
58.             tmp_list = []
59.             url_items = re.findall(imgreg, answer)
60.             for item in url_items: # 这里去掉得到的图片URL中的转义字符'\\'
61.                 image_url = item.replace("\\", "")
62.                 tmp_list.append(image_url)
63.             # 清理掉头像和去重 获取data-original的内容
64.             tmp_list = list(set(tmp_list)) # 去重
65.             for item in tmp_list:
66.                 if item.endswith('r.jpg'):
67.                     print item
68.                     image_urls.append(item)
69.         print 'size: %d, num : %d' % (size, len(image_urls))
70. if __name__ == '__main__':
71.     question_id = 26037846
72.     zhihu_url = "https://www.zhihu.com/question/{qid}".format(qid=question_id)
73.     path = 'zhihu_pic'
74.     mkdir(path) # 创建本地文件夹
75.     img_list = get_image_url(question_id, headers) # 获取图片的地址列表
76.     download_pic(img_list, path) # 保存图片