Fiddler 抓取页面截图
可以尝试用JSON解析，应该更方便（待解决）
[Python] 纯文本查看 复制代码 #!C:\Program Files\Python36 python
# -*- coding: UTF-8 -*-
"""
@author: 东方不败
QQ空间说说爬取程序
1. 登录QQ空间，获取cookie
2. 利用Fiddler抓取QQ空间说说的URL(Fiddler显示response直接解析成Json，但是自己尝试并未成功，带研究）
3. 利用requests库爬取页面数据
4. 利用re匹配页面内容，获取说说的createTime和content
5. 文件保存写入txt文件
"""
class Spider:
"""emotionSpider类"""
def __init__(self, cookie):
"""初始化"""
self.page = 0 # 记录爬取页数
self.counts = 0 # 记录爬取条数
self.url = "https://user.qzone.com" # 爬取的url
self.cookie = cookie # 登录cookie
def get(self, url):
"""利用requests爬取页面内容，并转化成str返回"""
import requests # 导入requests库
"""组合headers"""
headers = {
"Cookie": self.cookie,
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
}
self.url = url
# 特别提醒：verify=False参数必不可少，关闭HTTPS验证，否则可能报错(踩过的坑)
try:
response = requests.get(self.url, headers=headers, verify=False).content
html = str(response, 'utf-8')
return html
except Exception as err:
print("Error:" + err)
def analyse(self, content):
"""利用re匹配createTime和con"""
import re
con = re.findall("\"con\":\"([^\"]+)", content)
createTime = re.findall("\"createTime\":\"([^\"]+)", content)
emotion = tuple(zip(createTime, con))
return emotion
def saveEmotin(self, emotion):
"""保存emotion到TXT文件"""
for i in emotion:
# 参数encoding='utf-8'要带，避免Unicode编码解析保存(踩过的坑)
try:
with open("emotion.txt", "a+", encoding='utf-8') as f:
f.write(i[0])
f.write(":")
f.write(i[1])
f.write("\n")
self.counts += 1
print("已经保存 "+str(self.counts)+" 条说说")
except Exception as err:
print("Error:"+err)
self.page += 1
print("第"+str(self.page)+"页说说爬取完毕")
print("#"*50)
if __name__ == "__main__":
pos = 0
url = ":" # Fiddler 抓取的url
cookie = ":" # cookie
spider = Spider(cookie)
while True:
emotion = spider.analyse(spider.get(url))
if len(emotion) > 0:
spider.saveEmotin(emotion)
pos += 20 # 通过pos控制下一页
url = "https://user.qzone.qq.com/proxy/domain/taotao.qq.com/cgi-bin/emotion_cgi_msglist_v6?uin=65&ftype=0&sort=0&pos="+str(pos)+"&num=20"
else:
print("+"*50)
print("爬取结束!!!共计爬取"+str(spider.counts)+"条说说")
break
爬取成果
| 
[复制链接]
发表于 2019-9-11 18:11
|
发表于 2019-11-21 09:23
