Python爬取知乎内容
本帖最后由 liuye 于 2021-6-18 22:04 编辑题主的数据科学导论作业，关于舆情分析负责信息爬取。
可能会对大家有点帮助，如果有哪写的不太好的地方，希望可以告诉我
如果不想看，直接用的话把js代码命名为 g_encrypt.js 和python代码放在同一级目录就可以了（要搭建nodejs环境，具体可以参考Nodejs安装及环境配置 - 简书 (jianshu.com)）
import hashlib
import json
import re
import time
import pandas as pd
import execjs
import requests
import urllib.parse
def stampTotime(timestamp):
timeArray = time.localtime(timestamp)
otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
return otherStyleTime
def strToUrl(str):
return urllib.parse.quote(str)
def getDataFromKey(keyword):
keywordUrl= strToUrl(keyword)
url = f"/api/v4/search_v3?t=general&q={keywordUrl}&correction=1&offset=0&limit=40&lc_idx=0&show_all_topics=0"
referer = "https://www.zhihu.com/search?type=content&q="+keywordUrl
f = "+".join(["3_2.0", url, '"AFAaXJyCNBKPTmvMea6p4UCHCN4Ji2J2wk8=|1605541545"'])
fmd5 = hashlib.new('md5', f.encode()).hexdigest()
with open('g_encrypt.js', 'r') as f:
ctx1 = execjs.compile(f.read(), cwd=r'D:\program\node_modules')
encrypt_str = ctx1.call('b', fmd5)
session = requests.session()
session.headers = {
"user - agent": "Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 90.0.4430.93Safari / 537.36",
"referer": "https://www.zhihu.com/",
}
# "encodedParams": "=="},"
response = session.get("https://www.zhihu.com/search?type=content&q="+keywordUrl).text
x_ab_pb = re.findall('encodedParams":"(.*?)"},"trigge', response, re.DOTALL).replace("\\u002F","/")
headers = {
"authority": "www.zhihu.com",
"method": "GET",
"referer": referer,
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36",
"x-ab-pb": x_ab_pb,
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same - origin",
'cookie': 'd_c0 = "AFAaXJyCNBKPTmvMea6p4UCHCN4Ji2J2wk8=|1605541545";',
"x-ab-param": "tp_zrec = 1;top_test_4_liguangyi = 1;li_sp_mqbk = 0;li_edu_page = old;zr_expslotpaid = 1;tp_contents = 2;li_panswer_topic = 0;se_ffzx_jushen1 = 0;zr_slotpaidexp = 1;pf_adjust = 1;li_vip_verti_search = 0;tp_dingyue_video = 0;li_paid_answer_exp = 0;pf_noti_entry_num = 2;tp_topic_style = 0;qap_question_visitor = 0;qap_question_author = 0",
"x-api-version": "3.0.91",
"x-zse-83": "3_2.0",
"x-zse-86": "2.0_%s" % encrypt_str,
}
r = requests.get("https://www.zhihu.com" + url, headers=headers)
print(r.text)
return r.text
def getData(data_json,i):
title = data_json["data"]["highlight"]["title"]
description = data_json["data"]["highlight"]["description"]
content = data_json["data"]["object"]["content"]
voteup_count = data_json["data"]["object"]["voteup_count"]
comment_count = data_json["data"]["object"]["comment_count"]
created_time = stampTotime(int(data_json["data"]["object"]["created_time"]))
id = data_json["data"]["object"]["id"]
question = data_json["data"]["object"]["question"]["name"]
question_id = data_json["data"]["object"]["question"]["id"]
print((title,description,content,voteup_count,created_time,id,question,question_id))
return (title,description,content,voteup_count,comment_count,created_time,id,question,question_id)
def getAllData(keyword):
page_data = []
column_name = ('标题', '简介', '内容', '点赞数', '评论数','创建时间', '文章id', '问题', '问题id')
data_json=json.loads(getDataFromKey(keyword))
for i in range(0,30):
try:
page_data.append(dict(zip(column_name, getData(data_json,i))))
except:
print(f"索引异常，跳过本次{i}")
return page_data
def packDateToexcel(all_data):
df = pd.DataFrame(all_data)
df.to_excel('zhihu'+time.strftime("%Y-%m-%d-%H-%M",time.localtime(time.time()))+'.xlsx', index=False)
if __name__ == '__main__':
all_data = []
# 关键词
keywords = ["耐克", "Nike", "李宁", "LI-NING", "阿迪达斯", "adidas", "安踏", "ANTA", "新疆棉", "棉花", "新疆", "国产崛起"]
for keyword in keywords:
one_data = getAllData(keyword)
all_data += one_data
packDateToexcel(all_data)
js文件
const jsdom = require("jsdom");
const { JSDOM } = jsdom;
const dom = new JSDOM(`<!DOCTYPE html><p>Hello world</p>`);
window = dom.window;
document = window.document;
XMLHttpRequest = window.XMLHttpRequest;
var exports = {}
function t(e) {
return (t = "function" == typeof Symbol && "symbol" == typeof Symbol.A ? function(e) {
return typeof e
}
: function(e) {
return e && "function" == typeof Symbol && e.constructor === Symbol && e !== Symbol.prototype ? "symbol" : typeof e
}
)(e)
}
Object.defineProperty(exports, "__esModule", {
value: !0
});
var A = "2.0"
, __g = {};
function s() {}
function i(e) {
this.t = (2048 & e) >> 11,
this.s = (1536 & e) >> 9,
this.i = 511 & e,
this.h = 511 & e
}
function h(e) {
this.s = (3072 & e) >> 10,
this.h = 1023 & e
}
function a(e) {
this.a = (3072 & e) >> 10,
this.c = (768 & e) >> 8,
this.n = (192 & e) >> 6,
this.t = 63 & e
}
function c(e) {
this.s = e >> 10 & 3,
this.i = 1023 & e
}
function n() {}
function e(e) {
this.a = (3072 & e) >> 10,
this.c = (768 & e) >> 8,
this.n = (192 & e) >> 6,
this.t = 63 & e
}
function o(e) {
this.h = (4095 & e) >> 2,
this.t = 3 & e
}
function r(e) {
this.s = e >> 10 & 3,
this.i = e >> 2 & 255,
this.t = 3 & e
}
s.prototype.e = function(e) {
e.o = !1
}
,
i.prototype.e = function(e) {
switch (this.t) {
case 0:
e.r = this.i;
break;
case 1:
e.r = e.k
}
}
,
h.prototype.e = function(e) {
e.k = e.r
}
,
a.prototype.e = function(e) {
switch (this.t) {
case 0:
e.r = e.r + e.r;
break;
case 1:
e.r = e.r - e.r;
break;
case 2:
e.r = e.r * e.r;
break;
case 3:
e.r = e.r / e.r;
break;
case 4:
e.r = e.r % e.r;
break;
case 5:
e.r = e.r == e.r;
break;
case 6:
e.r = e.r >= e.r;
break;
case 7:
e.r = e.r || e.r;
break;
case 8:
e.r = e.r && e.r;
break;
case 9:
e.r = e.r !== e.r;
break;
case 10:
e.r = t(e.r);
break;
case 11:
e.r = e.rin e.r;
break;
case 12:
e.r = e.r > e.r;
break;
case 13:
e.r = -e.r;
break;
case 14:
e.r = e.r < e.r;
break;
case 15:
e.r = e.r & e.r;
break;
case 16:
e.r = e.r ^ e.r;
break;
case 17:
e.r = e.r << e.r;
break;
case 18:
e.r = e.r >>> e.r;
break;
case 19:
e.r = e.r | e.r;
break;
case 20:
e.r = !e.r
}
}
,
c.prototype.e = function(e) {
e.Q.push(e.C),
e.B.push(e.k),
e.C = e.r,
e.k = [];
for (var t = 0; t < this.i; t++)
e.k.unshift(e.f.pop());
e.g.push(e.f),
e.f = []
}
,
n.prototype.e = function(e) {
e.C = e.Q.pop(),
e.k = e.B.pop(),
e.f = e.g.pop()
}
,
e.prototype.e = function(e) {
switch (this.t) {
case 0:
e.u = e.r >= e.r;
break;
case 1:
e.u = e.r <= e.r;
break;
case 2:
e.u = e.r > e.r;
break;
case 3:
e.u = e.r < e.r;
break;
case 4:
e.u = e.r == e.r;
break;
case 5:
e.u = e.r != e.r;
break;
case 6:
e.u = e.r;
break;
case 7:
e.u = !e.r
}
}
,
o.prototype.e = function(e) {
switch (this.t) {
case 0:
e.C = this.h;
break;
case 1:
e.u && (e.C = this.h);
break;
case 2:
e.u || (e.C = this.h);
break;
case 3:
e.C = this.h,
e.w = null
}
e.u = !1
}
,
r.prototype.e = function(e) {
switch (this.t) {
case 0:
for (var t = [], n = 0; n < this.i; n++)
t.unshift(e.f.pop());
e.r = e.r(t, t);
break;
case 1:
for (var r = e.f.pop(), o = [], i = 0; i < this.i; i++)
o.unshift(e.f.pop());
e.r = e.r(o, o);
break;
case 2:
for (var a = [], c = 0; c < this.i; c++)
a.unshift(e.f.pop());
e.r = new e.r(a,a)
}
}
;
var k = function(e) {
for (var t = 66, n = [], r = 0; r < e.length; r++) {
var o = 24 ^ e.charCodeAt(r) ^ t;
n.push(String.fromCharCode(o)),
t = o
}
return n.join("")
};
function Q(e) {
this.t = (4095 & e) >> 10,
this.s = (1023 & e) >> 8,
this.i = 1023 & e,
this.h = 63 & e
}
function C(e) {
this.t = (4095 & e) >> 10,
this.a = (1023 & e) >> 8,
this.c = (255 & e) >> 6
}
function B(e) {
this.s = (3072 & e) >> 10,
this.h = 1023 & e
}
function f(e) {
this.h = 4095 & e
}
function g(e) {
this.s = (3072 & e) >> 10
}
function u(e) {
this.h = 4095 & e
}
function w(e) {
this.t = (3840 & e) >> 8,
this.s = (192 & e) >> 6,
this.i = 63 & e
}
function G() {
this.r = ,
this.C = 0,
this.Q = [],
this.k = [],
this.B = [],
this.f = [],
this.g = [],
this.u = !1,
this.G = [],
this.b = [],
this.o = !1,
this.w = null,
this.U = null,
this.F = [],
this.R = 0,
this.J = {
0: s,
1: i,
2: h,
3: a,
4: c,
5: n,
6: e,
7: o,
8: r,
9: Q,
10: C,
11: B,
12: f,
13: g,
14: u,
15: w
}
}
Q.prototype.e = function(e) {
switch (this.t) {
case 0:
e.f.push(e.r);
break;
case 1:
e.f.push(this.i);
break;
case 2:
e.f.push(e.k);
break;
case 3:
e.f.push(k(e.b))
}
}
,
C.prototype.e = function(A) {
switch (this.t) {
case 0:
var t = A.f.pop();
A.r = A.r;
break;
case 1:
var s = A.f.pop()
, i = A.f.pop();
A.r = i;
break;
case 2:
var h = A.f.pop();
A.r = eval(h)
}
}
,
B.prototype.e = function(e) {
e.r = k(e.b)
}
,
f.prototype.e = function(e) {
e.w = this.h
}
,
g.prototype.e = function(e) {
throw e.r
}
,
u.prototype.e = function(e) {
var t = this
, n = ;
e.k.forEach(function(e) {
n.push(e)
});
var r = function(r) {
var o = new G;
return o.k = n,
o.k = r,
o.v(e.G, t.h, e.b, e.F),
o.r
};
r.toString = function() {
return "() { }"
}
,
e.r = r
}
,
w.prototype.e = function(e) {
switch (this.t) {
case 0:
for (var t = {}, n = 0; n < this.i; n++) {
var r = e.f.pop();
t = r
}
e.r = t;
break;
case 1:
for (var o = [], i = 0; i < this.i; i++)
o.unshift(e.f.pop());
e.r = o
}
}
,
G.prototype.D = function(e) {
console.log(window.atob(e));
for (var t = window.atob(e), n = t.charCodeAt(0) << 8 | t.charCodeAt(1), r = [], o = 2; o < n + 2; o += 2)
r.push(t.charCodeAt(o) << 8 | t.charCodeAt(o + 1));
this.G = r;
for (var i = [], a = n + 2; a < t.length; ) {
var c = t.charCodeAt(a) << 8 | t.charCodeAt(a + 1)
, s = t.slice(a + 2, a + 2 + c);
i.push(s),
a += c + 2
}
this.b = i
}
,
G.prototype.v = function(e, t, n) {
for (t = t || 0,
n = n || [],
this.C = t,
"string" == typeof e ? this.D(e) : (this.G = e,
this.b = n),
this.o = !0,
this.R = Date.now(); this.o; ) {
var r = this.G;
if ("number" != typeof r)
break;
var o = Date.now();
if (500 < o - this.R)
return;
this.R = o;
try {
this.e(r)
} catch (e) {
this.U = e,
this.w && (this.C = this.w)
}
}
}
,
G.prototype.e = function(e) {
var t = (61440 & e) >> 12;
new this.J(e).e(this)
}
,
(new G).v("AxjgB5MAnACoAJwBpAAAABAAIAKcAqgAMAq0AzRJZAZwUpwCqACQACACGAKcBKAAIAOcBagAIAQYAjAUGgKcBqFAuAc5hTSHZAZwqrAIGgA0QJEAJAAYAzAUGgOcCaFANRQ0R2QGcOKwChoANECRACQAsAuQABgDnAmgAJwMgAGcDYwFEAAzBmAGcSqwDhoANECRACQAGAKcD6AAGgKcEKFANEcYApwRoAAxB2AGcXKwEhoANECRACQAGAKcE6AAGgKcFKFANEdkBnGqsBUaADRAkQAkABgCnBagAGAGcdKwFxoANECRACQAGAKcGKAAYAZx+rAZGgA0QJEAJAAYA5waoABgBnIisBsaADRAkQAkABgCnBygABoCnB2hQDRHZAZyWrAeGgA0QJEAJAAYBJwfoAAwFGAGcoawIBoANECRACQAGAOQALAJkAAYBJwfgAlsBnK+sCEaADRAkQAkABgDkACwGpAAGAScH4AJbAZy9rAiGgA0QJEAJACwI5AAGAScH6AAkACcJKgAnCWgAJwmoACcJ4AFnA2MBRAAMw5gBnNasCgaADRAkQAkABgBEio0R5EAJAGwKSAFGACcKqAAEgM0RCQGGAYSATRFZAZzshgAtCs0QCQAGAYSAjRFZAZz1hgAtCw0QCQAEAAgB7AtIAgYAJwqoAASATRBJAkYCRIANEZkBnYqEAgaBxQBOYAoBxQEOYQ0giQKGAmQABgAnC6ABRgBGgo0UhD/MQ8zECALEAgaBxQBOYAoBxQEOYQ0gpEAJAoYARoKNFIQ/zEPkAAgChgLGgkUATmBkgAaAJwuhAUaCjdQFAg5kTSTJAsQCBoHFAE5gCgHFAQ5hDSCkQAkChgBGgo0UhD/MQ+QACAKGAsaCRQCOYGSABoAnC6EBRoKN1AUEDmRNJMkCxgFGgsUPzmPkgAaCJwvhAU0wCQFGAUaCxQGOZISPzZPkQAaCJwvhAU0wCQFGAUaCxQMOZISPzZPkQAaCJwvhAU0wCQFGAUaCxQSOZISPzZPkQAaCJwvhAU0wCQFGAkSAzRBJAlz/B4FUAAAAwUYIAAIBSITFQkTERwABi0GHxITAAAJLwMSGRsXHxMZAAk0Fw8HFh4NAwUABhU1EBceDwAENBcUEAAGNBkTGRcBAAFKAAkvHg4PKz4aEwIAAUsACDIVHB0QEQ4YAAsuAzs7AAoPKToKDgAHMx8SGQUvMQABSAALORoVGCQgERcCAxoACAU3ABEXAgMaAAsFGDcAERcCAxoUCgABSQAGOA8LGBsPAAYYLwsYGw8AAU4ABD8QHAUAAU8ABSkbCQ4BAAFMAAktCh8eDgMHCw8AAU0ADT4TGjQsGQMaFA0FHhkAFz4TGjQsGQMaFA0FHhk1NBkCHgUbGBEPAAFCABg9GgkjIAEmOgUHDQ8eFSU5DggJAwEcAwUAAUMAAUAAAUEADQEtFw0FBwtdWxQTGSAACBwrAxUPBR4ZAAkqGgUDAwMVEQ0ACC4DJD8eAx8RAAQ5GhUYAAFGAAAABjYRExELBAACWhgAAVoAQAg/PTw0NxcQPCQ5C3JZEBs9fkcnDRcUAXZia0Q4EhQgXHojMBY3MWVCNT0uDhMXcGQ7AUFPHigkQUwQFkhaAkEACjkTEQspNBMZPC0ABjkTEQsrLQ==");
function b(e) {
console.log(e);
console.log(encodeURIComponent(e));
return __g._encrypt(encodeURIComponent(e))
};
大神，帮我看看这是哪里出现了错了，感激不尽：runfile('D:/学习/Python电子书/Python_Study/测试4.py', wdir='D:/学习/Python电子书/Python_Study')
Traceback (most recent call last):
File "<input>", line 1, in <module>
File "D:\软件\pycharm_205063\Pycharm_Professional_2021.1.3_Protable\plugins\python\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
pydev_imports.execfile(filename, global_vars, local_vars)# execute the script
File "D:\软件\pycharm_205063\Pycharm_Professional_2021.1.3_Protable\plugins\python\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "D:/学习/Python电子书/Python_Study/测试4.py", line 98, in <module>
one_data = getAllData(keyword)
File "D:/学习/Python电子书/Python_Study/测试4.py", line 77, in getAllData
data_json = json.loads(getDataFromKey(keyword))
File "D:/学习/Python电子书/Python_Study/测试4.py", line 30, in getDataFromKey
encrypt_str = ctx1.call('b', fmd5)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python310\lib\site-packages\execjs\_abstract_runtime_context.py", line 37, in call
return self._call(name, *args)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python310\lib\site-packages\execjs\_external_runtime.py", line 92, in _call
return self._eval("{identifier}.apply(this, {args})".format(identifier=identifier, args=args))
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python310\lib\site-packages\execjs\_external_runtime.py", line 78, in _eval
return self.exec_(code)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python310\lib\site-packages\execjs\_abstract_runtime_context.py", line 18, in exec_
return self._exec_(source)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python310\lib\site-packages\execjs\_external_runtime.py", line 88, in _exec_
return self._extract_result(output)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python310\lib\site-packages\execjs\_external_runtime.py", line 167, in _extract_result
raise ProgramError(value)
execjs._exceptions.ProgramError: SyntaxError: 语法错误
至今还没办法正常运行，需要pip install 好几个模块。
不知道哪里出错了，调试的时候告知是语法错误。
已经将js文件放到同一个目录，然后名字也符合要求，但是发现似乎不行，后来在源代码强制改为绝对目录才可以继续往下运行，
然后继续运行的时候就到了这一步告知语法错误，
encrypt_str = ctx1.call('b', fmd5)
不知道为何？
打算继续研究分析中。
现在大概看的源代码是找寻关键词
“”"耐克", "Nike", "李宁", "LI-NING", "阿迪达斯", "adidas", "安踏", "ANTA", "新疆棉", "棉花", "新疆", "国产崛起"“”
这几个在知乎的关键词。
感谢分享 好东西写的很好.我都不会js 用python 还写不好 哈哈 谢谢你提供 知乎现在不需要登录就 能 拿数据了嘛- - 感觉我落伍好久了 大佬可以发下你用到的模块信息吗 nastroy 发表于 2021-6-17 16:28
大佬可以发下你用到的模块信息吗
就是用到了那些模块。 感谢分享！！！！！！！ 很棒啊，拿了。 涉及到环境问题，不说清楚nodejs环境搭建会遇到很多坑。