快视频资源单线程爬取仅供学习
萌新第一次发帖，还请各位大佬多多关注！做了一个单线程的快视频网站资源爬取工具（虽然没什么卵用，但踩了很多坑），先直接上源码：
from lxml import etree
import os
import requests
import re
import json
headers = {
"Accept": "image/webp,image/apng,image/*,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Connection": "keep-alive",
"Cookie": BAIDUID=088E2BB15037D6D16EE53CF0C148A1B6:FG=1;BDUSS=0zR0o3TFVtNHphVzdIbTRDUXV1a0NXS1VIZkZBWUxJRXNUVGFFdFlCYXV4QlZlRVFBQUFBJCQAAAAAAAAAAAEAAAAKm01V19TQtrO1dngAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAK437l2uN-5dZE; BIDUPSID=088E2BB15037D6D16EE53CF0C148A1B6; PSTM=1575893178; ab_jid=9a9daf1234c49053f822b72cbcf254d38ccc; pgv_pvi=8645735424; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; H_PS_PSSID=1422_21119_30211_18560_30086_26350_22157; BDRCVFR=I67x6TjHwwYf0; delPer=0; PSINO=2; ZD_ENTRY=baidu",
"Referer": "https://haokan.baidu.com/",
"Sec-Fetch-Mode": "no-cors",
"Sec-Fetch-Site": "same-site",
"User-Agent": "Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/78.0.3904.108Safari/537.36."
} #获得headers
#################获得页面分类信息###########################
category_url="https://haokan.baidu.com/"
temp_data=etree.HTML(requests.get(url=category_url,headers=headers).content.decode('utf-8')).xpath('//*[@id="_page_data"]/text()')
temp_data=json.loads(re.findall('STATE__ =(.*?)\; document\.',temp_data))["tab"]
category={}
for eve_data in temp_data:
category.update({eve_data['name']: eve_data['page']})
print(category)
#################分别下载各分类的内容#################
for folder in category:
try:
os.makedirs("E:\\py\\tmp\\"+folder)
except:
pass
list_url="https://haokan.baidu.com/videoui/api/videorec?tab="+category+"&act=pcFeed&pd=pc&num=10"
list_data=json.loads(requests.get(list_url).content.decode("utf-8"))['data']['response']['videos']#原网址直接将视频源放在json中
print("begin download "+folder+"!")
for i,eve_data in enumerate(list_data):
# print(i+1,eve_data['title'],eve_data['play_url'])
try:
with open("/py/tmp/"+folder+"/"+eve_data['title']+".mp4","wb") as f:
f.write(requests.get(eve_data['play_url']).content)
print("\""+eve_data['title']+"\""+" has downloaded.")
except:
print(eve_data['title']+" timeout.")
很简单很简单的代码，仅供参考，有什么建议可以留言呀，我也好学习学习
页:
[1]