python 爬虫2880*1800 等超清妹子图 进程池高效率爬取
本帖最后由 17788210295 于 2019-8-22 14:18 编辑利用 pool.map 进程池 高效爬取大图片大图片的意思就是 最下面那张图....{:301_976:} 记的给个评分,评论哟{:301_997:}
编码格式为 gb2312
不说了上代码:
# coding=gb2312
import requests
from lxml import etree
from multiprocessing import Pool
import os
from time import sleep
import random
class Down_pic():
def __init__(self):
self.headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "max-age=0",
"Connection": "keep-alive",
"Host": "desk.zol.com.cn",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
}
self.main_url = 'http://desk.zol.com.cn'# 主页
self.dic = {}# 存放图片路径
self.count = 0# 总得图片数量
self.k = 0# 用来取datas数据切片
self.type = 5# 大分类取 5个
self.small_type_num = 4# 小分类取4个
self.tupian = 3# 每一个小分类下载的图片数量 (最多3个)
def get_tree(self, htlm):
tree = etree.HTML(htlm)
return tree
# 获取大分类
def get_type(self):
main_page = requests.get(self.main_url, headers=self.headers).text
tree = self.get_tree(main_page)
a_list = tree.xpath('//*[@id="main"]/dl/dd/a')
# 去掉全部这个分类
a_list.pop(0)
for a in a_list:
type_name = a.xpath('./text()')
type_url = self.main_url + a.xpath('./@href')
yield type_name, type_url
# 获取小分类
def get_small_type(self):
for type_name, type_url in self.get_type():
small_page = requests.get(type_url)
small_page.encoding = 'gb2312'
tree = self.get_tree(small_page.text)
# 获取小分类
small_name_list = tree.xpath('//a[@class="pic"]/span/@title')# 小分类取3个
small_url_list = tree.xpath('//a[@class="pic"]/@href')
self.dic = small_name_list
yield small_url_list
# 获取图片列表
def get_pic_list(self):
for pic_page_url in self.get_small_type():
for pic_url in pic_page_url:
url = self.main_url + pic_url
pic_page = requests.get(url=url).text
etree = self.get_tree(pic_page)
pic_list_url = etree.xpath('//*[@id="showImg"]/li/a/@href')
for pic_url in pic_list_url[:self.tupian]:# 每一个小分类 取几个图片
yield self.main_url + pic_url
# 获取图片尺寸网址页面
def get_size(self):
for pic_url in self.get_pic_list():
pic_page = requests.get(pic_url).text
etree = self.get_tree(pic_page)
try:
data_url = self.main_url + etree.xpath('//*[@id="tagfbl"]/a/@href')# 多数默认2880*1800
except Exception:
data_url = etree.xpath('//*[@id="bigImg"]/@src')
yield data_url
# 获取图片下载地址
def get_data(self):
for url in self.get_size():
data_page = requests.get(url).text
etree = self.get_tree(data_page)
try:
pic_data_url = etree.xpath('/html/body/img/@src')
except Exception:
pic_data_url = url
self.count += 1
yield pic_data_url
self.num = self.count
# 开启线程下载
def ppp(self):
print('开启线程')
pool = Pool(5)
datas = pool.map(self.download, )
pool.close()
pool.join()
for type_name in self.dic:
for small_name in self.dic:
path = type_name + '/' + small_name
path = path[:path.find('?')]
if not os.path.exists(path):
os.makedirs(path)
for data in datas:
name = small_name + str(random.randint(1, 1000))# 图片名
pa = path + '/' + name + '.jpg'
with open(pa, 'wb') as f:
f.write(data)
self.k += self.tupian
print('共下载:{}图片'.format(self.count))
def download(self, url):
# print('\r当前下载进度:{}%'.format((1 - self.num / self.count) * 100), end='')
data = requests.get(url=url).content
sleep(1)
return data
if __name__ == '__main__':
down = Down_pic()
down.ppp()
注:此文章所有内容仅供学习，不允许商用，如有侵权，请联系删除，谢谢。... SyntaxError: encoding problem: gb2312这个是什么问题呢？ Error: encoding problem: gb2312 临时小号not404 发表于 2019-8-2 17:54
不懂就问，为啥会这样啊
pip install lxml 安装包 非常适合初学者学习，多谢分享 看不懂 感觉很厉害的样子{:301_1009:} 这个代码怎么使用?能说一下吗 ygr121 发表于 2019-8-2 16:30
这个代码怎么使用?能说一下吗
python 运行就行了呀 有点不太懂 求教各位大佬复制代码前是要打开别的页面吗 厉害啊，楼主 看不懂，也不知道是干嘛的，但是觉得很厉害 不错 装个python跑跑试试看 嗯！可以，适合初学者。