python 爬虫2880*1800 等超清妹子图 进程池高效率爬取

17788210295

利用 pool.map 进程池 高效爬取大图片  大图片的意思就是 最下面那张图....   记的给个评分,评论哟
编码格式为 gb2312
不说了上代码:

[Python] 纯文本查看 复制代码

# coding=gb2312
import requests
from lxml import etree
from multiprocessing import Pool
import os
from time import sleep
import random


class Down_pic():
    def __init__(self):
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            "Host": "desk.zol.com.cn",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
        }
        self.main_url = 'http://desk.zol.com.cn'  # 主页
        self.dic = {}  # 存放图片路径
        self.count = 0  # 总得图片数量
        self.k = 0  # 用来取datas数据  切片
        self.type = 5  # 大分类取 5个
        self.small_type_num = 4  # 小分类取4个
        self.tupian = 3  # 每一个小分类下载的图片数量 (最多3个)


    def get_tree(self, htlm):
        tree = etree.HTML(htlm)
        return tree

    # 获取大分类
    def get_type(self):
        main_page = requests.get(self.main_url, headers=self.headers).text
        tree = self.get_tree(main_page)
        a_list = tree.xpath('//*[@id="main"]/dl[1]/dd/a')
        # 去掉全部这个分类
        a_list.pop(0)
        for a in a_list[0:self.type]:
            type_name = a.xpath('./text()')[0]
            type_url = self.main_url + a.xpath('./@href')[0]
            yield type_name, type_url

    # 获取小分类
    def get_small_type(self):
        for type_name, type_url in self.get_type():
            small_page = requests.get(type_url)
            small_page.encoding = 'gb2312'
            tree = self.get_tree(small_page.text)
            # 获取小分类
            small_name_list = tree.xpath('//a[@class="pic"]/span/@title')[0:self.small_type_num]  # 小分类取3个
            small_url_list = tree.xpath('//a[@class="pic"]/@href')[0:self.small_type_num]
            self.dic[type_name] = small_name_list
            yield small_url_list

    # 获取图片列表
    def get_pic_list(self):
        for pic_page_url in self.get_small_type():
            for pic_url in pic_page_url:
                url = self.main_url + pic_url
                pic_page = requests.get(url=url).text
                etree = self.get_tree(pic_page)
                pic_list_url = etree.xpath('//*[@id="showImg"]/li/a/@href')
                for pic_url in pic_list_url[:self.tupian]:  # 每一个小分类 取几个图片
                    yield self.main_url + pic_url

    # 获取图片尺寸网址页面
    def get_size(self):
        for pic_url in self.get_pic_list():
            pic_page = requests.get(pic_url).text
            etree = self.get_tree(pic_page)
            try:
                data_url = self.main_url + etree.xpath('//*[@id="tagfbl"]/a[2]/@href')[0]  # 多数默认2880*1800
            except Exception:
                data_url = etree.xpath('//*[@id="bigImg"]/@src')[0]
            yield data_url

    # 获取图片下载地址
    def get_data(self):
        for url in self.get_size():
            data_page = requests.get(url).text
            etree = self.get_tree(data_page)
            try:
                pic_data_url = etree.xpath('/html/body/img[1]/@src')[0]
            except Exception:
                pic_data_url = url
            self.count += 1
            yield pic_data_url
        self.num = self.count

    # 开启线程下载
    def ppp(self):
        print('开启线程')
        pool = Pool(5)
        datas = pool.map(self.download, [url for url in self.get_data()])
        pool.close()
        pool.join()
        for type_name in self.dic:
            for small_name in self.dic[type_name]:
                path = type_name + '/' + small_name
                path = path[:path.find('?')]
                if not os.path.exists(path):
                    os.makedirs(path)
                for data in datas[self.k:self.k + self.tupian]:
                    name = small_name + str(random.randint(1, 1000))  # 图片名
                    pa = path + '/' + name + '.jpg'
                    with open(pa, 'wb') as f:
                        f.write(data)
                self.k += self.tupian
        print('共下载:{}图片'.format(self.count))

    def download(self, url):
        # print('\r当前下载进度:{}%'.format((1 - self.num / self.count) * 100), end='')
        data = requests.get(url=url).content
        sleep(1)
        return data



if __name__ == '__main__':
    down = Down_pic()
    down.ppp()

注:此文章所有内容仅供学习，不允许商用，如有侵权，请联系删除，谢谢。...

you2012

SyntaxError: encoding problem: gb2312  这个是什么问题呢？

hncnxh

Error: encoding problem: gb2312

17788210295

临时小号not404 发表于 2019-8-2 17:54
不懂就问，为啥会这样啊

pip install lxml   安装包

c00kie

非常适合初学者学习，多谢分享

wsq5201314

看不懂 感觉很厉害的样子

ygr121

这个代码怎么使用?能说一下吗

17788210295

ygr121 发表于 2019-8-2 16:30
这个代码怎么使用?能说一下吗

python 运行就行了呀

丶苍渊丶

有点不太懂 求教各位大佬  复制代码前是要打开别的页面吗   

iyhui

厉害啊，楼主

zhong_xj

看不懂，也不知道是干嘛的，但是觉得很厉害

denverh

不错 装个python跑跑试试看

Franklichao

嗯！可以，适合初学者。