爬取图网的4K图片自动保存本地

龍魂小白

初学py爬虫，还在继续努力


这个是单进程爬取，下边有个进程池获取 太猛自己悠着点

import os.path
import random
import time

import requests
from lxml import etree
import webbrowser

webbrowser.open('https://pic.netbian.com')

ua = input("请输入浏览器的User-Agent，不会的请百度\n")

print('''例如：https://pic.netbian.com/4kmeinv
              https://pic.netbian.com/4kfengjing
              https://pic.netbian.com
''')
uuuu = input("请输入彼岸图的URL不带'/'\n")

headers = {
    'User-Agent': ua
}
url = uuuu

def huoqu(urll):
    url = urll
    # print(url)

    respones = requests.get(url, headers=headers)
    mg = respones.content.decode("gbk")

    tree = etree.HTML(mg)
    img_url = tree.xpath('//ul[@class="clearfix"]//li')
    # print(img_url)

    for a in img_url:
        c = a.xpath('./a//img/@alt')[0]
        img_mg = a.xpath('./a//img/@src')
        img_mgg = requests.get('https://pic.netbian.com' + ''.join(img_mg))

        if not os.path.exists('彼岸图网'):
            os.mkdir("彼岸图网")
        with open(f'彼岸图网/{c}.jpg', 'wb') as f:

            f.write(img_mgg.content)
            print(f'彼岸图网/{c}.jpg，保存成功')

def zongpage(url):
    url = url
    print(url)

    respones = requests.get(url, headers=headers)
    mg = respones.content.decode("gbk")

    tree = etree.HTML(mg)
    page = tree.xpath('//div[@class="page"]/a/text()')

    zongpage = page[-2]
    print('总共：' + zongpage + '页')
    return zongpage

zongpagee = zongpage(url)

for a in range(1, int(zongpagee)):

    if a == 1:
        huoqu(url)
    else:
        uu = f'{url}/index_{a}.html'
        print(uu)
        huoqu(uu)
        time.sleep(random.randint(2, 5))

这个是进程池获取

import os.path
import random
import time
import requests
from lxml import etree
import threading
from multiprocessing import Pool
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
url = 'https://pic.netbian.com'

def huoqu(urll):

    print('我已经运行了')

    # 发送HTTP请求，获取页面内容
    respones = requests.get(urll, headers=headers)
    mg = respones.content.decode("gbk")
    tree = etree.HTML(mg)
    # 使用XPath选择器提取图片URL和相关信息
    img_url = tree.xpath('//ul[@class="clearfix"]//li')

    for a in img_url:
        c = a.xpath('./a//img/@alt')[0]
        img_mg = a.xpath('./a//img/@src')
        img_mgg = requests.get('https://pic.netbian.com' + ''.join(img_mg))
        if not os.path.exists('彼岸图网'):
            os.mkdir("彼岸图网")
        with open(f'彼岸图网/{c}.jpg', 'wb') as f:
            f.write(img_mgg.content)
            print(f'彼岸图网/{c}.jpg，保存成功')

def zongpage(url):
    # 发送HTTP请求，获取页面内容
    respones = requests.get(url, headers=headers)
    mg = respones.content.decode("gbk")
    tree = etree.HTML(mg)
    # 使用XPath选择器提取总页数
    page = tree.xpath('//div[@class="page"]/a/text()')
    zongpage = page[-2]
    print('总共：' + zongpage + '页')
    return zongpage

zongpagee = zongpage(url)

if __name__ == '__main__':
    threads = []
#Pool(5)可以调整进程数量  不填则按照CPU最大核心数来并发不填会有死机风险，最好不要超过10
    pool = Pool(5)
    for a in range(1, int(zongpagee)):
        if a == 1:
            # 第一页直接调用huoqu函数
            huoqu(url)
        else:
            uu = f'{url}/index_{a}.html'
            print(uu)
            threads.append(uu)

    for i in range(1, int(zongpagee)):
        # 创建线程并启动
        pool.map(huoqu,threads)
        pool.close()

        # 随机延迟一段时间
        time.sleep(random.randint(2, 5))

    for t in threads:
        # 等待所有线程完成
        pool.join()
    print("完事")

秋海棠地图

对代码进行了一小点修改，可以运行了，但不能获取到4K的图片

[Python] 纯文本查看 复制代码

import os
import requests
from lxml import etree
from concurrent.futures import ThreadPoolExecutor

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
url = 'https://pic.netbian.com'

def clean_filename(filename):
    # 使用正则表达式将特殊字符替换为空格
    return re.sub(r'[\\/:*?"<>|\n\r\t]', ' ', filename)

def huoqu(urll):
    # 发送HTTP请求，获取页面内容
    respones = session.get(urll, headers=headers)
    mg = respones.content.decode("gbk")
    tree = etree.HTML(mg)
    # 使用XPath选择器提取图片URL和相关信息
    img_url = tree.xpath('//ul[@class="clearfix"]//li')

    for a in img_url:
        c = a.xpath('./a//img/@alt')[0]
        c = clean_filename(c)  # 使用clean_filename函数来清理文件名
        img_mg = a.xpath('./a//img/@src')
        img_mgg = session.get('https://pic.netbian.com' + ''.join(img_mg))
        if not os.path.exists('彼岸图网'):
            os.makedirs("彼岸图网")
        with open(os.path.join('彼岸图网', f'{c}.jpg'), 'wb') as f:
            f.write(img_mgg.content)
            print(f'彼岸图网/{c}.jpg，保存成功')

def zongpage(url):
    # 发送HTTP请求，获取页面内容
    respones = session.get(url, headers=headers)
    mg = respones.content.decode("gbk")
    tree = etree.HTML(mg)
    # 使用XPath选择器提取总页数
    page = tree.xpath('//div[@class="page"]/a/text()')
    zongpage = page[-2]
    print('总共：' + zongpage + '页')
    return zongpage

if __name__ == '__main__':
    import re  # 导入re模块
    session = requests.Session()
    zongpagee = zongpage(url)

    with ThreadPoolExecutor(max_workers=4) as executor:
        for a in range(1, int(zongpagee)):
            if a == 1:
                # 第一页直接调用huoqu函数
                huoqu(url)
            else:
                uu = f'{url}/index_{a}.html'
                print(uu)
                executor.submit(huoqu, uu)

    session.close()
    print("完事")

吖力锅

互相学习学习

echoaku

不错，支持一波

lingwushexi

学习学习

yu520

学习一下，感谢楼主分享

evanjhon

不错  收藏了  回头试下

jackxp2008

我才刚学python，学习了，多谢大佬分享。

normandie2008

都喜欢搞彼岸图片呢？

15705107305

nice !!!

jddgh3

感谢楼主分享，学习学习