python 异步协程下载图片

black8

今天用python抓取  https://wallhaven.cc/ 网站壁纸图片，加上windows自带ppt放映功能实现壁纸自动切换。




以下是异步下载代码，测试了一下，对比同步下载速度提神2-3倍

[Python] 纯文本查看 复制代码

import requests
import re
import os
import asyncio
import aiohttp
import aiofiles
import time
import os
import sys


def gethtml(url, encode):  # 获取网页源码
    r = requests.get(url)
    r.encoding = encode
    return r.text


def mkdir(path):  # 创建文件夹
    path = path.strip()
    isExists = os.path.exists(path)
    if not isExists:
        os.makedirs(path)


def geturl(url):  
    html = gethtml(url, 'utf-8')
    obj = re.compile(r'<a class="preview" href="(?P<url>.*?)"  target="_blank"  ></a>', re.S)
    result = obj.finditer(html)
    with open('image_path.txt', 'a') as f:
        for item in result:
            f.write(item.group('url') + '\n')


async def aio_download_pic(session, img_url):
    pic_name = img_url.split('/')[-1]
    img_url = get_data(img_url)
    print(img_url)
    async with session.get(img_url) as resp:
        async with aiofiles.open(save_path + pic_name + '.jpg', mode='wb') as f:
            await f.write(await resp.content.read())
    print(pic_name + "下载完毕")


def get_data(page_url):
    page_content = gethtml(page_url, 'utf-8')
    obj = re.compile(r'<img id="wallpaper" src="(?P<img_true_url>.*?)"', re.S)
    img_true_url = obj.search(page_content, re.S).group('img_true_url')
    return img_true_url


async def aio_download():
    tasks = []
    timeout = aiohttp.ClientTimeout(total=600)
    async with aiohttp.ClientSession(timeout=timeout) as session:
        async with aiofiles.open('image_path.txt', mode='r', encoding='utf-8') as f:
            async for line in f:
                print(line)
                url = line.strip()
                tasks.append(asyncio.create_task(aio_download_pic(session, url)))
            await asyncio.wait(tasks)


if __name__ == '__main__':
    base_url = 'https://wallhaven.cc/toplist?page='
    save_path = 'E:\\wall image\\'  # 保存位置

    now_path = os.path.abspath('') + '\\image_path.txt'
    if os.path.exists(now_path):
        os.system(f'del /Q {now_path}')

    for i in range(20, 30):
        geturl(base_url+str(i))
    """
        多线程下载图片
    """

    start_time = time.time()
    loop = asyncio.get_event_loop()
    loop.run_until_complete(aio_download())
    end_time = time.time()
    print(f'任务完成，耗时{end_time - start_time}')

dork

wallhaven.cc

double07

请问大佬们：异步与多线程区别在哪里?如何判断某爬虫代码最适合使用异步或多线程？