python 异步协程下载图片
本帖最后由 black8 于 2021-11-12 15:57 编辑今天用python抓取https://wallhaven.cc/ 网站壁纸图片，加上windows自带ppt放映功能实现壁纸自动切换。
以下是异步下载代码，测试了一下，对比同步下载速度提神2-3倍
import requests
import re
import os
import asyncio
import aiohttp
import aiofiles
import time
import os
import sys
def gethtml(url, encode):# 获取网页源码
r = requests.get(url)
r.encoding = encode
return r.text
def mkdir(path):# 创建文件夹
path = path.strip()
isExists = os.path.exists(path)
if not isExists:
os.makedirs(path)
def geturl(url):
html = gethtml(url, 'utf-8')
obj = re.compile(r'<a class="preview" href="(?P<url>.*?)"target="_blank"></a>', re.S)
result = obj.finditer(html)
with open('image_path.txt', 'a') as f:
for item in result:
f.write(item.group('url') + '\n')
async def aio_download_pic(session, img_url):
pic_name = img_url.split('/')[-1]
img_url = get_data(img_url)
print(img_url)
async with session.get(img_url) as resp:
async with aiofiles.open(save_path + pic_name + '.jpg', mode='wb') as f:
await f.write(await resp.content.read())
print(pic_name + "下载完毕")
def get_data(page_url):
page_content = gethtml(page_url, 'utf-8')
obj = re.compile(r'<img id="wallpaper" src="(?P<img_true_url>.*?)"', re.S)
img_true_url = obj.search(page_content, re.S).group('img_true_url')
return img_true_url
async def aio_download():
tasks = []
timeout = aiohttp.ClientTimeout(total=600)
async with aiohttp.ClientSession(timeout=timeout) as session:
async with aiofiles.open('image_path.txt', mode='r', encoding='utf-8') as f:
async for line in f:
print(line)
url = line.strip()
tasks.append(asyncio.create_task(aio_download_pic(session, url)))
await asyncio.wait(tasks)
if __name__ == '__main__':
base_url = 'https://wallhaven.cc/toplist?page='
save_path = 'E:\\wall image\\'# 保存位置
now_path = os.path.abspath('') + '\\image_path.txt'
if os.path.exists(now_path):
os.system(f'del /Q {now_path}')
for i in range(20, 30):
geturl(base_url+str(i))
"""
多线程下载图片
"""
start_time = time.time()
loop = asyncio.get_event_loop()
loop.run_until_complete(aio_download())
end_time = time.time()
print(f'任务完成，耗时{end_time - start_time}')
wallhaven.cc 请问大佬们：异步与多线程区别在哪里?如何判断某爬虫代码最适合使用异步或多线程？
页:
[1]