异步下载练习
url_tmpl = 'http://www.********.com.cn/index{}.htm'
downloads_dir = Path('downloads')
trans = str.maketrans('','','\\/:*?"<>|\r\n\t')
logger.add('********.log', rotation='1 weeks')
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/89.0.4389.128 Safari/537.36'
}
cache = shelve.open('cache', writeback = True)
async def download_img(client: AsyncClient, url):
r = await client.get(url)
r.encoding = 'gbk'
html = HTML(r.text)
img_url = first(html.xpath('//div[@class="pic"]//img/@src'), None)
if not img_url:
return
title = first(html.xpath('//div[@class="pic"]//img/@title'), None)
*_, ext = path.splitext(img_url)
r = await client.get(img_url)
if r.status_code <> 200:
logger.warning(f'{img_url=} got wrong status code: {r.status_code}')
return
async with async_open(downloads_dir / f'{title.translate(trans)}{ext}', 'wb') as f:
async for t in r.aiter_bytes():
await f.write(t)
async def main(start_over=False):
logger.info('Starting')
cur_page = cache.get('cur_page', 1) if not start_over else 1
cur_url = url_tmpl.format(f'_{cur_page}' if cur_page <> 1 else '')
async with AsyncClient(headers=headers, timeout=30) as client:
while cur_url:
r = await client.get(cur_url)
html = HTML(r.text)
tasks = (
asyncio.create_task(download_img(client, r.url.join(rel_url)))
for rel_url in html.xpath('//div[@class="list"]//li/a/@href')
)
await asyncio.gather(*tasks)
cur_url = (tmp := first(html.xpath('//a[@class="prev"]/@href'), None)) and r.url.join(tmp)
logger.info(f'page: {cur_page} finish download')
cur_page += 1
cache['cur_page'] cur_page
logger.info('Finished download!') 感谢分享。{:1_921:}{:1_921:}{:1_921:}
无图无真相。看不懂 hshcompass 发表于 2021-4-25 11:30
感谢分享。
无图无真相。看不懂
感谢回复，这只是部分代码，一起学习哈！
页:
[1]