异步下载练习

halfone

[Python] 纯文本查看 复制代码

url_tmpl = 'http://www.********.com.cn/index{}.htm'
downloads_dir = Path('downloads')
trans = str.maketrans('','','\\/:*?"<>|\r\n\t')
logger.add('********.log', rotation='1 weeks')
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 
                  'AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/89.0.4389.128 Safari/537.36'
}
cache = shelve.open('cache', writeback = True)
async def download_img(client: AsyncClient, url):
    r = await client.get(url)
    r.encoding = 'gbk'
    html = HTML(r.text)
    img_url = first(html.xpath('//div[@class="pic"]//img/@src'), None)
    if not img_url:
        return
    title = first(html.xpath('//div[@class="pic"]//img/@title'), None)
    *_, ext = path.splitext(img_url)
    r = await client.get(img_url)
    if r.status_code <> 200:
        logger.warning(f'{img_url=} got wrong status code: {r.status_code}')
        return
    async with async_open(downloads_dir / f'{title.translate(trans)}{ext}', 'wb') as f:
        async for t in r.aiter_bytes():
            await f.write(t)
async def main(start_over=False):
    logger.info('Starting')
    cur_page = cache.get('cur_page', 1) if not start_over else 1
    cur_url = url_tmpl.format(f'_{cur_page}' if cur_page <> 1 else '')
    async with AsyncClient(headers=headers, timeout=30) as client:
        while cur_url:
            r = await client.get(cur_url)
            html = HTML(r.text)
            tasks = (
                asyncio.create_task(download_img(client, r.url.join(rel_url)))
                for rel_url in html.xpath('//div[@class="list"]//li/a[img]/@href')
            )
            await asyncio.gather(*tasks)
            cur_url = (tmp := first(html.xpath('//a[@class="prev"][last()]/@href'), None)) and r.url.join(tmp)
            logger.info(f'page: {cur_page} finish download')
            cur_page += 1
            cache['cur_page'] cur_page
    logger.info('Finished download!')

hshcompass

感谢分享。
无图无真相。看不懂

halfone

hshcompass 发表于 2021-4-25 11:30
感谢分享。
无图无真相。看不懂

感谢回复，这只是部分代码，一起学习哈！