asyncio怎么做到生产者消费者呢
直接上代码吧。消费者这块我不知道怎么实现。大神帮我瞅瞅思路。import asyncio
import aiohttp
import queue
import pymysql
from proxy import proxy_pool
import random
async def crawl(session, data, html_queue, pool, proxy_auth):
url = 'http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'
async with session.post(url, data, proxy=random.choice(pool),proxy_auth=proxy_auth) as response:
html_queue.put(await response.json())
async def parse(html_queue, conn):
html = html_queue.get()
for info in html['list']:
name = info['EPS_NAME']
sn = info['PRODUCT_SN']
manager_name = info['QF_MANAGER_NAME']
issue_date = info['XC_DATE']
validity_date = info['XK_DATE']
cursor = conn.cursor()
sql = 'insert into yjzj(企业名称, 许可证编号, 发证机关, 有效期至, 发证日期) values(%s,%s,%s,%s,%s)'
cursor.execute(sql, (name, sn, manager_name, issue_date, validity_date))
conn.commit()
cursor.close()
async def main():
html_queue = queue.Queue()
pool = proxy_pool()
proxy_auth = aiohttp.BasicAuth('xxxx', 'xxxx')
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False), trust_env=True) as session:
formdata = [{
'on': 'true',
'page': i,
'pageSize': '15',
'productName': '',
'conditionType': '1',
'applyname': '',
'applysn': ''
} for i in range(1, 51)]
crawl_tasks =
await asyncio.wait(crawl_tasks)
conn = pymysql.connect(
host='xxxx',
port=3306,
user='xxxx',
password='xxxx',
db='text',
charset='utf8'
)
parse_tasks =
conn.close()
if __name__ == '__main__':
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
loop = asyncio.get_event_loop()
loop.run_until_complete(main()) 这不得看下操作系统 layuai 发表于 2021-8-21 10:35
这不得看下操作系统
啊什么操作系统 他这是python。我觉得，异步的请求应该用队列的方式，来通讯生产者、消费者。 Wxylkxy 发表于 2021-8-21 10:49
啊什么操作系统
操作系统知识就讲解了如何处理这些问题啊，看一看就好
页:
[1]