本帖最后由 ∫護着妳佉遠方 于 2022-1-31 15:40 编辑
使用进程池爬取星巴克的完整代码
[Python] 纯文本查看 复制代码 """
* @codding : utf-8
* @time : 2022/1/29 13:27:28
* @name : 09_多线程爬取.py
* @system : Windows 11
* @Types : pythonProject
* @software : PyCharm
* @author :
"""
import re
import time
import urllib.parse
import urllib.request
from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
def producer():
url = 'https://www.starbucks.com.cn/menu/'
head = {
'Cookie': 'ZHh6ku4z=AMCsJHZ-AQAAOcyxX6Yl2s20HjtkZfHK87MTPofIn6iYXJHzUvqd_HT1ZDMD|1|0'
'|36c0e1a8addcb91415dc5a8a4223425eca69e3d4',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, '
'like Gecko) Chrome/97.0.4692.99 Safari/537.36 Edg/97.0.1072.76'
}
# 请求对象定制
request = urllib.request.Request(url=url, headers=head)
# 模拟浏览器向服务器发送请求
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
# print(content)
soup = BeautifulSoup(content, 'lxml')
# 名称
# //ul[@class="grid padded-3 product"]//strong/text()
# 名称列表
name_list = soup.select('ul[class="grid padded-3 product"] strong')
# 图片列表.jpg
# //ul[@class="grid padded-3 product"]//div/@style
img_lisi = soup.select('ul[class="grid padded-3 product"] div')
tinydict = {}
for i in range(len(name_list)):
name = name_list[i]
aaa = name.get_text()
yyy = re.sub('/', ',', aaa)
img = img_lisi[i]
bbb = img.attrs.get('style')
ccc = re.findall(r'url\("(.*)"\)', bbb)[0]
ddd = 'https://www.starbucks.com.cn' + ccc
xx = {yyy: ddd}
tinydict.update(xx)
# print(list)
return tinydict
def consumer(tinydict):
name = tinydict[0]
img = tinydict[1]
# print(name, img)
f = open(name + '.jpg', 'wb')
d = requests.get(img).content
f.write(d)
f.close()
print('成功下载', name)
if __name__ == '__main__':
tinydict = producer()
# print(list)
pool = Pool(processes=4)
pool.map(consumer, tinydict.items())
|