zrq648022547 发表于 2022-8-16 23:40

利用爬虫爬取套图网站美女的所有写真（带多线程）

本帖最后由 zrq648022547 于 2022-8-16 23:43 编辑

======================================初次学习python，写的比较烂，暂时写了两种方式====指定某美女，爬取全部所有的图册                          ====指定图集首页地址，爬取单图册图片                  ====写的不好，烦请大神不吝赐教，在此谢过               ==
====================================
代码一：爬取指定美女图册
缺点：
1、虽然写入了多线程爬取，但是测试貌似还是比较慢；
2、爬取的图片名称不能按照序号命名
3、未处理反扒机制
# 多进程异步并发
import random
import requests
from bs4 import BeautifulSoup
import os
import time
import threading
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import concurrent.futures

USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
    "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    "UCWEB7.0.2.37/28/999",
    "NOKIA5700/ UCWEB7.0.2.37/28/999",
    "Openwave/ UCWEB7.0.2.37/28/999",
    "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
    # iPhone 6：
    "Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25",
]

headers = {
    'User-Agent': random.choice(USER_AGENTS),
    "referer": "https://www.xiurenji.vip/"
}


def get_item(item, main_title):
    title = item['title']
    threads = []
    item_links = ]
    # print(f'{title}>>>{item_links}')
    for item_link in item_links:
      threads.append(threading.Thread(target=get_images,args=(title, item_link, main_title)))
    for thread in threads:
      thread.start()
    for thread in threads:
      thread.join()


def get_images(title, item_link, main_title):
    try:
      item_res = requests.get(url=item_link, headers=headers,timeout=30)
      item_res.encoding = 'gzip'
      item_soup = BeautifulSoup(item_res.text, 'lxml')
      img_list = item_soup.select('.content_left p img')
      # print(img_list)
      folder = main_folder + '/' + main_title + '/' + title + '/'
      if not os.path.exists(folder):
            os.makedirs(folder)
      else:
            pass
      try:
            for img in img_list:
                img_link = start_url + img['src']
                # print(img_link)
                with open(folder + img_link.split('/')[-1], 'wb') as f:
                  starttime = time.time()
                  image = requests.get(url=img_link, headers=headers,timeout=30).content
                  f.write(image)
                  time.sleep(1)
                  endtime = time.time()
                  print(f'正在保存>>>{title}' + '>>>' + img_link.split('/')[-1] + '>>>用时%.3f'%(endtime - starttime), 'seconds')
      except IndexError:
            pass
      next = item_soup.select_one('.page a:last-of-type')
      if 'class="current"' in str(next):
            pass
      else:
            item_link = start_url + next['href']
            with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
                pool.submit(get_images,title, item_link, main_title)
    except IndexError:
      pass


if __name__ == '__main__':
    start = time.time()
    start_url = 'https://www.xrmn5.cc'# 网站根地址
    main_title = '秀人网'
    main_folder = "单项目2\\"# 主文件夹路径（请填入你自己的文件夹路径）
    main_url = 'https://www.xrmn5.cc/younisi.html'
    itemlist_res = requests.get(url=main_url, headers=headers,timeout=30)
    itemlist_res.encoding = 'gzip'
    itemlist_soup = BeautifulSoup(itemlist_res.text, 'lxml')
    itemlist = itemlist_soup.select('.list_n2 a')
    # print(itemlist)
    for item in itemlist:
      get_item(item, main_title)
    end = time.time()
    print('总共用时：',end - start,'seconds',end='')

代码二：爬取美女单图集
缺点：
1、文件夹名称需要手动指定
2、爬取的图册首页地址需要手动指定
3、爬取的图片和页面显示的图片数量不符合，有的页面图片漏掉了，不知道为啥
import requests
import parsel
import random
import os
import datetime
import time


starttime = datetime.datetime.now()
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
    "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    "UCWEB7.0.2.37/28/999",
    "NOKIA5700/ UCWEB7.0.2.37/28/999",
    "Openwave/ UCWEB7.0.2.37/28/999",
    "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
    # iPhone 6：
    "Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25",
]

headers = {
    'User-Agent': random.choice(USER_AGENTS),
    # 'Connection': 'close'
    "referer":"https://www.xrmn5.cc"
}

# 0.创建文件夹
directory = 'Vol.809_女神尤妮丝Egg红色轻透上衣配红短裙半脱露红色内衣诱惑写真60P\\'
if os.path.exists(directory):
    pass
else:
    os.mkdir(directory)
# 1.确定爬取的网站

for page in range(0, 30):
    try:
      if page >= 1:
            base_url = 'https://www.xrmn5.com/YouMi/2022/202211014_{}.html'.format(page)
      else:
            base_url = 'https://www.xrmn5.com/YouMi/2022/202211014.html'
      # print('==========正在爬取第{}页数据============='.format(page))
      # 2.发送请求
      response = requests.get(url=base_url, headers=headers)
      response.encoding = 'UTF-8'# 自动识别响应体的编码
      # print(response)
      html_data = response.text
      # print(html_data)
      # 解析详情页图片地址
      response_1 = requests.get(base_url, headers=headers).text
      html_1 = parsel.Selector(response_1)
      # print(html_1)
      # 解析图册中图片地址
      for i in range(1,4):
            img_list_1 = html_1.xpath('//*[@class="content_left"]/p/img[{}]/@src'.format(i)).extract_first()
            img_list = img_list_1.replace('uploadfile','Uploadfile')
            img_url = 'https://p.xrmn5.com/' + str(img_list)
            # print(img_list_1)
            # 请求图片地址
            img_data = requests.get(img_url, headers=headers).content
            img_name_1 = str(int(page) + 1)# 图片文件名称
            # print(img_name_1)
            # 4.数据保存
            with open(directory + img_name_1 + '-' + str(i) + '.jpg', 'wb') as f:
                f.write(img_data)
                time.sleep(10)
            print('#####################正在爬取第', page + 1, '页,第', int(i),'张图片#####################')
    except IndexError:
      continue
endtime = datetime.datetime.now()
print('####################下载完成，共用时', (endtime - starttime).seconds, '秒###################')

水上凌波 发表于 2022-8-17 00:03

https://www.3gbizhi.com/，能爬吗

rangersxiaoyan 发表于 2022-8-24 22:26

隐身三娃 发表于 2022-8-16 23:57

共同学习，图片漏掉和不能按照序号命名，问题多数是出在多线程上，我看到的问题有两点
第一：多线程未设置数量，会增加服务器的阻拦机率
第二：在爬取数据处理的时候，建议加上try来处理一些报错和数据的处理
个人意见，共同学习

icodeme 发表于 2022-8-17 00:18

可以，感谢大佬分享，希望继续完善{:1_893:}

decdeva 发表于 2022-8-17 01:41

哥们你是真滴秀

ysjd22 发表于 2022-8-17 08:05

ppt。视频素材爬取的软件有没有》？

wallacebai 发表于 2022-8-17 08:12

感谢分享， 学习了。

烟花非易冷 发表于 2022-8-17 08:19

爬取内容挺不错的，哈哈

guohuanxian 发表于 2022-8-17 08:27

感谢分享

excess1989 发表于 2022-8-17 08:32

牛逼老司机
页: [1] 2 3 4 5 6 7
查看完整版本: 利用爬虫爬取套图网站美女的所有写真（带多线程）


免责声明：
吾爱破解所发布的一切破解补丁、注册机和注册信息及软件的解密分析文章仅限用于学习和研究目的；不得将上述内容用于商业或者非法用途，否则，一切后果请用户自负。本站信息来自网络，版权争议与本站无关。您必须在下载后的24个小时之内，从您的电脑中彻底删除上述内容。如果您喜欢该程序，请支持正版软件，购买注册，得到更好的正版服务。如有侵权请邮件与我们联系处理。

Mail To:Service@52pojie.cn