本帖最后由 beyond1994 于 2022-7-20 23:06 编辑
大佬勿喷，记录一次使用selenium爬取自己想要的信息，里面的def函数不熟练和代码会有些乱，在这抛砖引玉望各位大佬指出更好的解决方法
[Python] 纯文本查看 复制代码 from selenium import webdriver
from pyquery import PyQuery as pq
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
TOTAL_PAGE = 24
TIME_OUT = 24
IDNEX_URL = 'http://bang.dangdang.com/books/bestsellers/1-{page}'
import csv
f = open('book.csv',mode='a',encoding='utf-8',newline='')
#表头
csv_writer = csv.DictWriter(f,fieldnames=[
'书名',
'评论',
'作者',
'出版时间',
'价格',
'原价',
'电子书价格',
'书连接'
])
csv_writer.writeheader() #写入表头
def index_page(page):
url = IDNEX_URL.format(page=page) #完善url
return url
def scrape_page(url):
option = ChromeOptions()
option.add_argument('--headless')
browser = webdriver.Chrome(options=option)
browser.get(url)
WebDriverWait(browser,TIME_OUT).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'.bang_list')))
html = browser.page_source
doc = pq(html)
ul = doc('.clearfix li').items()
for i in ul:
name = i('.name a').attr('title')
comment = i('.star a').text() #评论
author = i('.publisher_info a').attr('title') #作者和出版社
time = i('.publisher_info span').text() #出版时间
price = i('.price p:first-child .price_n').text() #原价
discount = i('.price p:first-child .price_r').text()
ebook = i('.price p:nth-child(2) .price_n').text()
detailrul = i('.name a').attr('href')
dict = {
'书名':name,
'评论':comment,
'作者':author,
'出版时间':time,
'价格':price,
'原价':discount,
'电子书价格':ebook,
'书连接':detailrul
}
csv_writer.writerow(dict)
def main():
for page in range(1, TOTAL_PAGE + 1):
urlindex = index_page(page)
test = scrape_page(urlindex)
print(test)
if __name__ == '__main__':
main() | 
[复制链接]
发表于 2022-7-20 23:04
|
发表于 2022-7-21 17:45
