python 小白，问一下动态参数的爬取
使用python 爬了一下苏宁图书分类下的全部图书，抓取图书的名称、图片、商家、评价，详细信息， 发现价格的标签是动态添加的DOM，不知道这个用scrapy 怎么能拿到。这个是Model
import scrapy
class SnbookItem(scrapy.Item):
# define the fields for your item here like:
classTitle = scrapy.Field()
classUrl = scrapy.Field()
class SnbookDetail(scrapy.Item):
# define the fields for your item here like:
goodsName = scrapy.Field()
goodsImg = scrapy.Field()
goodsPingjia = scrapy.Field()
shopName = scrapy.Field()
goodsUrl = scrapy.Field()
goodsPrice = scrapy.Field()
这个是爬取的代码
import scrapy
from scrapy.selector import Selector
from snbook.items import SnbookItem,SnbookDetail
import re
class SubookSpider(scrapy.Spider):
name = 'subook'
allowed_domains = ['suning.com']
start_urls = ['http://list.suning.com/0-502282-0.html']
# 获得所有图书一级分类数据
def parse(self, response):
list = response.xpath("//div[@class='all-class clearfix']//a").extract()
if list is not None:
for li in list:
item = SnbookItem()
data = Selector(text=li)
item["classUrl"] ='http:'+data.xpath("//a/@href").extract_first()
item["classTitle"] = data.xpath("//a/@title").extract_first()
yield scrapy.Request(
item['classUrl'],
callback= self.parse_book_list,
meta= {'item':item}
)
# 爬取二级分类的数据
def parse_book_list(self,response):
list = response.xpath("//div[@class='all-class clearfix']//a").extract()
if list is not None:
for li in list:
item = SnbookItem()
data = Selector(text=li)
item["classUrl"] = 'http:' + data.xpath("//a/@href").extract_first()
item["classTitle"] = data.xpath("//a/@title").extract_first()
yield scrapy.Request(
item['classUrl'],
callback=self.parse_book_detail,
meta={'item': item}
)
# 爬取分类的商品数据
def parse_book_detail(self,response):
#
detailItem = SnbookDetail()
detailItem['goodsName'] = response.xpath(
"//div[@class='product-listclearfix']//ul//li//div[@class='title-selling-point']//a//text()").extract_first()
detailItem['goodsImg'] = response.xpath(
"//div[@class='product-listclearfix']//ul//li//div[@class='res-img']//img/@src").extract_first()
detailItem['goodsImg'] = 'http:'+detailItem['goodsImg']
detailItem['goodsPingjia'] = response.xpath(
"//div[@class='product-listclearfix']//ul//li//div[@class='evaluate-old clearfix']//a//i//text()").extract_first()
detailItem['shopName'] = response.xpath(
"//div[@class='product-listclearfix']//ul//li//div[@class='store-stock']//a//text()").extract_first()
detailItem['goodsUrl'] = response.xpath(
"//div[@class='product-listclearfix']//ul//li//div[@class='res-img']//a/@href").extract_first()
detailItem['goodsUrl'] = 'http:' + detailItem['goodsUrl']
# detailItem['goodsPrice'] = response.xpath(
# "//div[@class='product-listclearfix']//ul//li//div[@class='price-box']")()
# print(detailItem['goodsPrice'])
yield detailItem
pattern = re.compile(r'共(\d+)页，')
pagecount = pattern.findall(response.body.decode())# 子类图书总页数
# print(pagecount)
if pagecount is not None:
# 拿出第一页的URL。拼接
next_url = response.xpath("//div[@class='search-page page-fruits clearfix']//a/@href").extract_first()
# print(next_url)
if next_url is not None:
for i in range(2,int(pagecount)):
pattern1 = re.compile(r'(-\d*.html)$')
pagecount1 = pattern1.findall(next_url)
# print(pagecount1)
str = '-{}.html'.format(i)
nextpage_url = 'http://list.suning.com'+next_url.replace(pagecount1,str)
# print(nextpage_url)
yield scrapy.Request(
nextpage_url,
callback=self.parse_book_detail
)
好像是要用scrapy splash这种库抓js动态渲染的内容，否则可能得上selunium? 我试试哈～
页:
[1]