[Python] 纯文本查看 复制代码 '''
爬取网址：学习猿地猿圈
爬取内容：文章标题，文章连接，作者，时间
工具：bs4,requests
结果：爬取到文件之中
'''
from bs4 import BeautifulSoup
import requests,json
#1.定义请求头和url
url = 'https://www.lmonkey.com/t'
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"
}
alldata = []
#2.请求获取数据
res = requests.get(url = url,headers = headers)
if res.status_code == 200:
#3.解析数据
soup = BeautifulSoup(res.text,'lxml')
#获取页面中所有的文章
divclass = soup.find_all( 'div',class_="list-group-item list-group-item-action p-06")
for i in divclass:
my_title = i.find('div',class_="topic_title mb-0 lh-180")
if my_title:
my_title = my_title.text.split("\n")[0]
my_url = i.a["href"]
my_time = i.span['title']
my_author = i.strong.a.text
print(my_author)
print(my_url)
print(my_time)
print(my_title)
temp = {'title':my_title,
'passage_url':my_url,
'time':my_time,
'author':my_author
}
alldata.append(temp)
#4.写入文件
with open('C:/Users/lsy/Desktop/lsy.json','w') as fp:
json.dump(alldata,fp)
| 
发表于 2020-5-31 15:33
