一个简单的例子
[Python] 纯文本查看 复制代码 import time
import requests
from lxml import etree
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
# proxies = {
# "http":"http://127.0.0.1:1080","https":"https://127.0.0.1:1080"
# }
def get_url(url):
r2=requests.get(url,headers=headers).url
with open('url.txt','a') as f:
f.write(r2+'\n')
if __name__ == '__main__':
page=0
requests.adapters.DEFAULT_RETRIES = 500
while True:
try:
if page == 5000:
break
url="https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=inurl:'asp?id=1'&pn=%s"%page
print(url)
r=requests.get(url,headers=headers)
html = etree.HTML(r.content)
pnnext = html.xpath('//*[@id="page"]/div/a[@class="n"]/@href') #获取下一页节点
if pnnext: #判断节点是否存在,存在即继续
div = html.xpath('//h3[@class="t"]/a/@href') #搜索结果
for i in div:
# print(i)
get_url(i) #写入url
else:
print('nopnnext')
break
page+=10
except Exception as e:
print(e) #输出错误
print(time.strftime("%H:%M:%S"))
time.sleep(3) |