爬取boss直聘
本帖最后由 wushaominkk 于 2020-1-6 09:11 编辑仅限学习！仅限学习！！仅限学习！！！
————————————————————————
实用性不高 数据处理的一般 只是学习
我是进入二级页面爬取的详细信息 其实在一级页面就能爬取大致的信息 但是一级页面没有公司的具体名字
cookie是我登录后获取的 你们自行更换自己的cookie 我已经把cookie删除掉了
有时爬到第5页会出现 爬不下来 我就加了个判断
如果出现异常 让你输入cookie1 和 cookie2 就手动刷新一下页面获取 然后输入
一级页面是：爬取的页面
二级页面是：职位点进去的详情
cookie1是一级页面的
cookie2是二级页面的
这俩个能不能用同一个 我没去实验 你可以自己试试
爬取一次要更换一次cookie 不然爬不出来
import requests
import re
import time
import random
from lxml import etree
# 详细页面的后缀
search_list = 0
# 页码
cd = 1
headers = {
# 填上你自己登录后的cookie
"cookie": "",
"referer": "https://www.zhipin.com/c101010100/?query=%E7%88%AC%E8%99%AB&page={}&ka=page-{}",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
}
headers_1 = {
# 填上你自己登录后的详情页的cookie
"cookie": "",
"referer": "https://www.zhipin.com/c101010100/?query=%E7%88%AC%E8%99%AB&page=3&ka=page-3",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
}
parameters = {
"query": "爬虫",
"page": "2",
"ka": "page-2"
}
# 匹配详细信息
def detailed_information(response):
# gs(公司),cs(城市),gzyq(岗位职责),rs(招聘人数),gz(工资),name(岗位名字),detail(技能要求)
try:
gs = re.findall('<h3>工商信息</h3>(.*?)<div class="level-list">', response, re.S).replace('<div class="name">',"").replace('</div>',"").replace('\n',"").replace(' ',"")
except:
gs="null"
html = etree.HTML(response)
try:
cs = html.xpath('//*[@id="macin"]/div/div/div/div/p/text()').replace("\n","").replace(" ","")
except:
cs="null"
try:
yq = html.xpath('//*[@id="main"]/div/div/div/div/div/div/text()')
except:
yq="null"
rs = random.randint(1, 10)
try:
gz = html.xpath('//*[@id="main"]/div/div/div/div/div/span/text()').replace("\n","").replace(" ","")
except:
gz = "null"
try:
name = html.xpath('//*[@id="main"]/div/div/div/div/div/h1/text()').replace("\n","").replace(" ","")
except:
name = "null"
if len(cs) == 0:
cs="null"
if len(gz) == 0:
cs="null"
if len(gz) == 0:
cs = "null"
if len(name) == 0:
cs = "null"
detail = ""
yq = "".join(yq)
yq = yq.replace("\n", "").replace(" ", "")
gzyq = ""
if len(yq) == 0:
gzyq = "null"
detail = "null"
else:
data = yq.split("：")
if len(data) == 1:
gzyq = data
detail = "null"
elif len(data) == 2:
gzyq = data
detail = "null"
elif len(data) == 3:
gzyq = data[:-4]
detail = data
else:
gzyq = "null"
detail = "null"
print(gs + "," + cs + "," + gzyq + "," + str(rs) + "," + gz + "," + name + "," + detail)
with open("Boss直聘.txt", "a", encoding="utf-8") as file:
file.write((gs + "," + cs + "," + gzyq + "," + str(rs) + "," + gz + "," + name + "," + detail))
file.write("\n")
def information(urls):
global search_list
parameters_1 = {"ka": "search_list_"+str(search_list)}
headers_1["referer"] = headers["referer"]
prefix = "https://www.zhipin.com/"
for url in urls:
url = prefix+url
search_list += 1
response = requests.get(url, headers=headers_1, params=parameters_1).text
detailed_information(response)
time.sleep(random.randint(2,4))
def position_url(response):
try:
urls = re.findall(' <a href="(/job_detail/.*?html)"', response, re.S)
return urls
except:
return 0
def main():
url = "https://www.zhipin.com/c101010100/"
# cd使用的是全局变量
for i in range(cd, 6):
parameters["page"] = str(i)
parameters["ka"] = "page-" + str(i)
if i < 1:
page = str(i - 1)
headers["referer"] = headers["referer"].format(page, page)
response = requests.get(url, headers=headers, params=parameters).text
urls = position_url(response)
if len(urls) == 0 and i == 5:
print("出现异常")
return i
print("第", i, "页")
time.sleep(random.randint(2, 4))
information(urls)
return 0
if __name__ == '__main__':
# 写标题
with open("Boss直聘.txt", "w", encoding="utf-8") as file:
file.write("公司名称,工作城市,工作要求,招聘人数,工资情况,name(岗位名称),detail")
file.write("\n")
while True:
result = main()
if result == 0:
print("完成")
break
else:
cd = result
headers["cookie"] = input("输入cookie1：")
headers_1["cookie"] = input("输入cookie2：")
continue
谢谢发表！一点小建议：将爬取的数据写到一个数据库（比如MongoDB）或者是写入到Excel就更好了，也方便下一步的数据处理分析！代码的话可以在代码框里输入，可读性会比较高。 大佬大佬，学习学习了 感谢作者，辛苦，新年快乐 感谢分享，新年快乐！ 厉害 学习了 建议写出到Excel中，不然纯txt的话，数据大了就不好读了 6666，感觉我已经离各位大佬越来越远了。。。不过人家反发爬虫吗？ 厉害
学习了！ 很好的爬虫实例，大佬牛皮