Python爬取C语言中文网教程生成PDF
本帖最后由 null119 于 2019-7-16 04:19 编辑修正代码,解决获取html/PDF无图片
代码演示获取地址：http://c.biancheng.net/python/
页面截图：
生成PDF截图：
import requests
from lxml import etree
import re
import os
import pdfkit
def gethtml(url,encode):
r = requests.get(url)
r.encoding = encode
return r.text
def writehtml(path,str):
f = open(path,'w+',encoding='utf-8')
f.write(str)
f.close
def validateTitle(title):
rstr = r"[\/\\\:\*\?\"\<\>\|]"
new_title = re.sub(rstr, "_", title)
return new_title
def mkdir(path):
path = path.strip()
isExists = os.path.exists(path)
if not isExists:
os.makedirs(path)
return True
else:
print('文件夹已存在，请检查后再试！')
return False
def getdata(url,pdf):
print('开始获取,请稍候...')
c_url = url.split('/') + '//' + url.split('/') + '/'
html = gethtml(url,'utf-8')
ehtml = etree.HTML(html)
urll1 = ehtml.xpath('//*[@id="contents"]/dd/a/@href')
till1 = ehtml.xpath('//*[@id="contents"]/dd/a/text()')
s = ehtml.xpath('//*[@id="contents"]/dd/span/text()')
folder=validateTitle(ehtml.xpath('//*[@id="contents"]/dt/a/text()'))
if mkdir(savepath+folder):
m = 0
txt=''
for i in urll1:
html = gethtml(c_url + i,'utf-8')
ehtml = etree.HTML(html)
strs = ehtml.xpath('//*[@id="article"]')[-1]
txtl1 = etree.tostring(strs, encoding="utf-8", pretty_print=True, method="html").decode("utf-8")
fname = validateTitle(s + ' ' + till1)
txtl1 = re.sub('<h1>.*?</h1>','<h1>'+fname+'</h1>',txtl1)
txtl1 = re.sub('src="/','src="'+c_url+'/',txtl1)
txt=txt+txtl1
#writehtml(savepath+folder+'\\'+fname+'.html', txtl1)#每个章节生成一个html文件
s1 = ehtml.xpath('//*[@id="contents"]/dl/dd/text()')
s2 = ehtml.xpath('//*[@id="contents"]/dl/dd/a/text()')
urll2=ehtml.xpath('//*[@id="contents"]/dl/dd/a/@href')
print(fname)
n=0
for j in urll2:
html=gethtml(c_url+j,'utf-8')
ehtml = etree.HTML(html)
strs = ehtml.xpath('//*[@id="arc-body"]')[-1]
txtl2 = etree.tostring(strs, encoding="utf-8", pretty_print=True, method="html").decode("utf-8")
fname = validateTitle(s1+' '+s2)
txtl2= re.sub('<h>','<h3>',txtl2)
txtl2 = re.sub('</h>', '</h3>', txtl2)
txtl2 = re.sub('src="/','src="'+c_url+'/',txtl2)
txtl2 = '<h2>' + fname + '</h2>'+txtl2
txt = txt + txtl2
#writehtml(savepath+folder+'\\'+fname + '.html', txtl2) #每个章节生成一个html文件
print(fname)
n+=1
m+=1
writehtml(savepath+folder+'\\'+folder + '.html',txt)
if pdf:
print('开始生成pdf,请稍候...')
path_wk = r'C:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe'# wkhtmltopdf安装位置
config = pdfkit.configuration(wkhtmltopdf=path_wk)
options = {
'page-size': 'A4',
'margin-top': '0.75in',
'margin-right': '0.75in',
'margin-bottom': '0.75in',
'margin-left': '0.75in',
'encoding': "UTF-8",
'outline': None
}
pdfkit.from_file(, savepath+folder+'\\'+folder+'.pdf',options=options,configuration=config)
print('任务完成!')
if __name__ == '__main__':
url = 'http://c.biancheng.net/python/' #获取教程url地址
savepath='C:\\' #保存位置
getdata(url,True) #后面True表示生成PDF，False不生成
说明：
代码中两处writehtml注释后仅生成一个html和一个Pdf文件
取消注释的话，每个章节将会生成一个html文件，如下：
另外，生成PDF使用了pdfkit库，关于这个库要说明一下，PIP安装后直接使用还是会报错的，还需要下载安装windows版本的wkhtmltopdf
下载地址：https://downloads.wkhtmltopdf.or ... .msvc2015-win64.exe
安装完成之后需要在代码中修改path_wk wkhtmltopdf.exe的安装路径，然后可以了 本帖最后由 itanium 于 2019-9-23 15:37 编辑
import requests
from lxml import etree
import re
import os
import pdfkit
def gethtml(url,encode):
r = requests.get(url)
r.encoding = encode
return r.text
def writehtml(path,str):
f = open(path,'w+',encoding='utf-8')
f.write(str)
f.close
def validateTitle(title):
rstr = r"[\/\\\:\*\"\<\>\|]"
new_title = re.sub(rstr, "_", title)
return new_title
def mkdir(path):
path = path.strip()
isExists = os.path.exists(path)
if not isExists:
os.makedirs(path)
return True
else:
print('文件夹已存在，请检查后再试！')
return False
def getdata(url,pdf):
print('开始获取,请稍候...')
c_url = url.split('/') + '//' + url.split('/') + '/'
html = gethtml(url,'utf-8')
ehtml = etree.HTML(html)
urll1 = ehtml.xpath('//*[@id="contents"]/dd/a/@href')
till1 = ehtml.xpath('//*[@id="contents"]/dd/a/text()')
s = ehtml.xpath('//*[@id="contents"]/dd/span/text()')
folder=validateTitle(ehtml.xpath('//*[@id="contents"]/dt/a/text()'))
if mkdir(savepath+folder):
m = 0
txt=''
for i in urll1:
html = gethtml(c_url + i,'utf-8')
ehtml = etree.HTML(html)
strs = ehtml.xpath('//*[@id="article"]')[-1]
txtl1 = etree.tostring(strs, encoding="utf-8", pretty_print=True, method="html").decode("utf-8")
fname = validateTitle(s + ' ' + till1)
txtl1 = re.sub('<h1>.*</h1>','<h1>'+fname+'</h1>',txtl1)
txtl1 = re.sub('src="/','src="'+c_url+'/',txtl1)
txt=txt+txtl1
#writehtml(savepath+folder+'\\'+fname+'.html', txtl1) #每个章节生成一个html文件
s1 = ehtml.xpath('//*[@id="contents"]/dl/dd/text()')
s2 = ehtml.xpath('//*[@id="contents"]/dl/dd/a/text()')
urll2=ehtml.xpath('//*[@id="contents"]/dl/dd/a/@href')
print(fname)
n=0
for j in urll2:
html=gethtml(c_url+j,'utf-8')
ehtml = etree.HTML(html)
strs = ehtml.xpath('//*[@id="arc-body"]')[-1]
txtl2 = etree.tostring(strs, encoding="utf-8", pretty_print=True, method="html").decode("utf-8")
fname = validateTitle(s1+' '+s2)
txtl2= re.sub('<h>','<h3>',txtl2)
txtl2 = re.sub('</h>', '</h3>', txtl2)
txtl2 = re.sub('src="/','src="'+c_url+'/',txtl2)
txtl2 = '<h2>' + fname + '</h2>'+txtl2
txt = txt + txtl2
#writehtml(savepath+folder+'\\'+fname + '.html', txtl2) #每个章节生成一个html文件
print(fname)
n+=1
m+=1
writehtml(savepath+folder+'\\'+folder + '.html',txt)
if pdf:
print('开始生成pdf,请稍候...')
path_wk = r'C:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe' # wkhtmltopdf安装位置
config = pdfkit.configuration(wkhtmltopdf=path_wk)
options = {
'page-size': 'A4',
'margin-top': '0.75in',
'margin-right': '0.75in',
'margin-bottom': '0.75in',
'margin-left': '0.75in',
'encoding': "UTF-8",
'outline': None
}
pdfkit.from_file(, savepath+folder+'\\'+folder+'.pdf',options=options,configuration=config)
print('任务完成!')
if __name__ == '__main__':
url = 'http://c.biancheng.net/shell/' #获取教程url地址
savepath='C:\\' #保存位置
getdata(url,True) #后面True表示生成PDF，False不生成
提示下面的错误,，那位大神看一下我那个地方弄错了，
D:\>python 1.py
开始获取,请稍候...
Traceback (most recent call last):
File "1.py", line 94, in <module>
getdata(url,True) #后面True表示生成PDF，False不生成
File "1.py", line 49, in getdata
fname = validateTitle(s + ' ' + till1)
IndexError: list index out of range
@null119楼主大大，这个python教程内容收费阿，能不能把PDF文件分享出来阿？{:1_893:} 同志你好，可否分享一下pdf，收费的，木得办法 亲测可用，装完python需要安装requests和lxml，就是页面中的图片没有下来，需要再研究下 页面中的图片没有下来 绵绵青山 发表于 2019-7-15 15:56
亲测可用，装完python需要安装requests和lxml，就是页面中的图片没有下来，需要再研究下
已修正,感谢反馈:handshake cocoan 发表于 2019-7-15 16:28
页面中的图片没有下来
已修正,感谢反馈 对于想保存一份本地副本的学习者很有用，谢谢！ 哇 支持一下 厉害 厉害 代码写的很漂亮 {:1_921:} 厉害！分享给朋友 感谢分享 感谢楼主的分享 支持下+1