可见即可抓，可视即可爬,cefsharp响应拦截
本帖最后由 TZ糖纸 于 2023-12-27 16:19 编辑先上图
这里用的语言是C#
使用的谷歌内核
直接在nuget
然后安装即可
创建一个winform项目，然后对cefsharp 进行配置
public partial class Form1 : Form {
ChromiumWebBrowser browser;
public void InitBrowser() {
CefSettings settings = new CefSettings();
// Note that if you get an error or a white screen, you may be doing something wrong !
// Try to load a local file that you're sure that exists and give the complete path instead to test
// for example, replace page with a direct path instead :
// String page = @"C:\Users\SDkCarlos\Desktop\afolder\index.html";
// String page = string.Format(@"{0}\html-resources\html\index.html", Application.StartupPath);
String url = "http://www.html5test.com";
// Initialize cef with the provided settings
Cef.Initialize(settings);
// Create a browser component
browser = new ChromiumWebBrowser(url);
// Add it to the form and fill it to the form window.
this.Controls.Add(browser);
browser.Dock = DockStyle.Fill;
// Allow the use of local resources in the browser
BrowserSettings browserSettings = new BrowserSettings();
browserSettings.FileAccessFromFileUrls = CefState.Enabled;
browserSettings.UniversalAccessFromFileUrls = CefState.Enabled;
browser.BrowserSettings = browserSettings;
}
public Form1() {
InitializeComponent();
InitBrowser();
}
}
然后要进行资源的拦截
创建文件ResourceRequestHandler对IResourceRequestHandler进行实现
修改GetResourceResponseFilter方法
public IResponseFilter GetResourceResponseFilter(IWebBrowser chromiumWebBrowser, IBrowser browser, IFrame frame, IRequest request, IResponse response) {
var url = new Uri(request.Url);
if (url.AbsoluteUri.Contains("douyin.com/aweme/v1/web/aweme/post"))
{
return DouYinResponseFilter.CreateFilter(request.Identifier.ToString());
}
return null;
}
修改OnResourceLoadComplete 方法
public void OnResourceLoadComplete(IWebBrowser chromiumWebBrowser, IBrowser browser, IFrame frame, IRequest request, IResponse response, UrlRequestStatus status, long receivedContentLength) {
var url = new Uri(request.Url);
if (url.AbsoluteUri.Contains("douyin.com/aweme/v1/web/aweme/post"))
{
var filter = DouYinResponseFilter.GetFileter(request.Identifier.ToString()) as DouYinResponseFilter;
if (filter != null)
{
var encode = !string.IsNullOrEmpty(response.Charset)
? Encoding.GetEncoding(response.Charset)
: Encoding.UTF8;
using (var read = new StreamReader(filter.GetStream(), encode))
{
var text = read.ReadToEnd();
Debug.WriteLine(text);//捕获的响应
}
}
}
}
DouYinResponseFilter 具体实现
internal class DouYinResponseFilter : IResponseFilter
{
private MemoryStream Stream;
public DouYinResponseFilter()
{
Stream = new MemoryStream();
}
public Stream GetStream()
{
Stream.Seek(0, SeekOrigin.Begin);
return Stream;
}
private static Dictionary<string, IResponseFilter> _dictionary = new Dictionary<string, IResponseFilter>();
public static IResponseFilter CreateFilter(string id)
{
var filter = new DouYinResponseFilter();
_dictionary = filter;
return filter;
}
public static IResponseFilter GetFileter(string id)
{
if (_dictionary.ContainsKey(id))
{
var filter = _dictionary;
_dictionary.Remove(id);
return filter;
}
return null;
}
public void Dispose()
{
}
public FilterStatus Filter(Stream dataIn, out long dataInRead, Stream dataOut, out long dataOutWritten)
{
try
{
if (dataIn == null || dataIn.Length == 0)
{
dataInRead = 0;
dataOutWritten = 0;
return FilterStatus.Done;
}
dataInRead = dataIn.Length;
dataOutWritten = Math.Min(dataInRead, dataOut.Length);
dataIn.CopyTo(dataOut);
dataIn.Seek(0, SeekOrigin.Begin);
byte[] bs = new byte;
dataIn.Read(bs, 0, bs.Length);
Stream.Write(bs, 0, bs.Length);
dataInRead = dataIn.Length;
dataOutWritten = dataIn.Length;
return FilterStatus.NeedMoreData;
}
catch (Exception ex)
{
dataInRead = dataIn.Length;
dataOutWritten = dataIn.Length;
return FilterStatus.Done;
}
}
public bool InitFilter()
{
return true;
}
}
拿到我们在OnResourceLoadComplete捕获的text(响应数据)
长这样
后边我们就可以拿着json数据进行处理
提取出来作品描述和作品链接
var item = JObject.Parse(data);
var aweme_list = item["aweme_list"];
foreach (var item1 in aweme_list)
{
var desc = item1["desc"];//作品标题
if (desc.ToString() == "")
{
desc= item1["aweme_id"];//没有作品标题使用作品的id
}
var playaddr = item1["video"]["play_addr"]["url_list"];//作品链接
}
然后就可以拿到这样的数据
然后自己写一个下载器
进行下载即可
然后就能看到下载的作品啦
使用教程
默认的下载位置在D:\XDownload
下载地址https://tangzhizzz.lanzouq.com/inYT21izum1i
TZ糖纸 发表于 2023-12-26 17:52
看我的新帖！
运行 xbrowser.exe 出错
问题签名:
问题事件名称: CLR20r3
问题签名 01: XBrowser.exe
问题签名 02: 1.0.0.0
问题签名 03: db51a119
问题签名 04: CefSharp.WinForms
问题签名 05: 120.1.110.0
问题签名 06: badc7076
问题签名 07: 1
问题签名 08: 0
问题签名 09: System.IO.FileLoadException
OS 版本: 6.1.7601.2.1.0.256.48
这个很强大啊 ，谢谢分享，学习一下源码 文西思密达 发表于 2023-12-19 14:53
这个很强大啊 ，谢谢分享，学习一下源码
源码自己写，主要的都贴出来了 感谢分享，思路学习了，成品能否分享！ 逐雅斋 发表于 2023-12-19 14:57
感谢分享，思路学习了，成品能否分享！
不白嫖哈 66666666666666666 这么厉害 PC端IDM，手机端1DM，汇报完毕。 上个成品就更方便了。 ss258b 发表于 2023-12-19 15:08
PC端IDM，手机端1DM，汇报完毕。
批量呢？？