C# 多线程缓存技术小说极速爬取

TZ糖纸

[C#] 纯文本查看 复制代码

internal class Program
{
    static async Task Main(string[] args)
    {

        var redis = Redis.GetInstance();

        Book book = new Book();
        book.Url = "http://www.tycqzw.net/113_113116/";
        await book.SetItems();

        await book.SetItemsContent();

        var items = book.Items.OrderBy(x => x.Path).ToList();
        
        StringBuilder stringBuilder = new StringBuilder();  

        foreach (var item in items)
        {
            stringBuilder.Append(item.Title+"\n");
            stringBuilder.Append(item.Content);
        }

        File.WriteAllText(book.Title+".txt", stringBuilder.ToString());

        Console.ReadKey();

    }
}

[C#] 纯文本查看 复制代码

public class Book
{

    public Book()
    {
        if (Items == null) Items = new List<BookItem>();
    }
    public string Title { get; set; }
    public string Url { get; set; }
    public List<BookItem> Items { get; set; }

    public void Add(BookItem item)
    {
        if (Items.Where(n => n.Path == item.Path).Count() > 0) return;
        Items.Add(item);
    }

    internal async Task SetItems()
    {
        var redis = Redis.GetInstance();

        var json = redis.Get(this.Title);
        if (!string.IsNullOrWhiteSpace(json))
        {
            this.Items = JsonConvert.DeserializeObject<List<BookItem>>(json);
            return;
        }

        var client = new RestClient(this.Url);
        var request = new RestRequest();
        var response = await client.ExecuteAsync(request);
        var content = response.Content;
        var parser = new HtmlParser();
        var doc = parser.ParseDocument(content);

        var title = doc.QuerySelector("#info h1").InnerHtml;
        var dda = doc.QuerySelectorAll("dd a");

        this.Title = title;
        Console.WriteLine(this.Title);
        foreach (IHtmlAnchorElement item in dda)
        {
            Items.Add(new BookItem() { Title = item.InnerHtml, Path = item.PathName });

        }
        redis.Set(this.Title, JsonConvert.SerializeObject(this.Items));

    }

    internal async Task SetItemsContent()
    {
        var index = 0;
        var redis = Redis.GetInstance();


        // 初始化信号量，允许的最大并发数为10
        SemaphoreSlim semaphore = new SemaphoreSlim(1000, 1000);
        // 任务列表
        List<Task> tasks = new List<Task>();

        async Task ProcessTask(BookItem bookItem)
        {
            await semaphore.WaitAsync();
            try
            {
                string mess = "";
                mess += bookItem.Title;
                await bookItem.SetContent();
                mess += "-获取完成";
                index++;
                if (index % 10 == 0)
                {
                    redis.Set(this.Title, JsonConvert.SerializeObject(this.Items));
                    mess += "-写入缓存";
                }
                Console.WriteLine(mess);
            }
            finally
            {
                // 释放信号量，允许下一个任务开始
                semaphore.Release();
            }
        }

        foreach (var item in this.Items)
        {
            if (item.Content == null)
            {

                tasks.Add(ProcessTask(item));
            }

        }

        // 等待所有任务完成
        await Task.WhenAll(tasks);

        redis.Set(this.Title, JsonConvert.SerializeObject(this.Items));
    }
}
public class BookItem
{
    public string Title { get; set; }
    public string Path { get; set; }
    public string Content { get; set; }

    internal async Task SetContent()
    {
        try
        {
            var client = new RestClient("http://www.tycqzw.net/" + Path);
            var request = new RestRequest();
            var response = await client.ExecuteAsync(request);
            var content = response.Content;
            var parser = new HtmlParser();
            var doc = parser.ParseDocument(content);
            IHtmlDivElement contentMain = doc.QuerySelector("#content") as IHtmlDivElement;
            this.Content = contentMain.TextContent;
        }
        catch (Exception)
        {

        }

    }
}

[C#] 纯文本查看 复制代码

public class Redis
{
    private static RedisClient redisClient;

    private Redis() { }

    public static RedisClient GetInstance()
    {
        if (redisClient == null) {
            redisClient = new RedisClient("127.0.0.1:6379,defaultDatabase=12");
        }

        return redisClient;
    }
}

yhzh

感谢分享。。

小小小酥

感谢大佬分享

SONGXINGJING520

难得看到C#的教程。支持

caisonglinlove

感谢分享，大神牛x，必须点赞

msmvc

看论坛多数是用python的，我用C Sharp

findevery

学习了！！！

81169009

不会用，大神谁帮忙打包下成品啊

xyj152

好东西，抽空要学习一下