Go语言 汽车之家精选论坛图片下载

Jocelyn123

[Golang] 纯文本查看 复制代码

// 精选网页地址
url := "https://club.autohome.com.cn/jingxuan/"

地址也可以更换成其它板块，你们懂的！需要手动创建path路径，路径必须存在。
下面是源码demo

[Golang] 纯文本查看 复制代码

package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"go_dev_idea/app/service"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func main()  {
	arr := GetAutoList()
	for _,v := range arr {
		DownloadAutoDetail("https:" + v)
		// 防止被盾
		time.Sleep(2 * time.Second)
	}
}

func GetAutoList() []string{
	// 精选
	url := "https://club.autohome.com.cn/jingxuan/"
	header := map[string]string{
		"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36",
	}
	byt := MicHeaderRequest(url,nil,header)
	doc, _ := goquery.NewDocumentFromReader(strings.NewReader(string(byt)))
	// tz-picture
	imgUrls := make([]string,0)
	doc.Find(".pic-box").Each(func(i int, s *goquery.Selection) {
		//解析<li>标签
		url,_ := s.Find("a").Attr("href")
		imgUrls = append(imgUrls,url)
	})
	return imgUrls
}

func MicHeaderRequest(url string,params,header map[string]string) []byte{
	client := &http.Client{Timeout: 5 * time.Second}
	newRequest, err := http.NewRequest("GET",url,nil)
	if err != nil {
		return nil
	}
	for k,v := range header {
		newRequest.Header.Set(k,v)
	}
	resp, err := client.Do(newRequest)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	return body
}

func DownloadAutoDetail(url string)  {
	header := map[string]string{
		"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36",
	}
	byt := service.MicHeaderRequest(url,nil,header)
	//fmt.Println(string(byt))
	doc, _ := goquery.NewDocumentFromReader(strings.NewReader(string(byt)))
	// tz-picture
	imgUrls := make([]string,0)
	doc.Find(".tz-figure").Each(func(i int, s *goquery.Selection) {
		//解析<li>标签
		url,_ := s.Find("img").Attr("data-src")
		imgUrls = append(imgUrls,url)
	})
	doc.Find(".tz-picture").Each(func(i int, s *goquery.Selection) {
		//解析<li>标签
		url,_ := s.Find("img").Attr("data-src")
		imgUrls = append(imgUrls,url)
	})
	title := strings.Replace(strings.Replace(doc.Find(".post-title").Text(), " ", "", -1), "\n", "", -1)
	// "F:/images/2021.5.8/" 需要自己手动创建 路劲必须存在
	path := "F:/images/2021.5.8/"+title
	CreatedDir(path)
	for k,v := range imgUrls {
		SaveImages("https:"+v,path+"/"+strconv.Itoa(k))
	}
}

func CreatedDir(path string) {
	os.Mkdir(path, os.ModePerm)
}

func SaveImages(url,path string){
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		fmt.Println(err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
	}
	_ = ioutil.WriteFile(path+".jpg", body, 0755)
}

七秒的记忆

小白不会  坐等成品   

RKCN

缺少东西吧，

go_dev_idea/app/service
这个是什么类啊？

Jocelyn123

RKCN 发表于 2021-7-7 22:24
缺少东西吧，

go_dev_idea/app/service

把64行 byt := service.MicHeaderRequest(url,nil,header) 换成 byt := MicHeaderRequest(url,nil,header)

RKCN

感谢，已经可以执行了。