golang 爬取壁纸和图片其他信息，提交保存入库
```package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
)
var (
path_url = `https://wall.alphacoders.com/by_sub_category.php?id=333944&name=Genshin%20Impact&page=`
pageStart int = 1 // 起始页
page int = 1 // 爬取页数
DownLoadPath string = "C:/Users/me/go/src/go_query/img/" // 图片下载存放地址
wg sync.WaitGroup
)
type imginfo struct {
Titlestring `json:"title"`
Url string `json:"url"`
Widthstring `json:"width"`
Height string `json:"height"`
Page int `json:"page"`
}
func main() {
fmt.Println("从第几页开始")
fmt.Scanln(&pageStart)
fmt.Println("扫描页数")
fmt.Scanln(&page)
for i := pageStart; i < pageStart+page; i++ {
wg.Add(1)
getBody(i)
}
wg.Wait()
fmt.Println("执行完毕,回车退出")
var input string
fmt.Scanln(&input)
}
//获取页面数据
func getBody(p int) {
resp, err := http.Get(path_url + strconv.Itoa(p))
if err != nil {
log.Fatalf("status code error: %s", err)
}
defer resp.Body.Close()
dom, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
log.Fatal(err)
}
dom.Find(".boxgrid").Each(func(i int, s *goquery.Selection) {
href, _ := s.Find("a").Attr("href")
title, _ := s.Find("a").Attr("title")
width, height := "0", "0"
_imginfo := imginfo{
Title:title,
Url: href,
Width:width,
Height: height,
Page: p,
}
getImgInfo(_imginfo)
})
wg.Done()
}
//获取图片详情
func getImgInfo(_imginfo imginfo) {
url := "https://wall.alphacoders.com" + _imginfo.Url
resp, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
dom, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
log.Fatal(err)
}
dom.Find(".center.img-container-desktop").Each(func(i int, s *goquery.Selection) {
href, _ := s.Find("a").Attr("href")
width, _ := s.Find("img").Attr("width")
height, _ := s.Find("img").Attr("height")
_imginfo.Url = href
_imginfo.Width = width
_imginfo.Height = height
handleImg(_imginfo)
})
}
// 处理图片方法
func handleImg(_imginfo imginfo) {
fileName := getFilename(_imginfo.Url)
result := DownloadFile(_imginfo, fileName)
if result {
fmt.Printf("已下载完毕: %s - %s - %s - %s \n", _imginfo.Url, _imginfo.Title, _imginfo.Width, _imginfo.Height)
} else {
fmt.Printf("下载失败 %v \n", _imginfo.Url)
}
}
// 截取url名字
func getFilename(url string) (filename string) {
lastIndex := strings.LastIndex(url, "/")
filename = url
timePrefix := strconv.Itoa(int(time.Now().UnixNano()))
filename = timePrefix + "_" + filename
return
}
//下载图片
func DownloadFile(_imginfo imginfo, filename string) bool {
var err error
resp, err := http.Get(_imginfo.Url)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
if _, err := os.Stat(DownLoadPath); os.IsNotExist(err) {
os.Mkdir(DownLoadPath, 0777)
os.Chmod(DownLoadPath, 0666)
}
filename = DownLoadPath + filename
err = ioutil.WriteFile(filename, bytes, 0666)
if err != nil {
return false
}
postFile(_imginfo, filename, "http://127.0.0.1:9999/wp-admin/admin-ajax.php")
return true
}
//数据提交
func postFile(_imginfo imginfo, filename string, target_url string) error {
body_buf := bytes.NewBufferString("")
body_writer := multipart.NewWriter(body_buf)
fh, err := os.Open(filename)
if err != nil {
fmt.Println("error opening file")
return err
}
defer fh.Close()
body_writer.WriteField("action", "upload_wallpaper")
body_writer.WriteField("width", _imginfo.Width)
body_writer.WriteField("height", _imginfo.Height)
body_writer.WriteField("title", _imginfo.Title)
body_writer.CreateFormFile("file", filename)
boundary := body_writer.Boundary()
close_buf := bytes.NewBufferString(fmt.Sprintf("\r\n--%s--\r\n", boundary))
request_reader := io.MultiReader(body_buf, fh, close_buf)
req, err := http.NewRequest("POST", target_url, request_reader)
if err != nil {
return err
}
req.Header.Add("Content-Type", "multipart/form-data; boundary="+boundary)
resp, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Printf("上传失败: %s\n", err)
return err
}
ret, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Printf("转换失败: %s\n", err)
return err
}
defer resp.Body.Close()
fmt.Printf("返回的状态码是： %v\n", resp.StatusCode)
fmt.Printf("返回的信息是： %v\n", string(ret))
return nil
}
``` 路过，支持 感谢分享。 感谢分享，很好用 学习一下
页:
[1]