Go 爬虫代理开发示例

1
package main
2

3
import (
4
    "net/url"
5
    "net/http"
6
    "bytes"
7
    "fmt"
8
    "io/ioutil"
9
)
10

11
// 代理服务器(产品官网 www.16yun.cn)
12
const ProxyServer = "t.16yun.cn:31111"
13

14
type ProxyAuth struct {
15
    Username string
16
    Password string
17
}
18

19
func (p ProxyAuth) ProxyClient() http.Client {
20

21
    var proxyURL *url.URL
22
    if p.Username != ""&& p.Password!="" {
23
        proxyURL, _ = url.Parse("http://" + p.Username + ":" + p.Password + "@" + ProxyServer)
24
    }else{
25
        proxyURL, _ = url.Parse("http://" + ProxyServer)
26
    }
27
    return http.Client{Transport: &http.Transport{Proxy:http.ProxyURL(proxyURL)}}
28
}
29

30
func main()  {
31

32

33
    targetURI := "https://httpbin.org/ip"
34

35

36
    // 初始化 proxy http client
37
    client := ProxyAuth{"username",  "password"}.ProxyClient()
38

39
    request, _ := http.NewRequest("GET", targetURI, bytes.NewBuffer([] byte(``)))
40

41
    // 设置Proxy-Tunnel
42
    // rand.Seed(time.Now().UnixNano())
43
    // tunnel := rand.Intn(10000)
44
    // request.Header.Set("Proxy-Tunnel", strconv.Itoa(tunnel) )
45

46
    response, err := client.Do(request)
47

48
    if err != nil {
49
        panic("failed to connect: " + err.Error())
50
    } else {
51
        bodyByte, err := ioutil.ReadAll(response.Body)
52
        if err != nil {
53
            fmt.Println("读取 Body 时出错", err)
54
            return
55
        }
56
        response.Body.Close()
57

58
        body := string(bodyByte)
59

60
        fmt.Println("Response Status:", response.Status)
61
        fmt.Println("Response Header:", response.Header)
62
        fmt.Println("Response Body:\n", body)
63
    }
64
}

1
package main
2

3
import (
4
    "crypto/tls"
5
    "fmt"
6
    "io/ioutil"
7
    "net"
8
    "net/http"
9
    "net/http/httputil"
10
    "net/url"
11
    "time"
12
)
13

14
func main() {
15

16
    for {
17
        // 代理服务器的用户名和密码
18
        proxyUsername := "username"
19
        proxyPassword := "password"
20
        // 代理服务器(产品官网 www.16yun.cn)
21
        // 代理服务器的 URL
22
        proxyURL, err := url.Parse(fmt.Sprintf("http://%s:%s@t.16yun.cn:31111", proxyUsername, proxyPassword))
23
        if err != nil {
24
            fmt.Println(err)
25
            return
26
        }
27
        // 添加自定义头部,
28
        //rand.Seed(time.Now().UnixNano())
29
        //tunnel := rand.Intn(10000)
30
        //proxyHeaders.Add("Proxy-Tunnel", strconv.Itoa(tunnel))
31

32
        proxyHeaders := http.Header{}
33
        // 设置为固定的数字 ，后面所有请求都会固定到一个IP上
34
        // proxyHeaders.Add("Proxy-Tunnel", "1")
35
        
36
        // 定制Transport
37
        tr := &http.Transport{
38
            Proxy:           http.ProxyURL(proxyURL),
39
            TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // 如果需要跳过证书验证
40
            // 自定义DialContext函数
41
            DialContext: (&net.Dialer{
42
                Timeout:   30 * time.Second,
43
                KeepAlive: 30 * time.Second,
44
                DualStack: true,
45
            }).DialContext,
46
            ProxyConnectHeader: proxyHeaders,
47
        }
48
        // 定制Client
49
        client := &http.Client{
50

51
            Transport: tr,
52
        }
53

54
        // 发起请求
55
        req, err := http.NewRequest("GET", "https://httpbin.org/ip", nil)
56
        if err != nil {
57
            fmt.Println(err)
58
            return
59
        }
60

61
        // 使用httputil.DumpRequest输出完整的请求
62
        dump, err := httputil.DumpRequestOut(req, true)
63
        if err != nil {
64
            fmt.Println(err)
65
            return
66
        }
67
        fmt.Println(string(dump))
68

69
        // 发送请求
70
        resp, err := client.Do(req)
71
        if err != nil {
72
            fmt.Println("error")
73
            fmt.Println(err)
74
            return
75
        }
76
        defer resp.Body.Close()
77
        // 读取响应体
78
        body, err := ioutil.ReadAll(resp.Body)
79
        if err != nil {
80
            fmt.Println(err)
81
            return
82
        }
83

84
        // 打印响应体
85
        fmt.Println("Response Body:", string(body))
86
        time.Sleep(time.Second * 1)
87
    }
88

89
}