Java 爬虫代理示例

1
import java.io.ByteArrayOutputStream;
2
import java.io.InputStream;
3
import java.net.Authenticator;
4
import java.net.HttpURLConnection;
5
import java.net.InetSocketAddress;
6
import java.net.PasswordAuthentication;
7
import java.net.Proxy;
8
import java.net.URL;
9
import java.util.Random;
10

11
class ProxyAuthenticator extends Authenticator {
12
    private String user, password;
13

14
    public ProxyAuthenticator(String user, String password) {
15
        this.user     = user;
16
        this.password = password;
17
    }
18

19
    protected PasswordAuthentication getPasswordAuthentication() {
20
        return new PasswordAuthentication(user, password.toCharArray());
21
    }
22
}
23

24
/**
25
 * 注意：下面代码仅仅实现HTTP请求链接，每一次请求都是无状态保留的，仅仅是这次请求是更换IP的，如果下次请求的IP地址会改变
26
 * 如果是多线程访问的话，只要将下面的代码嵌入到你自己的业务逻辑里面，那么每次都会用新的IP进行访问，如果担心IP有重复，
27
 * 自己可以维护IP的使用情况，并做校验。
28
 */
29
public class Demo {
30
    public static void main(String args[]) throws Exception {
31
        // Change in Java 8 Update 111 以上版本需要下面代码
32
        // System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "false");
33
        // System.setProperty("jdk.http.auth.proxying.disabledSchemes", "false");
34
        
35
        // 要访问的目标页面
36
        String targetUrl = "http://httpbin.org/ip";
37

38

39
        // 代理服务器(产品官网 www.16yun.cn)
40
        String proxyServer = "t.16yun.cn";
41
        int proxyPort      = 31111;
42

43
        // 代理验证信息
44
        String proxyUser  = "username";
45
        String proxyPass  = "password";
46

47
        try {
48
            URL url = new URL(targetUrl);
49

50
            Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));
51

52
            // 创建代理服务器地址对象
53
            InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
54
            // 创建HTTP类型代理对象
55
            Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
56

57
            // 设置通过代理访问目标页面
58
            HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
59
            
60
            // 设置KeepAlive
61
            // connection.setRequestProperty("Connection", "keep-alive");
62
            // connection.setRequestProperty("Keep-Alive", "timeout=5, max=100");                
63

64
            // 设置Proxy-Tunnel
65
            // Random random = new Random();
66
            // int tunnel = random.nextInt(10000);
67
            // connection.setRequestProperty("Proxy-Tunnel",String.valueOf(tunnel));
68

69
            // 解析返回数据
70
            byte[] response = readStream(connection.getInputStream());
71

72
            System.out.println(new String(response));
73
        } catch (Exception e) {
74
            System.out.println(e.getLocalizedMessage());
75
        }
76
    }
77

78
    /**
79
     * 将输入流转换成字符串
80
     *
81
     * @param inStream
82
     * @return
83
     * @throws Exception
84
     */
85
    public static byte[] readStream(InputStream inStream) throws Exception {
86
        ByteArrayOutputStream outSteam = new ByteArrayOutputStream();
87
        byte[] buffer = new byte[1024];
88
        int len = -1;
89

90
        while ((len = inStream.read(buffer)) != -1) {
91
            outSteam.write(buffer, 0, len);
92
        }
93
        outSteam.close();
94
        inStream.close();
95

96
        return outSteam.toByteArray();
97
    }
98
}

1
package htmlunit;
2

3
import org.apache.http.auth.AuthScope;
4
import org.apache.http.auth.UsernamePasswordCredentials;
5
import org.apache.http.client.CredentialsProvider;
6
import org.apache.http.impl.client.BasicCredentialsProvider;
7

8
import com.gargoylesoftware.htmlunit.BrowserVersion;
9
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
10
import com.gargoylesoftware.htmlunit.WebClient;
11
import com.gargoylesoftware.htmlunit.html.HtmlPage;
12

13
public class HtmlunitDemo {
14
    // 代理服务器(产品官网 www.16yun.cn)
15
    final static String proxyHost = "t.16yun.cn";
16
    final static Integer proxyPort = 31111;
17

18
    // 代理验证信息
19
    final static String proxyUser = "USERNAME";
20
    final static String proxyPass = "PASSWORD";
21

22
    public static void main(String[] args) {
23
        
24
        CredentialsProvider credsProvider = new BasicCredentialsProvider();
25
        credsProvider.setCredentials(
26
        
27
        new AuthScope(proxyHost, proxyPort),
28
        new UsernamePasswordCredentials(proxyUser, proxyPass));
29
  
30
        WebClient webClient = new WebClient(BrowserVersion.CHROME,proxyHost, proxyPort);
31
  
32
        webClient.setCredentialsProvider(credsProvider);
33
     
34
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
35
        webClient.getOptions().setJavaScriptEnabled(true);
36
        webClient.getOptions().setThrowExceptionOnScriptError(false);
37
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
38
        webClient.getOptions().setActiveXNative(false);
39
        webClient.getOptions().setCssEnabled(false);
40

41
        HtmlPage page = null;
42

43
        try {
44
            page = webClient.getPage("http://httpbin.org/ip");
45
        } catch (Exception e) {
46
            e.printStackTrace();
47
        } finally {
48
            webClient.close();
49
        }
50

51
        webClient.waitForBackgroundJavaScript(30000);
52

53
        
54
        String pageXml = page.asXml();
55

56
        System.out.println(pageXml);
57
    }
58
}

1

2
    import okhttp3.*;
3

4
    import java.io.IOException;
5
    import java.net.InetSocketAddress;
6
    import java.net.Proxy;
7
    import java.util.concurrent.TimeUnit;
8

9
    public class OkHttp {
10

11
        // 代理服务器(产品官网 www.16yun.cn)
12
        final static String proxyHost = "t.16yun.cn";
13
        final static Integer proxyPort = 31111;
14

15
        // 代理验证信息
16
        final static String proxyUser = "USERNAME";
17
        final static String proxyPass = "PASSWORD";
18

19
        static OkHttpClient client = null;
20

21
        static {
22
            Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
23

24
            Authenticator proxyAuthenticator = new Authenticator() {
25
                public Request authenticate(Route route, Response response) {
26
                    String credential = Credentials.basic(proxyUser, proxyPass);
27
                    return response.request().newBuilder()
28
                            .header("Proxy-Authorization", credential)
29
                            .build();
30
                }
31
            };
32

33
            client = new OkHttpClient().newBuilder()
34
                    .connectTimeout(5, TimeUnit.SECONDS)
35
                    .readTimeout(5, TimeUnit.SECONDS)
36
                    .proxy(proxy)
37
                    .proxyAuthenticator(proxyAuthenticator)
38
                    .connectionPool(new ConnectionPool(5, 1, TimeUnit.SECONDS))
39
                    .build();
40
        }
41

42
        public static Response doGet() throws IOException {
43
            // 要访问的目标页面
44
            String targetUrl = "http://httpbin.org/ip";
45

46
            Request request = new Request.Builder()
47
                    .url(targetUrl)
48
                    .build();
49
            Response response = client.newCall(request).execute();
50
            return response;
51
        }
52

53
        public static void main(String[] args) throws IOException {
54
            Response response1 = doGet();
55
            System.out.println("GET请求返回结果：");
56
            System.out.println(response1.body().string());
57
        }
58

59
    }

1
import org.json.JSONException;
2
import org.json.JSONObject;
3
import org.openqa.selenium.Platform;
4
import org.openqa.selenium.Proxy;
5
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
6
import org.openqa.selenium.remote.CapabilityType;
7
import org.openqa.selenium.remote.DesiredCapabilities;
8

9
import com.gargoylesoftware.htmlunit.DefaultCredentialsProvider;
10
import com.gargoylesoftware.htmlunit.WebClient;
11

12
public class HtmlUnitDriverProxyDemo
13
{
14
    // 代理验证信息
15
    final static String proxyUser = "username";
16
    final static String proxyPass = "password";
17

18
    // 代理服务器
19
    final static String proxyServer = "t.16yun.cn:31111";
20

21
    public static void main(String[] args) throws JSONException
22
    {
23
        HtmlUnitDriver driver = getHtmlUnitDriver();
24

25
        driver.get("https://httpbin.org/ip");
26

27
        String title = driver.getTitle();
28
        System.out.println(title);
29
    }
30

31
    public static HtmlUnitDriver getHtmlUnitDriver()
32
    {
33
        HtmlUnitDriver driver = null;
34

35
        Proxy proxy = new Proxy();
36
        
37
        proxy.setHttpProxy(proxyServer);
38

39
        DesiredCapabilities capabilities = DesiredCapabilities.htmlUnit();
40
        capabilities.setCapability(CapabilityType.PROXY, proxy);
41
        capabilities.setJavascriptEnabled(true);
42
        capabilities.setPlatform(Platform.WIN8_1);
43

44
        driver = new HtmlUnitDriver(capabilities) {
45
            @Override
46
            protected WebClient modifyWebClient(WebClient client) {
47
                DefaultCredentialsProvider creds = new DefaultCredentialsProvider();
48
                creds.addCredentials(proxyUser, proxyPass);
49
                client.setCredentialsProvider(creds);
50
                return client;
51
            }
52
        };
53

54
        driver.setJavascriptEnabled(true);
55

56
        return driver;
57
    }
58
}              

1
import org.json.JSONException;
2
import org.json.JSONObject;
3
import org.openqa.selenium.Platform;
4
import org.openqa.selenium.Proxy;
5
import org.openqa.selenium.firefox.FirefoxDriver;
6
import org.openqa.selenium.firefox.FirefoxProfile;
7
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
8
import org.openqa.selenium.remote.CapabilityType;
9
import org.openqa.selenium.remote.DesiredCapabilities;
10

11
import com.gargoylesoftware.htmlunit.DefaultCredentialsProvider;
12
import com.gargoylesoftware.htmlunit.WebClient;
13

14
public class FirefoxDriverProxyDemo
15
{
16
    // 代理隧道验证信息
17
    final static String proxyUser = "username";
18
    final static String proxyPass = "password";
19

20
    // 代理服务器
21
    final static String proxyHost = "t.16yun.cn";
22
    final static int proxyPort = 31111;
23

24
    final static String firefoxBin = "C:/Program Files/Mozilla Firefox/firefox.exe";
25

26
    public static void main(String[] args) throws JSONException
27
    {
28
        System.setProperty("webdriver.firefox.bin", firefoxBin);
29

30
        FirefoxProfile profile = new FirefoxProfile();
31
        
32
        profile.setPreference("network.proxy.type", 1);
33
        
34
        
35
        profile.setPreference("network.proxy.http", proxyHost);
36
        profile.setPreference("network.proxy.http_port", proxyPort);
37

38
        profile.setPreference("network.proxy.ssl", proxyHost);
39
        profile.setPreference("network.proxy.ssl_port", proxyPort);
40

41
        profile.setPreference("username", proxyUser);
42
        profile.setPreference("password", proxyPass);
43

44
        
45
        profile.setPreference("network.proxy.share_proxy_settings", true);
46

47
        
48
        profile.setPreference("network.proxy.no_proxies_on", "localhost");
49

50
        
51
        FirefoxDriver driver = new FirefoxDriver(profile);
52
    }
53
}