HttpClient 客户端编程

HttpClient 提供高效的，最新的，功能丰富的支持HTTP协议的客户端编程工具，并且支持HTTP协议最新版本和建议

HttpClient可以模拟浏览器请求第三方站点url，然后响应获取网页数据。

引入依赖

  4.0.0
  com.httpclient
  HttpClientDemo
  0.0.1-SNAPSHOT
  
  
  
  	
	    org.apache.httpcomponents
	    httpclient
	    4.5.2
	
	
	
	    commons-io
	    commons-io
	    2.5
	

  

获得httpClient客户端实例，执行get和post请求

post可以提交form

get直接url

execute 发送请求

import java.io.IOException;

import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class HelloWorld {

	public static void main(String[] args) {
		CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
		HttpGet httpGet=new HttpGet("http://www.tuicool.com/"); // 创建httpget实例
		CloseableHttpResponse response=null;
		try {
			response=httpClient.execute(httpGet); // 执行http get请求
		} catch (ClientProtocolException e) { // http协议异常
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) { // io异常
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		HttpEntity entity=response.getEntity(); // 获取返回实体
		try {
			System.out.println("网页内容："+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
		} catch (ParseException e) { // 解析异常
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) { // io异常
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		try {
			response.close(); // response关闭
		} catch (IOException e) {  // io异常
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		try {
			httpClient.close(); // httpClient关闭
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}

抓取网页

设置请求头消息 User-Agent 模拟浏览器

获取响应内容类型 Content-Type 响应状态 Status(200正常，403拒绝，500服务器报错，400未找到页面)

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class Demo {

	public static void main(String[] args)throws Exception {
		CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
		HttpGet httpGet=new HttpGet("http://www.tuicool.com/"); // 创建httpget实例
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
		CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
		System.out.println("Status:"+response.getStatusLine().getStatusCode());
        HttpEntity entity=response.getEntity(); // 获取返回实体
		System.out.println("网页内容："+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
	    System.out.println("Content-Type:"+entity.getContentType().getValue());
		response.close(); // response关闭
		httpClient.close(); // httpClient关闭
	}
}

HttpClient 通过流抓取图片

import java.io.File;
import java.io.InputStream;

import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

public class Demo1 {

	public static void main(String[] args)throws Exception {
		CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
		HttpGet httpGet=new HttpGet("http://www.java1234.com/gg/dljd4.gif"); // 创建httpget实例
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
		CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
		HttpEntity entity=response.getEntity(); // 获取返回实体
		if(entity!=null){
			System.out.println("ContentType:"+entity.getContentType().getValue());
			InputStream inputStream=entity.getContent();
			FileUtils.copyToFile(inputStream, new File("C://dljd4.gif"));
		}
		response.close(); // response关闭
		httpClient.close(); // httpClient关闭
	}
}

HttpClient 使用代理ip

代理ip种类，透明代理，匿名代理，混淆代理，高匿代理

一般用高匿代理

HttpHost  代理主机

custom().setProxy(proxy).build() 设置代理ip返回构造

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class Demo01 {

	public static void main(String[] args)throws Exception {
		CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
		HttpGet httpGet=new HttpGet("http://www.tuicool.com/"); // 创建httpget实例
		HttpHost proxy=new HttpHost("175.155.213.235", 9999);
		RequestConfig config=RequestConfig.custom().setProxy(proxy).build();
		httpGet.setConfig(config);
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
		CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
		HttpEntity entity=response.getEntity(); // 获取返回实体
		System.out.println("网页内容："+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
		response.close(); // response关闭
		httpClient.close(); // httpClient关闭
	}
}

Http连接超时和读取超时

 

import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class Demo {

	public static void main(String[] args)throws Exception {
		CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
		HttpGet httpGet=new HttpGet("http://central.maven.org/maven2/"); // 创建httpget实例
		RequestConfig config=RequestConfig.custom()
				.setConnectTimeout(10000) // 设置连接超时时间 10秒钟
				.setSocketTimeout(20000) // 设置读取超时时间10秒钟
				.build();
		httpGet.setConfig(config);
		httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
		CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
		HttpEntity entity=response.getEntity(); // 获取返回实体
		System.out.println("网页内容："+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
		response.close(); // response关闭
		httpClient.close(); // httpClient关闭
	}
}