比尔云BierYun--阿里云最新优惠活动
阿里云优惠码丨阿里云代金券

HttpClient在java中的使用

HttpClient在java中的使用http://www.bieryun.com/1011.html

项目中一直在使用HttpClient,版本是3.6的,负责维护的同事离职后,就没有更新过,在这次项目改版中决定对这块进行升级到4.3版本,查阅了一些资料写了部分出来,还不是很完善,后期有时间在更新

这里对httpclient如何获取原页面编码进行了一些处理,感觉还不是很完善,后期想到在处理吧

具体的理论东西就不说了,代码中注释都有一些说明。

直接上代码

[java] view plain copy

  1. /**
  2.   * 项目名:NewsCrawlerV3.1
  3.   * 文件名:InfoLoad.java
  4.   * 作者:zhouyh
  5.   * 时间:2015-10-14 上午08:46:47
  6.   * 描述:TODO(用一句话描述该文件做什么) 
  7.   */
  8. package com.boryou.module.load;
  9. import java.io.IOException;
  10. import java.io.InputStream;
  11. import java.io.InterruptedIOException;
  12. import java.net.MalformedURLException;
  13. import java.net.URI;
  14. import java.net.URISyntaxException;
  15. import java.net.URL;
  16. import java.net.UnknownHostException;
  17. import java.nio.charset.Charset;
  18. import java.security.KeyManagementException;
  19. import java.security.KeyStoreException;
  20. import java.security.NoSuchAlgorithmException;
  21. import java.security.cert.CertificateException;
  22. import java.security.cert.X509Certificate;
  23. import java.util.Random;
  24. import java.util.regex.Matcher;
  25. import java.util.regex.Pattern;
  26. import java.util.zip.GZIPInputStream;
  27. import javax.net.ssl.SSLContext;
  28. import javax.net.ssl.SSLException;
  29. import org.apache.http.Header;
  30. import org.apache.http.HeaderElement;
  31. import org.apache.http.HttpEntity;
  32. import org.apache.http.HttpEntityEnclosingRequest;
  33. import org.apache.http.HttpHost;
  34. import org.apache.http.HttpRequest;
  35. import org.apache.http.client.ClientProtocolException;
  36. import org.apache.http.client.HttpRequestRetryHandler;
  37. import org.apache.http.client.config.CookieSpecs;
  38. import org.apache.http.client.config.RequestConfig;
  39. import org.apache.http.client.methods.CloseableHttpResponse;
  40. import org.apache.http.client.methods.HttpGet;
  41. import org.apache.http.client.protocol.HttpClientContext;
  42. import org.apache.http.conn.ConnectTimeoutException;
  43. import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
  44. import org.apache.http.entity.ContentType;
  45. import org.apache.http.impl.client.CloseableHttpClient;
  46. import org.apache.http.impl.client.HttpClients;
  47. import org.apache.http.impl.client.LaxRedirectStrategy;
  48. import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
  49. import org.apache.http.protocol.HttpContext;
  50. import org.apache.http.ssl.SSLContextBuilder;
  51. import org.apache.http.ssl.TrustStrategy;
  52. import org.apache.http.util.ByteArrayBuffer;
  53. import com.boryou.constant.Constant;
  54. import com.boryou.util.Common;
  55. /**
  56.  * 类名: InfoLoad
  57.  * 包名: com.boryou.module.load
  58.  * 作者: zhouyh
  59.  * 时间: 2015-10-14 上午08:46:47
  60.  * 描述: 下载基础类,共监控、下载等模块使用 
  61.  */
  62. public class InfoLoad {
  63.     // 创建httpclient连接池
  64.     private PoolingHttpClientConnectionManager httpClientConnectionManager = null;
  65.     /******单例模式声明开始******/
  66.     //类初始化时,自动实例化,饿汉单例模式
  67.     private static final InfoLoad infoLoad = new InfoLoad();
  68.     /**
  69.      * 
  70.      * 方法名:getInfoLoadInstance
  71.      * 作者:zhouyh
  72.      * 创建时间:2015-10-14 上午08:59:54
  73.      * 描述:单例的静态方法,返回InfoLoad的实例
  74.      * @return
  75.      */
  76.     public static InfoLoad getInfoLoadInstance(){
  77.         return infoLoad;
  78.     }
  79.     /******单例模式声明结束******/
  80.     /**
  81.      * 私有的构造函数
  82.      */
  83.     private InfoLoad(){
  84.         //初始化httpClient
  85.         initHttpClient();
  86.     }
  87.     /**
  88.      * 
  89.      * 方法名:initHttpClient
  90.      * 作者:zhouyh
  91.      * 创建时间:2015-10-14 上午11:00:30
  92.      * 描述:创建httpclient连接池,并初始化httpclient
  93.      */
  94.     public void initHttpClient(){
  95.         //创建httpclient连接池
  96.         httpClientConnectionManager = new PoolingHttpClientConnectionManager();
  97.         //设置连接池最大数量
  98.         httpClientConnectionManager.setMaxTotal(Constant.HTTPCLIENT_CONNECTION_COUNT);
  99.         //设置单个路由最大连接数量
  100.         httpClientConnectionManager.setDefaultMaxPerRoute(Constant.HTTPCLIENT_MAXPERROUTE_COUNT);
  101.     }
  102.     //请求重试机制
  103.     HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() {
  104.         public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
  105.             if (executionCount >= 3) {
  106.                 // 超过三次则不再重试请求
  107.                 return false;
  108.             }
  109.             if (exception instanceof InterruptedIOException) {
  110.                 // Timeout
  111.                 return false;
  112.             }
  113.             if (exception instanceof UnknownHostException) {
  114.                 // Unknown host
  115.                 return false;
  116.             }
  117.             if (exception instanceof ConnectTimeoutException) {
  118.                 // Connection refused
  119.                 return false;
  120.             }
  121.             if (exception instanceof SSLException) {
  122.                 // SSL handshake exception
  123.                 return false;
  124.             }
  125.             HttpClientContext clientContext = HttpClientContext.adapt(context);
  126.             HttpRequest request = clientContext.getRequest();
  127.             boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
  128.             if (idempotent) {
  129.                 // Retry if the request is considered idempotent
  130.                 return true;
  131.             }
  132.             return false;
  133.         }
  134.     };
  135.     /**
  136.      * 
  137.      * 方法名:getHttpClient
  138.      * 作者:zhouyh
  139.      * 创建时间:2016-2-18 下午01:23:32
  140.      * 描述:多线程调用时,需要创建自己的httpclient
  141.      * @return
  142.      */
  143.     public CloseableHttpClient getHttpClient(){
  144.         // 创建全局的requestConfig
  145.         RequestConfig requestConfig = RequestConfig.custom()
  146.                 .setConnectTimeout(Constant.HTTPCLIENT_CONNECT_TIMEOUT)
  147.                 .setSocketTimeout(Constant.HTTPCLIENT_SOCKET_TIMEOUT)
  148.                 .setCookieSpec(CookieSpecs.BEST_MATCH).build();
  149.         // 声明重定向策略对象
  150.         LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();
  151.         CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(httpClientConnectionManager)
  152.                                                     .setDefaultRequestConfig(requestConfig)
  153.                                                     .setRedirectStrategy(redirectStrategy)
  154.                                                     .setRetryHandler(myRetryHandler)
  155.                                                     .build();
  156.         return httpClient;
  157.     }
  158.     /**
  159.      * 
  160.      * 方法名:loadForString
  161.      * 作者:zhouyh
  162.      * 创建时间:2015-10-14 下午02:22:19
  163.      * 描述:根据传入的url获取下载信息
  164.      * @param url
  165.      * @param type
  166.      * @return
  167.      */
  168.     public static String loadForString(String urlString, int type){
  169.         String src = “”;
  170.         if(null==urlString || urlString.isEmpty() || !urlString.startsWith(“http”)){//如果urlString为null或者urlString为空,或urlString非http开头,返回src空值
  171.             return src;
  172.         }
  173.         //创建response
  174.         CloseableHttpResponse response = null;
  175.         HttpGet httpGet = null;
  176.         urlString = urlString.trim();//防止传入的urlString首尾有空格
  177.         //转化String url为URI,解决url中包含特殊字符的情况
  178.         try {
  179.             URL url = new URL(urlString);
  180.             URI uri = new URI(url.getProtocol(), url.getHost(), url.getPath(), url.getQuery(), null);
  181.             httpGet = new HttpGet(uri);
  182.             //针对https采用SSL的方式创建httpclient
  183. //          if(urlString.startsWith(“https”)){
  184. //              httpClient = createSSLClientDefault();
  185. //              System.setProperty (“jsse.enableSNIExtension”, “false”);
  186. //          }
  187.             //设置请求头
  188.             httpGet.addHeader(“Accept”,“*/*”);
  189. //          httpGet.addHeader(“Content-Type”,”application/x-www-form-urlencoded; charset=UTF-8″);
  190.             httpGet.addHeader(“Connection”,“keep-alive”);
  191.             httpGet.addHeader(“Accept-Encoding”“gzip, deflate”);
  192.             //设置USER_AGENT
  193.             Random random = new Random();
  194.             int randomInt = random.nextInt(4);
  195.             System.err.println(randomInt);
  196.             httpGet.addHeader(“User-Agent”, Constant.USER_AGENT[randomInt]);
  197.             //此处的代理暂时注释
  198. //          String[] proxys = Constant.HTTPCLIENT_PROXY[randomInt].split(“\\s+”);
  199. //          //添加代理
  200. //          HttpHost proxy = new HttpHost(proxys[0].trim(), Integer.parseInt(proxys[1].trim()), “http”);
  201. //          RequestConfig config = RequestConfig.custom().setProxy(proxy).build();
  202. //          httpGet.setConfig(config);  
  203.             //执行请求      
  204.             try {
  205.                 if(urlString.startsWith(“https”)){
  206.                     System.setProperty (“jsse.enableSNIExtension”“false”);
  207.                     response = createSSLClientDefault().execute(httpGet);
  208.                 }else{
  209.                     response = infoLoad.getHttpClient().execute(httpGet);
  210.                 }
  211.             } catch (Exception e) {
  212.                 e.printStackTrace();
  213.             }
  214.             //得到响应状态码
  215.             int statuCode = response.getStatusLine().getStatusCode();
  216.             //根据状态码进行逻辑处理
  217.             switch (statuCode){
  218.             case 200:
  219.                 //获得响应实体
  220.                 HttpEntity entity = response.getEntity();
  221.                 /**
  222.                  * 仿浏览器获取网页编码
  223.                  * 浏览器是先从content-type的charset(响应头信息)中获取编码,
  224.                  * 如果获取不了,则会从meta(HTML里的代码)中获取charset的编码值
  225.                  */
  226.                 //第一步–>处理网页字符编码
  227.                 String charset = null;
  228.                 ContentType contentType = null;
  229.                 contentType = ContentType.getOrDefault(entity);
  230.                 Charset charsets = contentType.getCharset();
  231.                 if(null != charsets){
  232.                     charset = charsets.toString();
  233.                 }
  234.                 //判断返回的数据流是否采用了gzip压缩
  235.                 Header header = entity.getContentEncoding();
  236.                 boolean isGzip = false;
  237.                 if(null != header){
  238.                     for(HeaderElement headerElement : header.getElements()){
  239.                         if(headerElement.getName().equalsIgnoreCase(“gzip”)){
  240.                             isGzip = true;
  241.                         }
  242.                     }
  243.                 }
  244.                 //获得响应流
  245.                 InputStream inputStream = entity.getContent();
  246.                 ByteArrayBuffer buffer = new ByteArrayBuffer(4096);
  247.                 byte[] tmp = new byte[4096];
  248.                 int count;
  249.                 if(isGzip){//如果采用了Gzip压缩,则进行gizp压缩处理
  250.                     GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
  251.                     while((count=gzipInputStream.read(tmp)) != –1){
  252.                         buffer.append(tmp, 0, count);
  253.                     }
  254.                 }else{//处理非gzip格式的数据
  255.                     while((count=inputStream.read(tmp)) != –1){
  256.                         buffer.append(tmp, 0, count);
  257.                     }
  258.                 }
  259.                 //第二步—>如果第一步contenttyp未获取到编码,这里从meta标签中获取
  260.                 if(null==charset || “”.equals(charset) || “null”.equals(charset)
  261.                         || “zh-cn”.equalsIgnoreCase(charset)){
  262.                     charset = getCharsetFromMetaTag(buffer, urlString);
  263.                 }
  264.                 //根据获取的字符编码转为string类型
  265.                 src = new String(buffer.toByteArray(), charset);
  266.                 //替换特殊编码
  267.                 src = replaceStr(src);
  268.                 //转化Unicode编码格式]
  269.                 src = Common.decodeUnicode(src);
  270. //              System.out.println(src);
  271.                 break;
  272.             case 400:
  273.                 System.out.println(“下载400错误代码,请求出现语法错误” + urlString);
  274.                 //TODO 要进行判断是列表页还是正文页下载,再去修改数据库,下同
  275.                 //TODO 此处添加对mongodb数据库的操作,将该url的isStart改为0,暂时不在进行监控,后续根据模板状态为0的进行修改
  276.                 break;
  277.             case 403:
  278.                 System.out.println(“下载403错误代码,资源不可用” + urlString);
  279.                 //TODO 此处添加对mongodb数据库的操作,将该url的isStart改为0,暂时不在进行监控,后续根据模板状态为0的进行修改
  280.                 break;
  281.             case 404:
  282.                 System.out.println(“下载404错误代码,无法找到指定资源地址” + urlString);
  283.                 //TODO 此处添加对mongodb数据库的操作,将该url的isStart改为0,暂时不在进行监控,后续根据模板状态为0的进行修改
  284.                 break;
  285.             case 503:
  286.                 System.out.println(“下载503错误代码,服务不可用” + urlString);
  287.                 //TODO 此处添加对mongodb数据库的操作,将该url的isStart改为0,暂时不在进行监控,后续根据模板状态为0的进行修改
  288.                 break;
  289.             case 504:
  290.                 System.out.println(“下载504错误代码,网关超时” + urlString);
  291.                 //TODO 此处添加对mongodb数据库的操作,将该url的isStart改为0,暂时不在进行监控,后续根据模板状态为0的进行修改
  292.                 break;
  293.             }
  294.         } catch (MalformedURLException e) {
  295.             //执行URL url = new URL()的异常
  296.             e.printStackTrace();
  297.         } catch (URISyntaxException e) {
  298.             //执行URI uri = new URI()的异常
  299.             e.printStackTrace();
  300.         } catch (ClientProtocolException e) {
  301.             // 执行httpClient.execute(httpGet)的异常
  302.             e.printStackTrace();
  303.         } catch (IOException e) {
  304.             // 执行httpClient.execute(httpGet)的异常
  305.             e.printStackTrace();
  306.         } finally{
  307.             if(response != null){
  308.                 try {
  309.                     response.close();
  310.                 } catch (IOException e) {
  311.                     e.printStackTrace();
  312.                 }
  313.             }
  314.             httpGet.abort();    //结束后关闭httpGet请求
  315.             /**
  316.              * httpclient的链接有线程池管理,这里不用直接关闭
  317.              */
  318. //          try {//关闭连接
  319. //              httpClient.close();
  320. //          } catch (IOException e) {
  321. //              e.printStackTrace();
  322. //          }   
  323.         }
  324.         return src;
  325.     }
  326.     /**
  327.      * 
  328.      * 方法名:createSSLClientDefault
  329.      * 作者:zhouyh
  330.      * 创建时间:2015-10-14 下午03:03:30
  331.      * 描述:针对https采用SSL的方式创建httpclient
  332.      * @return
  333.      */
  334.     public static CloseableHttpClient createSSLClientDefault(){
  335.         try {
  336.             SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(nullnew TrustStrategy(){
  337.             //信任所有
  338.             public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
  339.                 return true;
  340.             }}).build();
  341.             SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
  342.             return HttpClients.custom().setSSLSocketFactory(sslsf).build();
  343.         } catch (KeyManagementException e) {
  344.             e.printStackTrace();
  345.         } catch (NoSuchAlgorithmException e) {
  346.             e.printStackTrace();
  347.         } catch (KeyStoreException e) {
  348.             e.printStackTrace();
  349.         }
  350.         return  HttpClients.createDefault();
  351.     }
  352.     /**
  353.      * 
  354.      * 方法名:getCharsetFromMetaTag
  355.      * 作者:zhouyh
  356.      * 创建时间:2015-10-14 下午05:23:08
  357.      * 描述:从meta标签中获取编码格式
  358.      * @param buffer
  359.      * @param url
  360.      * @return
  361.      */
  362.     public static String getCharsetFromMetaTag(ByteArrayBuffer buffer,String url){
  363.         String charset = null;
  364.         String regEx = Constant.CHARSET_REGEX;
  365.         Pattern p = Pattern.compile(regEx,
  366.                 Pattern.CASE_INSENSITIVE);
  367.         Matcher m = p.matcher(new String(buffer.toByteArray()));
  368.         boolean result = m.find();
  369.         if (result) {
  370.             if (m.groupCount() == 1) {
  371.                 charset = m.group(1);
  372.             }
  373.             System.err.println(“网页 中的编码:” + charset + “\t url:” + url);
  374.         } else {
  375.             //出现未匹配的编码,先赋值为gbk
  376.             charset = “gbk”;
  377.             System.out.println(“字符编码未匹配到 : “ + url);
  378.         }
  379.         return charset;
  380.     }
  381.     /**
  382.      * 
  383.      * 方法名:replaceStr
  384.      * 作者:zhouyh
  385.      * 创建时间:2015-10-14 下午05:33:01
  386.      * 描述:替换原网页中的特殊字符
  387.      * @param src
  388.      * @return
  389.      */
  390.     public static String replaceStr(String src){
  391.         if (src == null || “”.equals(src)) return null;
  392.         src = src.replaceAll(“<!–““”);
  393.         src = src.replaceAll(“–>”“”);
  394.         src = src.replaceAll(“<““<“);
  395.         src = src.replaceAll(“>”“>”);
  396.         src = src.replaceAll(“”“, “\“”);
  397.         src = src.replaceAll(” “” “);
  398.         src = src.replaceAll(“&”“&”);
  399.         return src;
  400.     }
  401.     /**
  402.      * 方法名:main
  403.      * 作者:zhouyh
  404.      * 创建时间:2015-10-14 上午08:46:47
  405.      * 描述:main方法
  406.      * @param args
  407.      */
  408.     public static void main(String[] args) {
  409.         // TODO Auto-generated method stub
  410.         Random random = new Random();
  411.         int randomInt = random.nextInt(4);
  412.         System.out.println(randomInt);
  413. //      InfoLoad.getInfoLoadInstance().loadForString(“http://weixin.sogou.com/remind/doc_list.php?callback=jQuery111006747886650961997_1446517725478&from=web&uid=B31F8214DA30BE47F750B8BE2BF0E4AA%40qq.sohu.com&start=0&num=20&wordid=237&clear=1&_=1446517725480”, 0);
  414.         InfoLoad.getInfoLoadInstance().loadForString(“http://www.xinli001.com/info”0);
  415.     }
  416. }

[java] view plain copy

这里用到了IO常用的工具类IOUtils,详细了解可以参考这篇文章[file]IO常用工具类IOUtils(Java读文件、写文件、打Zip包)http://www.bieryun.com/1003.html

未经允许不得转载:比尔云 » HttpClient在java中的使用
分享到: 更多 (0)

评论 抢沙发

  • 昵称 (必填)
  • 邮箱 (必填)
  • 网址

强烈推荐

高性能SSD云服务器ECS抗攻击,高可用云数据库RDS