如图：
首先需要一个Http工具类：HttpHandle
package org.coody.robot.util;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
public class HttpHandle {
public final static String POST="POST";
public final static String GET="GET";
public final static String HEAD="HEAD";
public final static String PUT="PUT";
public final static String CONNECT="CONNECT";
public final static String OPTIONS="OPTIONS";
public final static String DELETE="DELETE";
public final static String PATCH="PATCH";
public final static String PROPFIND="PROPFIND";
public final static String PROPPATCH="PROPPATCH";
public final static String MKCOL="MKCOL";
public final static String COPY="COPY";
public final static String MOVE="MOVE";
public final static String LOCK="LOCK";
public final static String UNLOCK ="UNLOCK";
public final static String TRACE="TRACE";
public final static String HTTP_GENERAL="HTTP_GENERAL";
public final static String HTTP_JSON="HTTP_JSON";
public HttpConfig config=new HttpConfig();
public HttpConfig getConfig() {
return config;
}
public void setConfig(HttpConfig config) {
this.config = config;
}
public static class HttpConfig{
private boolean allowRedirects=true;
private String cookie="";
private String encode="UTF-8";
private int timeOut=15;
private String httpModule=HTTP_GENERAL;
private Map<String, String> headerMap=new HashMap<String, String>();
public void setEncode(String encode) {
this.encode = encode;
}
public void setTimeOut(int timeOut) {
this.timeOut = timeOut;
}
public void setCookie(String cookie) {
this.cookie = cookie;
}
public void setHeaderMap(Map<String, String> headerMap) {
this.headerMap = headerMap;
}
//设置Header头部
public void setRequestProperty(String fieldName,String value){
headerMap.put(fieldName, value);
}
//是否开启Gzip
public void setGzip(boolean isGzip){
if(isGzip){
headerMap.put("Accept-Encoding", "gzip, deflate, sdch");
return;
}
headerMap.put("Accept-Encoding", "*");
}
//是否保持连接
public void setKeepAlive(boolean keepAlive){
if(keepAlive){
headerMap.put("Connection", "keep-alive");
return;
}
headerMap.put("Connection", "close");
}
//是否允许重定向
public void allowRedirects(boolean allowRedirects){
this.allowRedirects=allowRedirects;
}
}
private HttpURLConnection createConnectionGeneral(String url) {
try {
HttpURLConnection conn = (HttpURLConnection) new URL(url)
.openConnection();
conn.addRequestProperty("Referer", getDomain(url));
conn.addRequestProperty(
"Accept",
"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*");
conn.addRequestProperty("Content-type",
"application/x-www-form-urlencoded");
conn.addRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
return conn;
} catch (Exception e) {
return null;
}
}
private HttpURLConnection createConnectionJson(String url) {
try {
HttpURLConnection conn = (HttpURLConnection) new URL(url)
.openConnection();
conn.addRequestProperty("Referer", getDomain(url));
conn.addRequestProperty(
"Accept",
"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*");
conn.addRequestProperty("Content-type",
"application/x-www-form-urlencoded");
conn.addRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
return conn;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
//获取默认来源地址
public static String getDomain(String urlStr){
try {
URI uri=new URI(urlStr);
String result=uri.getScheme()+"://"+uri.getHost();
if(uri.getPort()>0&&uri.getPort()!=80){
result+=("/"+uri.getPort());
}
if(!result.endsWith("/")){
result+="/";
}
return result;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
//合并Cookie
private static String mergeCookie(String oldCookie, String newCookie) {
if (newCookie == null) {
return oldCookie;
}
Map<String, String> cookieMap = new HashMap<String, String>();
String[] cookTmp = null;
String[] cookieTab = null;
StringBuilder valueTmp = new StringBuilder();
String[] cookies = { oldCookie, newCookie };
for (String currCookie : cookies) {
if (StringUtil.isNullOrEmpty(currCookie)) {
continue;
}
cookieTab = currCookie.split(";");
for (String cook : cookieTab) {
cookTmp = cook.split("=");
if (cookTmp.length < 2) {
continue;
}
valueTmp = new StringBuilder();
for (int i = 1; i < cookTmp.length; i++) {
valueTmp.append(cookTmp[i]);
if (i < cookTmp.length - 1) {
valueTmp.append("=");
}
}
if (StringUtil.findNull(cookTmp[0], valueTmp) > -1) {
continue;
}
cookieMap.put(cookTmp[0], valueTmp.toString());
}
}
valueTmp = new StringBuilder();
for (String key : cookieMap.keySet()) {
valueTmp.append(key).append("=").append(cookieMap.get(key));
valueTmp.append(";");
}
return valueTmp.toString();
}
private HttpURLConnection getConnection(String url) {
if(config.httpModule.equals(HTTP_GENERAL)){
return createConnectionGeneral(url);
}
if(config.httpModule.equals(HTTP_JSON)){
return createConnectionJson(url);
}
return null;
}
public HttpEntity Get(String url){
return Conn(url, GET, null);
}
public HttpEntity Post(String url,String data){
return Conn(url, POST, data);
}
public HttpEntity Conn(String url, String method,
String postData){
if(url.contains(" ")){
url=url.replace(" ", "%20");
}
HttpURLConnection conn = getConnection(url);
if (conn == null) {
return null;
}
if (!StringUtil.isNullOrEmpty(config.headerMap)) {
for (String key : config.headerMap.keySet()) {
conn.setRequestProperty(key, config.headerMap.get(key));
key = null;
}
}
if(!config.allowRedirects){
conn.setInstanceFollowRedirects(false);
}
if (!StringUtil.isNullOrEmpty(config.cookie)) {
conn.setRequestProperty("Cookie", config.cookie);
}
try {
conn.setRequestMethod(method);
if (method.equalsIgnoreCase(POST)||method.equalsIgnoreCase(PUT)) {
conn.setDoOutput(true);
byte [] postByte=postData.getBytes(config.encode);
conn.setRequestProperty("Content-Length", String.valueOf(postByte.length));
conn.getOutputStream().write(postByte);
conn.connect();
conn.getOutputStream().flush();
conn.getOutputStream().close();
}
} catch (Exception e) {
e.printStackTrace();
}
conn.setConnectTimeout(config.timeOut*1000);
InputStream ins = null;
HttpEntity hEntity = new HttpEntity();
String key = "";
StringBuilder cookie = new StringBuilder();
try {
Integer status=conn.getResponseCode();
if (status !=HttpURLConnection.HTTP_OK) {
ins=conn.getErrorStream();
}else{
ins=conn.getInputStream();
}
hEntity.setCode(conn.getResponseCode());
Map<String,String> headMap=new HashMap<String, String>();
for (int i = 1; (key = conn.getHeaderFieldKey(i)) != null; i++) {
headMap.put(key, conn.getHeaderField(key));
if (key.equalsIgnoreCase("set-cookie")) {
try {
cookie.append(conn.getHeaderField(i).replace("/", ""));
} catch (Exception e) {
}
}
}
config.cookie = mergeCookie(config.cookie, cookie.toString());
byte[] b = toByte(ins);
if((headMap.get("Content-Encoding")!=null && headMap.get("Content-Encoding").contains("gzip"))||(conn.getRequestProperty("Accept-Encoding")!=null&&conn.getRequestProperty("Accept-Encoding").contains("gzip"))){
b = GZIPUtils.uncompress(b);
}
hEntity.setEncode(config.encode);
hEntity.setBye(b);
hEntity.setCookie(config.cookie);
hEntity.setHeadMap(headMap);
} catch (Exception e) {
e.printStackTrace();
}finally{
try {
ins.close();
} catch (Exception e2) {
}
}
return hEntity;
}
private byte[] toByte(InputStream ins) {
if(ins==null){
return null;
}
ByteArrayOutputStream swapStream = null;
try {
swapStream = new ByteArrayOutputStream();
byte[] buff = new byte[1024];
int rc = 0;
while ((rc = ins.read(buff, 0, 1024)) > 0) {
swapStream.write(buff, 0, rc);
}
return swapStream.toByteArray();
} catch (Exception e) {
e.printStackTrace();
return null;
} finally {
try {
swapStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
其次需要一个Http响应对象类：HttpEntity
package org.coody.robot.util;
import java.util.HashMap;
import java.util.Map;
public class HttpEntity {
private String html;
private byte[] bye;
private String cookie;
private Integer code=-1;
private Map<String,String> headMap;
public Map<String, String> getHeadMap() {
return headMap;
}
public void setHeadMap(Map<String, String> headMap) {
this.headMap = headMap;
}
private String encode="UTF-8";
public String getHtml() {
try {
if(html!=null){
return html;
}
if(bye==null){
return null;
}
String str= new String(bye, encode);
html=str;
return str;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public String getHtml(boolean isGzip) {
try {
if(bye==null){
return null;
}
String str= new String(GZIPUtils.uncompress(bye), encode);
return str;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public String getEncode() {
return encode;
}
public void setEncode(String encode) {
this.encode = encode;
}
public void setHtml(String html) {
this.html = html;
}
public Integer getCode() {
return code;
}
public void setCode(Integer code) {
this.code = code;
}
public String getCookie() {
return cookie;
}
public void setCookie(String cookie) {
this.cookie = cookie;
}
public byte[] getBye() {
return bye;
}
public void setBye(byte[] bye) {
this.bye = bye;
}
public Map<String, String> getCookieMap() {
if (cookie == null) {
return null;
}
Map<String, String> cookieMap = new HashMap<String, String>();
String[] cookies = cookie.split(";");
for (String cook : cookies) {
String[] tmps = cook.split("=");
if (tmps.length >= 2) {
String cookieValue = "";
for (int i = 1; i < tmps.length; i++) {
cookieValue += tmps[i];
if (i < tmps.length-1) {
cookieValue += "=";
}
}
cookieMap.put(tmps[0].trim(), cookieValue.trim());
}
}
return cookieMap;
}
}
某些网站是有Gzip压缩的，需要一个Gzip压缩类GzipUtils
package org.coody.robot.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/**
*
* @author wenqi5
*
*/
public class GZIPUtils {
public static final String GZIP_ENCODE_UTF_8 = "UTF-8";
/**
* 字符串压缩为GZIP字节数组
*
* @param str
* @return
*/
public static byte[] compress(String str) {
try {
return compress(str.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
return null;
}
}
/**
* 字符串压缩为GZIP字节数组
*
* @param str
* @param encoding
* @return
*/
public static byte[] compress(byte[] data) {
if (data == null || data.length == 0) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream gzip = null;
try {
gzip = new GZIPOutputStream(out);
gzip.write(data);
} catch (IOException e) {
}finally {
try {
gzip.close();
} catch (Exception e2) {
}
try {
out.close();
} catch (Exception e2) {
}
}
return out.toByteArray();
}
/**
* GZIP解压�?
*
* @param bytes
* @return
*/
public static byte[] uncompress(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayInputStream in = new ByteArrayInputStream(bytes);
try {
GZIPInputStream ungzip = new GZIPInputStream(in);
byte[] buffer = new byte[256];
int n;
while ((n = ungzip.read(buffer)) >= 0) {
out.write(buffer, 0, n);
}
} catch (IOException e) {
} finally {
try {
in.close();
} catch (Exception e2) {
}
try {
out.close();
} catch (Exception e2) {
}
}
return out.toByteArray();
}
}
以上类均依赖一个StringUtil，笔者比较懒，也没有拆分出来
package org.coody.robot.util;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
//import oracle.sql.CLOB;
public class StringUtil {
public static Integer[] getIntegerParas(Object[] objs) {
if (isNullOrEmpty(objs)) {
return null;
}
Integer[] ints = new Integer[objs.length];
for (int i = 0; i < objs.length; i++) {
try {
ints[i] = Integer.valueOf(objs[i].toString());
} catch (Exception e) {
}
}
return ints;
}
/**
* 生成指定数目字符串按分隔符分�?
*
* @param baseStr
* @param mosaicChr
* @param size
* @return
*/
public static String getByMosaicChr(String baseStr, String mosaicChr, Integer size) {
List<String> list = new ArrayList<String>();
for (int i = 0; i < size; i++) {
if (isNullOrEmpty(baseStr)) {
continue;
}
list.add(baseStr);
}
return collectionMosaic(list, mosaicChr);
}
/**
* 根据分割符将字符串分割成String数组
*
* @param src
* 源字符串
* @param separator
* 分隔�??
* @return String数组
*/
public static String[] splitToStringArray(String src, String separator) {
Vector<String> splitArrays = new Vector<String>();
int i = 0;
int j = 0;
while (i <= src.length()) {
j = src.indexOf(separator, i);
if (j < 0) {
j = src.length();
}
splitArrays.addElement(src.substring(i, j));
i = j + 1;
}
int size = splitArrays.size();
String[] array = new String[size];
System.arraycopy(splitArrays.toArray(), 0, array, 0, size);
return array;
}
/**
* 根据分割符将字符串分割成Integer数组
*
* @param src
* 源字符串
* @param separator
* 分隔�??
* @return Integer数组
*/
public static Integer[] splitToIntgArray(String src, String separator) {
String[] arr = splitToStringArray(src, separator);
Integer[] intArr = new Integer[arr.length];
for (int i = 0; i < arr.length; i++) {
intArr[i] = Integer.valueOf(arr[i]);
}
return intArr;
}
/**
* 根据分隔符将字符串分割成int数组
*
* @param src
* 源字符串
* @param separator
* 分隔�??
* @return int数组
*/
public static int[] splitToIntArray(String src, String separator) {
String[] arr = splitToStringArray(src, separator);
int[] intArr = new int[arr.length];
for (int i = 0; i < arr.length; i++) {
intArr[i] = Integer.parseInt(arr[i]);
}
return intArr;
}
public static String getInPara(Integer size) {
return getByMosaicChr("?", ",", size);
}
public static String textCutCenter(String allTxt, String firstTxt, String lastTxt) {
try {
String tmp = "";
int n1 = allTxt.indexOf(firstTxt);
if (n1 == -1) {
return "";
}
tmp = allTxt.substring(n1 + firstTxt.length(), allTxt.length());
int n2 = tmp.indexOf(lastTxt);
if (n2 == -1) {
return "";
}
tmp = tmp.substring(0, n2);
return tmp;
} catch (Exception e) {
return "";
}
}
public static List<String> textCutCenters(String allTxt, String firstTxt, String lastTxt) {
try {
List<String> results = new ArrayList<String>();
while(allTxt.contains(firstTxt)){
int n = allTxt.indexOf(firstTxt);
allTxt=allTxt.substring(n+firstTxt.length(), allTxt.length());
n=allTxt.indexOf(lastTxt);
if(n==-1){
return results;
}
String result=allTxt.substring(0, n);
results.add(result);
allTxt=allTxt.substring(n+firstTxt.length(), allTxt.length());
}
return results;
} catch (Exception e) {
return null;
}
}
public static String convertToUnicode(String source) {
String result = "";
char[] chrs = source.toCharArray();
for (int i = 0; i < chrs.length; i++) {
result += "&#" +Character.codePointAt(chrs, i);
}
return result;
}
public static Integer toInteger(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Integer.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}
public static String toString(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return String.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}
public static Double toDouble(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Double.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}
public static Float toFloat(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Float.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}
public static Long toLong(Object obj) {
if (isNullOrEmpty(obj)) {
return null;
}
try {
return Long.valueOf(obj.toString());
} catch (Exception e) {
return null;
}
}
public static Integer getRanDom(int start, int end) {
return (int) (Math.random() * (end - start + 1)) + start;
}
public static float getRanDom(Float start, Float end) {
String str = String.valueOf(start);
String[] tabs = str.split("\\.");
Integer startLength = 1;
if (tabs.length == 2) {
startLength = tabs[1].length();
}
str = String.valueOf(end);
tabs = str.split("\\.");
Integer endLength = 1;
if (tabs.length == 2) {
endLength = tabs[1].length();
}
if (endLength > startLength) {
startLength = endLength;
}
start = (float) (start * Math.pow(10, startLength));
end = (float) (end * Math.pow(10, startLength));
return (float) (getRanDom(start.intValue(), end.intValue()) / Math.pow(10, startLength));
}
public static String replaceBlank(String str) {
String dest = "";
if (str != null) {
Pattern p = Pattern.compile("\\s*|\t|\r|\n");
Matcher m = p.matcher(str);
dest = m.replaceAll("");
}
return dest;
}
public static Boolean isMatcher(String val, String matcher) {
Pattern p = Pattern.compile(matcher);
Matcher m = p.matcher(val);
return m.matches();
}
public static boolean isMobile(String mobile) {
if (isNullOrEmpty(mobile)) {
return false;
}
Pattern p = Pattern.compile("^((13[0-9])|(15[^4,\\D])|(17[^4,\\D])|(18[0,5-9]))\\d{8}$");
Matcher m = p.matcher(mobile);
return m.matches();
}
public static boolean isLegal(String str) {
if (isNullOrEmpty(str)) {
return false;
}
Pattern p = Pattern.compile("[A-Za-z0-9_]{3,16}");
Matcher m = p.matcher(str);
return m.matches();
}
public static boolean isEmail(String email) {
if (isNullOrEmpty(email)) {
return false;
}
Pattern p = Pattern.compile(
"^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$");
Matcher m = p.matcher(email);
return m.matches();
}
public static boolean isMd5(String md5) {
if (isNullOrEmpty(md5)) {
return false;
}
Pattern p = Pattern.compile("[A-Za-z0-9_]{16,40}");
Matcher m = p.matcher(md5);
return m.matches();
}
public static boolean isAllNull(Object... obj) {
if (obj == null || obj.length == 0) {
return true;
}
for (int i = 0; i < obj.length; i++) {
if (!isNullOrEmpty(obj[i])) {
return false;
}
}
return true;
}
public static boolean isAllNull(List<Object> objs) {
return isAllNull(objs.toArray());
}
/**
* 把一个数组按照分隔符拼接成字符串
*
* @param 数组参数
* @param 分隔�?
* @return
*/
public static String collectionMosaic(Object[] objs, String mosaicChr) {
if (isNullOrEmpty(objs)) {
return null;
}
List<Object> objList = Arrays.asList(objs);
return collectionMosaic(objList, mosaicChr);
}
/**
* 把一个数组按照分隔符拼接成字符串
*
* @param 数组参数
* @param 分隔�?
* @return
*/
public static String collectionMosaic(int[] intObjs, String mosaicChr) {
Object[] objs = new Object[intObjs.length];
for (int i = 0; i < intObjs.length; i++) {
objs[i] = String.valueOf(intObjs[i]);
}
return collectionMosaic(objs, mosaicChr);
}
/**
* 把一个或多个字符串按照分隔符拼接成字符串
*
* @param 数组参数
* @param 分隔�?
* @return
*/
public static String collectionMosaic(String mosaicChr, Object... objs) {
List<Object> objList = Arrays.asList(objs);
return collectionMosaic(objList, mosaicChr);
}
/**
* 把一个集合按照分隔符拼接成字符串
*
* @param 集合参数
* @param 分隔�?
* @return 字符�?
*/
public static String collectionMosaic(List<?> objs, String mosaicChr) {
if (objs == null || objs.isEmpty()) {
return null;
}
StringBuilder sb = new StringBuilder();
int i = 0;
for (Object obj : objs) {
if (isNullOrEmpty(obj)) {
continue;
}
sb.append(obj);
if (i < objs.size() - 1) {
sb.append(mosaicChr);
}
i++;
}
return sb.toString();
}
/**
* 生成指定数目字符串按分隔符分�?
*
* @param baseStr
* @param mosaicChr
* @param size
* @return
*/
public static String getStringSByMosaicChr(String baseStr, String mosaicChr, Integer size) {
List<String> list = new ArrayList<String>();
for (int i = 0; i < size; i++) {
if (isNullOrEmpty(baseStr)) {
continue;
}
list.add(baseStr);
}
return collectionMosaic(list, mosaicChr);
}
/**
* 按照分隔符分�?,得到字符串集�?
*
* @param text
* 原字符串
* @param mosaiChr
* 分隔�?
* @return list
*/
public static List<String> splitByMosaic(String text, String mosaiChr) {
if (text == null || mosaiChr == null) {
return null;
}
String[] tab = text.split(mosaiChr);
List<String> list = new ArrayList<String>();
for (int i = 0; i < tab.length; i++) {
if (isNullOrEmpty(tab[i])) {
continue;
}
list.add(tab[i]);
}
return list;
}
/**
* 按照分隔符分�?,得到字符串集�?
*
* @param text
* 原字符串
* @param mosaiChr
* 分隔�?
* @return list
*/
public static List<Integer> splitByMosaicInteger(String text, String mosaiChr) {
if (text == null || mosaiChr == null) {
return null;
}
String[] tab = text.split(mosaiChr);
List<Integer> list = new ArrayList<Integer>();
for (int i = 0; i < tab.length; i++) {
if (isNullOrEmpty(tab[i])) {
continue;
}
try {
list.add(Integer.valueOf(tab[i]));
} catch (Exception e) {
}
}
return list;
}
/**
* 按照分隔符分�?,得到字符串集�?
*
* @param text
* 原字符串
* @param mosaiChr
* 分隔�?
* @return list
*/
public static Integer[] splitByMosaicIntegers(String text, String mosaiChr) {
if (text == null || mosaiChr == null) {
return null;
}
String[] tab = text.split(mosaiChr);
Integer[] list = new Integer[tab.length];
for (int i = 0; i < tab.length; i++) {
if (isNullOrEmpty(tab[i])) {
continue;
}
try {
list[i] = Integer.valueOf(tab[i]);
} catch (Exception e) {
}
}
return list;
}
public static List<String> doMatcher(String context, String pat) {
try {
List<String> images = new ArrayList<String>();
Integer index = 0;
Pattern pattern = Pattern.compile(pat, Pattern.DOTALL);
Matcher matcher = pattern.matcher(context);
String tmp = null;
while (matcher.find(index)) {
tmp = matcher.group(0);
index = matcher.end();
if (StringUtil.isNullOrEmpty(tmp)) {
continue;
}
images.add(tmp);
}
return images;
} catch (Exception e) {
return null;
}
}
public static String doMatcherFirst(String context, String pat) {
List<String> strs = doMatcher(context, pat);
if (StringUtil.isNullOrEmpty(strs)) {
return null;
}
return strs.get(0);
}
public static boolean isNullOrEmpty(Object obj) {
try {
if (obj == null)
return true;
if (obj instanceof CharSequence) {
return ((CharSequence) obj).length() == 0;
}
if (obj instanceof Collection) {
return ((Collection<?>) obj).isEmpty();
}
if (obj instanceof Map) {
return ((Map<?, ?>) obj).isEmpty();
}
if (obj instanceof Object[]) {
Object[] object = (Object[]) obj;
if (object.length == 0) {
return true;
}
boolean empty = true;
for (int i = 0; i < object.length; i++) {
if (!isNullOrEmpty(object[i])) {
empty = false;
break;
}
}
return empty;
}
return false;
} catch (Exception e) {
return true;
}
}
public static Integer findNull(Object... objs) {
if (isNullOrEmpty(objs)) {
return 0;
}
for (int i = 0; i < objs.length; i++) {
if (isNullOrEmpty(objs[i])) {
return i;
}
}
return -1;
}
public static boolean hasNull(Object... objs) {
return findNull(objs)>-1;
}
// 判断是否为数�?
public static Boolean isNumber(String str) {
if (isNullOrEmpty(str)) {
return false;
}
try {
Integer.valueOf(str);
return true;
} catch (Exception e) {
return false;
}
}
public static String argsToString(String[] args) {
StringBuilder sb = new StringBuilder();
for (String tmp : args) {
sb.append(tmp);
}
return sb.toString();
}
// 字符串意义分�?
public static String[] splitString(String str) {
if (isNullOrEmpty(str)) {
return null;
}
String[] finalStrs = new String[str.length()];
for (int i = 0; i < str.length(); i++) {
finalStrs[i] = str.substring(i, i + 1);
}
return finalStrs;
}
public static String getString(Object... objs) {
if (isNullOrEmpty(objs)) {
return "";
}
StringBuilder sb = new StringBuilder();
for (Object obj : objs) {
if (isNullOrEmpty(obj)) {
sb.append("null");
}
sb.append(String.valueOf(obj));
}
return sb.toString();
}
public static String stringSort(String str) {
if (isNullOrEmpty(str)) {
return "";
}
String[] strs = splitString(str);
Arrays.sort(strs);
return argsToString(strs);
}
/**
* 集合碰撞
*
* @param needList
* �?要的集合
* @param actualList
* 当前实际集合
* @return 缺少的元�?
*/
public static List<?> collisionList(List<?> needList, List<?> actualList) {
List<Object> list = new ArrayList<Object>();
for (Object o : needList) {
if (actualList.contains(o)) {
continue;
}
list.add(o);
}
if (isNullOrEmpty(list)) {
return null;
}
return list;
}
public static List<Long> integerListToLong(List<Integer> ids) {
if (isNullOrEmpty(ids)) {
return null;
}
List<Long> list = new ArrayList<Long>();
for (Integer id : ids) {
list.add(Long.valueOf(id));
}
return list;
}
/**
* List碰撞取缺�?
*
* @param allList
* 理论应该出现的List
* @param conflictList
* 实际出现的List
* @return 丢失的List
*/
public static List<?> listConflict(List<?> allList, List<?> conflictList) {
if (isNullOrEmpty(allList)) {
return null;
}
if (isNullOrEmpty(conflictList)) {
return allList;
}
List<Object> list = new ArrayList<Object>();
for (Object obj : allList) {
if (conflictList.contains(obj)) {
continue;
}
list.add(obj);
}
if (isNullOrEmpty(list)) {
return null;
}
return list;
}
public static Integer bambooParse(Integer... prs) {
Integer prSum = 0;
for (Integer pr : prs) {
prSum += pr;
}
Integer random = getRanDom(1, prSum);
prSum = 0;
for (int i = 0; i < prs.length; i++) {
prSum += prs[i];
if (random <= prSum) {
return i;
}
}
return 0;
}
public static Integer SumInteger(Integer... sums) {
if (isNullOrEmpty(sums)) {
return -1;
}
Integer total = 0;
for (Integer tmp : sums) {
total += tmp;
}
return total;
}
/**
* 概率算法
*
* @param chances
* 各成员概率权�?
* @return 权重下标
*/
public static Integer getBambooIndex(Integer... chances) {
if (isNullOrEmpty(chances)) {
return -1;
}
Integer total = SumInteger(chances);
Integer random = getRanDom(1, total);
total = new Integer(0);
for (int i = 0; i < chances.length; i++) {
total += chances[i];
if (random <= total) {
return i;
}
}
return -1;
}
public static List<?> removeEmpty(List<?> list) {
if (StringUtil.isNullOrEmpty(list)) {
return null;
}
List<Object> newList = new ArrayList<Object>(list.size());
for (Object obj : list) {
if (isNullOrEmpty(obj)) {
continue;
}
newList.add(obj);
}
if (isNullOrEmpty(newList)) {
return null;
}
return newList;
}
public static Integer getBambooIndex(Float... chanceSources) {
if (isNullOrEmpty(chanceSources)) {
return -1;
}
Float[] chances = Arrays.copyOf(chanceSources, chanceSources.length);
Integer smallLength = 0;
for (Float f : chances) {
String str = String.valueOf(f);
String[] tabs = str.split("\\.");
if (tabs.length != 2) {
continue;
}
smallLength = tabs[1].length();
}
if (smallLength > 0) {
Integer multiple = Double.valueOf(Math.pow(10, smallLength)).intValue();
for (int i = 0; i < chances.length; i++) {
chances[i] = chances[i] * multiple;
}
}
Integer[] chanceInts = new Integer[chances.length];
for (int i = 0; i < chances.length; i++) {
chanceInts[i] = chances[i].intValue();
}
return getBambooIndex(chanceInts);
}
public static Float floatCut(Float f1, Float f2) {
BigDecimal b1 = new BigDecimal(Float.toString(f1));
BigDecimal b2 = new BigDecimal(Float.toString(f2));
return b1.subtract(b2).floatValue();
}
/**
* 获取网址后缀
*
* @param url
* @return
*/
public static String getSuffix(String url) {
if (isNullOrEmpty(url)) {
return "";
}
String[] tab = url.split("\\.");
if (tab.length > 1) {
return tab[tab.length - 1];
}
return "";
}
}
为了方便我们使用，特意为百度云防护的网站封装了一个工具类RobotHttpHandle，维护了Cookie机制
package org.coody.robot.rote;
import java.util.Date;
import javax.script.Invocable;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import org.coody.robot.util.HttpEntity;
import org.coody.robot.util.HttpHandle;
import org.coody.robot.util.StringUtil;
public class RobotHttpHandle {
public static String cookie="";
public RobotHttpHandle(){
}
private HttpEntity initCookie(String url){
try {
String baseURL=HttpHandle.getDomain(url);
HttpHandle http=new HttpHandle();
http.config.setRequestProperty("If-Modified-Since", new Date().toString());
http.config.setRequestProperty("Cache-Control", "max-age=0");
http.config.setRequestProperty("Upgrade-Insecure-Requests", "1");
http.config.setKeepAlive(true);
HttpEntity entity = http.Get(baseURL);
System.out.println(entity.getCookie());
String html = entity.getHtml();
String temp = html.replace(" ", "");
String jschl_vc = StringUtil.textCutCenter(temp, "jschl_vc\"value=\"", "\"");
String pass = StringUtil.textCutCenter(temp, "pass\"value=\"", "\"");
String funcCode = StringUtil.textCutCenter(html, "setTimeout(function(){", "f.submit();");
funcCode = funcCode.replace("a.value", "a");
funcCode = funcCode.replace(" ", " ");
String[] tabs = funcCode.split("\n");
funcCode = tabs[1];
funcCode += "\r\nt=\"" + baseURL + "\";";
funcCode += "\r\nr = t.match(/https?:\\/\\//)[0];";
funcCode += "\r\nt = t.substr(r.length);";
funcCode += "\r\nt = t.substr(0, t.length - 1);";
funcCode += tabs[8];
funcCode += "\r\n return a;";
funcCode = "function jschl_answer(){\r\n" + funcCode + "\r\n}";
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("js");
engine.eval(funcCode);
Invocable invocable = (Invocable) engine;
Double jschl_answer = (Double) invocable.invokeFunction("jschl_answer");
url=baseURL+"/cdn-cgi/l/chk_jschl?jschl_vc="+jschl_vc+"&pass="+pass+"&jschl_answer="+jschl_answer.intValue();
http.config.allowRedirects(false);
System.out.println(url);
Thread.sleep(3800l);
http.config.setGzip(true);
entity=http.Get(url);
cookie=entity.getCookie();
if(!cookie.contains("cf_clearance")){
return null;
}
return entity;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
public HttpEntity Get(String url){
if(cookie!=null&&!"".equals(cookie)){
loadCookie(url);
}
HttpHandle http=new HttpHandle();
http.config.setRequestProperty("If-Modified-Since", new Date().toString());
http.config.setRequestProperty("Cache-Control", "max-age=0");
http.config.setRequestProperty("Upgrade-Insecure-Requests", "1");
http.config.setKeepAlive(true);
http.config.setCookie(cookie);
HttpEntity entity=http.Get(url);
if(entity.getCode()!=200){
loadCookie(url);
http.config.setCookie(cookie);
entity=http.Get(url);
}
return entity;
}
public void loadCookie(String url){
cookie=null;
HttpEntity entity=initCookie(url);
while(entity==null){
entity=initCookie(url);
}
}
public static void main(String[] args) throws NoSuchMethodException, ScriptException, InterruptedException {
HttpEntity entity=new RobotHttpHandle().Get("http://www.myexception.cn/");
System.out.println(entity.getHtml());
}
}
使用方式：
HttpEntity entity=new RobotHttpHandle().Get("http://www.myexception.cn/");
System.out.println(entity.getHtml());
如图：
