最近做个产品要用爬虫获得信息，可是这个网页要模拟浏览器输出，我参考了网上很多setRequestProperty里面User-Agent都写得Mozilla/5.0结果还是不能访问，最后研究了一下写出来了。
public static String URLPostConnReturnHtml(String ip){
URL url = null ;
HttpURLConnection conn = null ;
BufferedReader br = null ;
InputStreamReader isr = null ;
InputStream is = null ;
OutputStream os = null ;
OutputStreamWriter writer ;
int responseCode ;
String readerLine = "" ;
StringBuffer htmlCode = new StringBuffer() ;
String htmlCodeStr = "" ;
try {
url = new URL("http://www.yougetsignal.com/tools/whois-lookup/php/get-whois-lookup-json-data.php") ;
conn= (HttpURLConnection)url.openConnection() ;
conn.setConnectTimeout(TIME_OUT) ;
conn.setReadTimeout(TIME_OUT) ;
//设置可以输出
conn.setDoOutput(true) ;
//设置可以读取默认是true
conn.setDoInput(true) ;
//设置不适用缓存，因为我们的数据是实时更新的
conn.setUseCaches(false) ;
//设置请求方式为post，默认是get
conn.setRequestMethod("POST") ;
//设置所有的http连接自动处理重定向
HttpURLConnection.setFollowRedirects(true) ;
//设置本次连接自动处理重定向
conn.setInstanceFollowRedirects(true) ;
//模拟ios 谷歌浏览器访问
conn.setRequestProperty("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11") ;
//conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded") ;
os = conn.getOutputStream() ;
writer = new OutputStreamWriter(os ,"utf-8") ;
writer.write("remoteAddress="+ip) ;
writer.flush() ;
writer.close() ;
responseCode= conn.getResponseCode() ;
//TODO:如果得到返回html代码，则读取
if(responseCode == 200){
is = conn.getInputStream() ;
isr = new InputStreamReader(is) ;
br = new BufferedReader(isr) ;
readerLine = br.readLine() ;
while(readerLine != null){
htmlCode.append(readerLine) ;
readerLine = br.readLine() ;
}
}else{
System.out.println(errorResponseStr + responseCode);
}
htmlCodeStr = htmlCode.toString() ;
} catch (Exception e) {
e.printStackTrace() ;
RETRY_COUNT++;
System.out.println(errorNetConnStr + RETRY_COUNT);
if (RETRY_COUNT < RETRY_COUNT_BREAK) {
htmlCodeStr = URLPostConnReturnHtml(ip);
}else{
System.out.println("网络连接失败！");
}
}finally{
try {
if(br!=null){
br.close() ;
}
if(isr!=null){
isr.close() ;
}
if(isr!=null){
is.close() ;
}
if(os!=null){
os.close() ;
}
} catch (IOException e) {
// TODO Auto-generated catch block
System.out.println("IO流关闭错误--->");
e.printStackTrace();
}
if(conn!=null){
conn.disconnect() ;
}
}
System.out.println(htmlCodeStr);
return htmlCodeStr ;
}