Java指定URLページの内容を取得する
3997 ワード
指定したURLページの内容をキャプチャし、そのページのjsonデータを取得してデータベースに保存します.
ダイレクトコード:
ダイレクトコード:
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Map;
public class GetURLContent {
public static String getURLContent(String requestUrl,Map data,String method) throws IOException{
HttpURLConnection conn = null;
BufferedReader br = null;
StringBuffer sb = new StringBuffer();
String content = null;
//GET url
if(null == method || "GET" == method){
requestUrl = requestUrl+"?"+urlEncode(data);
}
try {
URL url = new URL(requestUrl);
conn = (HttpURLConnection) url.openConnection();
//
if("GET" == method || null == method){
conn.setRequestMethod("GET");
}else{
conn.setRequestMethod("POST");
// URL
conn.setDoOutput(true);
}
//
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
//
conn.setUseCaches(false);
// ,
conn.setConnectTimeout(1000);
// ,
conn.setReadTimeout(1000);
// 。setFollowRedirects
conn.setInstanceFollowRedirects(false);
//
conn.connect();
// post
if(null != data && "POST" == method){
DataOutputStream out = new DataOutputStream(conn.getOutputStream());
out.writeBytes(urlEncode(data));
}
//
br = new BufferedReader(new InputStreamReader(conn.getInputStream(),"UTF-8"));
String strContent = null;
while((strContent = br.readLine()) != null){
sb.append(strContent);
}
content = sb.toString();
} catch (MalformedURLException e) {
e.printStackTrace();
}finally{
//
if(null != br){
br.close();
}
if(null != conn){
conn.disconnect();
}
}
return content;
}
// map
public static String urlEncode(Map data) {
if(data == null){
return "";
}else{
StringBuilder sb = new StringBuilder();
for (Map.Entry i : data.entrySet()) {
try {
sb.append(i.getKey()).append("=").append(URLEncoder.encode(i.getValue() + "", "UTF-8")).append("&");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
return sb.toString();
}
}
public static void main(String[] args) {
Map data = new HashMap();
data.put("1", "111");
data.put("2", "222");
try {
String content = GetURLContent .getURLContent("https://baidu.com", data, "GET");
System.out.println(content);
} catch (Exception e) {
e.printStackTrace();
}
}
}