使用JAVA抓取网页数据

一、使用 HttpClient 抓取网页数据

public String getHtml(String htmlurl) throws IOException {		StringBuffer sb = new StringBuffer();		String acceptEncoding = "";		/* 1.生成 HttpClinet 对象并设置参数 */		HttpClient httpClient = new HttpClient();		GetMethod method = new GetMethod(htmlurl);		int statusCode;		try {			statusCode = httpClient.executeMethod(method);			// 判断访问的状态码			if (statusCode != HttpStatus.SC_OK) {				return null;			} else {				if (method.getResponseHeader("Content-Encoding") != null)					acceptEncoding = method.getResponseHeader(							"Content-Encoding").getValue();				if (acceptEncoding.toLowerCase().indexOf("gzip") > -1) {					// 建立gzip解压工作流					InputStream is;					is = method.getResponseBodyAsStream();					GZIPInputStream gzin = new GZIPInputStream(is);					InputStreamReader isr = new InputStreamReader(gzin, Charset.forName(CHARSET)); // 设置读取流的编码格式，自定义编码					java.io.BufferedReader br = new java.io.BufferedReader(isr);					String tempbf;					while ((tempbf = br.readLine()) != null) {						if(StringUtils.isNotBlank(tempbf)){							sb.append(tempbf);						}					}					isr.close();					gzin.close();					System.out.println(sb);				} else {					InputStreamReader isr;					isr = new InputStreamReader(							method.getResponseBodyAsStream(), CHARSET);					java.io.BufferedReader br = new java.io.BufferedReader(isr);					String tempbf;					while ((tempbf = br.readLine()) != null) {						if(StringUtils.isNotBlank(tempbf)){							sb.append(tempbf);						}					}					isr.close();				}			}		} catch (HttpException e) {			e.printStackTrace();		} catch (IOException e) {			e.printStackTrace();		}		method.abort();		method.releaseConnection();		return sb.toString();	}

海外公司注册、海外银行开户、跨境平台代入驻、VAT、EPR等知识和在线办理：https://www.xlkjsw.com

原标题：使用JAVA抓取网页数据

关键词：JAVA