【java】抓取页面内容，提取链接（此方法可以http get无需账号密码的请求）

 package 网络编程;

 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.net.URL;

 public class TestBaidu {
     public static void main(String[] args) throws IOException {
         URL url=new URL("http://www.baidu.com");
         /*此方法会有乱码输出
         InputStream is=url.openStream();
         byte[] b=new byte[1024];
         int len=0;
         while((len=is.read(b))!=-1){
             System.out.println(new String(b,0,len));
         }
         */
         BufferedReader br=new BufferedReader(new InputStreamReader(url.openStream(),"utf-8"));
         BufferedWriter bw=new BufferedWriter(new OutputStreamWriter(new FileOutputStream("baidu.html"),"utf-8"));
         String str=null;
         while((str=br.readLine())!=null){
             bw.append(str);
             bw.newLine();
         }
             //System.out.print(str);
         bw.flush();
         bw.close();
         br.close();
     }
 }

抓取页面内容

 package 网络编程;

 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.Charset;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 public class Get163URL {
     public static void main(String[] args) throws IOException {
         URL url=new URL("http://www.163.com");
         BufferedReader br=new BufferedReader(new InputStreamReader(url.openStream(),Charset.forName("gbk")));
         StringBuffer sb=new StringBuffer();
         String tmp=null;
         while((tmp=br.readLine())!=null){
             sb.append(tmp);
         }
 //        System.out.println(sb.toString());
         Pattern p=Pattern.compile("\"(http:\\/\\/.+?)\"");
         Matcher m=p.matcher(sb);
         while(m.find())
             System.out.println(m.group(1));
     }
 }

提出链接

---

 public class WikiDownload {
     static final String name = "username";
     static final String pwd = "password";

     public static void main(String[] args){
         CookieManager manager = new CookieManager();
         CookieHandler.setDefault(manager);
         String wikiUrl = "http://wiki.xxxxx.org/pages/viewpage.action?pageId=71709153";
         String loginUrl = "http://wiki.xxxxx.org/login.action?os_destination=%2Fpages%2Fviewpage.action%3FpageId%3D71709153";
         try{
             URL url = new URL(loginUrl);
             HttpURLConnection connection = (HttpURLConnection)url.openConnection();
             connection.setRequestProperty("accept", "*/*");
             connection.setRequestProperty("connection", "Keep-Alive");
             connection.setRequestProperty("user-agent",
                     "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36");
             String line;
             connection.setDoInput(true);
             connection.setDoOutput(true);
             connection.setUseCaches(false);
             connection.setRequestMethod("POST");
             try(OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream())){
                 writer.write("os_username=" + name
                         +"&os_password="+ pwd
                         + "&login=%E7%99%BB%E5%BD%95&os_destination="
                         + URLEncoder.encode(wikiUrl.split("http://wiki.xxxxx.org")[0],"utf-8"));
             }
             try(InputStreamReader reader = new InputStreamReader(connection.getInputStream())){
                 BufferedReader in = new BufferedReader(reader);
                 StringBuilder result= new StringBuilder("");
                 while ((line = in.readLine()) != null) {
                     result.append("\n");
                     result.append(line);
                 }
                 System.out.println(result);
             }
         }catch (Exception e){
             e.printStackTrace();
         }

     }

 }

获取需要登录的网页