简单的java爬虫程序

进来公司要做爬虫，框架都不会用，就按照网上的python原理用java实现间的程序用到HTMLParser包

package my.url.test;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

public class MyUrl {

private URL url ;
    private File file;
    private static String dirName;
    private static int depth = 0;
    private static int number = 0;
    private static HttpURLConnection urlConnectin ;

    public static void main(String[] args){
        try {
            String [] urlNames = new String[]{
                    "http://www.ebiotrade.com/newsf/",
            };
            init(urlNames);
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        } catch (ParserException e) {
            e.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void init(String[] urlNames) throws Exception{
        for(int i=0;i<urlNames.length;i++){
            MyUrl myUrl = new MyUrl(urlNames[i]);
            myUrl.connect();
            myUrl.html2txt(urlConnectin,dirName);
        }
    }


    public MyUrl(String urlString) throws URISyntaxException, ParserException, IOException{
        url= new URL(urlString);
        if(url.getPath().equals("") || url.getPath().equals("/")){
            dirName = url.toURI().getHost();
        }else{
            dirName = url.toURI().getHost()+""+url.toURI().getPath();
        }

CreateDir(dirName);
    }

    private void CreateDir(String dirName) throws URISyntaxException{
        //创建目录
        String[] dirNames = dirName.split("/");
        String temp = "";
        for(int i=0;i<((dirNames.length == 4)?dirNames.length - 1:dirNames.length);i++){
            temp += dirNames[i]+"/";
            File dir = new File("c:/"+ temp);
            if(dir.exists() == false){
                System.out.println("创建文件夹:"+dirNames[i]);
                dir.mkdir();
            }
        }
    }

    private void connect() throws Exception{
        urlConnectin = (HttpURLConnection)url.openConnection();
        urlConnectin.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
        urlConnectin.setRequestProperty("Accept","image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-powerpoint, application/vnd.ms-excel, application/msword, */*");
        urlConnectin.setRequestProperty("Accept-Language", "zh-cn");
        urlConnectin.setRequestProperty("UA-CPU", "x86");
        //urlConnectin.setRequestProperty("Accept-Encoding", "gzip");//为什么没有deflate呢
        urlConnectin.setRequestProperty("Content-type", "text/html");
        urlConnectin.setRequestProperty("Connection", "close");
        urlConnectin.setUseCaches(false);//不要用cache，用了也没有什么用，因为我们不会经常对一个链接频繁访问。（针对程序）
        //urlConnectin.setConnectTimeout(10 * 1000);
        //urlConnectin.setReadTimeout(10 * 1000);
        urlConnectin.setDoOutput(true);
        urlConnectin.setDoInput(true);
        if(depth == 0){
            analyzeHTML(urlConnectin,"gb2312");
        }

}

    private void readAndWrite(HttpURLConnection urlConnectin,String fileName) throws IOException, ParserException, URISyntaxException{
        if(url.getPath().equals("")){
            file = new File("c:/"+dirName+"/index.html");
        }else{
            file = new File("c:/"+dirName);
        }
        if(file.exists() == false){
            number ++;
            System.out.println("下载:"+number);
            System.out.println(file.getPath());
            file.createNewFile();
            FileWriter fw = new FileWriter(file);
            BufferedWriter bw = new BufferedWriter(fw);
            String inputLine;
            BufferedReader in = new BufferedReader( new InputStreamReader(urlConnectin.getInputStream()));

            while ((inputLine = in.readLine()) != null) {
                bw.write(inputLine);
                bw.newLine();
                bw.flush();
              }

}
}

//解析出连接
    private void analyzeHTML(HttpURLConnection urlConnectin,String encode) throws Exception{
        depth = 1;
        Parser parser = new Parser(urlConnectin);
        parser.setEncoding(encode);
        NodeFilter filter = new AndFilter(new TagNameFilter("a"),new HasAttributeFilter("class","TDcenter"));
        NodeList nodeList = parser.parse(filter);
        NodeIterator it = nodeList.elements();
        while(it.hasMoreNodes()){
            Node node = it.nextNode();
            init(new String[]{((LinkTag)node).getLink()});
        }
    }

//下载图片
    private void downloadImageAnalyzeHtml(URLConnection urlConnectin,String encode) throws Exception{
        Parser parser = new Parser(urlConnectin);
        parser.setEncoding(encode);
        NodeFilter filter = new TagNameFilter("img");
        NodeList nodeList = parser.parse(filter);
        NodeIterator it = nodeList.elements();
        while(it.hasMoreNodes()){
            Node node = it.nextNode();
            if(((ImageTag)node).getImageURL().toString().startsWith("http") == false){
                String tempDirectory = url.getHost()+((ImageTag)node).getImageURL().replace("file:", "");
                CreateDir(tempDirectory);
                URLConnectionDownloader.download("http://"+tempDirectory,"c:/"+tempDirectory);
            }
        }
    }

//解析成txt文件
    private void html2txt(HttpURLConnection urlConnectin,String dirName) throws Exception{
        File dirTxt = new File("c:/"+dirName+".txt");
        if(dirTxt.exists() == false){
            Parser parser = new Parser(urlConnectin);
            parser.setEncoding("gb2312");

            NodeFilter filter =new OrFilter(new NodeFilter[]{new TagNameFilter("p"),new HasAttributeFilter("class","MsoNormal"),new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","newsf"))});
            NodeList nodeList = parser.parse(filter);
            NodeIterator it = nodeList.elements();
            FileWriter fw = new FileWriter(dirTxt);
            BufferedWriter bw = new BufferedWriter(fw);
            if(it.hasMoreNodes()){
                dirTxt.createNewFile();
                System.out.println("创建文件："+dirTxt);
            }
            while(it.hasMoreNodes()){
                Node node = it.nextNode();
                  bw.write(node.toHtml());
                  bw.newLine();
                  bw.flush();
            }
            bw.close();
            downloadImageAnalyzeHtml((dirTxt.toURI().toURL().openConnection()),"gb2312");
        }
    }

}

//以下类来源网络

import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
/**
* 使用URLConnection下载文件或图片并保存到本地。
*
* @author 老紫竹(laozizhu.com)
*/
public class URLConnectionDownloader {
public static void main(String[] args) throws Exception {
   download("http://www.cnbeta.com/images/cnlogo.gif", "cnlogo.gif");
}
/**
* 下载文件到本地
*
* @param urlString
*          被下载的文件地址
* @param filename
*          本地文件名
* @throws Exception
*           各种异常
*/
public static void download(String urlString, String filename) throws Exception {
   // 构造URL
   URL url = new URL(urlString);
   // 打开连接
   URLConnection con = url.openConnection();
   // 输入流
   InputStream is = con.getInputStream();
   // 的数据缓冲
   byte[] bs = new byte[1024] ;
   // 读取到的数据长度
   int len;
   // 输出的文件流
   OutputStream os = new FileOutputStream(filename);
   // 开始读取
   while ((len = is.read(bs)) != -1) {
     os.write(bs, 0, len);
   }
   // 完毕，关闭所有链接
   os.close();
   is.close();
}
}

简单的java爬虫程序相关推荐

java网络爬虫0基础_简单的java爬虫程序（入门）
首先做好准备工作: 了解正则表达式中基本的Select的用法: 此处引用楼主查到的资料 http://www.tuicool.com/articles/ZnyMvu 第一个爬虫:无验证码的简单爬虫楼 ...
超简单的java爬虫
最简单的爬虫,不需要设定代理服务器,不需要设定cookie,不需要http连接池,使用httpget方法,只是为了获取html代码... 好吧,满足这个要求的爬虫应该是最基本的爬虫了.当然这也是做复杂 ...
一个简单的python爬虫程序
#简介在每次论文被拒再投的过程中,都需要查询最近的与自己论文相关的会议列表.每到这种情况,我一遍采用的是遍历会伴www.myhuiban.com的网站,然后逐个查看会议,关注的有三点,投稿日期,cc ...
一个简单的Java抽奖程序
文章目录需求背景设计思路代码实现定义奖品及中奖概率执行抽奖中奖率测试测试结果数据本文逻辑思想比较简单,旨在了解后端如何设计抽奖以及控制抽奖概率. 需求背景现在奖品池有如下奖品: 序号 ...
写一个简单的Java界面程序
写一个简单的Java界面程序有时候未免想写一些有界面的java小程序练练手,那么如何写一个比较好看的界面话程序呢?下面小编就带你一步一步来搭建这个小洋房. 实现界面化编程要用到的一个主要包impor ...
一个简单的Java应用程序
下面看一个最简单的Java应用程序,其功能:发送一条消息到控制台窗口中. 源代码如下: public class FirstSample { public static void main(Strin ...
日记本java代码_简单的JAVA日记本程序源代码
[实例简介] 一个入门级的JAVA程序源代码,界面绝对赞,很Q的日记本,原理不复杂,代码注释非常详尽清晰,一看就懂.适合初学者. [实例截图] [核心代码] JAVA日记本程序 └── yang ├─ ...
java applet配置_最简单的Java Applet程序
1.配置好java的开发环境 2.用Eclipse新建一个Java Project,其他选择默认的配置. 3.新建一个类,在SuperClass选择Browser,查找Java Applet,选择后确 ...
第一次使用Eclipse：编写简单的Java小程序
通过前部分的学习,了解了Java的安装和配置,那么从现在开始,要开始自己着手编写Java程序,学习一门编程语言,学会编写的第一个程序一般都是写一个输出"hello World!"语 ...

简单的java爬虫程序

简单的java爬虫程序相关推荐

最新文章

热门文章