网络内容爬取、文件操作、KMP匹配

1 importjava.net.MalformedURLException;2 importjava.net.URL;3 importjava.net.URLConnection;4 importjava.util.ArrayList;5 import java.io.*;6 importjava.util.Scanner;7

8 classGetHtmlData {9 privateString url;10 GetHtmlData(){//构造函数

11 }12 URL readurl;13 String bq(String pt) {14 String ps;15 ps=newString();16 if(pt.charAt(0)!='h'&&pt.charAt(1)!='t'&&pt.charAt(2)!='t'&&pt.charAt(3)!='p') {17 ps="https://"+pt;18 returnps;19 }20 returnpt;21 }22 String GetUrl(String pt,boolean check) throwsIOException {23 if(check==true)24 pt=bq(pt);//补全网址

25 url=newString();26 url=pt;//init url

27 readurl=new URL(url);//url字符串构建URL类

28 URLConnection connection = readurl.openConnection();//初始化连接

29 connection.setRequestProperty("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11");30 connection.connect();//开始连接

31 String line;32 String web_data;33 web_data=newString();34 line=newString();35 //输入流

36 BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(),"GBK"));37 while((line=in.readLine())!=null) {//java 中 null 为小写

38 web_data+=line;39 }40 returnweb_data;41 }42

43 String GetUrl_utf(String pt,boolean check) throwsIOException {44 if(check==true)45 pt=bq(pt);//补全网址

46 url=newString();47 url=pt;//init url

48 readurl=new URL(url);//url字符串构建URL类

49 URLConnection connection = readurl.openConnection();//初始化连接

50 connection.setRequestProperty("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11") ;51 connection.connect();//开始连接

52 String line;53 String web_data;54 web_data=newString();55 line=newString();56 //输入流

57 BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(),"UTF-8"));58 while((line=in.readLine())!=null) {//java 中 null 为小写

59 web_data+=line;60 }61 returnweb_data;62 }63 }64 classWFILE {65 FileWriter fp;66 PrintWriter outfp;67 boolean OpenFile(String m,String f) {//将s内容输出至文件目录m下的f文件,wdata为文件内容

68 File catalogue = new File(m);//目录

69 try{70 if (!catalogue.exists()) {71 boolean ok=catalogue.mkdir();72 if(!ok)//如果失败返回失败

73 returnok;74 }75 }76 catch(Exception e) {//输出异常

77 System.out.println("在构建目录时异常,目录名为: "+m);78 e.printStackTrace();79 }80 String lj=m+"\\"+f;81 File fe=newFile(lj);82 try{83 if(!fe.exists()) {84 boolean ok2=fe.createNewFile();85 if(!ok2)86 returnok2;87 }88 fp = newFileWriter(fe);89 outfp = newPrintWriter(fp);90 return true;91 }92 catch(Exception e){93 System.out.println("在构建文件时异常");94 e.printStackTrace();95 }96 return false;97 }98 booleanCloseFile() {99 if(fp==null)100 return false;101 else{102 try{103 fp.close();104 } catch(IOException e) {105 //TODO Auto-generated catch block

106 System.out.println("关闭文件错误");107 e.printStackTrace();108 }109

110 }111 return true;112 }113 booleanprint(T tdata){114 if (fp==null||outfp==null)115 return false;116 outfp.print(tdata);117 return true;118 }119 booleanprintln(T tdata){120 if (fp==null||outfp==null)121 return false;122 outfp.println(tdata);123 return true;124 }125 }126

127 classKMP {128 intnextp[];129 intn,m;130 String pattern;131 String text;132 intcc;133 intpp;134 KMP(){135 cc=0;136 pp=0;137 last=-1;138 begin='\0';139 }140 ArrayListans;141 booleanSet_substring(String pattern) {142 this.pattern=pattern;143 if(pattern==null)144 return false;145 int n=pattern.length();146 this.n=n;147 nextp=new int[n+10];148 for(int i=0;i0) {153 j =nextp[j];154 if(pattern.charAt(j)==pattern.charAt(i)) {155 nextp[i+1]=j+1;156 break;157 }158 }159 }160 return true;161 }162 voidfind_substring(String text) {163 ans=new ArrayList();164 this.text=text;165 int m=text.length();166 this.m=m;167 for(int i= 0,j=0;i0) {173 try{174 j=nextp[j];175 }176 catch(Exception e) {177 }178 if(text.charAt(i)==pattern.charAt(j)) {179 ++j;180 break;181 }182 }183 }184 if(j>=n) {185 ans.add(i);186 }187 }188 }189 void Set_min(int t) {//Set the min index

190 cc=t;191 }192

193 charbegin;194 intlast;195 void set_last(intt) {196 last=t;197 }198 void Set_begin(chart) {199 begin =t;200 }201

202 ArrayList get_data(chars) {203 chara[];204 ArrayList zans = new ArrayList();205 a=new char[m];206 for(int i=0;ilast)212 break;213 if(begin!='\0') {214 while( ss < m && text.charAt(ss) !=begin)215 ss++;216 if(ss==m)217 continue;218 else{219 w=ss;220 }221 }222 int t=0,ok=0;223 if(w+1<=cc)224 continue;225 for(int j=w+1;j

237 a[t]=0;238 if(t==0)239 continue;240 charb[];241 b=new char[t];242 for(int i1=0;i1