JAVA写的网页爬虫爬取必应词典的词汇音标

为了建立一个英语词汇音标库，需要从网上搜索词汇音标，由于有10万多个词汇需要处理，所以做个这个单词音标爬虫爬取必应网络词典网页上的单词音标。

package bingword;import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javafx.application.Application;
import javafx.beans.value.ChangeListener;
import javafx.beans.value.ObservableValue;
import javafx.concurrent.Worker;
import javafx.event.ActionEvent;
import javafx.event.EventHandler;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.Button;
import javafx.scene.control.Label;
import javafx.scene.control.TextArea;
import javafx.scene.control.TextField;
import javafx.scene.control.Tooltip;
import javafx.scene.layout.GridPane;
import javafx.scene.text.Font;
import javafx.scene.web.WebEngine;
import javafx.scene.web.WebView;
import javafx.stage.FileChooser;
import javafx.stage.Stage;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;public class BingWord extends Application {private final GridPane grid = new GridPane();private final TextField inputText = new TextField();private final Button btn = new Button();private final Button btn1 = new Button();private final Button btn2 = new Button();private final Button btnPause = new Button();private final WebView browser = new WebView();private final WebEngine webEngine = browser.getEngine();private final TextArea textArea = new TextArea();private final Label info = new Label();private int recordPosition = 1;private String currentWord = "";private Connection connection = null;private Statement statement;private ResultSet rs;private int maxNumber = 1;private Boolean pause = false;@Overridepublic void start(Stage primaryStage) throws SQLException {//程序启动后就连接好SQLite数据库，或者新建一个dictionary.db数据库文件connection = DriverManager.getConnection("jdbc:sqlite:dictionary.db");statement = connection.createStatement();statement.setQueryTimeout(30);statement.executeUpdate("CREATE TABLE IF NOT EXISTS wordlist (id INTEGER PRIMARY KEY AUTOINCREMENT, word TEXT, pronunciation1 TEXT, pronunciation2 TEXT)");System.out.println("dictionary.db数据库已经连接");//文本框供手动设置开始行位置，默认值为第一行开始inputText.setText("1");//查找必应词典音标的启动按钮Tooltip tooltip=new Tooltip();tooltip.setText("请在右侧输入框内填入开始查找的行号");tooltip.setFont(new Font("Arial", 20));     btn.setTooltip(tooltip);btn.setText("开始查找音标行");btn.setOnAction(new EventHandler<ActionEvent>() {@Overridepublic void handle(ActionEvent event) {//重启pause = false;//获取文本框中设定的起始行位置recordPosition = Integer.parseInt(inputText.getText());try {//查询记录总条数rs = statement.executeQuery("SELECT count(*) FROM wordlist;");while (rs.next()) {maxNumber = rs.getInt(1);System.out.println("数据库中总单词量：" + maxNumber);}//单词音标搜索程序search();} catch (SQLException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);}}});//将EXCEL导入到SQLITE数据库中Tooltip tooltip1=new Tooltip();tooltip1.setText("将EXCEL表中第一列单词导入到SQLite数据库中");tooltip1.setFont(new Font("Arial", 20));          btn1.setTooltip(tooltip1);        btn1.setText("Excel -> SQLite");btn1.setOnAction(new EventHandler<ActionEvent>() {@Overridepublic void handle(ActionEvent event) {System.out.println("将Excel中的单词导入dictionary.db数据库的wordlist表中");FileChooser fileChooser = new FileChooser();fileChooser.setTitle("打开EXCEL文件");fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("EXCEL", "*.xls"));File file = fileChooser.showOpenDialog(primaryStage);if (file == null) {return;}String filePath = file.getPath();System.out.println(filePath);try {//导入EXCEL文件FileInputStream inputExcel = new FileInputStream(filePath);HSSFWorkbook wb = new HSSFWorkbook(inputExcel);//获取SHEET的张数int sheetCount = wb.getNumberOfSheets();System.out.println("导入的EXCEL表总共有工作表张数：" + sheetCount);//循环操作各个工作表for (int i = 0; i < sheetCount; i++) {HSSFSheet sheet = wb.getSheetAt(i);int rowCount = sheet.getLastRowNum();System.out.println("sheet" + i + "总共有" + rowCount + "行");for (int j = 0; j <= rowCount; j++) {HSSFRow row = sheet.getRow(j);if (row != null) {if (row.getCell(0) != null) {row.getCell(0).setCellType(1);//循环读取各个工作表第一列String cellValue = row.getCell(0).getStringCellValue().trim();//先查询看单词是否存在rs = statement.executeQuery("SELECT * FROM wordlist WHERE word='" + cellValue + "';");System.out.println("SELECT * FROM wordlist WHERE word='" + cellValue + "';");if (rs.next()) {//如果单词已经存在则无需重复插入} else {//如果单词不存在就将该单词插入到单词列表中statement.executeUpdate("INSERT INTO wordlist (word) VALUES('" + cellValue + "');");System.out.println("INSERT INTO wordlist (word) VALUES('" + cellValue + "');");}}}}}info.setText("EXCEL中的数据已经导入数据库完毕");//记得打开的文件要及时关闭wb.close();inputExcel.close();} catch (FileNotFoundException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);} catch (IOException | SQLException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);}}});//将数据写入EXCEL文件保存,注意EXCEL字体需要设置为IPAPANNEW，否则音标会乱码Tooltip tooltip2=new Tooltip();tooltip2.setText("完整的单词列表和单词音标将从数据库输出到EXCEL表中");tooltip2.setFont(new Font("Arial", 20));  btn2.setTooltip(tooltip2);        btn2.setText("SQLite -> Excel");btn2.setOnAction(new EventHandler<ActionEvent>() {@Overridepublic void handle(ActionEvent event) {System.out.println("将SQLITE数据库中的数据导出到EXCEL文件中");FileChooser fileChooser = new FileChooser();fileChooser.setTitle("保存到EXCEL文件");fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("EXCEL", "*.xls"));fileChooser.setInitialFileName("wordlist.xls");File file = fileChooser.showSaveDialog(primaryStage);if (file == null) {return;}String filePath = file.getPath();System.out.println(filePath);try {//获取总记录数目maxNumberrs = statement.executeQuery("SELECT count(*) FROM wordlist;");while (rs.next()) {maxNumber = rs.getInt(1);System.out.println("数据库中总单词量：" + maxNumber);}//计算总共需要多少个工作表来存数据int maxRowNumber = 65536;int sheetCount = (int) (Math.floor(maxNumber / (maxRowNumber + 1)) + 1);//目标EXCEL文件输出流FileOutputStream outputExcel = new FileOutputStream(filePath);//创建EXCEL表HSSFWorkbook wb = new HSSFWorkbook();//循环创建各个工作表和单元格for (int i = 0; i < sheetCount; i++) {HSSFSheet sheet = wb.createSheet("wordlist" + i);//设置列宽sheet.setColumnWidth(1, 6400);sheet.setColumnWidth(2, 6400);sheet.setColumnWidth(3, 6400);for (int j = 0; j < maxRowNumber; j++) {HSSFRow row = sheet.createRow(j);HSSFCell cell0 = row.createCell(0);HSSFCell cell1 = row.createCell(1);HSSFCell cell2 = row.createCell(2);HSSFCell cell3 = row.createCell(3);cell0.setCellType(1);cell1.setCellType(1);cell2.setCellType(1);cell3.setCellType(1);}}//将数据库记录存入Excel各个对应单元格rs = statement.executeQuery("SELECT * FROM wordlist;");while (rs.next()) {int currentLineNumber = 0;String currentWordSpell = "";String currentPronunciation1 = "";String currentPronunciation2 = "";currentLineNumber = rs.getInt(1);if (rs.getString(2) != null) {currentWordSpell = rs.getString(2).trim();}if (rs.getString(3) != null) {currentPronunciation1 = rs.getString(3).trim();}if (rs.getString(4) != null) {currentPronunciation2 = rs.getString(4).trim();}//计算当前记录应该写入第几个工作表第几行int targetSheetNumber = (int) (Math.floor(currentLineNumber / (maxRowNumber + 1)));int targetRowNumber = (currentLineNumber - 1) % maxRowNumber;//将数据库的记录写入相应单元格中Row currentRow = wb.getSheetAt(targetSheetNumber).getRow(targetRowNumber);currentRow.getCell(0).setCellValue(currentLineNumber + "");currentRow.getCell(1).setCellValue(currentWordSpell);currentRow.getCell(2).setCellValue(currentPronunciation1);currentRow.getCell(3).setCellValue(currentPronunciation2);}//写入文件并关闭wb.write(outputExcel);wb.close();outputExcel.close();info.setText("数据已经保存到文件 " + filePath);} catch (FileNotFoundException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);} catch (IOException | SQLException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);}}});//暂停按钮设置Tooltip tooltip3=new Tooltip();tooltip3.setText("暂停查找循环");tooltip3.setFont(new Font("Arial", 20));    btnPause.setTooltip(tooltip3);        btnPause.setText("Pause");btnPause.setOnAction(new EventHandler<ActionEvent>() {@Overridepublic void handle(ActionEvent event) {pause = true;}});//WEBVIEW浏览器监听加载成功事件webEngine.getLoadWorker().stateProperty().addListener(new ChangeListener<Worker.State>() {@Overridepublic void changed(ObservableValue<? extends Worker.State> ov,Worker.State oldState, Worker.State newState) {//如果点击了暂停按钮则跳出程序循环if (pause == true) {return;}//每次浏览器加载完毕则执行操作if (newState == Worker.State.SUCCEEDED) {//获取浏览器中网页的HTMLString docContent = webEngine.getDocument().getDocumentElement().getTextContent();String resultString = "";//用正则表达式搜索获取HTML中的美式音标Pattern patternUS = Pattern.compile("美\\S\\[[^\\[\\]]+\\]");Matcher matcherUS = patternUS.matcher(docContent);if (matcherUS.find()) {resultString = matcherUS.group(0).replaceAll("[美\\s\\[\\]]", "");//把单个单引号替换成两个单引号避免SQL报错resultString = resultString.replaceAll("'", "''");System.out.println("PronunciationUS:" + resultString);info.setText("PronunciationUS: " + resultString);try {//如果找到了音标则写入数据库statement.executeUpdate("UPDATE wordlist SET pronunciation1='" + resultString + "' WHERE word='" + currentWord + "';");System.out.println("UPDATE wordlist SET pronunciation1='" + resultString + "' WHERE word='" + currentWord + "';");} catch (SQLException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);}} else {System.out.println("NO MATCH");info.setText("NO MATCH");}//用正则表达式搜索获取HTML中的英式音标Pattern patternUK = Pattern.compile("英\\S\\[[^\\[\\]]+\\]");Matcher matcherUK = patternUK.matcher(docContent);if (matcherUK.find()) {resultString = matcherUK.group(0).replaceAll("[英\\s\\[\\]]", "");//把单个单引号替换成两个单引号避免SQL报错resultString = resultString.replaceAll("'", "''");System.out.println("PronunciationUK: " + resultString);info.setText("PronunciationUK: " + resultString);try {//如果找到了音标则写入数据库statement.executeUpdate("UPDATE wordlist SET pronunciation2='" + resultString + "' WHERE word='" + currentWord + "';");System.out.println("UPDATE wordlist SET pronunciation2='" + resultString + "' WHERE word='" + currentWord + "';");} catch (SQLException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);}} else {System.out.println("NO MATCH");info.setText("NO MATCH");}//进入查询接龙式循环recordPosition++;inputText.setText(recordPosition + "");if (recordPosition > maxNumber) {info.setText("大功告成");return;}try {//继续查询程序search();} catch (SQLException ex) {Logger.getLogger(BingWord.class.getName()).log(Level.SEVERE, null, ex);}}}});//整理控件布局并显示到场景中grid.add(btn, 0, 0);grid.add(inputText, 1, 0);grid.add(btn1, 2, 0);grid.add(btn2, 3, 0);grid.add(btnPause, 4, 0);grid.add(browser, 0, 1, 5, 1);grid.add(info, 0, 3, 5, 1);grid.setPadding(new Insets(10, 10, 10, 10));grid.setVgap(10);grid.setHgap(10);browser.setPrefSize(800, 600);Scene scene = new Scene(grid, 600, 400);scene.widthProperty().addListener(new ChangeListener() {@Overridepublic void changed(ObservableValue observable,Object oldValue, Object newValue) {Double width = (Double) newValue;browser.setPrefWidth(width - 20);}});scene.heightProperty().addListener(new ChangeListener() {@Overridepublic void changed(ObservableValue observable,Object oldValue, Object newValue) {Double height = (Double) newValue;browser.setPrefHeight(height - 50);}});primaryStage.setTitle("必应英汉词典音标爬虫 Benny");primaryStage.setScene(scene);primaryStage.show();}public static void main(String[] args) {launch(args);}public void search() throws SQLException {//按单词列表序号依次开始在必应词典网站中请求查询各个单词rs = statement.executeQuery("SELECT * FROM wordlist WHERE id=" + recordPosition);if (rs.next()) {currentWord = rs.getString(2);System.out.println("当前正在处理的单词是：" + currentWord);webEngine.load("http://cn.bing.com/dict/search?q=" + currentWord);}}}

JAVA写的网页爬虫爬取必应词典的词汇音标相关推荐

java用爬虫爬一个页面_使用Java写一个简单爬虫爬取单页面
使用Java爬虫爬取人民日报公众号页面图片使用Java框架Jsoup和HttpClient实现,先看代码爬取目标页面 1.使用Maven构建一个普通Java工程加入依赖: org.jsoup j ...
教你如何使用Java代码从网页中爬取数据到数据库中——网络爬虫精华篇
文章目录 1:网络爬虫介绍 2:HttpClients类介绍 2.1 HttpGet参数问题 2.2 HttpPost参数问题 2.3 连接池技术问题 3:Jsoup介绍 4:动手实践如何抓取网页上数 ...
python爬虫爬取必应每日高清壁纸
python爬虫爬取必应每日高清壁纸一.简介二.使用的环境三.网页分析 1.分析网页每一页url形式以及总页数 2.网页重要信息收集 3.在源码中寻找所需信息的位置四.代码实现五.运行爬虫 ...
java爬虫拉勾网_[Java教程]node.js爬虫爬取拉勾网职位信息
[Java教程]node.js爬虫爬取拉勾网职位信息 0 2017-03-14 00:00:21 简介用node.js写了一个简单的小爬虫,用来爬取拉勾网上的招聘信息,共爬取了北京.上海.广州.深圳 ...
用Python写一个网络爬虫爬取网页中的图片
写一个爬虫爬取百度贴吧中一个帖子图片网址:壁纸用谷歌浏览器的开发工具检查网页,可以发现其每一张图片都有如下格式 <img class="BDE_Image" src=&q ...
基于Jsoup的Java爬虫-爬取必应壁纸网站的壁纸（Java静态壁纸爬虫实例）
准备阶段 1.必应壁纸网站:https://bing.ioliu.cn(爬取对象网站) 2.Jsoup包下载地址:https://jsoup.org/download(以下代码需要用到该包,记得导入包 ...
java 百度贴吧爬虫,爬取百度贴吧帖子
依然是参考教程 Python爬虫实战一之爬取百度贴吧帖子.作者崔庆才写了很多关于Python爬虫的文章,大家有兴趣的话可以去他的个人博客静觅学习. 这次将爬取的数据写入了文本文件中,并用到了re模块中 ...
python爬虫下载电影百度文档_写一个python 爬虫爬取百度电影并存入mysql中
目标是利用python爬取百度搜索的电影在类型地区年代各个标签下电影的名字评分和图片连接以及电影连接首先我们先在mysql中建表 create table liubo4( id in ...
python自动换壁纸_Python爬取必应每日图片并实现Windows壁纸自动切换
不知道大家是否对每日一成不变的壁纸感到厌倦呢?反正对于我个人来说,如果每天打开电脑映入眼帘的都是不同的画面,那么科研热情都会被充分激发,从而提高自己的劳动生产力. 原来使用的是Deepin系统,自己写 ...

JAVA写的网页爬虫爬取必应词典的词汇音标

JAVA写的网页爬虫爬取必应词典的词汇音标相关推荐

最新文章

热门文章