如何构建一个大数据量的搜索引擎

构建一个大数据量的搜索引擎,数据很重要,数据来源在哪里呢?一方面可以从站内结构化数据库导入,如MySQL,Oracle等数据库,构建一个站内搜索引擎,提高查询速度.另一方面构建一个分布式爬虫,每天定时抓取数据,不断地添加到索引库.典型地如百度,谷歌等全文检索引擎.

我们现在要做的就是第二种东西.说难不难,show That

1.定义一个实体,与索引库的type数据字段名一致.

package com.ytdx.entity;import java.io.Serializable;/*** ES 索引对应实体* @author lhy**/
public class Blob implements Serializable {private Integer  id;        //文章idprivate String   title;     //文章标题private String   describe;  //描述private String   url;       //文章路径private String   ImageUrl;  //图片路径private String   postInfo;  //发布信息public Integer getId() {return id;}public void setId(Integer id) {this.id = id;}public String getTitle() {return title;}public void setTitle(String title) {this.title = title;}public String getDescribe() {return describe;}public void setDescribe(String describe) {this.describe = describe;}public String getUrl() {return url;}public void setUrl(String url) {this.url = url;}public String getImageUrl() {return ImageUrl;}public void setImageUrl(String imageUrl) {ImageUrl = imageUrl;}public String getPostInfo() {return postInfo;}public void setPostInfo(String postInfo) {this.postInfo = postInfo;}@Overridepublic String toString() {return "Blob [id=" + id + ", title=" + title + ", describe=" + describe + ", url=" + url + ", ImageUrl="+ ImageUrl + ", postInfo=" + postInfo + "]";}}

2.ES服务操作的工具类

package com.ytdx.util;import java.net.InetAddress;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;/*** ES服务器操作工具类* @author lhy* @date  2018.04.20*/
public class EsUtil {private Client client;private static String name = "127.0.0.1";private static Integer port = 9300;
//  private static ObjectMapper objectMapper = new ObjectMapper(); static Settings set = Settings.builder().put("cluster.name", "elasticsearch").build();//.put("client.transport.sniff",false)./*** 本地客户端连接ES服务器* @return*/public static Client EsConnect(){TransportClient client = null;try {client = new PreBuiltTransportClient(set).addTransportAddress(new TransportAddress(InetAddress.getByName(name), port));System.out.println("ES服务器连接成功!");return client;} catch (Exception e) {// TODO Auto-generated catch blockSystem.out.println("ES服务器连接失败!");e.printStackTrace();}return client;}/*** 关闭ES连接的客户端* @param client*/public static void closeClient(Client client){if(client != null){client.close();System.out.println("Client已关闭!");}}
}

3.索引库的批量操作

package com.ytdx.es;import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.XContentFactory;import com.ytdx.entity.Blob;
public class BulkIndex {private Client client;private String index = "home";private String type = "blob";public BulkIndex(Client client) {super();this.client = client;}//批量添加索引public void BulkAddBlob(Blob blob) throws Exception {BulkRequestBuilder builder = client.prepareBulk();builder.add(client.prepareIndex(index, type).setSource(XContentFactory.jsonBuilder().startObject().field("title", blob.getTitle()).field("describe", blob.getDescribe()).field("url", blob.getUrl()).field("ImageUrl", blob.getImageUrl()).field("postInfo", blob.getPostInfo()).endObject()));BulkResponse response = builder.execute().actionGet();
//      for (BulkItemResponse item : response.getItems()) {
//          System.out.println("你的批量操作Response信息为: " + item.getResponse());
//      }// 4.错误信息日志读取if (response.hasFailures()) {// 可在这里对于失败请求进行处理for (BulkItemResponse item : response.getItems()) {if (item.isFailed()) {System.out.println("失败信息:--------" + item.getFailureMessage());}}}}//批量删除索引库public void BulkDelIndex() throws Exception {DeleteIndexResponse Response = client.admin().indices().prepareDelete(index).execute().actionGet();System.out.println("索引删除成功!");}//批量创建结构化索引库public void BulkAdd() throws Exception {BulkRequestBuilder builder = client.prepareBulk();builder.add(client.prepareIndex(index, type).setSource(XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("id").field("type", "integer").field("store", "yes").endObject().startObject("title").field("type", "text").field("store", "yes").endObject().startObject("describe").field("type", "text").field("store", "yes").endObject().startObject("url").field("type", "text").field("store", "yes").endObject().startObject("ImageUrl").field("type", "text").field("store", "yes").endObject().startObject("postInfo").field("type", "text").field("store", "yes").endObject().endObject().endObject()));BulkResponse response = builder.execute().actionGet();System.out.println("索引库创建成功!");}
}

4.分布式爬虫定时抓取数据,构建大数据量的索引库

package com.ytdx.task;import org.elasticsearch.client.Client;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;import com.ytdx.entity.Blob;
import com.ytdx.es.BulkIndex;
import com.ytdx.util.EsUtil;@Component("FetchDataTask")
public class FetchData {@Scheduled(cron = "0 30 12 ? * *") // 每天上午12:30抓取一次数据public void TimeTask() throws Exception {Client client = EsUtil.EsConnect();BulkIndex bulk = new BulkIndex(client);bulk.BulkDelIndex();             //先删除之前的索引库bulk.BulkAdd();                  //结构化重建索引库String URL = "https://www.cnblogs.com/";Fetch(client,URL); // 先抓取首页数据for(int l=2;l<=200; l++){       //抓取从第2页到200页的数据String url = "https://www.cnblogs.com/#p";url += l;Fetch(client,url);}System.out.println("系统抓取博客数据成功!");}public void Fetch(Client client,String URL) throws Exception {
//      Client client = EsUtil.EsConnect();BulkIndex bulk = new BulkIndex(client);Document doc = Jsoup.connect(URL).get();Element e = doc.getElementById("post_list");Elements es = e.children();for (int i = 0; i < es.size(); i++) {Element nodes = es.get(i);Element item = nodes.getElementsByClass("post_item_body").first();String title = item.getElementsByTag("h3").get(0).getElementsByTag("a").text();String describe = item.getElementsByTag("p").text();String url = item.getElementsByTag("h3").get(0).getElementsByTag("a").attr("href");String ImageUrl = "";if (item.getElementsByTag("p").get(0).getElementsByTag("a").size() >= 1) {ImageUrl = item.getElementsByTag("p").get(0).getElementsByTag("a").get(0).getElementsByTag("img").attr("src");}String postInfo = item.getElementsByClass("post_item_foot").first().text();//添加抓取数据到索引库Blob blob = new Blob();blob.setTitle(title);blob.setDescribe(describe);blob.setUrl(url);blob.setImageUrl(ImageUrl);blob.setPostInfo(postInfo);bulk.BulkAddBlob(blob);}}
}

5.后台的数据检索功能实现

package com.ytdx.es;import java.util.ArrayList;
import java.util.List;
import java.util.Map;import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;import com.ytdx.entity.Blob;public class EsQuery {private Client client;private String index = "home";private String type = "blob";public EsQuery(Client client) {super();this.client = client;}/*** 将查询后获得的response转成list * @param client* @param response* @return*/public List<Blob> responseToList(Client client, SearchResponse response) {SearchHits hits = response.getHits();
//      List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();List<Blob> list = new ArrayList<Blob>();for (int i = 0; i < hits.getHits().length; i++) {Map<String, Object> map = hits.getAt(i).getSourceAsMap();Blob blob = new Blob();if(map.containsKey("title")){blob.setTitle(map.get("title").toString());}else if(map.containsKey("describe")){blob.setDescribe(map.get("describe").toString());}else if(map.containsKey("url")){blob.setUrl(map.get("url").toString());}else if(map.containsKey("ImageUrl")){blob.setImageUrl(map.get("ImageUrl").toString());}else if(map.containsKey("postInfo")){blob.setPostInfo(map.get("postInfo").toString());}list.add(blob);}return list;}/*** 返回全部索引数据* @return*/public List<Blob> getAll() {SearchResponse response = client.prepareSearch(index).setTypes(type) // 设置索引类型.setQuery(QueryBuilders.matchAllQuery()).setSize(30).setScroll(TimeValue.timeValueMinutes(30)).execute().actionGet();return responseToList(client, response);}/*** 对title和describe进行检索* @param values* @return*/public List<Blob> getFilter(String values) {String filter1 = "title";String filter2 = "describe";HighlightBuilder hiBuilder = new HighlightBuilder();hiBuilder.preTags("<span style=\"color:red\">");         //检索关键词高亮显示hiBuilder.postTags("</span>");hiBuilder.field(filter1);hiBuilder.preTags("<span style=\"color:red\">");hiBuilder.postTags("</span>");hiBuilder.field(filter2);SearchResponse response = client.prepareSearch(index).setTypes(type).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(QueryBuilders.multiMatchQuery(values, filter1, filter2)).setFrom(0).setSize(10)          //数据分页显示.setScroll(TimeValue.timeValueMinutes(30)) // 设置过期时间为30分钟.setExplain(true).highlighter(hiBuilder).execute().actionGet();return responseToList(client, response);}}

6.最后用SSM接口映射到页面,就不写了.看看具体检索效果.搜索的数据达到6万多条数据检索,毫秒级返回,搜索速度可见非同一般.

如何构建一个大数据量的搜索引擎相关推荐

一个大数据量表访问优化--联动下拉框查询优化
问题描述有一数据表(产品标签表,每个产品一个唯一的SN,每月100万左右),查询界面上有2个联动下拉框,[规格]____,[批次]______ 用户选择一个规格后(目前200来个规格),列出该规格下达 ...
如何知道一个大数据解决方案是否适合您的组织
作者:Divakar Mysore,Shrikant Khupat,Shweta Jain 来源:IBM 发布于:2014-12-10 简介在确定投资大数据解决方案之前,评估可用于分析的数据:通过分 ...
vue渲染大量数据如何优化_大数据量场景下的Vue性能优化
性能优化最常见的落脚点是在网络和dom上,但是在大数据量的场景下,由于Vue本身的特性,可能会造成js运行层面的性能问题,这篇文章讨论的就是针对这一部分的性能优化方案. 模拟一个大数据量的场景 // ...
大数据量表中，增加一个NOT NULL的新列
这次,发布清洗列表功能,需要对数据库进行升级.MailingList表加个IfCleaning字段,所有的t_User*表加个IfCleaned字段. 脚本如下对所有的t_User表执行 alter ...
用Python实现一个大数据搜索引擎
用Python实现一个大数据搜索引擎搜索是大数据领域里常见的需求.Splunk和ELK分别是该领域在非开源和开源领域里的领导者.本文利用很少的Python代码实现了一个基本的数据搜索功能,试图让大家 ...
【转】解决WCF大数据量传输，System.Net.Sockets.SocketException: 远程主机强迫关闭了一个现有的连接...
开发中所用的数据需要通过WCF进行数据传输,结果就遇到了WCF大量传输问题也就是提示System.Net.Sockets.SocketException: 远程主机强迫关闭了一个现有的连接网上解决 ...
从另一个角度看大数据量处理利器：布隆过滤器
思路:从简单的排序谈到BitMap算法,再谈到数据去重问题,谈到大数据量处理利器:布隆过滤器. 情景1:对无重复的数据进行排序 @给定数据(2,4,1,12,9,7,6)如何对它排序? 方法1:基本的 ...
教你如何用 Python 来实现一个大数据搜索引擎
搜索是大数据领域里常见的需求.Splunk和ELK分别是该领域在非开源和开源领域里的领导者.本文利用很少的Python代码实现了一个基本的数据搜索功能,试图让大家理解大数据搜索的基本原理. 布隆过滤器 ...
MySQL数据库如何解决大数据量存储问题
FROM http://blog.csdn.net/likika2012/article/details/38816037 各位高手您们好,我最近接手公司里一个比较棘手的问题,关于如何利用MySQL存 ...

如何构建一个大数据量的搜索引擎

如何构建一个大数据量的搜索引擎相关推荐

最新文章

热门文章