引用说明
本文参考狂神说狂神的视频以及文章,大家可以支持下狂神(狂神ElasticSearch文章地址、狂神ElasticSearch视频地址)
版本说明
本文涉及一下软件以及版本的声明(以下下载地址除ik之外,都是国内华为镜像)
软件 | 版本 |
---|
elasticSearch | 7.6.1 | elasticSearch-head-master | 7.6.1 | kibana | 7.6.1 | elasticsearch-analysis-ik1 | 7.6.1 | jdk | 8 (最低要求) |
本文源码
本文源码放到gitee仓库上了,地址
一、springboot整合连接Es做基本操作
创建springboot项目
过程省略
导入依赖
注意因为elasticSearch涉及到版本问题,所以注意自定义版本依赖(es是什么版本,就用什么版本的jar包,在如下指定)
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.5</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>cn.es</groupId>
<artifactId>elasticSearch_api</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>elasticSearch_api</name>
<description>elasticSearch_api</description>
<properties>
<java.version>1.8</java.version>
<!--自定义版本依赖-->
<elasticsearch.version>7.6.1</elasticsearch.version>
</properties>
<dependencies>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.76</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>
写一个高级客户端的配置类
package cn.es.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.Node;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class ESConfig {
@Bean
public RestHighLevelClient restHighLevelClient(){
RestHighLevelClient restHighLevelClient=new RestHighLevelClient(RestClient.builder(new HttpHost("localhost",9200,"http")));
return restHighLevelClient;
}
}
这样就将高级客户端注入到bean容器中了,待会就可以使用该高级客户端,对es进行操作了。
创建一个测试用的实体类
package cn.es.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class User {
private int id;
private String name;
private int age;
}
编写配置文件
由于我的es是在本机,并且没有设置任何的密码,所以不需要配置,需要配置的可以在application.yml(properties)中进行配置
测试类
此时目录结构如下:
因为涉及到bean容器的问题,请使用springboot默认的测试包,进行测试,防止bean注入不进去。 测试类如下:
package cn.es;
import cn.es.pojo.User;
import com.alibaba.fastjson.JSON;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
@SpringBootTest
class ElasticSearchApiApplicationTests {
@Autowired
RestHighLevelClient client;
@Test
void contextLoads() {
}
@Test
void createIndex() throws IOException {
CreateIndexRequest createIndexRequest= new CreateIndexRequest("blog_demo");
CreateIndexResponse createIndexResponse= client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
System.out.println(createIndexResponse);
}
@Test
void getIndex() throws IOException {
GetIndexRequest request= new GetIndexRequest("blog_demo");
boolean flag_exist= client.indices().exists(request,RequestOptions.DEFAULT);
System.out.println(flag_exist);
}
@Test
void deleteIndex() throws IOException {
DeleteIndexRequest request= new DeleteIndexRequest("blog_demo");
AcknowledgedResponse response= client.indices().delete(request,RequestOptions.DEFAULT);
System.out.println(response.isAcknowledged());
}
@Test
void addDocument() throws IOException {
User user=new User(1,"小明",18);
IndexRequest indexRequest= new IndexRequest("blog_demo");
indexRequest.id(String.valueOf(user.getId()));
indexRequest.timeout(TimeValue.timeValueSeconds(1));
indexRequest.timeout("1s");
indexRequest.source(JSON.toJSONString(user), XContentType.JSON);
IndexResponse response= client.index(indexRequest,RequestOptions.DEFAULT);
System.out.println(response.toString());
System.out.println(response.status());
}
@Test
void existDocument() throws IOException {
GetRequest getRequest=new GetRequest("blog_demo","1");
getRequest.fetchSourceContext(new FetchSourceContext(false));
getRequest.storedFields("_none_");
boolean flag=client.existsSource(getRequest,RequestOptions.DEFAULT);
System.out.println(flag);
}
@Test
void getDocument() throws IOException {
GetRequest getRequest=new GetRequest("blog_demo","1");
GetResponse getResponse =client.get(getRequest,RequestOptions.DEFAULT);
System.out.println(getResponse.toString());
System.out.println(getResponse.getSourceAsString());
}
@Test
void updateDocument() throws IOException {
User user=new User(1,null,81);
UpdateRequest request=new UpdateRequest("blog_demo","1");
request.timeout("1s");
request.doc(JSON.toJSONString(user), XContentType.JSON);
UpdateResponse updateResponse =client.update(request,RequestOptions.DEFAULT);
System.out.println(updateResponse.toString());
}
@Test
void deleteDocument() throws IOException {
DeleteRequest request=new DeleteRequest("blog_demo","1");
request.timeout("1s");
DeleteResponse deleteResponse =client.delete(request,RequestOptions.DEFAULT);
System.out.println(deleteResponse.toString());
}
@Test
void insertBulkRequest() throws IOException {
BulkRequest request=new BulkRequest();
request.timeout("10s");
ArrayList<User> arrayList=new ArrayList<>();
arrayList.add(new User(10,"zs",18));
arrayList.add(new User(11,"lisi",21));
arrayList.add(new User(12,"ww",16));
arrayList.add(new User(13,"zl",15));
arrayList.add(new User(14,"hh",16));
arrayList.add(new User(15,"ll",14));
for (User user : arrayList) {
request.add(
new IndexRequest("blog_demo")
.id(String.valueOf(user.getId()))
.source(JSON.toJSONString(user),XContentType.JSON));
}
BulkResponse response= client.bulk(request,RequestOptions.DEFAULT);
System.out.println(response.hasFailures());
}
@Test
void searchRequest() throws IOException {
SearchRequest searchRequest=new SearchRequest("blog_demo");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("name", "zs");
searchSourceBuilder.query(termQueryBuilder);
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
System.out.println(JSON.toJSONString(response.getHits()));
System.out.println("-----------------------");
for (SearchHit hit : response.getHits().getHits()) {
System.out.println(hit.getSourceAsMap());
}
}
}
二、编写jd全局搜索例子
原本想写一个jd一样的全局搜索的例子,但是由于时间仓促,没找到合适的模板,所以直接用postman测试,请忽略前端页面的缺少
创建项目
创建过程省略
引入jar包
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.5</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>cn.es</groupId>
<artifactId>elasticSearch_jd</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>elasticSearch_jd</name>
<description>elasticSearch_jd</description>
<properties>
<java.version>1.8</java.version>
<!--自定义版本依赖-->
<elasticsearch.version>7.6.1</elasticsearch.version>
</properties>
<dependencies>
<!--解析网页-->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.10.2</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.76</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>
因为要使用jsoup爬取jd上的商品数据,所以需要jsoup的jar包
封装一个抓取jd的商品数据的工具类
package cn.es.util;
import cn.es.pojo.Content;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
public class HtmlParseUtil {
public static void main(String[] args) throws IOException {
System.out.println(HtmlParseUtil.getList("华为"));
}
public static List<Content> getList(String keyword) throws IOException {
String url = "https://search.jd.com/Search?keyword="+keyword;
Document document = Jsoup.parse(new URL(url), 30000);
Element element = document.getElementById("J_goodsList");
Elements elements = document.getElementsByTag("li");
List<Content> contents=new ArrayList<>();
int i=0;
for (Element el:elements){
if (el.attr("class").equalsIgnoreCase("gl-item")) {
String img = el.getElementsByTag("img").eq(0).attr("data-lazy-img");
String price = el.getElementsByClass("p-price").eq(0).text();
String title = el.getElementsByClass("p-name").eq(0).text();
contents.add(new Content(++i,title,img,price));
}
}
return contents;
}
}
想要抓取其他的网页,可以根据自己的需求修改页面地址,以及前端标签的关系
封装一个es操作的工具类
package cn.es.util;
import ch.qos.logback.core.net.SyslogOutputStream;
import cn.es.pojo.Content;
import com.alibaba.fastjson.JSON;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestParam;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public class EsUtil {
public static boolean createIndex(RestHighLevelClient client,String index) throws IOException {
GetIndexRequest request= new GetIndexRequest(index);
boolean flag_exist= client.indices().exists(request,RequestOptions.DEFAULT);
if (flag_exist){
return true;
}
CreateIndexRequest createIndexRequest= new CreateIndexRequest(index);
CreateIndexResponse createIndexResponse= client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
return createIndexResponse.isAcknowledged();
}
public static boolean insertListEs(RestHighLevelClient client,String key,String index) throws IOException {
BulkRequest request = new BulkRequest();
request.timeout("10s");
List<Content> contents = HtmlParseUtil.getList(key);
if (createIndex(client,index)) {
for (Content content : contents) {
request.add(
new IndexRequest(index)
.source(JSON.toJSONString(content), XContentType.JSON));
}
BulkResponse response = client.bulk(request, RequestOptions.DEFAULT);
return !response.hasFailures();
}
return false;
}
public static List<Map<String,Object>> searchEs(RestHighLevelClient client,String index,String key,int pageNum,int pageSize) throws IOException {
List<Map<String,Object>> list=new ArrayList<>();
SearchRequest searchRequest=new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
int begin=(pageNum-1)*pageSize;
searchSourceBuilder.from(begin);
searchSourceBuilder.size(pageSize);
MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("title", key);
searchSourceBuilder.query(matchQueryBuilder);
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
System.out.println(response);
for (SearchHit hit : response.getHits().getHits()) {
System.out.println(hit.getSourceAsMap());
Map<String, Object> map = hit.getSourceAsMap();
System.out.println(map);
list.add(map);
}
return list;
}
public static List<Map<String,Object>> searchEsHighlight(RestHighLevelClient client,String index,String key,int pageNum,int pageSize) throws IOException {
List<Map<String,Object>> list=new ArrayList<>();
SearchRequest searchRequest=new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
int begin=(pageNum-1)*pageSize;
searchSourceBuilder.from(begin);
searchSourceBuilder.size(pageSize);
MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("title", key);
HighlightBuilder highlightBuilder=new HighlightBuilder();
highlightBuilder.field("title");
highlightBuilder.preTags("<p class='light' style='color:red'>");
highlightBuilder.postTags("</p>");
searchSourceBuilder.highlighter(highlightBuilder);
highlightBuilder.requireFieldMatch(false);
searchSourceBuilder.query(matchQueryBuilder);
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
for (SearchHit hit : response.getHits().getHits()) {
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
HighlightField title = highlightFields.get("title");
Map<String, Object> map = hit.getSourceAsMap();
if (title!=null){
Text[] fragments = title.fragments();
String newTitle="";
for (Text fragment : fragments) {
newTitle+=fragment;
}
hit.getSourceAsMap().put("title",newTitle);
}
list.add(map);
}
return list;
}
}
编写控制器
package cn.es.controller;
import cn.es.util.EsUtil;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@RestController
public class EsController {
@Autowired
RestHighLevelClient client;
String index="jd_index";
@GetMapping("/insertEs/{key}")
public Boolean insertEs(@PathVariable String key){
Boolean flag=false;
try {
flag= EsUtil.insertListEs(client,key,index);
} catch (IOException e) {
e.printStackTrace();
}
return flag;
}
@GetMapping("/searchEs/{key}/{pageNum}/{pageSize}")
public List<Map<String,Object>> searchEs(@PathVariable String key, @PathVariable int pageNum, @PathVariable int pageSize) throws IOException {
List<Map<String,Object>> list=EsUtil.searchEs(client,index,key,pageNum,pageSize);
return list;
}
@GetMapping("/searchEsHighlight/{key}/{pageNum}/{pageSize}")
public List<Map<String,Object>> searchEsHighlight(@PathVariable String key, @PathVariable int pageNum, @PathVariable int pageSize) throws IOException {
List<Map<String,Object>> list=EsUtil.searchEsHighlight(client,index,key,pageNum,pageSize);
return list;
}
}
编写启动类
注意扫描servlet组件的注解
package cn.es;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.servlet.ServletComponentScan;
@SpringBootApplication
@ServletComponentScan
public class ElasticSearchJdApplication {
public static void main(String[] args) {
SpringApplication.run(ElasticSearchJdApplication.class, args);
}
}
三、测试
启动该项目
测试插入数据
从jd网页上抓取指定关键词的数据到索引库,如下:
发送请求: 查看数据 一切ok
查询数据(不高亮)
发送请求,如下:
查询数据(高亮)
发送请求: 具体查询的逻辑、高亮的规则,以及高亮的样式还有很多,可以另外拓展!!!
本文结束!!!
|