[大数据]ElasticSearch - 编程实现

ElasticSearch - 编程实现

ElasticSearch版本：5.6.8

1. ElasticSearch 编程操作

1.1 创建工程，导入坐标

pom.xml

<dependencies>
    <dependency>
        <groupId>org.elasticsearch</groupId>
        <artifactId>elasticsearch</artifactId>
        <version>5.6.8</version>
    </dependency>
    
    <dependency>
        <groupId>org.elasticsearch.client</groupId>
        <artifactId>transport</artifactId>
        <version>5.6.8</version>
    </dependency>
    
    <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-to-slf4j</artifactId>
        <version>2.9.1</version>
    </dependency>
    
    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-api</artifactId>
        <version>1.7.24</version>
    </dependency>
    
    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-simple</artifactId>
        <version>1.7.21</version>
    </dependency>
    
    <dependency>
        <groupId>log4j</groupId>
        <artifactId>log4j</artifactId>
        <version>1.2.17</version>
    </dependency>
    
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
        <scope>test</scope>
    </dependency>
</dependencies>

1.2 创建索引 index

代码

@Test
public void createIndex() throws Exception {
    // 1. 配置
    Settings settings = Settings.builder().put("cluster.name","murphy-application").build();
    // 2. 客户端
    TransportClient client = new PreBuiltTransportClient(settings);
    client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("192.168.2.180"),9300));
    // 3. 创建索引 - 使用API
    client.admin().indices().prepareCreate("index_hello").get();
    // 4. 释放资源 - 关闭client
    client.close();
}

运行结果

1.3 创建映射 mapping

代码

/**
 * 创建映射
 * @throws Exception
 */
@Test
public void setMappings() throws Exception {
    // 1. 配置 - 创建Client连接对象
    Settings settings = Settings.builder().put("cluster.name","murphy-application").build();
    // 2. 客户端
    TransportClient client = new PreBuiltTransportClient(settings);
    client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("192.168.2.180"),9300));

    // 添加映射
    /**
     * 格式：
     *      "mappings" : {
     *          "article" : {
     *              "dynamic" : "false",
     *              "properties" : {
     *                  "id" : {"type": "string"},
     *                  "content" : { "type" : "string" },
     *                  "author" : { "type" : "string" }
     *              }
     *          }
     *      }
     */
    XContentBuilder builder = XContentFactory.jsonBuilder()
            .startObject()
            .startObject("article")
            .startObject("properties")
            .startObject("id")
            .field("type","integer").field("store","yes")
            .endObject()
            .startObject("title")
            .field("type","string").field("store","yes").field("analyzer","ik_smart")
            .endObject()
            .startObject("content")
            .field("type","string").field("store","yes").field("analyzer","ik_smart")
            .endObject()
            .endObject()
            .endObject()
            .endObject();
    // 创建映射
    client.admin().indices().preparePutMapping("index_hello").setType("article").setSource(builder).get();
    // 4. 释放资源 - 关闭client
    client.close();
}

运行结果

1.4 建立文档 document

1.4.1 建立文档（通过XContentBuilder）

代码

/**
 * 使用XContentBuilder构建Document对象
 * @throws Exception
 */
@Test
public void testAddDocument() throws Exception {
    // 创建一个client对象
    // 创建一个文档对象
    XContentBuilder builder = XContentFactory.jsonBuilder()
            .startObject()
            .field("id",21)
            .field("title","近日以来，北方的秋天悄然来临..")
            .field("content","北方入秋速度明显加快，多地降温幅度最多可达10度")
            .endObject();
    // 把文档对象添加到索引库
    client.prepareIndex()
            // 设置索引名称
            .setIndex("index_hello")
            // 设置type
            .setType("article")
            // 设置文档的ID，如果不设置的话，自动生成一个ID
            .setId("2")
            // 设置文档信息
            .setSource(builder)
            // 执行操作
            .get();
    client.close();
}

运行结果

1.4.2 建立文档（使用Jackson转换实体）

1. 创建Article实体

/**
 * 文章 - 实体类
 *
 * @author murphy
 */
public class Article {
    private Integer id;
    private String title;
    private String content;
}

2. 添加Jackson坐标依赖

<dependency>
    <groupId>com.fasterxml.jackson.core</groupId>
    <artifactId>jackson-core</artifactId>
    <version>2.12.3</version>
</dependency>

<dependency>
    <groupId>com.fasterxml.jackson.core</groupId>
    <artifactId>jackson-databind</artifactId>
    <version>2.12.3</version>
</dependency>

<dependency>
    <groupId>com.fasterxml.jackson.core</groupId>
    <artifactId>jackson-annotations</artifactId>
    <version>2.12.3</version>
</dependency>

3. 代码实现

/**
 * 使用Jackson构建Document对象
 * @throws Exception
 */
@Test
public void testAddDocument2() throws Exception {
    // 创建一个Article对象
    Article article = new Article();
    // 设置对象的属性
    article.setId(22);
    article.setTitle("ElasticSearch是一个基于Lucene的搜索服务器");
    article.setContent("它提供了一个分布式多用户能力的全文搜索引擎，基于RESTful web接口。 Elasticsearch是用Java开发的，并作为Apache许可条款下的开放源码发布，是当前流行的企业级搜索引擎。设计用于云计算中，能够达到实时搜索，稳定，可靠，快速，安装使用方便。");
    // 使用article对象转换成json格式的字符串
    ObjectMapper objectMapper = new ObjectMapper();
    String jsonDocument = objectMapper.writeValueAsString(article);
    System.out.println(jsonDocument);
    // 使用client对象把文档写入索引库
    client.prepareIndex("index_hello","article","3")
            .setSource(jsonDocument, XContentType.JSON)
            .get();
    // 关闭客户端
    client.close();
}

4. 运行结果

1.5 查询文档操作

测试用例

1.5.1 termQuery

/**
 * termQuery
 * @throws Exception
 */
@Test
public void testQueryByTerm() throws Exception {
    // 1. 构建QueryBuilder - 参数1:要搜索的字段 / 参数2:要搜索的关键词
    QueryBuilder queryBuilder = QueryBuilders.termQuery("title","重磅");
    // 2. 执行查询得到结果
    SearchResponse searchResponse = client.prepareSearch("index_hello")
            .setTypes("article")
            .setQuery(queryBuilder)
            .get();
    // 3. 处理结果
    SearchHits searchHits = searchResponse.getHits();
  	// 查询结果的总记录数
    System.out.println("总行数：" + searchHits.getTotalHits());
		// 查询结果列表
    Iterator<SearchHit> iterator = searchHits.iterator();
    while (iterator.hasNext()) {
        SearchHit searchHit = iterator.next();
        // Source -> Document的JSON输出
        System.out.println(searchHit.getSourceAsString());
      	// 取文档的属性
        System.out.println("--- 文档属性 ---");
        Map<String, Object> document = searchHit.getSource();
        System.out.println(document.get("id"));
        System.out.println(document.get("title"));
        System.out.println(document.get("content"));
    }
  	// 关闭client
    client.close();
}

1.5.2 QueryString

/**
 * QueryString
 * @throws Exception
 */
@Test
public void testQueryByTerm() throws Exception {
    // 1. 构建QueryBuilder
    QueryBuilder queryBuilder = QueryBuilders.queryStringQuery("微信").defaultField("title");
    // 2. 执行查询得到结果	
    SearchResponse searchResponse = client.prepareSearch("index_hello")
            .setTypes("article")
            .setQuery(queryBuilder)
            .get();
    // 3. 处理结果
    SearchHits searchHits = searchResponse.getHits();
  	// 查询结果的总记录数
    System.out.println("总行数：" + searchHits.getTotalHits());
		// 查询结果列表
    Iterator<SearchHit> iterator = searchHits.iterator();
    while (iterator.hasNext()) {
        SearchHit searchHit = iterator.next();
        // Source -> Document的JSON输出
        System.out.println(searchHit.getSourceAsString());
      	// 取文档的属性
        System.out.println("--- 文档属性 ---");
        Map<String, Object> document = searchHit.getSource();
        System.out.println(document.get("id"));
        System.out.println(document.get("title"));
        System.out.println(document.get("content"));
    }
  	// 关闭client
    client.close();
}

1.5.3 MatchQuery

/**
 * MatchQuery
 * @throws Exception
 */
@Test
public void testQueryByMatch() throws Exception {
    // 1. 构建QueryBuilder - 参数1:要搜索的字段 / 参数2:要搜索的关键词
    QueryBuilder queryBuilder = QueryBuilders.matchQuery("title","微信");
    // 2. 执行查询得到结果
    SearchResponse searchResponse = client.prepareSearch("index_hello")
            .setTypes("article")
            .setQuery(queryBuilder)
            .get();
    // 3. 处理结果
    SearchHits searchHits = searchResponse.getHits();
    // 查询结果的总记录数
    System.out.println("总行数：" + searchHits.getTotalHits());
    // 查询结果列表
    Iterator<SearchHit> iterator = searchHits.iterator();
    while (iterator.hasNext()) {
        SearchHit searchHit = iterator.next();
        // Source -> Document的JSON输出
        System.out.println(searchHit.getSourceAsString());
        // 取文档的属性
        System.out.println("--- 文档属性 ---");
        Map<String, Object> document = searchHit.getSource();
        System.out.println(document.get("id"));
        System.out.println(document.get("title"));
        System.out.println(document.get("content"));
    }
    // 关闭client
    client.close();
}

1.5.4 重构查询方法

// 提取
private void search(QueryBuilder queryBuilder) {
    // 1. 执行查询得到结果
    SearchResponse searchResponse = client.prepareSearch("index_hello")
            .setTypes("article")
            .setQuery(queryBuilder)
            .get();
    // 2. 处理结果
    SearchHits searchHits = searchResponse.getHits();
    System.out.println("总行数：" + searchHits.getTotalHits());

    Iterator<SearchHit> iterator = searchHits.iterator();
    while (iterator.hasNext()) {
        SearchHit searchHit = iterator.next();
        // Source -> Document的JSON输出
        System.out.println(searchHit.getSourceAsString());
        System.out.println("--- 文档属性 ---");
        Map<String, Object> document = searchHit.getSource();
        System.out.println(document.get("id"));
        System.out.println(document.get("title"));
        System.out.println(document.get("content"));
    }
    // 关闭client
    client.close();
}

1.5.5 使用文章ID查询文档

/**
 * QueryById
 * @throws Exception
 */
@Test
public void testQueryById() throws Exception {
    // 1. 构建QueryBuilder - 可以查询多个ID
    QueryBuilder queryBuilder = QueryBuilders.idsQuery().addIds("4","5","6");
    // 2. 执行查询得到结果
    search(queryBuilder);
}

1.5.6 查询文章分页操作

/**
 * 分页查询
 * @throws Exception
 */
@Test
public void testQueryByMatchAll() throws Exception {
    // 1. 构建QueryBuilder
    QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();

    // 2. 执行查询得到结果
    SearchResponse searchResponse = client.prepareSearch("index_hello")
            .setTypes("article")
            .setQuery(queryBuilder)
            // 设置分页信息
            .setFrom(0)
            // 每页显示的行数
            .setSize(5)
            .get();
    // 3. 处理结果
    SearchHits searchHits = searchResponse.getHits();
    System.out.println("总行数：" + searchHits.getTotalHits());

    Iterator<SearchHit> iterator = searchHits.iterator();
    while (iterator.hasNext()) {
        SearchHit searchHit = iterator.next();
        // Source -> Document的JSON输出
        System.out.println(searchHit.getSourceAsString());
        System.out.println("--- 文档属性 ---");
        Map<String, Object> document = searchHit.getSource();
        System.out.println(document.get("id"));
        System.out.println(document.get("title"));
        System.out.println(document.get("content"));
    }
    // 关闭client
    client.close();
}

1.6 查询结果高亮操作

1.6.1 高亮显示

在进行关键字搜索时，搜索出的内容中的关键字会显示不同的颜色，称之为高亮。

1.6.2 高亮显示的HTML分析

通过开发者工具查看高亮数据的html代码实现：

• ElasticSearch可以对查询出的内容中关键字部分进行标签和样式的设置，但是你需要告诉ElasticSearch使用什么标签对高亮关键字进行包裹。

1.6.3 高亮代码实现

@Test
public void testSearchHighlight() throws Exception {
    // 创建一个client对象 - 查询对象
//        QueryBuilder queryBuilder = QueryBuilders.matchQuery("title","北方");
    QueryBuilder queryBuilder = QueryBuilders.multiMatchQuery("北方","title","content");

    HighlightBuilder highlightBuilder = new HighlightBuilder();
    highlightBuilder.field("title");
    highlightBuilder.field("content");
    highlightBuilder.preTags("<em>");
    highlightBuilder.postTags("</em");

    // 2. 执行查询得到结果
    SearchResponse searchResponse = client.prepareSearch("index_hello")
            .setTypes("article")
            .setQuery(queryBuilder)
      			// 设置高亮信息
            .highlighter(highlightBuilder)
            .get();
    // 3. 处理结果
    SearchHits searchHits = searchResponse.getHits();
    System.out.println("总行数：" + searchHits.getTotalHits());

    Iterator<SearchHit> iterator = searchHits.iterator();
    while (iterator.hasNext()) {
        SearchHit searchHit = iterator.next();
        // Source -> Document的JSON输出
        System.out.println(" -- 文档内容 -- ");
        System.out.println(searchHit.getSourceAsString());
        // 取title高亮显示的结果
        System.out.println(" -- 高亮结果 -- ");
        Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
        for (Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
            System.out.println(entry.getKey() + ":\t" + Arrays.toString(entry.getValue().getFragments()));
        }
    }
    // 关闭client
    client.close();
}

2. Spring Data ElasticSearch

2.1 简介

2.1.1 Spring Data

Spring Data是一个用于简化数据库访问，并支持云服务的开源框架。其主要目标是使得对数据的访问变得方便快捷，并支持map-reduce框架和云计算数据服务。 Spring Data可以极大的简化JPA的写法， 可以在几乎不用写实现的情况下，实现对数据的访问和操作。除了CRUD外，还包括如分页、排序等一些常用的功能。

Spring Data的官网：http://projects.spring.io/spring-data/

Spring Data常用的功能模块如下：

2.1.2 Spring Data ElasticSearch

Spring Data ElasticSearch基于spring data API简化elasticSearch操作，将原始操作elasticSearch的客户端API进行封装。Spring Data为Elasticsearch项目提供集成搜索引擎。Spring Data Elasticsearch POJO的关键功能区域为中心的模型与Elastichsearch交互文档和轻松地编写一个存储库数据访问层。

官方网站：http://projects.spring.io/spring-data-elasticsearch/

2.2 入门案例

查询用例

1. 导入Spring Data ElasticSearch坐标

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.murphy</groupId>
    <artifactId>SpringData-Demo</artifactId>
    <version>1.0-SNAPSHOT</version>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.1.16.RELEASE</version>
        <relativePath/>
    </parent>

    <properties>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

2. 启动配置器文件

@SpringBootApplication
public class EsApplication {
    public static void main(String[] args) {
        SpringApplication.run(EsApplication.class, args);
    }
}

spring:
  data:
    elasticsearch:
      cluster-name: murphy-application	
      cluster-nodes: 192.168.2.170:9300

3. 编写实体类

/**
 * 文档 - 实体类
 *
 * @author murphy
 */
@Document(indexName = "murphy_blog", type = "article")
public class Article {
    @Id
    @Field(type = FieldType.Long, store = true)
    private Long id;
    @Field(type = FieldType.Text, store = true, analyzer = "ik_smart")
    private String title;
    @Field(type = FieldType.Text, store = true, analyzer = "ik_smart")
    private String content;

    @Override
    public String toString() {
        return "Article{" +
                "id=" + id +
                ", title='" + title + '\'' +
                ", content='" + content + '\'' +
                '}';
    }

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String getContent() {
        return content;
    }

    public void setContent(String content) {
        this.content = content;
    }
}

4. 编写Dao

• 方法命名规则查询的基本语法：findBy + 属性 + 关键词 + 连接符

/**
 * 范型 - 存储 <实体类, 主键>
 * @author murphy
 */
public interface ArticleDao extends ElasticsearchRepository<Article, Long> {

    public List<Article> findByTitle(String title);

    public List<Article> findByTitleLike(String title);

    public List<Article> findByTitleOrContent(String title, String content);

    public List<Article> findByTitleOrContent(String title, String content, Pageable pageable);

}

5. 创建测试类

/**
 * 测试类
 *
 * @author murphy
 */
@RunWith(SpringRunner.class)
@SpringBootTest
public class EsApplicationTest {

    @Autowired
    private ArticleDao articleDao;

    @Autowired
    private ElasticsearchTemplate template;

    /**
     * 创建索引
     */
    @Test
    public void createIndex() {
        // 创建索引 + mapping
        template.createIndex(Article.class);

        // 配置mapping - template.putMapping(Article.class)
    }

    /**
     * 新建文档对象
     */
    @Test
    public void addDocument() throws Exception {
        for (int i = 1; i <= 20; i++) {
            // 创建一个 Article 对象
            Article article = new Article();
            article.setId(new Long(i));
            article.setTitle("近日以来，北方的秋天悄然来临.." + i);
            article.setContent("北方入秋速度明显加快，多地降温幅度最多可达10度" + i);
            // 把文档写入索引库
            articleDao.save(article);
        }
    }

    /**
     * 删除文档对象
     */
    @Test
    public void deleteDocumentById() throws Exception {
        // 根据ID删除
        articleDao.deleteById(3l);
        // 全部删除 - articleDao.deleteAll();
    }

    /**
     * 查询全部文档对象
     */
    @Test
    public void findAll() {
        articleDao.findAll().forEach(System.out :: println);
    }

    /**
     * 根据ID查询文档对象
     */
    @Test
    public void findById() {
        System.out.println(articleDao.findById(1l));
    }

    /**
     * 根据Title查询文档对象
     */
    @Test
    public void findByTitle() {
        articleDao.findByTitle("秋天").forEach(System.out :: println);
        System.out.println("---");
        // Like - 可以进行分词查询
        articleDao.findByTitleLike("美丽秋天").forEach(System.out :: println);
    }

    /**
     * 根据Title或Content查询文档对象
     */
    @Test
    public void findByTitleOrContent() {
        articleDao.findByTitleOrContent("秋天","无此").forEach(System.out :: println);
    }

    /**
     * 根据Title分页查询
     */
    @Test
    public void findByTitlePage() {
        Pageable pageable = PageRequest.of(1,5);
        articleDao.findByTitleOrContent("秋天","北方", pageable).forEach(System.out :: println);
    }

    /**
     * 本地构建的查询
     * @throws Exception
     */
    @Test
    public void testNativeSearchQuery() throws Exception {
        NativeSearchQuery query = new NativeSearchQueryBuilder()
                .withQuery(QueryBuilders.queryStringQuery("北方").defaultField("title"))
                .withPageable(PageRequest.of(1,5))
                .build();
        template.queryForList(query, Article.class).forEach(System.out :: println);
    }
}