[Java知识库] Elasticsearch(五)—

开发: C++知识库 Java知识库 JavaScript Python PHP知识库人工智能区块链大数据移动开发嵌入式开发工具数据结构与算法开发测试游戏开发网络协议系统运维
教程: HTML教程 CSS教程 JavaScript教程 Go语言教程 JQuery教程 VUE教程 VUE3教程 Bootstrap教程 SQL数据库教程 C语言教程 C++教程 Java教程 Python教程 Python3教程 C#教程
数码: 电脑笔记本显卡显示器固态硬盘硬盘耳机手机 iphone vivo oppo 小米华为单反装机图拉丁

-> Java知识库 -> Elasticsearch(五)——集成SpringBoot -> 正文阅读

[Java知识库]Elasticsearch(五)——集成SpringBoot

狂神说——集成SpringBoot

一、项目搭建

项目地址
链接：https://pan.baidu.com/s/1k18l3wwZD5VHuDrMX28pUQ
提取码：k4ak

创建项目工程

在这里插入图片描述

删除多余文件：
工程文件

esconfig

package com.cjw.stuesjd.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration //xml文件
public class esConfig {

    //spring <bean id="restHighLevelClient" class="RestHighLevelClient"/>
    @Bean
    public RestHighLevelClient restHighLevelClient() {
        //如果是集群则构建多个，否则构建一个即可
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("localhost", 9200, "http")));
                       // new HttpHost("localhost", 9201, "http")));
        return client;
    }
}

ContentController

package com.cjw.stuesjd.controller;
import com.cjw.stuesjd.service.ContentService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
  前端请求编写
 */
@Controller
public class ContentController {
    @Autowired
    ContentService contentService;
    @GetMapping("/parse/{keyword}")
    @ResponseBody
    public Boolean test(@PathVariable String keyword) throws IOException {
        Boolean parseContext = contentService.parseContext(keyword);
        System.out.println("parseContext = " + parseContext);
        return parseContext;
    }
    @GetMapping("/search/{keyword}/{pageNO}/{pageSize}")
    @ResponseBody
    public List<Map<String, Object>> search(@PathVariable("keyword") String keyword,@PathVariable("pageNO") int pageNo,@PathVariable int pageSize) throws IOException {
        List<Map<String, Object>> list = contentService.serchPageBuilder(keyword, pageNo, pageSize);
        return list;
    }
}

indexController

@Controller
public class indexController {
    @GetMapping({"/","/index"})
    public String index() {
        return "index";
    }
}

Content

//@Builder
@Data
@NoArgsConstructor
@AllArgsConstructor
@Component
public class Content {
    private String title;
    private String img;
    private String price;
}

ContentService

package com.cjw.stuesjd.service;

import com.alibaba.fastjson.JSON;
import com.cjw.stuesjd.pojo.Content;
import com.cjw.stuesjd.utils.HtmlParseUtil;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.naming.directory.SearchResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * 业务编写
 */
@Service
public class ContentService {

    @Autowired
    private RestHighLevelClient restHighLevelClient;

/*    public static void main(String[] args) throws IOException {
        new ContentService().parseContext("java");
    }*/

    /**
     * 1. 解析数据放入es索引中
      */

    public Boolean parseContext(String keywords) throws IOException {
        List<Content> contents = new HtmlParseUtil().parseJD(keywords);
        //把查询到的数据放入es索引中
        BulkRequest bulkRequest = new BulkRequest("jd_goods");
        bulkRequest.timeout("2m");
        for (int i = 0; i < contents.size(); i++) {
            bulkRequest.add(new IndexRequest("jd_goods")
            .source(JSON.toJSONString(contents.get(i)), XContentType.JSON));
        }
        BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        return !bulk.hasFailures();
    }

    //2. 获取这些数据实现搜索功能
    public List<Map<String, Object>> serchPage(String keyword, int pageNo, int pageSize) throws IOException {
        if(pageNo <=1) {
            pageNo = 1;
        }
        //条件搜索
         SearchRequest jd_goods = new SearchRequest("jd_goods");
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        TermQueryBuilder termQuery = QueryBuilders.termQuery("title", keyword);
        searchSourceBuilder.query(termQuery);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.from(pageNo);
        searchSourceBuilder.size(pageSize);

        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.field("title");
        highlightBuilder.preTags("<span style='color:red'>");
        highlightBuilder.postTags("</span>");
        highlightBuilder.requireFieldMatch(false);//匹配第一个即可
        searchSourceBuilder.highlighter(highlightBuilder);
        jd_goods.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(jd_goods, RequestOptions.DEFAULT);
        Map<String, Object> map = new HashMap<>();
        ArrayList<Map<String,Object>> list = new ArrayList();
        for (SearchHit hit : search.getHits()) {
            map = hit.getSourceAsMap();
            //System.out.println("hit = " + hit);
            list.add(map);
        }
        return list;
    }

    //2. 获取这些数据实现搜索功能
    public List<Map<String, Object>> serchPageBuilder(String keyword, int pageNo, int pageSize) throws IOException {
        if(pageNo <=1) {
            pageNo = 1;
        }
        //条件搜索
        SearchRequest jd_goods = new SearchRequest("jd_goods");
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        TermQueryBuilder termQuery = QueryBuilders.termQuery("title", keyword);
        searchSourceBuilder.query(termQuery);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.from(pageNo);
        searchSourceBuilder.size(pageSize);

        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.requireFieldMatch(true);
        highlightBuilder.field("title");
        highlightBuilder.preTags("<span style='color:red'>");
        highlightBuilder.postTags("</span>");
       // highlightBuilder.requireFieldMatch(false);//匹配第一个即可
       // highlightBuilder.numOfFragments(0);
        searchSourceBuilder.highlighter(highlightBuilder);
        jd_goods.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(jd_goods, RequestOptions.DEFAULT);
        Map<String, Object> map = new HashMap<>();
        ArrayList<Map<String,Object>> list = new ArrayList();
        for (SearchHit hit : search.getHits()) {

            //
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            System.out.println("highlightFields = " + highlightFields);
            HighlightField title_high = highlightFields.get("title");
            String fr = "";
            for (Text fragment : title_high.fragments()) {
                System.out.println("fragment = " + fragment);
                fr = fragment.toString();
                map.put("fragment", JSON.toJSONString(fragment.toString()));

            }
            System.out.println("fr = " + fr);
            // map.put("fragment", JSON.toJSONString(fragment));
            map.put("fr", JSON.toJSONString(fr));
            //System.out.println("title_high_______fragments = " + title_high.fragments().toString());
            map = hit.getSourceAsMap();
            //System.out.println("hit = " + hit);
            list.add(map);
        }
        return list;
    }

}

HtmlParseUtil

package com.cjw.stuesjd.utils;

import com.cjw.stuesjd.pojo.Content;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

public class HtmlParseUtil {
    public static void main(String[] args) throws IOException {
        new HtmlParseUtil().parseJD("中文").forEach(System.out::println);
        //等价于for增强
      /*  for (User user : users) {
            System.out.println(user);
        }*/
    }

    public List<Content> parseJD(String keyword) throws IOException{
        //获取请求
        //前提：需要联网，ajax不能获取到
        String url = "https://search.jd.com/Search?keyword=" + keyword;
        //解析网页  ,(jsoup 返回的对象就是浏览器document对象)
        Document document = Jsoup.parse(new URL(url), 30000);
        //所有你在js中可以使用的方法，在这里都可以用。
        Element j_goodsList = document.getElementById("J_goodsList");
        //  System.out.println(j_goodsList.html());
        //获取所有的li元素
        Elements li = j_goodsList.getElementsByTag("li");
        //list
        ArrayList<Content> objects = new ArrayList<>();
        //获取元素中的内容
        for (Element element : li) {
            // System.out.println("element = " + element);
           /* Elements img = element.getElementsByTag("img").eq(0);
            System.out.println("img = " + img);*/
            Content content = new Content();
            //关于这种图片很多网站，图片都是懒加载的
            String img_src = element.getElementsByTag("img").eq(0).attr("data-lazy-img");
            //System.out.println("img_src = " + img_src);
            String price = element.getElementsByClass("p-price").eq(0).text();
            //System.out.println("price = " + price);
            String title = element.getElementsByClass("p-name").eq(0).text();
            //System.out.println("title = " + title);
            content.setImg(img_src);
            content.setPrice(price);
            content.setTitle(title);
            objects.add(content);
        }
        return objects;
    }
}

StuEsJdApplication

@SpringBootApplication
public class StuEsJdApplication {

	public static void main(String[] args) {
		SpringApplication.run(StuEsJdApplication.class, args);
	}

}

application.properties

server.port=9090
#关闭thymeleaf的缓存
spring.thymeleaf.cache=false

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.2.5.RELEASE</version>
		<relativePath/> <!-- lookup parent from repository -->
	</parent>
	<groupId>com.cjw</groupId>
	<artifactId>stu-es-jd</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>stu-es-jd</name>
	<description>Demo project for Spring Boot</description>
	<properties>
		<java.version>1.8</java.version>
		<!--自定义-->
		<elasticsearch.version>7.6.1</elasticsearch.version>
	</properties>
	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-thymeleaf</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-devtools</artifactId>
			<scope>runtime</scope>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-configuration-processor</artifactId>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>

		<dependency>
			<groupId>com.alibaba</groupId>
			<artifactId>fastjson</artifactId>
			<version>1.2.62</version>
		</dependency>
  	<!--解析网页可以使用jsoup-->
		<!--jsoup 不能爬取电影，tika爬取电影-->
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.10.2</version>
		</dependency>

		<dependency>
			<groupId>com.thetransactioncompany</groupId>
			<artifactId>Java-property-utils</artifactId>
			<version>1.9</version>
		</dependency>
		<dependency>
			<groupId>com.thetransactioncompany</groupId>
			<artifactId>cors-filter</artifactId>
			<version>1.7</version>
		</dependency>
	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
				<configuration>
					<excludes>
						<exclude>
							<groupId>org.projectlombok</groupId>
							<artifactId>lombok</artifactId>
						</exclude>
					</excludes>
				</configuration>
			</plugin>
		</plugins>
	</build>

</project>

二、处理ES版本问题

发现版本问题
在pom.xml中配置相应的版本
配置之后要进行重新导入

三、爬虫

数据问题？数据库获取，消息对列中获取中，都可以成为数据源，爬虫！

爬取数据：(获取请求返回的页面信息，筛选出我们想要的数据就可以了)

https://search.jd.com/Search?keyword=java
jsoup包！

public class HtmlParseUtil {
    public static void main(String[] args) throws IOException {
        new HtmlParseUtil().parseJD("中文").forEach(System.out::println);
        //等价于for增强
      /*  for (User user : users) {
            System.out.println(user);
        }*/
    }

    public List<Content> parseJD(String keyword) throws IOException{
        //获取请求
        //前提：需要联网，ajax不能获取到
        String url = "https://search.jd.com/Search?keyword=" + keyword;
        //解析网页  ,(jsoup 返回的对象就是浏览器document对象)
        Document document = Jsoup.parse(new URL(url), 30000);
        //所有你在js中可以使用的方法，在这里都可以用。
        Element j_goodsList = document.getElementById("J_goodsList");
        //  System.out.println(j_goodsList.html());
        //获取所有的li元素
        Elements li = j_goodsList.getElementsByTag("li");
        //list
        ArrayList<Content> objects = new ArrayList<>();
        //获取元素中的内容
        for (Element element : li) {
            // System.out.println("element = " + element);
           /* Elements img = element.getElementsByTag("img").eq(0);
            System.out.println("img = " + img);*/
            Content content = new Content();
            //关于这种图片很多网站，图片都是懒加载的
            String img_src = element.getElementsByTag("img").eq(0).attr("data-lazy-img");
            //System.out.println("img_src = " + img_src);
            String price = element.getElementsByClass("p-price").eq(0).text();
            //System.out.println("price = " + price);
            String title = element.getElementsByClass("p-name").eq(0).text();
            //System.out.println("title = " + title);
            content.setImg(img_src);
            content.setPrice(price);
            content.setTitle(title);
            objects.add(content);
        }
        return objects;
    }
}