[大数据]关于搜索elasticsearch的数据条数大于10000的坑 max_result_window的解决方案

关于这个问题我主要从5.6.x版本和7.10.x版本分析：

• 5.6.x版本：只需要一步即可

{
    "settings": {
      "index": {     
        "number_of_shards": "1",
        "max_result_window": "1000000",
        "analysis": {
          "analyzer": {
            "keep_none_chinese_in_joined_full_pinyin_analyzer": {
              "tokenizer": "keep_none_chinese_in_joined_full_pinyin"
            }
          },
          "tokenizer": {
            "keep_none_chinese_in_joined_full_pinyin": {
              "keep_joined_full_pinyin": "true",
              "lowercase": "true",
              "none_chinese_pinyin_tokenize": "false",
              "keep_none_chinese_in_joined_full_pinyin": "true",
              "keep_original": "false",
              "keep_first_letter": "false",
              "keep_separate_first_letter": "false",
              "trim_whitespace": "false",
              "type": "pinyin",
              "keep_none_chinese": "true",
              "keep_full_pinyin": "false"
            }
          }
        },
        "number_of_replicas": "1"
        
      }
  }
}

以上是我在5.6.16的es中设置的setting， 其中参数 "max_result_window": "1000000" 的设置 返回最大1000000条数据。如果你需要更大的可以设置成Integer.MAX_VALUE;

看看效果：

GET test/_search
{
  "size": 0, 
  "query": {
    "match_all": {}
  }
}
------
{
  "took": 0,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 29084,
    "max_score": 0,
    "hits": []
  }
}

可以看到total ：??29084

• 7.10.0 版本：需要两步才能实现

第一步：设置setting

{
  "settings": {
    "analysis": {
      "filter": {
        "my_synonym": {
          "type": "synonym",
          "synonyms_path": "analysis/synonym.txt"
        }
      },
      "analyzer": {
        "keep_none_chinese_in_joined_full_pinyin_analyzer": {
          "tokenizer": "keep_none_chinese_in_joined_full_pinyin"
        },
        "my_ik_max_word": {
          "tokenizer": "ik_max_word",
          "filter": [
            "my_synonym"
          ]
        },
        "my_ik_smart": {
          "tokenizer": "ik_smart",
          "filter": [
            "my_synonym"
          ]
        }
      },
      "normalizer": {
        "lowercase": {
          "type": "custom",
          "filter": [
            "lowercase"
          ]
        }
      },
      "tokenizer": {
        "keep_none_chinese_in_joined_full_pinyin": {
          "type": "pinyin",
          "keep_first_letter": false,
          "keep_separate_first_letter": false,
          "keep_none_chinese_in_joined_full_pinyin": true,
          "keep_joined_full_pinyin": true,
          "keep_none_chinese": true,
          "none_chinese_pinyin_tokenize": false,
          "keep_full_pinyin": false,
          "keep_original": false,
          "trim_whitespace": false,
          "lowercase": true
        }
      }
    },
    "index": {
      "max_result_window": 1000000,
      "number_of_shards": 1,
      "number_of_replicas": 1
    }
  },
  "mappings": {
    "dynamic_templates": [
      {
        "strings": {
          "match_mapping_type": "string",
          "mapping": {
            "type": "text",
            "fields": {
              "synonym_ik": {
                "type": "text",
                "store": false,
                "analyzer": "my_ik_max_word",
                "search_analyzer": "ik_smart"
              },
              "ik": {
                "type": "text",
                "store": false,
                "analyzer": "ik_max_word",
                "search_analyzer": "ik_smart"
              },
              "raw": {
                "type": "keyword",
                "normalizer": "lowercase"
              },
              "py": {
                "type": "text",
                "store": false,
                "fielddata": "true",
                "analyzer": "keep_none_chinese_in_joined_full_pinyin_analyzer"
              }
            }
          }
        }
      }
    ]
  }
}

同5.6.16一样需要设置 "max_result_window": "1000000" 我看看查询：

GET test/_search
{
  "size": 0, 
  "query": {
  "match_all": {}
  }
}
-------
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  }
}

我们查询返回最大total依然是es的默认值10000，注意此时：relation是gte 大于的意思；

所以我们需要：

GET test/_search
{
  "size": 0, 
  "track_total_hits": true, 
  "query": {
  "match_all": {}
  }
}
-----------
{
  "took" : 0,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 29087,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  }
}

返回的total是29087满足需求了，此时的relation是eq 等于的意思；

ok?