ES开发中知识积累

/ Annabelle
目录
Tips
  • _id可查，无论该字段是否在mapping中定义，属于ES内嵌字段。等同于ids查询。
  • keyword类型范围搜索不生效。
  • query_string 只适用于keyword 与 text 类型的字段。
  • query_string 的保留字符 + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ / 需要进行转义。\(1\+1\)\=2
  • text 不支持聚合和精准term查询
  • object 与 nested 区别：均支持数组对象，但object中每个对象的相同属性存储在一起，导致了同一个对象中的各个属性之间的依赖关系丢失。nested则可以保持属性之间的关联关系。
  • scroll 用于深度分页，可以用于大量数据迁移。想要速度可以根据 _doc 排序
  • search_after 也可用于解决 from+size 解决不了的情况

q:city:长沙

AND updated_at:["2019-06-19 17:08:05" TO "2019-12-19 17:08:05"]

sort:updated_at:desc

_source_include:updated_at,s_ip_start

_id:7953017778080

Body Search
size
"size": 10,
source
"_source": [
   "ip",
   "port",
   "created_at",
   "updated_at"
],

"_source": {
    "excludes": [
      "web_body",
      "banner_info",
      "http_head"
      "cpe_info"
    ]
},

"_source": {
    "includes": [
      ""
    ]
},
explain
"explain": true,
must
{
  "_source": [
    "ip",
    "port",
    "created_at",
    "updated_at"
  ],
  "sort": {
    "updated_at": "desc"
  },
  "size": 10,
  "query": {
    "bool": {
      "must": [],
      "must_not": []
    }
  }
}
must + should
{
  "_source": [
    "web_title"
  ],
  "size": 10,
  "query": {
    "bool": {
      "must": [
        {
          "bool": {
            "should": []
          }
        }
      ]
    }
  }
}
exists
{
  "exists": {
    "field": "user_id"
  }
}

命中至少有一个非null值的文档!!!!!!!!!!!!!

以下不能够被命中：

  • null
  • []
  • 不存在该字段

{ "user": null }
{ "user": [] }
{ "user": [null] }
{ "foo": "bar" }

null_value mapping

如果在mapping中定义了字段的null值（例如定义为未知） , 如果文档A的user字段对应值为null那么文档A将会被查询命中。

range

{
    "range": {
      "updated_at": {
        "gte": "2019-09-02 00:00:00",
        "lt": "2019-09-09 00:00:00"
      }
    }
}
term

{
  "term": {
    "record_address": "xxxx"
  }
}
terms

{
  "terms": {
    "severity": [
      "超危",
      "高危"
    ]
  }
}
ids
{
  "ids": {
    "type": "ip_search",
    "values": [
      "234696858480idle.alicdn.com.danuoyi.tbcache.com"
    ]
  }
}
aggs
"aggs": {
      "ip_count": {
        "cardinality": {
          "field": "iplong"
        }
      }
 }

申明聚合(aggs)
--------聚合名称
------------聚合方式
------------申明次级聚合(aggs)
----------------聚合名称
--------------------聚合方式

script aggs

"aggs": {
      "AGGSNAME": {
        "terms": {
          "script": {
            "inline": "doc['ip'].value  + ':' +  doc['port'].value",
            "lang": "painless"
          }
        }
      }
    }
date aggs
{
  "size": 0,
  "aggs": {
    "group_by_month": {
      "date_histogram": {
        "field": "created_at",
        "interval": "month",
        "format": "yyyy-MM"
      }
    }
  }
}
nested aggs

"aggs": {
    "aggs_name_http": {
      "nested": {
        "path": "http_head_map"
      },
      "aggs": {
        "aggs_name_server": {
          "terms": {
            "field": "http_head_map.server",
            "size": 20,
            "exclude": [
              ""
            ]
          }
        }
      }
    }
  }
nested query

{
  "nested": {
    "path": "http_head_map",
    "query": {
      "exists": {
        "field": "http_head_map.content_length"
      }
    }
  }
}
query_string
{
  "query_string": {
    "query": "*HTTP\\/1.1?200?OK\\\r\\\nSet\\-Cookie\\:?xcvi*",
    "fields": [
      "http_head"
    ],
    "default_operator": "AND",
    "split_on_whitespace": false
  }
}
aggs cardinality

{
  "aggs": {
    "ip_count": {
      "cardinality": {
        "field": "iplong"
      }
    }
  }
}
aggs top_hits

"aggs": {
      "info": {
        "top_hits": {
          "size": 1
        }
      }
    }
}
aggs date_histogram

    "aggs":
    {
        "group_by_time":
        {
            "date_histogram":
            {
                "field": "updated_at",
                "interval": "1h",
                "format": "yyyy-MM-dd HH"
            }
        }
    }