### 1\. term filter 與keyword
測試數據
~~~
POST /forum/article/_bulk
{ "index": { "_id": 1 }}
{ "articleID" : "XHDK-A-1293-#fJ3", "userID" : 1, "hidden": false, "postDate": "2017-01-01" }
{ "index": { "_id": 2 }}
{ "articleID" : "KDKE-B-9947-#kL5", "userID" : 1, "hidden": false, "postDate": "2017-01-02" }
{ "index": { "_id": 3 }}
{ "articleID" : "JODL-X-1937-#pV7", "userID" : 2, "hidden": false, "postDate": "2017-01-01" }
{ "index": { "_id": 4 }}
{ "articleID" : "QQPX-R-3956-#aD8", "userID" : 2, "hidden": true, "postDate": "2017-01-02" }
~~~
1. 匹配text
~~~
GET forum/_search
{
"query": {
"constant_score": { # 固定分值，就不用計算相關度，過濾唄
"filter": {
"term": {
"articleID": "KDKE-B-9947-#kL5"
}
},
"boost": 1.2
}
}
}
~~~
> articleID被分詞成：kdke,b,9947,kl5，而term過濾是精準過濾，KDKE-B-9947-#kL5不會被分詞，還是用KDKE-B-9947-#kL5去查詢過濾，所以這條過濾不能得到document。
結果啥也沒搜到
~~~
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 0,
"max_score": null,
"hits": []
}
}
~~~
2. 再用articleID.keyword去過濾
keyword參考elasticsearch基礎 type=keyword
~~~
GET forum/_search
{
"query": {
"constant_score": {
"filter": {
"term": {
"articleID.keyword": "KDKE-B-9947-#kL5"
}
},
"boost": 1.2 # 固定分值，不指定默認為1
}
}
}
~~~
結果
~~~
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 1.2,
"hits": [
{
"_index": "forum",
"_type": "article",
"_id": "2",
"_score": 1.2,
"_source": {
"articleID": "KDKE-B-9947-#kL5",
"userID": 1,
"hidden": false,
"postDate": "2017-01-02"
}
}
]
}
}
~~~
或者這么寫
~~~
GET forum/_search
{
"query": {
"bool": {
"filter": {
"term": {
"articleID.keyword": "KDKE-B-9947-#kL5"
}
}
}
}
}
~~~
## 2\. 組合查詢
~~~
GET forum/_search
{
"query": {
"bool": {
"should": [
{"term": {"postDate":"2017-01-01"}},
{"term": {"articleID":"QQPX-R-3956-#aD8"}} # 文檔被分詞，查詢條件（不被分詞）無法精準匹配到QQPX-R-3956-#aD8的文檔
]
}
}
}
~~~
~~~
GET forum/_search
{
"query": {
"bool": {
"should": [
{"term": {"postDate":"2017-01-01"}},
{"term": {"articleID.keyword":"QQPX-R-3956-#aD8"}} # text 字段keyword屬性，不分詞，查詢到QQPX-R-3956-#aD8
]
}
}
}
~~~
## 3\. terms 查詢
1. 查找articleID是QQPX-R-3956-#aD8 或者KDKE-B-9947-#kL5的文檔
~~~
GET forum/_search
{
"query": {
"terms": {
"articleID.keyword": [
"QQPX-R-3956-#aD8",
"KDKE-B-9947-#kL5"
]
}
}
}
~~~
## 4\. range
1. range查詢
~~~
GET forum/_search
{
"query": {
"range": {
"view_cnt": {
"gte": 50,
"lte": 90
}
}
}
}
~~~
2. range過濾
~~~
GET forum/_search
{
"query": {
"constant_score": {
"filter": {
"range": {
"view_cnt": {
"gte": 50,
"lte": 90
}
}
}
}
}
}
~~~
* 同樣的效果range過濾比range查詢要快
## 5\. 分組聚合
穿件index和映射
~~~
PUT /tvs
{
"mappings": {
"sales": {
"properties": {
"price": {
"type": "long"
},
"color": {
"type": "keyword"
},
"brand": {
"type": "keyword"
},
"sold_date": {
"type": "date"
}
}
}
}
}
~~~
數據
~~~
POST /tvs/sales/_bulk
{ "index": {}}
{ "price" : 1000, "color" : "紅色", "brand" : "長虹", "sold_date" : "2016-10-28" }
{ "index": {}}
{ "price" : 2000, "color" : "紅色", "brand" : "長虹", "sold_date" : "2016-11-05" }
{ "index": {}}
{ "price" : 3000, "color" : "綠色", "brand" : "小米", "sold_date" : "2016-05-18" }
{ "index": {}}
{ "price" : 1500, "color" : "藍色", "brand" : "TCL", "sold_date" : "2016-07-02" }
{ "index": {}}
{ "price" : 1200, "color" : "綠色", "brand" : "TCL", "sold_date" : "2016-08-19" }
{ "index": {}}
{ "price" : 2000, "color" : "紅色", "brand" : "長虹", "sold_date" : "2016-11-05" }
{ "index": {}}
{ "price" : 8000, "color" : "紅色", "brand" : "三星", "sold_date" : "2017-01-01" }
{ "index": {}}
{ "price" : 2500, "color" : "藍色", "brand" : "小米", "sold_date" : "2017-02-12" }
~~~
1. 按照color分組
~~~
GET /tvs/_search
{
"size": 0, # 只顯示聚合結果
"aggs": {
"group_by_color": {
"terms": {
"field": "color",
"size": 10000 # 控制返回結果數量
}
}
}
}
~~~
得到
~~~
"buckets": [
{
"key": "紅色",
"doc_count": 4
},
{
"key": "綠色",
"doc_count": 2
},
{
"key": "藍色",
"doc_count": 2
}
]
}
}
~~~
2. 求每種顏色電視銷售價格的平均值
~~~
GET /tvs/_search
{
"size": 0, # 只顯示聚合結果
"aggs": {
"group_by_color": {
"terms": {
"field": "color",
"size": 1000
},
"aggs": {
"avg_price": {
"avg": {
"field": "price"
}
}
}
}
}
}
~~~
color》brand》avg
~~~
GET tvs/_search
{
"size": 0,
"aggs": {
"group_by_color": {
"terms": {
"field": "color",
"size": 10
},
"aggs": {
"group_by_brand": {
"terms": {
"field": "brand",
"size": 10
},
"aggs": {
"avg_price": {
"avg": {
"field": "price"
}
}
}
}
}
}
}
}
~~~
3. 求顏色下的價格最高,最小值，價格總數
~~~
GET /tvs/_search
{
"size": 0,
"aggs": {
"group_by_color": {
"terms": {
"field": "color",
"size": 10000
},
"aggs": {
"max_price": {
"max": {
"field": "price"
}
},
"min_price":{
"min": {
"field": "price"
}
},
"sum_price":{
"sum": {
"field": "price"
}
}
}
}
}
}
~~~
得到
~~~
"buckets": [
{
"key": "紅色",
"doc_count": 4,
"max_price": {
"value": 8000
},
"min_price": {
"value": 1000
},
"sum_price": {
"value": 13000
}
},
{
"key": "綠色",
"doc_count": 2,
"max_price": {
"value": 3000
},
"min_price": {
"value": 1200
},
"sum_price": {
"value": 4200
}
},
{
"key": "藍色",
"doc_count": 2,
"max_price": {
"value": 2500
},
"min_price": {
"value": 1500
},
"sum_price": {
"value": 4000
}
~~~
## 6\. histogram
1. 求以2000為步長，電視的銷售總價
~~~
GET /tvs/_search
{
"size": 0,
"aggs": {
"group_by_price": {
"histogram": {
"field": "price",
"interval": 2000
},
"aggs": {
"NAME": {
"sum": {
"field": "price"
}
}
}
}
}
}
~~~
得到
~~~
"group_by_price": {
"buckets": [
{
"key": 0,
"doc_count": 3,
"NAME": {
"value": 3700
}
},
{
"key": 2000,
"doc_count": 4,
"NAME": {
"value": 9500
}
},
{
"key": 4000,
"doc_count": 0,
"NAME": {
"value": 0
}
},
{
"key": 6000,
"doc_count": 0,
"NAME": {
"value": 0
}
},
{
"key": 8000,
"doc_count": 1,
"NAME": {
"value": 8000
}
}
]
~~~
2. 求每個月的銷售總價
## 7\. 求季度每個品牌的銷售額
~~~
GET tvs/_search
{
"size": 0,
"aggs": {
"sales": {
"date_histogram": {
"field": "sold_date",
"interval": "quarter",
"format": "yyyy-MM-dd",
"min_doc_count": 0,
"extended_bounds":{
"min":"2016-01-01",
"max":"2017-12-12"
}
},
"aggs": {
"group_price_month": {
"terms": {
"field": "brand"
},
"aggs": {
"sales_sum": {
"sum": {
"field": "price"
}
}
}
}
}
}
}
}
~~~
~~~
},
{
"key_as_string": "2016-07-01",
"key": 1467331200000,
"doc_count": 2,
"group_price_month": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "TCL",
"doc_count": 2,
"sales_sum": {
"value": 2700
}
}
]
}
},
{
"key_as_string": "2016-10-01",
"key": 1475280000000,
"doc_count": 3,
"group_price_month": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "長虹",
"doc_count": 3,
"sales_sum": {
"value": 5000
}
}
]
}
},
{
~~~
## 8\. filter與aggression
查找電視價格大于2500的銷售平均價格
首先把價格大于2500的電視過濾出來，在聚合
~~~
GET /tvs/_search
{
"size": 0,
"query": {
"constant_score": {
"filter": {
"range": {
"price": {
"gt": 2500
}
}
}
}
},
"aggs": {
"avg_lgt1200": {
"avg": {
"field": "price"
}
}
}
}
~~~
得到
~~~
"hits": []
},
"aggregations": {
"avg_lgt1200": {
"value": 5500
}
}
}
~~~
## 9\. 最近一個月某品牌的銷售總價
~~~
GET /tvs/_search
{
"size": 0,
"query": {
"constant_score": {
"filter": {
"term": {
"brand": "長虹"
}
}
}
},
"aggs": {
"recent_moth":{
"filter": { # 在這里過濾，因為只在查詢的結果中過濾，性能最優
"range": {
"sold_date": {
"gte": "now-30d" # now-30d 從現在開始，減30天
}
}
},
"aggs": {
"sum_price": {
"sum": {
"field": "price"
}
}
}
}
}
}
~~~
得到
~~~
"aggregations": {
"recent_moth": {
"doc_count": 3,
"sum_price": {
"value": 5000
}
}
}
}
~~~
## 10\. 聚合排序
1. 一個桶排序
~~~
GET /tvs/_search
{
"size": 0,
"aggs": {
"groupbycolor": {
"terms": {
"field": "color",
"order": {
"avg_price": "asc" # 指定按照avg_price排序
}
},
"aggs": {
"avg_price": {
"avg": {
"field": "price"
}
}
}
}
}
}
~~~
2. 2個桶排序
~~~
GET /tvs/sales/_search
{
"size": 0,
"aggs": {
"group_by_color": {
"terms": {
"field": "color",
"order": {
"_count": "asc" # 大桶按照doc count升序
}
},
"aggs": {
"group_by_brand": {
"terms": {
"field": "brand",
"order": {
"avg_price": "desc" # 小桶降序
}
},
"aggs": {
"avg_price": {
"avg": {
"field": "price"
}
}
}
}
}
}
}
}
~~~
## 11\. 網站訪問時間統計
~~~
PUT /website
{
"mappings": {
"logs":{
"properties": {
"latency":{
"type": "long"
},
"province":{
"type": "keyword"
},
"timestamp":{
"type": "date"
}
}
}
}
}
~~~
~~~
POST /website/logs/_bulk
{ "index": {}}
{ "latency" : 105, "province" : "江蘇", "timestamp" : "2016-10-28" }
{ "index": {}}
{ "latency" : 83, "province" : "江蘇", "timestamp" : "2016-10-29" }
{ "index": {}}
{ "latency" : 92, "province" : "江蘇", "timestamp" : "2016-10-29" }
{ "index": {}}
{ "latency" : 112, "province" : "江蘇", "timestamp" : "2016-10-28" }
{ "index": {}}
{ "latency" : 68, "province" : "江蘇", "timestamp" : "2016-10-28" }
{ "index": {}}
{ "latency" : 76, "province" : "江蘇", "timestamp" : "2016-10-29" }
{ "index": {}}
{ "latency" : 101, "province" : "新疆", "timestamp" : "2016-10-28" }
{ "index": {}}
{ "latency" : 275, "province" : "新疆", "timestamp" : "2016-10-29" }
{ "index": {}}
{ "latency" : 166, "province" : "新疆", "timestamp" : "2016-10-29" }
{ "index": {}}
{ "latency" : 654, "province" : "新疆", "timestamp" : "2016-10-28" }
{ "index": {}}
{ "latency" : 389, "province" : "新疆", "timestamp" : "2016-10-28" }
{ "index": {}}
{ "latency" : 302, "province" : "新疆", "timestamp" : "2016-10-29" }
~~~
~~~
需求：比如有一個網站，記錄下了每次請求的訪問的耗時，需要統計tp50，tp90，tp99
tp50：50%的請求的耗時最長在多長時間
tp90：90%的請求的耗時最長在多長時間
tp99：99%的請求的耗時最長在多長時間
~~~
~~~
GET /website/_search
{
"aggs": {
"latency_percentiles": {
"percentiles": {
"field": "latency",
"percents": [
50,
99
]
}
}
}
}
~~~
得到
~~~
},
"aggregations": {
"latency_percentiles": {
"values": {
"50.0": 108.5, # 50%的請求在108以內
"99.0": 624.8500000000001 # 99%的請求在624以內
}
}
}
}
~~~
2. 統計各省的訪問情況
* 一個aggs內可以包含一個aggs
* 一個aggs可以包含多個不同的metric（指標統計：平均值，最大值，，，），但是記得起名字
~~~
GET /website/logs/_search
{
"size": 0,
"aggs": {
"group_by_province": {
"terms": {
"field": "province"
},
"aggs": {
"latency_percentiles": {
"percentiles": {
"field": "latency",
"percents": [
50,
95,
99
]
}
},
"latency_avg": {
"avg": {
"field": "latency"
}
}
}
}
}
}
~~~
得到，新疆的網絡不行啊
~~~
"buckets": [
{
"key": "新疆",
"doc_count": 6,
"latency_avg": {
"value": 314.5
},
"latency_percentiles": {
"values": {
"50.0": 288.5,
"95.0": 587.75,
"99.0": 640.75
}
}
},
{
"key": "江蘇",
"doc_count": 6,
"latency_avg": {
"value": 89.33333333333333
},
"latency_percentiles": {
"values": {
"50.0": 87.5,
"95.0": 110.25,
"99.0": 111.65
}
}
}
]
}
}
}
~~~
## 12\. percentile\_ranks(求某數值所占比例)
> SLA：就是你提供的服務的標準
> 我們的網站的提供的訪問延時的SLA，確保所有的請求100%，都必須在200ms以內，大公司內，一般都是要求100%在200ms以內
> 如果超過1s，則需要升級到A級故障，代表網站的訪問性能和用戶體驗急劇下降
> 需求：在200ms以內的，有百分之多少，在1000毫秒以內的有百分之多少，percentile ranks metric
> 這個percentile ranks，其實比pencentile還要常用
> 按照品牌分組，計算，電視機，售價在1000占比，2000占比，3000占比
~~~
GET /website/_search
{
"size": 0,
"aggs": {
"groupby_province": {
"terms": {
"field": "province"
},
"aggs": {
"percentile_s": {
"percentile_ranks": {
"field": "latency",
"values": [
100,
300,
500
]
}
}
}
}
}
}
~~~
得到
~~~
"buckets": [
{
"key": "新疆",
"doc_count": 6,
"percentile_s": {
"values": {
"100.0": 8.076923076923077,
"300.0": 53.3625730994152,
"500.0": 65.31446540880503
}
}
},
{
"key": "江蘇",
"doc_count": 6,
"percentile_s": {
"values": {
"100.0": 46.42857142857142,
"300.0": 100,
"500.0": 100
}
}
}
]
}
}
}
~~~
> 新疆：
> 響應時間小于100毫秒占46.42%
> 響應時間小于300毫秒占53.42%
> 響應時間小于500毫秒占65.42%
## 13\. 文檔模型
1. 建立索引，已經文件路徑分詞器
~~~
PUT /fs
{
"settings": {
"analysis": {
"analyzer": {
"paths": {
"tokenizer": "path_hierarchy"
}
}
}
}
}
~~~
path\_hierarchy：路徑分詞器
2. 創建映射
~~~
PUT /fs/_mapping/file
{
"properties": {
"name": {
"type": "keyword"
},
"path": {
"type": "keyword", # 不分詞的路徑，用于精準匹配
"fields": {
"tree": {
"type": "text",
"analyzer": "paths" # 分詞子field，用于所有路徑的搜索
}
}
}
}
}
~~~
3. 插入一條數據
~~~
PUT /fs/file/1
{
"name": "README.txt",
"path": "/workspace/projects/helloworld",
"contents": "這是我的第一個elasticsearch程序"
}
~~~
4. 搜索文件
5. 查找/workspace/projects/helloworld路徑下的elasticsearch文件
~~~
GET /fs/file/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"contents": "elasticsearch"
}
},
{
"constant_score": {
"filter": {
"term": {
"path": "/workspace/projects/helloworld"
}
}
}
}
]
}
}
}
~~~
~~~
"hits": [
{
"_index": "fs",
"_type": "file",
"_id": "1",
"_score": 1.284885,
"_source": {
"name": "README.txt",
"path": "/workspace/projects/helloworld",
"contents": "這是我的第一個elasticsearch程序"
}
}
]
}
}
~~~
5. 查找/workspace目錄下的所有elasticsearch文件
~~~
GET /fs/file/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"contents": "elasticsearch"
}
},
{
"constant_score": {
"filter": {
"term": {
"path.tree": "/workspace" # 用可分詞的field，過濾所有路徑
}
}
}
}
]
}
}
}
~~~
- springcloud
- springcloud的作用
- springboot服務提供者和消費者
- Eureka
- ribbon
- Feign
- feign在微服務中的使用
- feign充當http請求工具
- Hystrix 熔斷器
- Zuul 路由網關
- Spring Cloud Config 分布式配置中心
- config介紹與配置
- Spring Cloud Config 配置實戰
- Spring Cloud Bus
- gateway
- 概念講解
- 實例
- GateWay
- 統一日志追蹤
- 分布式鎖
- 1.redis
- springcloud Alibaba
- 1. Nacos
- 1.1 安裝
- 1.2 特性
- 1.3 實例
- 1. 整合nacos服務發現
- 2. 整合nacos配置功能
- 1.4 生產部署方案
- 環境隔離
- 原理講解
- 1. 服務發現
- 2. sentinel
- 3. Seata事務
- CAP理論
- 3.1 安裝
- 分布式協議
- 4.熔斷和降級
- springcloud與alibba
- oauth
- 1. abstract
- 2. oauth2 in micro-service
- 微服務框架付費
- SkyWalking
- 介紹與相關資料
- APM系統簡單對比(zipkin,pinpoint和skywalking)
- server安裝部署
- agent安裝
- 日志清理
- 統一日志中心
- docker安裝部署
- 安裝部署
- elasticsearch 7.x
- logstash 7.x
- kibana 7.x
- ES索引管理
- 定時清理數據
- index Lifecycle Management
- 沒數據排查思路
- ELK自身組件監控
- 多租戶方案
- 慢查詢sql
- 日志審計
- 開發
- 登錄認證
- 鏈路追蹤
- elk
- Filebeat
- Filebeat基礎
- Filebeat安裝部署
- 多行消息Multiline
- how Filebeat works
- Logstash
- 安裝
- rpm安裝
- docker安裝Logstash
- grok調試
- Grok語法調試
- Grok常用表達式
- 配置中常見判斷
- filter提取器
- elasticsearch
- 安裝
- rpm安裝
- docker安裝es
- 使用
- 概念
- 基礎
- 中文分詞
- 統計
- 排序
- 倒排與正排索引
- 自定義dynamic
- 練習
- nested object
- 父子關系模型
- 高亮
- 搜索提示
- kibana
- 安裝
- docker安裝
- rpm安裝
- 整合
- 收集日志
- 慢sql
- 日志審計s
- 云
- 分布式架構
- 分布式鎖
- Redis實現
- redisson
- 熔斷和降級