Skip to content

Instantly share code, notes, and snippets.

@honzakral

honzakral/.gitignore Secret

Last active December 17, 2015 21:29
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save honzakral/0a0895814d783516ed0f to your computer and use it in GitHub Desktop.
Save honzakral/0a0895814d783516ed0f to your computer and use it in GitHub Desktop.
Download ZIP
Elasticsearch tutorial
Raw
.gitignore
load_stack
data
bulk.json
elasticat
.*.swp
Raw
00-README.rst

Elasticsearch tutorial

Requirements

Preparation

Run:

BULK_FILE=$PWD/bulk.json ES_INDEX=dba-stack load_stack/load_stack.sh $PWD/data

From this repo. This should generate a bulk.json that can be re-used during the presentation (second run of this script won't have to generate it from the xml).

At this moment it's good idea to get kibana up and running and load the dashboard, verify that it's sane and displays correct data.

Clear all your indices:

curl -X DELETE http://localhost:9200

You should be set to go, enjoy!

Raw
00-setup.sh
curl -XDELETE http://localhost:9200/demo?pretty
Raw
01-index.sh
curl -XPUT http://localhost:9200/demo/question/1?pretty -d '{
"title": "How does Elasticsearch work?",
"author": "John",
"date": "2013-05-30"
}'
echo
curl -XPUT http://localhost:9200/demo/question/2?pretty -d '{
"title": "How does Lucene work?",
"author": "John",
"date": "2013-04-01"
}'
echo
curl -XPUT http://localhost:9200/demo/question/3?pretty -d '{
"title": "Does Elasticsearch support fuzziness?",
"author": "Mary",
"date": "2013-05-28"
}'
echo
curl -XPOST http://localhost:9200/demo/_refresh?pretty
Raw
02a-plain-search.sh
curl -XGET http://localhost:9200/demo/_search?pretty
Raw
02b-keyword-search.sh
curl -XGET 'http://localhost:9200/demo/_search?pretty&q=elasticsearch'
Raw
02c-query-string.sh
curl -XGET 'http://localhost:9200/demo/_search?pretty&q=(title:elasticsearch^10+OR+author:john)+AND+date:>=2013-05-25'
Raw
03-simple-dsl.sh
curl -XGET http://localhost:9200/demo/_search?pretty -d '{
"query": {
"filtered": {
"query": {
"bool": {
"should": [
{"match": {"title": {"query": "elasticsearch", "boost": 10}}},
{"match": {"author": "john"}}
]
}
},
"filter": {
"range": {
"date": {"from": "2013-05-25"}
}
}
}
}
}'
Raw
04a-cluster-state.sh
curl -XGET http://localhost:9200/_cluster/state?pretty
Raw
04b-index-with-3-shards.sh
curl -XPUT http://localhost:9200/shards_demo?pretty -d '{
"index": {
"number_of_shards": 3,
"number_of_replicas": 1
}
}'
Raw
04c-populate-new-index.sh
curl -XPUT http://localhost:9200/shards_demo/question/3?pretty -d '{
"title": "Does Elasticsearch suport multi-tenancy?",
"author": "Jane",
"date": "2013-03-20"
}'
echo
curl -XPOST http://localhost:9200/shards_demo/_refresh?pretty
Raw
04d-cross-index-search.sh
curl -XGET 'http://localhost:9200/demo,shards_demo/_search?pretty&q=elasticsearch'
Raw
04e-add-replicas.sh
curl -XPUT http://localhost:9200/shards_demo/_settings?pretty -d '{
"index": {
"number_of_replicas" : 2
}
}'
Raw
04f-cluster-health.sh
curl -XGET http://localhost:9200/_cluster/health?pretty
Raw
05a-analyze-plain.sh
curl -XGET 'http://localhost:9200/_analyze?pretty&text=Help+Elasticsearch+not+working'
Raw
05b-analyze-split.sh
curl -XGET 'http://localhost:9200/_analyze?pretty&text=Help+Elasticsearch+not+working&tokenizer=whitespace'
Raw
05c-analyze-split-lowercase.sh
curl -XGET 'http://localhost:9200/_analyze?pretty&text=Help+Elasticsearch+not+working&tokenizer=whitespace&filters=lowercase'
Raw
05d-analyze-split-lowercase-snoball.sh
curl -XGET 'http://localhost:9200/_analyze?pretty&text=Help+Elasticsearch+not+working&tokenizer=whitespace&filters=lowercase,snowball'
Raw
05e-analyze-snowball-shortcut.sh
curl -XGET 'http://localhost:9200/_analyze?pretty&text=Help+Elasticsearch+not+working&analyzer=snowball'
Raw
06a-get-mapping.sh
curl -XGET http://localhost:9200/demo/_mapping?pretty
Raw
06b-put-mapping.sh
curl -XPUT http://localhost:9200/dba-stack?pretty -d '{
"mappings": {
"question": {
"properties": {
"owner": {
"properties": {
"display_name": {
"type": "multi_field",
"fields": {
"display_name_raw": {"type" : "string", "analyzer" : "keyword"},
"display_name": {"type" : "string"}
}
}
}
},
"tags": {"type" : "string", "analyzer" : "keyword"},
"comments": {"type": "nested"}
}
}
}
}'
Raw
07-load-data.sh
BULK_FILE=$PWD/bulk.json ES_INDEX=dba-stack load_stack/load_stack.sh $PWD/data
Raw
08-search-stack.sh
curl -XGET http://localhost:9200/dba-stack/_search?pretty
echo
curl -XGET http://localhost:9200/dba-stack/question/_count?pretty&q=title:postgresql
echo
curl -XGET http://localhost:9200/dba-stack/question/_count?pretty&q=title:postgres
echo
Raw
09a-search-postgres-question.sh
curl -XGET http://localhost:9200/dba-stack/question,answer/_search?pretty -d '{
"query": {
"multi_match": {"fields": ["title^10", "body"], "query": "postgres postgresql"}
}
}'
Raw
09b-search-recent-questions.sh
curl -XGET http://localhost:9200/dba-stack/question,answer/_search?pretty -d '{
"query": {
"filtered": {
"query": {
"multi_match": {"fields": ["title^10", "body"], "query": "postgres postgresql"}
},
"filter": {
"range": {"creation_date": {"from": "2012-01-01"}}
}
}
}
}'
Raw
09c-search-czech-posts.sh
curl -XGET http://localhost:9200/dba-stack/question,answer/_search?pretty -d '{
"query": {
"filtered": {
"query": {
"bool": {
"must": [
{"match": {"owner.location": "czech"}},
{"multi_match": {"fields": ["title^10", "body"], "query": "postgres postgresql"}}
]
}
},
"filter": {
"range": {"creation_date": {"from": "2012-01-01"}}
}
}
}
}'
Raw
09d-search-limit-fields.sh
curl -XGET http://localhost:9200/dba-stack/question,answer/_search?pretty -d '{
"query": {
"filtered": {
"query": {
"bool": {
"must": [
{"match": {"owner.location": "czech"}},
{"multi_match": {"fields": ["title^10", "body"], "query": "postgres postgresql"}}
]
}
},
"filter": {
"range": {"creation_date": {"from": "2012-01-01"}}
}
}
},
"fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"]
}'
Raw
09e-search-custom-score.sh
curl -XGET http://localhost:9200/dba-stack/question,answer/_search?pretty -d '{
"query": {
"custom_score": {
"query": {
"filtered": {
"query": {
"bool": {
"must": [
{"match": {"owner.location": "czech"}},
{"multi_match": {"fields": ["title^10", "body"], "query": "postgres postgresql"}}
]
}
},
"filter": {
"range": {"creation_date": {"from": "2012-01-01"}}
}
}
},
"script": "_score * doc[\"rating\"].value"
}
},
"fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"]
}'
Raw
09f-search-highlight.sh
curl -XGET http://localhost:9200/dba-stack/question,answer/_search?pretty -d '{
"query": {
"custom_score": {
"query": {
"filtered": {
"query": {
"bool": {
"must": [
{"match": {"owner.location": "czech"}},
{"multi_match": {"fields": ["title^10", "body"], "query": "postgres postgresql"}}
]
}
},
"filter": {
"range": {"creation_date": {"from": "2012-01-01"}}
}
}
},
"script": "_score * doc[\"rating\"].value"
}
},
"fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"],
"highlight": {
"fields": {
"title": {
"fragment_size" : 50
},
"body": {
"fragment_size" : 50
}
}
}
}'
Raw
09g-search-facets.sh
curl -XGET http://localhost:9200/dba-stack/question,answer/_search?pretty -d '{
"query": {
"custom_score": {
"query": {
"filtered": {
"query": {
"bool": {
"must": [
{"match": {"owner.location": "czech"}},
{"multi_match": {"fields": ["title^10", "body"], "query": "postgres postgresql"}}
]
}
},
"filter": {
"range": {"creation_date": {"from": "2012-01-01"}}
}
}
},
"script": "_score * doc[\"rating\"].value"
}
},
"fields": ["title", "owner.location", "owner.display_name", "tags", "creation_date", "rating"],
"highlight": {
"fields": {
"title": {
"fragment_size" : 50
},
"body": {
"fragment_size" : 50
}
}
},
"facets": {
"tags": {"terms": {"field": "tags"}},
"frequency": {"date_histogram": {"field": "creation_date", "interval": "month"}},
"comment_stats": {"statistical": {"field": "comment_count"}}
}
}'
Raw
kibana-dash.json
{
"title": "DBA StackExchange",
"rows": [
{
"title": "Options",
"height": "50px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"loading": false,
"error": false,
"span": 5,
"editable": true,
"group": [
"default"
],
"type": "timepicker",
"mode": "since",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"5d"
],
"timespan": "5d",
"timefield": "creation_date",
"index": "dba-stack/question",
"defaultindex": "stack",
"index_interval": "none",
"refresh": {
"enable": false,
"interval": 30,
"min": 3
},
"timeformat": "",
"time": {
"from": "08/20/2012 18:38:29",
"to": "05/29/2013 18:59:58",
"index": [
"dba-stack/question"
]
}
},
{
"loading": false,
"error": false,
"span": 3,
"editable": true,
"group": [
"default"
],
"type": "dashcontrol",
"save": {
"gist": false,
"elasticsearch": true,
"local": true,
"default": true
},
"load": {
"gist": true,
"elasticsearch": true,
"local": true
},
"hide_control": false,
"elasticsearch_size": 20,
"elasticsearch_saveto": "kibana-int",
"temp": true,
"temp_ttl": "30d"
}
]
},
{
"title": "Query",
"height": "50px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"loading": false,
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "stringquery",
"label": "Search",
"query": "*",
"size": 100,
"sort": [
"_score",
"desc"
],
"multi": false,
"multi_arrange": "horizontal"
}
]
},
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"loading": false,
"span": 8,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"query": [
{
"query": "*",
"label": "*"
}
],
"interval": "1d",
"show": [
"points",
"lines",
"legend",
"x-axis",
"y-axis"
],
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"fill": 0,
"linewidth": 2,
"bars": true,
"stack": true,
"points": false,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"mode": "count",
"value_field": null,
"auto_int": true,
"resolution": 100,
"percentage": false,
"title": "Frequency"
},
{
"loading": false,
"error": false,
"span": 2,
"editable": true,
"group": [
"default"
],
"type": "pie",
"query": {
"query": "*",
"field": "tags"
},
"size": 10,
"exclude": [],
"donut": true,
"tilt": false,
"legend": true,
"labels": true,
"mode": "terms",
"default_field": "DEFAULT",
"spyable": true,
"title": "Tags"
},
{
"loading": false,
"span": 2,
"editable": true,
"group": [
"default"
],
"type": "hits",
"mode": "markdown",
"content": "",
"style": {},
"query": [
{
"query": "comment_count:[0 TO 2]",
"label": "0-2"
},
{
"query": "comment_count:[2 TO 4]",
"label": "2-4"
},
{
"query": "comment_count:[4 TO *]",
"label": "4 -"
}
],
"arrangement": "horizontal",
"chart": "bar",
"counter_pos": "above",
"donut": false,
"tilt": false,
"labels": true,
"title": "Comments"
}
]
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"loading": false,
"error": false,
"span": 2,
"editable": true,
"group": [
"default"
],
"type": "fields",
"style": {},
"arrange": "vertical",
"micropanel_position": "right",
"sort": [
"creation_date",
"desc"
],
"query": "*"
},
{
"loading": false,
"error": false,
"span": 10,
"editable": true,
"group": [
"default"
],
"type": "table",
"query": "*",
"interval": "1y",
"show": [
"bars",
"y-axis",
"x-axis",
"legend"
],
"fill": 3,
"overflow": "min-height",
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"size": 50,
"pages": 10,
"offset": 0,
"sort": [
"creation_date",
"desc"
],
"style": {
"font-size": "9pt"
},
"fields": [
"title",
"owner.display_name",
"tags",
"comment_count",
"answer_count"
],
"sortable": true,
"highlight": [],
"header": true,
"paging": true
}
]
}
],
"editable": true,
"last": null
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment