March 20, 2014
Elasticsearch Rosetta Stone
I’ve recently done some work with Elasticsearch. It’s a great platform, but I still found it pretty hard to get started. It wasn’t obvious what was what. Perhaps I didn’t read the docs closely enough. Anyhow, I wrote up some examples demonstrating how to get started with Elasticsearch.
Quick Intro
You’ll need to have an elasticsearch running to run these examples. Just
download elasticsearch,
unzip it, and run bin/elasticsearch. No need to use anything but the defaults.
Some vocabulary: Elasticsearch stores documents with types in indexes (noun).
Approximately, document : type : index :: row : table : database.
To add a new document, you index (verb) it.
Since elasticsearch uses a REST api, most client libraries reflect this. Most methods require arguments for index, type, and another argument representing the request body, usually some sort of JSON-esque data structure.
Examples
Ok, enough of that. I wrote examples in Ruby, Python, Javascript, and PHP, because people seem to be into that sort of thing. You can find all the samples on Github here
Ruby
require 'elasticsearch'
describe Elasticsearch do
it "can connect to an elasticsearch server" do
# Non-java clients connect to Elasticsearch via the REST api,
# which runs on port 9200 by default.
client = Elasticsearch::Client.new({
host: 'localhost',
port: 9200
})
end
it "can add arbitrary documents to the index" do
client = Elasticsearch::Client.new
# Elasticsearch can store arbitrary documents
post = {
title: "My Document",
body: "Hello world."
}
# Documents in elasticsearch have an INDEX and a
# TYPE. You can also specify an ID here if you
# want to generate your own.
client.index({
index: "blog", # Think "database"
type: "post", # Think "table"
body: post
})
end
describe "search" do
let(:post_1) do
{
"title" => "Hello World",
"body" => "This is a post"
}
end
let(:post_2) do
{
"title" => "Camp Grenada",
"body" => "Hello mother, hello father."
}
end
before(:each) do
@client = Elasticsearch::Client.new
# Drop the whole blog index for testing purposes
@client.delete_by_query index: "blog", body: {query: {"match_all" => {}}}
[post_1, post_2].each do |post|
@client.index index: "blog", type: "post", body: post
end
# If the index is not refreshed, we won't find our documents right away.
@client.perform_request "POST", "/blog/_refresh"
end
it "returns both results when searching for 'hello'" do
query = {
term: { # Search for a given term
_all: "hello" # in any field (_all is a special wildcard)
}
}
search_result = @client.search({
index: "blog",
type: "post",
body: {query: query}
})
# The search result contains some info about itself (this is not exhaustive)
search_result.should include({
"timed_out" => false,
"_shards" => {
"total" => 5,
"successful" => 5,
"failed" => 0
},
})
# The search hits are in "hits"
search_result.should include "hits"
# "hits" contains some metadata
search_result["hits"].should include(
"total", "max_score", "hits"
)
# "Camp Grenada" comes first because it has more hellos
search_result["hits"]["total"].should eq(2)
search_result["hits"]["hits"][0]["_source"].should eq(post_2)
search_result["hits"]["hits"][1]["_source"].should eq(post_1)
end
it "returns one result when searching for 'Hello' in the title" do
query = {
term: { # Search for a given term
title: "hello" # in the title
}
}
search_result = @client.search({
index: "blog",
type: "post",
body: {query: query}
})
search_result["hits"]["total"].should eq(1)
end
end
end
Python
from unittest import TestCase, main
from elasticsearch import Elasticsearch
class TestElasticsearch(TestCase):
def setUp(self):
self.client = Elasticsearch()
def test_connect_to_server(self):
"""Non-java clients connect to Elasticsearch via the REST api,
which runs on port 9200 by default."""
Elasticsearch(
host="localhost",
port=9200
)
def test_index_document(self):
"""Elasticsearch can store arbitrary documents"""
post = {
"title": "My Document",
"body": "Hello world."
}
# Documents in elasticsearch have an INDEX and a
# TYPE. You can also specify an ID here if you
# want to generate your own.
self.client.index(
index="blog", # Think "database"
doc_type="post",
body=post
)
def test_search_all(self):
"""Elasticsearch can search for your documents in many ways.
"""
post_1 = {
"title": "Hello World",
"body": "This is a post"
}
post_2 = {
"title": "Camp Grenada",
"body": "Hello mother, hello father."
}
self.client.delete_by_query(
index="blog",
body={'query': {"match_all": {}}}
)
for post in [post_1, post_2]:
self.client.index(
index="blog",
doc_type="post",
body=post)
# If the index is not refreshed, we won't find our documents
# right away.
self.client.indices.refresh(index="blog")
query = {
'term': { # Search for a given term
'_all': 'hello' # in any field (_all is a special wildcard)
}
}
search_result = self.client.search(
index="blog",
doc_type="post",
body={'query': query}
)
# The search result contains some info about itself
# (this is not exhaustive)
self.assertFalse(search_result['timed_out'])
self.assertEqual(
{'total': 5, 'successful': 5, 'failed': 0},
search_result['_shards'])
assert "hits" in search_result
assert "total" in search_result['hits']
assert "max_score" in search_result['hits']
assert "hits" in search_result['hits'] # Where the actual hits live
# "Camp Grenada" comes first because it has more hellos
self.assertEqual(2, search_result['hits']['total'])
self.assertEqual(post_2, search_result['hits']['hits'][0]['_source'])
self.assertEqual(post_1, search_result['hits']['hits'][1]['_source'])
# We can search by field, too
query = {
'term': { # Search for a given term
'title': 'hello' # in any field (_all is a special wildcard)
}
}
search_result = self.client.search(
index="blog",
doc_type="post",
body={'query': query}
)
self.assertEqual(1, search_result['hits']['total'])
self.assertEqual(post_1, search_result['hits']['hits'][0]['_source'])
if __name__ == "__main__":
main()
Javascript
var elasticsearch = require('elasticsearch');
// Non-java clients connect to Elasticsearch via the REST api,
// which runs on port 9200 by default.
var client = new elasticsearch.Client({
host: 'localhost:9200'
});
// Elasticsearch can store arbitrary documents
post = {
title: "My Document",
body: "Hello world."
};
// Documents in elasticsearch have an INDEX and a
// TYPE. You can also specify an ID here if you
// want to generate your own.
client.index({
index: "blog", // Think "database"
type: "post", // Think "table"
body: post
}).then(function(){
// Drop the whole blog index for testing purposes
return client.deleteByQuery({
index: "blog",
body: {query: {"match_all": {}}}
});
}).then(function(){
post_1 = {
"title": "Hello World",
"body": "This is a post"
};
return client.index({index: "blog", type: "post", body: post_1});
}).then(function(){
post_2 = {
"title": "Camp Grenada",
"body": "Hello mother, hello father."
};
return client.index({index: "blog", type: "post", body: post_2});
}).then(function(){
// If the index is not refreshed, we won't find our documents right away.
return client.indices.refresh({index: "blog"});
}).then(function(){
query = {
term: { // Search for a given term
_all: "hello" // in any field (_all is a special wildcard)
}
};
returned = 0;
return client.search({
index: "blog",
type: "post",
body: {query: query}
});
}).then(function(search_result){
console.assert(!search_result.timed_out);
console.assert(search_result._shards.total === 5);
console.assert(search_result._shards.successful === 5);
console.assert(search_result._shards.failed === 0);
console.assert(search_result.hits.total === 2);
actual = search_result.hits.hits[0]._source;
console.assert(actual.title == post_2.title);
console.assert(actual.body == post_2.body);
actual = search_result.hits.hits[1]._source;
console.assert(actual.title == post_1.title);
console.assert(actual.body == post_1.body);
}).then(function(){
query = {
term: { // Search for a given term
title: "hello" // in the title
}
};
return client.search({
index: "blog",
type: "post",
body: {query: query}
});
}).then(function(search_result){
console.assert(search_result.hits.total === 1);
console.log("Ok.");
}, function(e){
console.log("FAILED");
});
PHP
<?php
require 'vendor/autoload.php';
# Non-java clients connect to Elasticsearch via the REST api,
# which runs on port 9200 by default."""
$client = new Elasticsearch\Client(array(
'hosts' => ['localhost:9200']
));
# Elasticsearch can store arbitrary documents
$post = array(
"title" => "My Document",
"body" => "Hello world."
);
# Documents in elasticsearch have an INDEX and a
# TYPE. You can also specify an ID here if you
# want to generate your own.
$client->index(array(
'index' => 'blog', # Think "database"
'type' => "post", # Think "table"
'body' => $post
));
# Delete that post
$client->deleteByQuery(array(
"index" => "blog",
"body" => array(
'query' => array(
"match_all" => array()))));
$post_1 = array(
"title" => "Hello World",
"body" => "This is a post"
);
$post_2 = array(
"title" => "Camp Grenada",
"body" => "Hello mother, hello father."
);
# Index those documents
$client->index(array(
'index' => 'blog',
'type' => "post",
'body' => $post_1));
$client->index(array(
'index' => 'blog',
'type' => "post",
'body' => $post_2));
# If the index is not refreshed, we won't find our documents
# right away.
$client->indices()->refresh(array("index" => "blog"));
$query = array(
'term' => array( # Search for a given term
'_all' => 'hello' # in any field (_all is a special wildcard)
)
);
$search_result = $client->search(array(
"index" => "blog",
"type" => "post",
"body" => array('query' => $query)
));
# The search result contains some info about itself
# (this is not exhaustive)
assert(!$search_result['timed_out']);
assert($search_result['_shards']['total'] === 5);
assert($search_result['_shards']['successful'] === 5);
assert($search_result['_shards']['failed'] === 0);
assert($search_result["hits"]);
assert($search_result["hits"]["total"] === 2);
assert($search_result["hits"]["hits"][0]['_source'] == $post_2);
assert($search_result["hits"]["hits"][1]['_source'] == $post_1);
echo "Ok.\n";