第一节 单Elasticsearch单input部署
1、文件数据进入Elasticsearch
1.1、配置文件
test_elasticsearch.conf
input
{
file
{
path =>"/home/anbc/test/logstash-tutorial.log"
start_position =>beginning
ignore_older => 0
}
}
filter
{
grok
{
match =>{"message" => "%{COMBINEDAPACHELOG}"}
}
}
output
{
stdout
{
}
elasticsearch
{
}
}
1.2、运行Elasticsearch
执行下面命令：
/opt/elasticsearch/bin/elasticsearch
了解更详细的Elasticsearch安装配置信息，参见《第二章 Elasticsearch使用手册》中的《第二节 安装》
1.3、测试命令
执行命令：
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/test_elasticsearch.conf --configtest
执行结果：
1.4、执行命令
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/test_elasticsearch.conf
如果获得更详细的调试信息，可以增加参数--debug 或 --verbose如下所示：
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/test_elasticsearch.conf --debug
1.5、验证数据是否存储成功
添加成功后，索引名称为：logstash-2016.06.08，类型名称为：logs
通过查询Elasticsearch中的索引信息来判断，执行命令
curl 'localhost:9200/_cat/indices?v'
返回信息中可以看到，ElasticSearch中增加了一个名为“logstash-2016.06.08”的索引：
发送的请求：
curl -XGET 'localhost:9200/logstash-2016.06.08/_search?q=response=200'
返回的结果：
{
"took": 1246,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 98,
"max_score": 5.3242445,
"hits": [
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19T9IHRPCebL71F",
"_score": 5.3242445,
"_source": {
"message": "83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] \"GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1\" 200 52878 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.486Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "83.149.9.216",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:13:45 +0000",
"verb": "GET",
"request": "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png",
"httpversion": "1.1",
"response": "200",
"bytes": "52878",
"referrer": "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19U9IHRPCebL71K",
"_score": 0.17486966,
"_source": {
"message": "83.149.9.216 - - [04/Jan/2015:05:13:46 +0000] \"GET /presentations/logstash-monitorama-2013/images/nagios-sms5.png HTTP/1.1\" 200 78075 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.536Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "83.149.9.216",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:13:46 +0000",
"verb": "GET",
"request": "/presentations/logstash-monitorama-2013/images/nagios-sms5.png",
"httpversion": "1.1",
"response": "200",
"bytes": "78075",
"referrer": "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19U9IHRPCebL71N",
"_score": 0.17486966,
"_source": {
"message": "83.149.9.216 - - [04/Jan/2015:05:13:47 +0000] \"GET /presentations/logstash-monitorama-2013/images/logstashbook.png HTTP/1.1\" 200 54662 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.546Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "83.149.9.216",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:13:47 +0000",
"verb": "GET",
"request": "/presentations/logstash-monitorama-2013/images/logstashbook.png",
"httpversion": "1.1",
"response": "200",
"bytes": "54662",
"referrer": "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19U9IHRPCebL71Q",
"_score": 0.17486966,
"_source": {
"message": "83.149.9.216 - - [04/Jan/2015:05:13:47 +0000] \"GET /presentations/logstash-monitorama-2013/images/1983_delorean_dmc-12-pic-38289.jpeg HTTP/1.1\" 200 220562 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.585Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "83.149.9.216",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:13:47 +0000",
"verb": "GET",
"request": "/presentations/logstash-monitorama-2013/images/1983_delorean_dmc-12-pic-38289.jpeg",
"httpversion": "1.1",
"response": "200",
"bytes": "220562",
"referrer": "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"",
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19U9IHRPCebL71T",
"_score": 0.17486966,
"_source": {
"message": "83.149.9.216 - - [04/Jan/2015:05:13:53 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.610Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "83.149.9.216",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:13:53 +0000",
"verb": "GET",
"request": "/favicon.ico",
"httpversion": "1.1",
"response": "200",
"bytes": "3638",
"referrer": "\"-\"",
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19U9IHRPCebL71Y",
"_score": 0.17486966,
"_source": {
"message": "93.114.45.13 - - [04/Jan/2015:05:14:33 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.657Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "93.114.45.13",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:14:33 +0000",
"verb": "GET",
"request": "/favicon.ico",
"httpversion": "1.1",
"response": "200",
"bytes": "3638",
"referrer": "\"-\"",
"agent": "\"Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19U9IHRPCebL71a",
"_score": 0.17486966,
"_source": {
"message": "93.114.45.13 - - [04/Jan/2015:05:14:33 +0000] \"GET /images/web/2009/banner.png HTTP/1.1\" 200 52315 \"http://www.semicomplete.com/style2.css\" \"Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.684Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "93.114.45.13",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:14:33 +0000",
"verb": "GET",
"request": "/images/web/2009/banner.png",
"httpversion": "1.1",
"response": "200",
"bytes": "52315",
"referrer": "\"http://www.semicomplete.com/style2.css\"",
"agent": "\"Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19V9IHRPCebL71d",
"_score": 0.17486966,
"_source": {
"message": "66.249.73.185 - - [04/Jan/2015:05:15:23 +0000] \"GET / HTTP/1.1\" 200 37932 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.708Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "66.249.73.185",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:15:23 +0000",
"verb": "GET",
"request": "/",
"httpversion": "1.1",
"response": "200",
"bytes": "37932",
"referrer": "\"-\"",
"agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19V9IHRPCebL71g",
"_score": 0.17486966,
"_source": {
"message": "110.136.166.128 - - [04/Jan/2015:05:16:22 +0000] \"GET /reset.css HTTP/1.1\" 200 1015 \"http://www.semicomplete.com/projects/xdotool/\" \"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.731Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "110.136.166.128",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:16:22 +0000",
"verb": "GET",
"request": "/reset.css",
"httpversion": "1.1",
"response": "200",
"bytes": "1015",
"referrer": "\"http://www.semicomplete.com/projects/xdotool/\"",
"agent": "\"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0\""
}
},
{
"_index": "logstash-2016.06.08",
"_type": "logs",
"_id": "AVUvp19V9IHRPCebL71k",
"_score": 0.17486966,
"_source": {
"message": "123.125.71.35 - - [04/Jan/2015:05:16:31 +0000] \"GET /blog/tags/release HTTP/1.1\" 200 40693 \"-\" \"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)\"",
"@version": "1",
"@timestamp": "2016-06-08T10:55:41.740Z",
"path": "/home/anbc/test/logstash-tutorial_1.log",
"host": "localhost.localdomain",
"clientip": "123.125.71.35",
"ident": "-",
"auth": "-",
"timestamp": "04/Jan/2015:05:16:31 +0000",
"verb": "GET",
"request": "/blog/tags/release",
"httpversion": "1.1",
"response": "200",
"bytes": "40693",
"referrer": "\"-\"",
"agent": "\"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)\""
}
}
]
}
}