部署elk平台

2018-04-25 16:12:43 浏览数 (1)

说明

对于ELK部署使用而言，下面是一个再常见不过的架构了

  1. Redis：接收用户日志的消息队列。
  2. Logstash：做日志解析，统一成JSON输出给Elasticsearch。
  3. Elasticsearch：实时日志分析服务的核心技术，一个schemaless，实时的数据存储服务，通过index组织数据，兼具强大的搜索和统计功能。
  4. Kibana：基于Elasticsearch的数据可视化组件，超强的数据可视化能力是众多公司选择ELK stack的重要原因。

系统环境

代码语言：javascript复制
[root@localhost ~]# cat /etc/redhat-release 
CentOS Linux release 7.3.1611 (Core)

软件下载

代码语言：javascript复制
wget https://download.elastic.co/logstash/logstash/logstash-2.3.2.tar.gz
wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.2/elasticsearch-2.3.2.tar.gz
wget https://download.elastic.co/kibana/kibana/kibana-4.5.0-linux-x64.tar.gz
wget https://github.com/antirez/redis/archive/3.2.0.tar.gz

nginx

代码语言：javascript复制
#安装nginx
yum install nginx -y
#启动nginx
nginx
#查看端口
netstat -nplt | grep nginx
#修改nginx访问日志
[root@localhost ~]# vi /etc/nginx/nginx.conf
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
[root@localhost ~]# vi /etc/nginx/conf.d/default.conf 
#定义nginx日志使用的格式，以及日志文件的位置
   access_log  /var/log/nginx/access.log  main;
#重启nginx
nginx -s reload

redis

代码语言：javascript复制
#安装依赖
yum install tcl -y
yum groupinstall "Development Tools" "Compatibility Libraries" -y
#安装redis
tar xvf 3.2.0.tar.gz -C /usr/local
cd /usr/local
mv redis-3.2.0 redis
cd redis
make
make test
make install
mkdir /etc/redis
cp redis/redis.conf /etc/redis
#配置redis
[root@localhost ~]# vim /etc/redis/redis.conf
daemonize yes
启动redis
[root@localhost ~]# redis-server /etc/redis/redis.conf
[root@localhost ~]# netstat -nltp | grep redis
tcp        0      0 127.0.0.1:6379          0.0.0.0:*               LISTEN      32357/redis-server

java

代码语言：javascript复制
安装java
yum install java-1.8.0-openjdk -y

Logstash

代码语言：javascript复制
安装Logstash
tar xvf logstash-2.3.2.tar.gz -C /usr/local/
#在终端中，像下面这样运行命令来启动 Logstash 进程：
$ /usr/local/logstash-2.3.2/bin/logstash -e 'input{stdin{}}output{stdout{codec=>rubydebug}}'
hello world

$ /usr/local/logstash-2.3.2/bin/logstash -e 'input{stdin{}}output{stdout{codec=>rubydebug}}'
hello world
执行完命令，然后你会发现终端在等待你的输入。没问题，敲入hello world，然后回车，logstash会返回以下结果！
{
       "message" => "hello world",
      "@version" => "1",
    "@timestamp" => "2017-11-04T11:50:27.036Z",
          "host" => "localhost.localdomain"
}
编写Shipper角色的配置文件：logstash_shipper.conf
$ cat /usr/local/logstash-2.3.2/etc/logstash_shipper.conf
input {
      file {
          type => "nginx_access log"                  #这里定义的是日志文件名；
          path => ["/var/log/nginx/access.log"]       #这里定义的是日志文件路径；
      }
}
output {
      redis {
          host => "localhost"      #redis主机地址，这里是本机；
          port => 6379             #redis端口号；
          data_type => "list"      #使用redis队列模式；  
          key => "logstash:redis"  #队列通道的名称；
      }
}
编写indexer角色的配置文件：logstash_indexer.conf
cat /usr/local/logstash-2.3.2/etc/logstash_indexer.conf
input {
      redis {                        #去redis队列取数据；
          host => "localhost"        #连接redis服务器；
          port => 6379               #连接redis端口；
          data_type => "list"        #数据类型；
          key => "logstash:redis"    #队列名称；
      }
}
output {
    elasticsearch {                             #Logstash输出到elasticsearch；
      hosts => ["localhost"]                    #elasticsearch为本地；
      index => "logstash-nginx-%{ YYYY.MM.dd}"  #创建索引；
      document_type => "nginx"                  #文档类型；
      workers => 1                              #进程数量；
      flush_size => 20000
      idle_flush_time => 10
 }
}
#启动
nohup /usr/local/logstash-2.3.2/bin/logstash -f /usr/local/logstash-2.3.2/etc/logstash_shipper.conf &
nohup /usr/local/logstash-2.3.2/bin/logstash -f /usr/local/logstash-2.3.2/etc/logstash_indexer.conf

elastcearch

代码语言：javascript复制
tar xvf elasticsearch-2.3.2.tar.gz -C /usr/local
#创建用户
useradd elk
chown elk.elk -R /usr/local/elastcsearch.2.3.2
#修改elasticsearch配置文件
cat /usr/local/elasticsearch-2.3.2/config/elasticsearch.yml
cluster.name: elasticsearch
node.name: node1 
node.box_type: stale
path.data: ['/data/elasticsearch']
path.logs: /var/log/elasticsearch/
network.host: 0.0.0.0
index.number_of_replicas: 0
#创建elasticsearch需要的数据目录和日志目录。
mkdir -p /data/elasticsearch 
mkdir -p /var/log/elasticsearch/ 
chown elk.elk /data/elasticsearch/ -R 
chown elk.elk /var/log/elasticsearch/ -R
#启动
nohup sudo -u elk /usr/local/elasticsearch-2.3.2/bin/elasticsearch &
#访问
[root@localhost ~]# curl http://128.0.0.71:9200/
{
  "name" : "node1",
  "cluster_name" : "elasticsearch",
  "version" : {
    "number" : "2.3.2",
    "build_hash" : "b9e4a6acad4008027e4038f6abed7f7dba346f94",
    "build_timestamp" : "2016-04-21T16:03:47Z",
    "build_snapshot" : false,
    "lucene_version" : "5.5.0"
  },
  "tagline" : "You Know, for Search"
}

Elasticsearch安装head插件

代码语言：javascript复制
/usr/local/elasticsearch-2.3.2/bin/plugin install mobz/elasticsearch-head
查看插件
/usr/local/elasticsearch-2.3.2/bin/plugin list

kibana

代码语言：javascript复制
tar xvf kibana-4.5.0-linux-x64.tar.gz -C /usr/local/
启动
nohup /usr/local/kibana-4.5.0-linux-x64/bin/kibana &
系统架构 es2

0 人点赞