首页 技术 正文

ELK (Elasticsearch+Logstash+Kibana)部署

技术 2022年11月17日
0 收藏 340 点赞 4,120 浏览 6716 个字

部署机器:

服务端：
dev-server    X.X.X.X      ( logstash-1.5.4，elasticsearch-1.7.1，kibana-4.1.1 )

客户端:
dev-client    X.X.X.X        (logstash-forwarder-0.4.0-1)

需求:将客户端访问日志(nginx log)在ELK中展示。
安装ELK：
服务端:
设置FQDN(创建SSL证书的时候需要配置FQDN)：

[root@dev-client ~]# hostname
dev-client
[root@dev-client ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4
X.X.X.X      elk.test.com elk

安装Java 1.8:

[root@dev-server elk]# cat /etc/redhat-release
CentOS release 6.6 (Final)
[root@dev-server elk]# yum install java-1.8.0-openjdk.x86_64
[root@dev-server elk]# java -version
openjdk version "1.8.0_65"
OpenJDK Runtime Environment (build 1.8.0_65-b17)
OpenJDK 64-Bit Server VM (build 25.65-b01, mixed mode)

安装   elasticsearch-1.7.1:
#下载安装

[root@dev-server elk]# wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.noarch.rpm

#启动相关服务

[root@dev-server elk]# /etc/init.d/elasticsearch start
[root@dev-server elk]# /etc/init.d/elasticsearch stop

#查看elasticsearch配置文件

[root@dev-server elk]# rpm -qc elasticsearch
/etc/elasticsearch/elasticsearch.yml
/etc/elasticsearch/logging.yml
/etc/init.d/elasticsearch
/etc/sysconfig/elasticsearch
/usr/lib/sysctl.d/elasticsearch.conf
/usr/lib/systemd/system/elasticsearch.service
/usr/lib/tmpfiles.d/elasticsearch.conf

#查看端口使用情况

[root@dev-server elk]# netstat -tunlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       PID/Program name
tcp        0      0 :::9300                     :::*                        LISTEN      14585/java
tcp        0      0 :::9200                     :::*                        LISTEN      14585/java

安装Kibana 4.1.1:
#下载tar包

[root@dev-server elk]# wget https://download.elastic.co/kibana/kibana/kibana-4.1.1-linux-x64.tar.gz

#解压

[root@dev-server elk]# pwd
/data1/elk
[root@dev-server elk]# tar xf kibana-4.1.1-linux-x64.tar.gz
[root@dev-server elk]# ln -s /data1/elk/kibana-4.1.1-linux-x64 kibana

#创建kibana服务

[root@dev-server elk]# cat /etc/init.d/kibana
#!/bin/bash
### BEGIN INIT INFO
# Provides:          kibana
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Runs kibana daemon
# Description: Runs the kibana daemon as a non-root user
### END INIT INFO# Process name
NAME=kibana
DESC="Kibana4"
PROG="/etc/init.d/kibana"# Configure location of Kibana bin
KIBANA_BIN=/data1/elk/kibana/bin               #注意路径# PID Info
PID_FOLDER=/var/run/kibana/
PID_FILE=/var/run/kibana/$NAME.pid
LOCK_FILE=/var/lock/subsys/$NAME
PATH=/bin:/usr/bin:/sbin:/usr/sbin:$KIBANA_BIN
DAEMON=$KIBANA_BIN/$NAME# Configure User to run daemon process
DAEMON_USER=root
# Configure logging location
KIBANA_LOG=/var/log/kibana.log# Begin Script
RETVAL=0if [ `id -u` -ne 0 ]; then
        echo "You need root privileges to run this script"
        exit 1
fi# Function library
. /etc/init.d/functionsstart() {
        echo -n "Starting $DESC : "pid=`pidofproc -p $PID_FILE kibana`
        if [ -n "$pid" ] ; then
                echo "Already running."
                exit 0
        else
        # Start Daemon
if [ ! -d "$PID_FOLDER" ] ; then
                        mkdir $PID_FOLDER
                fi
daemon --user=$DAEMON_USER --pidfile=$PID_FILE $DAEMON 1>"$KIBANA_LOG" 2>&1 &
                sleep 2
                pidofproc node > $PID_FILE
                RETVAL=$?
                [[ $? -eq 0 ]] && success || failure
echo
                [ $RETVAL = 0 ] && touch $LOCK_FILE
                return $RETVAL
        fi
}reload()
{
    echo "Reload command is not implemented for this service."
    return $RETVAL
}stop() {
        echo -n "Stopping $DESC : "
        killproc -p $PID_FILE $DAEMON
        RETVAL=$?
echo
        [ $RETVAL = 0 ] && rm -f $PID_FILE $LOCK_FILE
}case "$1" in
  start)
        start
;;
  stop)
        stop
        ;;
  status)
        status -p $PID_FILE $DAEMON
        RETVAL=$?
        ;;
  restart)
        stop
        start
        ;;
  reload)
reload
;;
  *)
# Invalid Arguments, print the following message.
        echo "Usage: $0 {start|stop|status|restart}" >&2
exit 2
        ;;
esac

#修改启动权限

[root@dev-server elk]# chmod +x /etc/init.d/kibana

#启动kibana服务

[root@dev-server elk]# /etc/init.d/kibana start
[root@dev-server elk]# /etc/init.d/kibana status

#查看端口

[root@dev-server elk]# netstat -tunlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       PID/Program name
tcp        0      0 0.0.0.0:5601(默认，可以改为80)        0.0.0.0:        LISTEN      15128/node

安装logstash 1.5.4
#下载安装

[root@dev-server elk]# wget https://download.elastic.co/logstash/logstash/packages/centos/logstash-1.5.4-1.noarch.rpm
[root@dev-server elk]# yum localinstall logstash-1.5.4-1.noarch.rpm

#设置ssl，之前设置的FQDN是elk.test.com

[root@dev-server tls]# pwd
/etc/pki/tls
[root@dev-server tls]# openssl req -subj '/CN=elk.test.com/' -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt[root@dev-server certs]# pwd
/etc/pki/tls/certs
[root@dev-server certs]# ls -l logstash-forwarder.crt
-rw-r--r-- 1 root root 1103 Nov 23 22:46 logstash-forwarder.crt

#创建一个01-logstash-initial.conf文件

[root@dev-server conf.d]# cat 01-logstash-initial.conf
input {
  lumberjack {
    port => 5000
    type => "logs"
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}filter {
  if [type] == "nginx" {
    grok {
       match => { "message" => "%{NGINXACCESS}" }
    }
  }
}output {
  elasticsearch {
     index => "zabbix-access-%{+YYYY.MM.dd}"
     host => localhost
     }
  stdout { codec => rubydebug }
}

#启动logstash服务

[root@dev-server conf.d]# /etc/init.d/logstash start
[root@dev-server conf.d]# /etc/init.d/logstash stop

#查看端口

[root@dev-server conf.d]# netstat -tunlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       PID/Program name
tcp        0      0 :::9301                     :::*              LISTEN      4381/java
tcp        0      0 :::5000                     :::*              LISTEN      4381/java

#启动客户端logstash（后面会讲解客户端）

[root@dev-client ~]# /etc/init.d/logstash-forwarder start
[root@dev-client ~]# /etc/init.d/logstash-forwarder status

#访问kibana
http://XXXXX.XXX
 
#增加节点和客户端配置一样,注意同步证书

/etc/pki/tls/certs/logstash-forwarder.crt

客户端安装logstash-forwarder :
#安装客户端

[root@dev-client opt]# wget https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder-0.4.0-1.x86_64.rpm
[root@dev-client opt]# yum localinstall logstash-forwarder-0.4.0-1.x86_64.rpm

#查看配置文件

[root@dev-client opt]# rpm -qc logstash-forwarder
/etc/logstash-forwarder.conf

#备份配置文件

[root@dev-client opt]# cp /etc/logstash-forwarder.conf /etc/logstash-forwarder.conf.save

#编辑配置文件

[root@dev-client opt]# cat /etc/logstash-forwarder.conf
{
  "network": {
    "servers": [ "elk.test.com:5000" ],    "ssl ca": "/etc/pki/tls/certs/logstash-forwarder.crt",    "timeout": 15
  },  "files": [
    {
      "paths": [
        "/var/log/nginx/log/zabbix.access.log"
      ],
      "fields": { "type": "nginx" }
    }
  ]
}

配置日志规则
#服务端增加patterns

[root@dev-server ]# mkdir /opt/logstash/patterns/[root@dev-server patterns]# cat nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:remote_addr} - - \[%{HTTPDATE:time_local}\] "%{WORD:method} %{URIPATH:path}(?:%{URIPARAM:param})? HTTP/%{NUMBER:httpversion}" %{INT:status} %{INT:body_bytes_sent} %{QS:http_referer} %{QS:http_user_agent}

#修改logstash权限

[root@dev-server patterns]# chown  -R logstash:logstash /opt/logstash/patterns/

#修改服务端配置

[root@dev-server patterns]# cat /etc/logstash/conf.d/01-logstash-initial.conf
input {
  lumberjack {
    port => 5000
    type => "logs"
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}filter {
  if [type] == "nginx" {
    grok {
       match => { "message" => "%{NGINXACCESS}" }
    }
  }
}output {
  elasticsearch {
     index => "zabbix-access-%{+YYYY.MM.dd}"
     host => localhost
     }
  stdout { codec => rubydebug }
}

修改Kibana端口

[root@dev-server config]# pwd
/data1/elk/kibana/config
[root@dev-server config]# cat kibana.yml | grep port
# Kibana is served by a back end server. This controls which port to use.
#port: 5601
port: 80
 

访问Kibana
http://XXX.XXX

点赞 340

贡献者

上一篇： 封装php redis缓存操作类
下一篇： 20165302Exp0 Kali安装 Week1
相关推荐
python开发_常用的python模块及安装方法
python开发_常用的python模块及安装方法
adodb：我们领导推荐的数据库连接组件bsddb3：BerkeleyDB的连接组件Cheetah-1.0：我比较喜欢这个版本的cheeta…
技术
日期：2022-11-24 点赞：878 阅读：9,105
Educational Codeforces Round 11 C. Hard Process 二分
Educational Codeforces Round 11 C. Hard Process 二分
C. Hard Process题目连接：http://www.codeforces.com/contest/660/problem/CDes…
技术
日期：2022-11-24 点赞：807 阅读：5,582
下载Ubuntn 17.04 内核源代码
下载Ubuntn 17.04 内核源代码
zengkefu@server1:/usr/src$ uname -aLinux server1 4.10.0-19-generic #21…
技术
日期：2022-11-24 点赞：569 阅读：6,429
可用Active Desktop Calendar V7.86 注册码序列号
可用Active Desktop Calendar V7.86 注册码序列号
可用Active Desktop Calendar V7.86 注册码序列号Name: www.greendown.cn Code: &nb…
技术
日期：2022-11-24 点赞：733 阅读：6,200
Android调用系统相机、自定义相机、处理大图片
Android调用系统相机、自定义相机、处理大图片
Android调用系统相机和自定义相机实例本博文主要是介绍了android上使用相机进行拍照并显示的两种方式，并且由于涉及到要把拍到的照片显…
技术
日期：2022-11-24 点赞：512 阅读：7,836
Struts的使用
Struts的使用
一、Struts2的获取　　Struts的官方网站为：http://struts.apache.org/　　下载完Struts2的jar包,…
技术
日期：2022-11-24 点赞：671 阅读：4,919
Lomu

贡献者

这家伙很懒，只想把你留下。

文章
56
收藏
0
人气
179,888
粉丝
进主页
文章展示
  • [ 技术 ] python开发_常用的python模块及安装方法
  • [ 技术 ] Educational Codeforces Round 11 C. Hard Process 二分
  • [ 技术 ] 下载Ubuntn 17.04 内核源代码
  • [ 技术 ] 可用Active Desktop Calendar V7.86 注册码序列号
  • [ 技术 ] Android调用系统相机、自定义相机、处理大图片
  • [ 技术 ] Struts的使用
    • 个人收藏笔记记录
    我的笔记
    首页 首页 菜单 首页 首页