日志分析系统ELK（业务日志）
所有日志文件目录在/data/visitlog下，按日期分子目录
1，上传所有所需包至服务器
elasticsearch-1.7.0.zip
jdk-7u79-linux-x64.rpm
kibana-4.1.1-linux-x64.tar.gz
logstash-1.5.3.tar.gz
nginx-1.8.0.tar.gz
2,安装jdk
rpm -ivh nginx-1.8.0.tar.gz
echo export JAVA_HOME=/usr/java/jdk1.7.0_79/ >> /etc/profile
echo export PATH=$JAVA_HOME/bin:$PATH >> /etc/profile
echo export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar >> /etc/profile
source /etc/profile
3，部署elk
tar xzf kibana-4.1.1-linux-x64.tar.gz -C /data/elk/
tar xzf logstash-1.5.3.tar.gz -C /data/elk/
unzip elasticsearch-1.7.0.zip
mv elasticsearch-1.7.0 /data/elk/elasticsearch
cd /data/elk
mv kibana-4.1.1-linux-x64 kibana
mv logstash-1.5.3 logstash
4,配置logstash
mkdir /data/elk/logstash/etc
vim /data/elk/logstash/etc/logs.conf
input {
file {
path => ["/data/visitlog/**/*.log" ]
#start_position => "beginning" #start_position为从何处导入日志，不配置的情况下默认为从开启服务时开始导入生成的日志，beginning为将目录中所有日志导入
}
}
output {
stdout { codec=> dots }
elasticsearch {host => "localhost" }
}
5,启动服务
nohup /data/elk/kibana/bin/kibana &
nohup /data/elk/elasticsearch/bin/elasticsearch &
nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/etc/log.conf &
6，查看是否有9200 9300 5601端口启动
7，登录kibana查看
http://IP:5601
8，nginx反向代理与认证登录配置
yum install pcre-devel zlib-devel -y
tar xzf nginx-1.8.0.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx
make && make install
vim /etc/init.d/nginx
#!/bin/bash
# nginx Startup script for the Nginx HTTP Server
# this script create it by ruijie. at 2014.02.26
# if you find any errors on this scripts,please contact ruijie.
# and send mail to ruijie at gmail dot com.
# [email protected]
### BEGIN INIT INFO
# Provides: nginx
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: starts nginx
# Description: starts the nginx server
### END INIT INFO
nginxd=/usr/local/nginx/sbin/nginx
nginx_config=/usr/local/nginx/conf/nginx.conf
nginx_pid=/usr/local/nginx/logs/nginx.pid
RETVAL=0
prog="nginx"
[ -x $nginxd ] || exit 0
# Start nginx daemons functions.
start() {
if [ -e $nginx_pid ] && netstat -tunpl | grep nginx &> /dev/null;then
echo "nginx already running...."
exit 1
fi
echo -n $"Starting $prog!"
$nginxd -c ${nginx_config}
RETVAL=$?
echo
[ $RETVAL = 0 ] && touch /var/lock/nginx
return $RETVAL
}
# Stop nginx daemons functions.
stop() {
echo -n $"Stopping $prog!"
$nginxd -s stop
RETVAL=$?
echo
[ $RETVAL = 0 ] && rm -f /var/lock/nginx
}
# reload nginx service functions.
reload() {
echo -n $"Reloading $prog!"
#kill -HUP `cat ${nginx_pid}`
$nginxd -s reload
RETVAL=$?
echo
}
# See how we were called.
case "$1" in
start)
start
;;
stop)
stop
;;
reload)
reload
;;
restart)
stop
start
;;
*)
echo $"Usage: $prog {start|stop|restart|reload|help}"
exit 1
esac
exit $RETVAL
chmod +x /etc/init.d/nginx
mkdir /usr/local/nginx/conf/conf.d
nginx.conf http模块中添加include /usr/local/nginx/conf/conf.d/*.conf; 注释掉server模块
vim /usr/local/nginx/conf/conf.d/kibana.conf
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/kibana.access.log main;
error_log logs/kibana.error.log;
location / {
#root html;
#index index.html index.htm;
auth_basic "secret";
auth_basic_user_file /usr/local/nginx/passwd.db;
proxy_pass http://127.0.0.1:5601/;
proxy_set_header Cookie $http_cookie;
#proxy_cookie_path /vga/ /;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}
htpasswd -c /usr/local/nginx/passwd.db admin #admin为登录用户
chmod 777 passwd.db
service nginx start
9，现在可直接使用IP地址来登录kibana，提示输入用户名密码