nginx日志导入elasticsearch的方法示例

å°nginxæ¥å¿éè¿filebeatæ¶éåä¼ å¥logstashï¼ç»è¿logstashå¤çååå¥elasticsearchãfilebeatåªè´è´£æ¶éå·¥ä½ï¼logstashå®ææ¥å¿çæ ¼å¼åï¼æ°æ®çæ¿æ¢ï¼æå ï¼ä»¥åå°æ¥å¿åå¥elasticsearchåçç´¢å¼çå建ã

1ãéç½®nginxæ¥å¿æ ¼å¼

log_format main    '$remote_addr $http_x_forwarded_for [$time_local] $server_name $request ' 
            '$status $body_bytes_sent $http_referer ' 
            '"$http_user_agent" '
            '"$connection" '
            '"$http_cookie" '
            '$request_time '
            '$upstream_response_time';

2ãå®è£éç½®filebeatï¼å¯ç¨nginx module

tar -zxvf filebeat-6.2.4-linux-x86_64.tar.gz -C /usr/local
cd /usr/local;ln -s filebeat-6.2.4-linux-x86_64 filebeat
cd /usr/local/filebeat

å¯ç¨nginx模å

./filebeat modules enable nginx

æ¥ç模å

./filebeat modules list

å建éç½®æ件

vim /usr/local/filebeat/blog_module_logstash.yml
filebeat.modules:
- module: nginx
 access:
  enabled: true
  var.paths: ["/home/weblog/blog.cnfol.com_access.log"]
 #error:
 # enabled: true
 # var.paths: ["/home/weblogerr/blog.cnfol.com_error.log"]


output.logstash:
 hosts: ["192.168.15.91:5044"]

å¯å¨filebeat

./filebeat -c blog_module_logstash.yml -e

3ãéç½®logstash

tar -zxvf logstash-6.2.4.tar.gz /usr/local
cd /usr/local;ln -s logstash-6.2.4 logstash
å建ä¸ä¸ªnginxæ¥å¿çpiplineæ件
cd /usr/local/logstash

logstashåç½®ç模æ¿ç®å½

vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns

ç¼è¾ grok-patterns æ·»å ä¸ä¸ªæ¯æå¤ipçæ­£å

FORWORD (?:%{IPV4}[,]?[ ]?)+|%{WORD}

å®æ¹grok

http://grokdebug.herokuapp.com/patterns#

å建logstash piplineéç½®æ件

#input {
# stdin {}
#}
# ä»filebeatæ¥åæ°æ®
input {
 beats {
 port => 5044
 host => "0.0.0.0"
 }
}

filter {
 # æ·»å ä¸ä¸ªè°è¯çå¼å³
 mutate{add_field => {"[@metadata][debug]"=>true}}
 grok {
 # è¿æ»¤nginxæ¥å¿
 #match => { "message" => "%{NGINXACCESS_TEST2}" }
 #match => { "message" => '%{IPORHOST:clientip} # (?<http_x_forwarded_for>[^\#]*) # \[%{HTTPDATE:[@metadata][webtime]}\] # %{NOTSPACE:hostname} # %{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion} # %{NUMBER:response} # (?:%{NUMBER:bytes}|-) # (?:"(?:%{NOTSPACE:referrer}|-)"|%{NOTSPACE:referrer}|-) # (?:"(?<http_user_agent>[^#]*)") # (?:"(?:%{NUMBER:connection}|-)"|%{NUMBER:connection}|-) # (?:"(?<cookies>[^#]*)") # %{NUMBER:request_time:float} # (?:%{NUMBER:upstream_response_time:float}|-)' }
 #match => { "message" => '(?:%{IPORHOST:clientip}|-) (?:%{TWO_IP:http_x_forwarded_for}|%{IPV4:http_x_forwarded_for}|-) \[%{HTTPDATE:[@metadata][webtime]}\] (?:%{HOSTNAME:hostname}|-) %{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion} %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{NOTSPACE:referrer}|-)"|%{NOTSPACE:referrer}|-) %{QS:agent} (?:"(?:%{NUMBER:connection}|-)"|%{NUMBER:connection}|-) (?:"(?<cookies>[^#]*)") %{NUMBER:request_time:float} (?:%{NUMBER:upstream_response_time:float}|-)' }
    match => { "message" => '(?:%{IPORHOST:clientip}|-) %{FORWORD:http_x_forwarded_for} \[%{HTTPDATE:[@metadata][webtime]}\] (?:%{HOSTNAME:hostname}|-) %{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion} %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{NOTSPACE:referrer}|-)"|%{NOTSPACE:referrer}|-) %{QS:agent} (?:"(?:%{NUMBER:connection}|-)"|%{NUMBER:connection}|-) %{QS:cookie} %{NUMBER:request_time:float} (?:%{NUMBER:upstream_response_time:float}|-)' }
 }
 # å°é»è®¤ç@timestamp(beatsæ¶éæ¥å¿çæ¶é´)çå¼èµå¼ç»æ°å­æ®µ@read_tiimestamp
 ruby { 
 #code => "event.set('@read_timestamp',event.get('@timestamp'))"
 #å°æ¶åºæ¹ä¸ºä¸8åº
 code => "event.set('@read_timestamp',event.get('@timestamp').time.localtime + 8*60*60)"
 }
 # å°nginxçæ¥å¿è®°å½æ¶é´æ ¼å¼å
 # æ ¼å¼åæ¶é´ 20/May/2015:21:05:56 +0000
 date {
 locale => "en"
 match => ["[@metadata][webtime]","dd/MMM/yyyy:HH:mm:ss Z"]
 }
 # å°byteså­æ®µç±å­ç¬¦ä¸²è½¬æ¢ä¸ºæ°å­
 mutate {
 convert => {"bytes" => "integer"}
 }
 # å°cookieå­æ®µè§£ææä¸ä¸ªjson
 #mutate {
 # gsub => ["cookies",'\;',',']
 #} 
 # å¦ææ使ç¨å°cdnå éhttp_x_forwarded_forä¼æå¤ä¸ªipï¼ç¬¬ä¸ä¸ªipæ¯ç¨æ·çå®ip
 if[http_x_forwarded_for] =~ ", "{
     ruby {
         code => 'event.set("http_x_forwarded_for", event.get("http_x_forwarded_for").split(",")[0])'
        }
    }
 # 解æipï¼è·å¾ipçå°çä½ç½®
 geoip {
 source => "http_x_forwarded_for"
 # # åªè·åipçç»çº¬åº¦ãå½å®¶ãåå¸ãæ¶åº
 fields => ["location","country_name","city_name","region_name"] 
 }
 # å°agentå­æ®µè§£æ,è·å¾æµè§å¨ãç³»ç»çæ¬ç­å·ä½ä¿¡æ¯
 useragent {
 source => "agent"
 target => "useragent"
 }
 #æå®è¦å é¤çæ°æ®
 #mutate{remove_field=>["message"]}
 # æ ¹æ®æ¥å¿å设置索å¼åçåç¼
 ruby {
 code => 'event.set("@[metadata][index_pre]",event.get("source").split("/")[-1])'
 } 
 # å°@timestamp æ ¼å¼å为2019.04.23
 ruby {
 code => 'event.set("@[metadata][index_day]",event.get("@timestamp").time.localtime.strftime("%Y.%m.%d"))'
 }
 # 设置è¾åºçé»è®¤ç´¢å¼å
 mutate {
 add_field => {
  #"[@metadata][index]" => "%{@[metadata][index_pre]}_%{+YYYY.MM.dd}"
  "[@metadata][index]" => "%{@[metadata][index_pre]}_%{@[metadata][index_day]}"
 }
 }
 # å°cookieså­æ®µè§£ææjson
# mutate {
# gsub => [
#  "cookies", ";", ",",
#  "cookies", "=", ":"
# ]
# #split => {"cookies" => ","}
# }
# json_encode {
# source => "cookies"
# target => "cookies_json"
# }
# mutate {
# gsub => [
#  "cookies_json", ',', '","',
#  "cookies_json", ':', '":"'
# ]
# }
# json {
# source => "cookies_json"
# target => "cookies2"
# }
 # å¦ægrok解æå­å¨é误ï¼å°é误ç¬ç«åå¥ä¸ä¸ªç´¢å¼
 if "_grokparsefailure" in [tags] {
 #if "_dateparsefailure" in [tags] {
 mutate {
  replace => {
  #"[@metadata][index]" => "%{@[metadata][index_pre]}_failure_%{+YYYY.MM.dd}"
  "[@metadata][index]" => "%{@[metadata][index_pre]}_failure_%{@[metadata][index_day]}"
  }
 }
 # å¦æä¸å­å¨é误就å é¤message
 }else{
 mutate{remove_field=>["message"]}
 }
}

output {
 if [@metadata][debug]{
 # è¾åºå°rubydebuyg并è¾åºmetadata
 stdout{codec => rubydebug{metadata => true}}
 }else{
 # å°è¾åºå容转æ¢æ "."
 stdout{codec => dots} 
 # å°è¾åºå°æå®çes
 elasticsearch {
  hosts => ["192.168.15.160:9200"]
  index => "%{[@metadata][index]}"
  document_type => "doc"
 } 
 }
}

å¯å¨logstash

nohup bin/logstash -f test_pipline2.conf &

以ä¸å°±æ¯æ¬æçå¨é¨å容ï¼å¸æ对大家ç学习ææ帮å©ï¼ä¹å¸æ大家å¤å¤æ¯æèæ¬ä¹å®¶ã