awk 日志解析

cat test.txt test1.txt | sort | uniq -d
vimdiff tmp1 tem2

cat tmp1 tmp2 | uniq -d

awk '{print $7}' log.2 | sort | uniq > log2.log
合并文件
cat file1.out file2.out ＞ file2.out 
cat promition.success.log.2011-08-15-* > promition.success.log

gamelogin.log 16日

awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3  "|" $4}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' | awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' | sort| uniq -c |awk 'BEGIN {FS=" "}; {print $3 " " $1 " " $5 " " $4 " " $2 }' | sort -r -n > tmp
-----------------------------------------------------------------------
awk 'BEGIN {FS="|"}; {print $2 "|" $3  "|" $4}' gamelogin.log > tmp.1

awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' tmp.1 > tmp.2

awk 'BEGIN {FS="|"}; {print $2 "|" $3  "|" $4}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' > tmp.3

awk 'BEGIN {FS="|"}; {print $1 " " $2 " " $3 }' tmp.3 | sort| uniq -c > tmp.4

awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > tmp.5

awk '{ sub(/^[ \t]+/,""); print }' tmp.5 > tmp.7
awk 'gsub(/^ *| *$/,"")'

awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}' tmp.5  > tmp.6

sort tmp.5  > tmp.6
-----------------------------------------------------------------
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > tmp.5

awk 'BEGIN {FS="|"}; {print $2}' tmp.5 | uniq | sort > tmp-uid

===================================================================
promition.success.log 15日

awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $2 " " $3  " " $4 " " $5 " "  $8 " "substr($1,index($1, " ") + 1, 2) }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $2 " " $3 " " $1 " " $5 " " $4 " " $6 " " $7 " " $8 " " $9}'| sort -r -n >log.1


awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort >log.1

awk 'BEGIN {FS="|"}; {print $2}' log.1 | uniq | sort > log-uid

//多个文件合并为一行：
// paste glogin-uid-count.log prom-suc-uid-count.log log-rate.log > flux.log
----------------------------------------------------------------------
算登录率
cat log-uid tmp-uid | sort | uniq -d|wc -l >test.log

最终整理如下：

awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log
awk 'BEGIN {FS="|"}; {print $2}' glogin.log | uniq | sort > glogin-uid
awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log | uniq | sort > prom-suc-uid
计算前一天的次登录率
cat glogin-uid | wc -l  > tmp-glog-prom-uid.log
cat prom-suc-uid | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid glogin-uid | sort | uniq -d|wc -l >> tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
单个文件合并为一行
计算登录率
awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2}' tmp-flux-rate.log > flux-rate.log
rm -f tmp*

-----------------------------------------------------------------
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log
awk 'BEGIN {FS="|"}; {print $2}' glogin.log | uniq | sort > glogin-uid
awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log | uniq | sort > prom-suc-uid
cat glogin-uid | wc -l  > tmp-glog-prom-uid.log
cat prom-suc-uid | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid glogin-uid | sort | uniq -d|wc -l >> tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2}' tmp-flux-rate.log > flux-rate.log
rm -f tmp*

结果 532853|36842|15950|0.43293

----------------------------------
简化版