三阶段--elk日志分析（2）Logstash和ES和Kibana

1
2
83.149.9.216 ‐ ‐ [04/Jan/2015:05:13:42 +0000] "GET /presentations/logstash‐monitorama‐ 2013/imageskibana‐search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash‐monitorama‐2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
input {
    file {
		path => ["/var/log/httpd.log"]
		start_position => "beginning"
	}
}
filter {
	grok {
		match => { "message" => "%{COMBINEDAPACHELOG}" }
	}
	mutate {
	##重写字段，rename重命名名称，把clientip改成cip。想定义哪个字段改哪个
		rename => {
			"clientip" => "cip"
			}
	}
	##删除字段remove_field,如message input_type @..等
	mutate { remove_field => ["message","input_type","@version","fields"] }
}
output {
	stdout { }
}

1
2
在这里插入代码片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
input {
    file {
		path => ["/var/log/httpd.log"]
		start_position => "beginning"
	}
}
filter {
	grok {
		match => { "message" => "%{COMBINEDAPACHELOG}" }
	}
	geoip {
		source => "clientip"
	}

}
output {
	stdout { }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
input {
#    file {
#		path => ["/var/log/httpd.log"]
#		start_position => "beginning"
#	}
	beats {
		port => 5044
	}
}
filter {
	grok {
		match => { "message" => "%{COMBINEDAPACHELOG}" }
	}
	#mutate {
	#	rename => {
	#		"clientip" => "cip"
	#		}
	#}
	#mutate { remove_field => ["message","input_type","@version","fields"] }
	#geoip {
	#	source => "clientip"
	#}
	
}
output {
	stdout { }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch		#获取key
vim /etc/yum.repos.d/es.repo		#创建yum仓库
[elasticsearch]
name=Elasticsearch repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG‐KEY‐elasticsearch
enabled=0
autorefresh=1 
type=rpm‐md

yum install -y elasticsearch		#安装es

##另外两台机器：
vim /etc/yum.repos.d/es.repo		#创建yum仓库
[elasticsearch]
name=Elasticsearch repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=0
gpgkey=https://artifacts.elastic.co/GPG‐KEY‐elasticsearch
enabled=1
autorefresh=1 
type=rpm‐md
##然后安装
yum install -y elasticsearch
##过程很慢。因为我网不行，所以是下载的tar包，然后上传解压的。

##安装完成后
systemctl daemon-reload		#重新装载es程序
systemctl enable elasticsearch.service		#开机启动
systemctl start elasticsearch.service		#启动es
netstat -anpt		#查看开放的端口，当出现9200和9300的时候，es是正常启动的

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
sysctl -w vm.max_map_count=262144 > /etc/sysctl.conf		#修改内核参数，设置内存映射
sysctl -p		#
vim /etc/security/limits.conf		#设置文件描述符数量
tail /etc/security/limits.conf 
#@faculty        soft    nproc           20
#@faculty        hard    nproc           50
#ftp             hard    nproc           0
#@student        -       maxlogins       4

# End of file
* soft nofile 65536
* hard nofile 131072
* soft nproc 2048
* hard nproc 4096

1
2
3
4
5
6
7
8
9
10
11
12
cluster.name: elk 		#cluster.name集群名称
node.name: ela1		#注意这里，节点名称，2、3节点的改成自己的名字，别的地方都一样 
node.data: true 		#指定节点是否为数据节点，数据节点包含并管理索引的一部分
network.host: 0.0.0.0 		#监听哪个网卡
http.port: 9200 		#监听端口，对外的
discovery.seed_hosts: #官方指定写法如下，3台机器一样 指定集群成员，用于主动发现他们，要把所有的机器都写进来，包括本机
  ‐ ela1 #节点1主机名称 
  ‐ 192.168.122.106:9300 #节点2的ip加端口 9300是对内的
  ‐ 192.168.122.218 #节点3的ip 
cluster.initial_master_nodes: ["ela1", "ela2", "ela3"]		#指定有资格称为master的节点，这里是所有的都由资格

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
input {
        beats {
                port => 5044
        }
}
filter {
        grok {
                match => { "message" => "%{COMBINEDAPACHELOG}" }
                remove_field => [ "message" ]
        }
        geoip {
                source => "clientip"
        }
}
output {
        stdout { 
                code => rubydebug
        }
        elasticesearch {
                hosts => [ "10.9.30.108:9200","10.9.30.13:9200","10.9.30.182:9200" ]
        }

}

1
2
3
4
5
6
7
8
9
10
11
12
vim ./kibana/config/kibana.yml
server.port: 5601 #改 
server.host: "0.0.0.0" #改 
# 用于连接到 ES 集群的地址和端口 
elasticsearch.hosts: ["http://ela1:9200"] #改 

# 日志文件路径 
# logging.dest: stdout 
logging.dest: /var/log/kibana/kibana.log #改 
# 设置页面的字体为中文 
i18n.locale: "zh‐CN" #改