Compare Pastes
Differences between the pastes
#127881 (14.10.2019 14:36)
and
#145477 (22.09.2020 22:16).
1 | input { | |
2 | beats { | |
3 | port => 5044 | |
4 | } | |
5 | } | |
6 | ||
7 | filter { | |
8 | grok { | |
9 | match => { "message" => "%{SYSLOGTIMESTAMP} %{SYSLOGHOST} %{DATA:program}(?:\[%{POSINT}\])?: %{GREEDYDATA:message}" } | |
10 | overwrite => "message" | |
11 | } | |
12 | } | |
13 | ||
14 | ||
15 | filter { | |
16 | # grok log lines by program name (listed alpabetically) | |
17 | if [program] =~ /^postfix.*\/anvil$/ { | |
18 | grok { | |
19 | patterns_dir => "/etc/logstash/patterns" | |
20 | match => [ "message", "^%{POSTFIX_ANVIL}$" ] | |
21 | tag_on_failure => [ "_grok_postfix_anvil_nomatch" ] | |
22 | add_tag => [ "_grok_postfix_success" ] | |
23 | } | |
24 | } else if [program] =~ /^postfix.*\/bounce$/ { | |
25 | grok { | |
26 | patterns_dir => "/etc/logstash/patterns" | |
27 | match => [ "message", "^%{POSTFIX_BOUNCE}$" ] | |
28 | tag_on_failure => [ "_grok_postfix_bounce_nomatch" ] | |
29 | add_tag => [ "_grok_postfix_success" ] | |
30 | } | |
31 | } else if [program] =~ /^postfix.*\/cleanup$/ { | |
32 | grok { | |
33 | patterns_dir => "/etc/logstash/patterns" | |
34 | match => [ "message", "^%{POSTFIX_CLEANUP}$" ] | |
35 | tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ] | |
36 | add_tag => [ "_grok_postfix_success" ] | |
37 | } | |
38 | } else if [program] =~ /^postfix.*\/dnsblog$/ { | |
39 | grok { | |
40 | patterns_dir => "/etc/logstash/patterns" | |
41 | match => [ "message", "^%{POSTFIX_DNSBLOG}$" ] | |
42 | tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ] | |
43 | add_tag => [ "_grok_postfix_success" ] | |
44 | } | |
45 | } else if [program] =~ /^postfix.*\/error$/ { | |
46 | grok { | |
47 | patterns_dir => "/etc/logstash/patterns" | |
48 | match => [ "message", "^%{POSTFIX_ERROR}$" ] | |
49 | tag_on_failure => [ "_grok_postfix_error_nomatch" ] | |
50 | add_tag => [ "_grok_postfix_success" ] | |
51 | } | |
52 | } else if [program] =~ /^postfix.*\/local$/ { | |
53 | grok { | |
54 | patterns_dir => "/etc/logstash/patterns" | |
55 | match => [ "message", "^%{POSTFIX_LOCAL}$" ] | |
56 | tag_on_failure => [ "_grok_postfix_local_nomatch" ] | |
57 | add_tag => [ "_grok_postfix_success" ] | |
58 | } | |
59 | } else if [program] =~ /^postfix.*\/master$/ { | |
60 | grok { | |
61 | patterns_dir => "/etc/logstash/patterns" | |
62 | match => [ "message", "^%{POSTFIX_MASTER}$" ] | |
63 | tag_on_failure => [ "_grok_postfix_master_nomatch" ] | |
64 | add_tag => [ "_grok_postfix_success" ] | |
65 | } | |
66 | } else if [program] =~ /^postfix.*\/pickup$/ { | |
67 | grok { | |
68 | patterns_dir => "/etc/logstash/patterns" | |
69 | match => [ "message", "^%{POSTFIX_PICKUP}$" ] | |
70 | tag_on_failure => [ "_grok_postfix_pickup_nomatch" ] | |
71 | add_tag => [ "_grok_postfix_success" ] | |
72 | } | |
73 | } else if [program] =~ /^postfix.*\/pipe$/ { | |
74 | grok { | |
75 | patterns_dir => "/etc/logstash/patterns" | |
76 | match => [ "message", "^%{POSTFIX_PIPE}$" ] | |
77 | tag_on_failure => [ "_grok_postfix_pipe_nomatch" ] | |
78 | add_tag => [ "_grok_postfix_success" ] | |
79 | } | |
80 | } else if [program] =~ /^postfix.*\/postdrop$/ { | |
81 | grok { | |
82 | patterns_dir => "/etc/logstash/patterns" | |
83 | match => [ "message", "^%{POSTFIX_POSTDROP}$" ] | |
84 | tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ] | |
85 | add_tag => [ "_grok_postfix_success" ] | |
86 | } | |
87 | } else if [program] =~ /^postfix.*\/postscreen$/ { | |
88 | grok { | |
89 | patterns_dir => "/etc/logstash/patterns" | |
90 | match => [ "message", "^%{POSTFIX_POSTSCREEN}$" ] | |
91 | tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ] | |
92 | add_tag => [ "_grok_postfix_success" ] | |
93 | } | |
94 | } else if [program] =~ /^postfix.*\/qmgr$/ { | |
95 | grok { | |
96 | patterns_dir => "/etc/logstash/patterns" | |
97 | match => [ "message", "^%{POSTFIX_QMGR}$" ] | |
98 | tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ] | |
99 | add_tag => [ "_grok_postfix_success" ] | |
100 | } | |
101 | } else if [program] =~ /^postfix.*\/scache$/ { | |
102 | grok { | |
103 | patterns_dir => "/etc/logstash/patterns" | |
104 | match => [ "message", "^%{POSTFIX_SCACHE}$" ] | |
105 | tag_on_failure => [ "_grok_postfix_scache_nomatch" ] | |
106 | add_tag => [ "_grok_postfix_success" ] | |
107 | } | |
108 | } else if [program] =~ /^postfix.*\/sendmail$/ { | |
109 | grok { | |
110 | patterns_dir => "/etc/logstash/patterns" | |
111 | match => [ "message", "^%{POSTFIX_SENDMAIL}$" ] | |
112 | tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ] | |
113 | add_tag => [ "_grok_postfix_success" ] | |
114 | } | |
115 | } else if [program] =~ /^postfix.*\/smtp$/ { | |
116 | grok { | |
117 | patterns_dir => "/etc/logstash/patterns" | |
118 | match => [ "message", "^%{POSTFIX_SMTP}$" ] | |
119 | tag_on_failure => [ "_grok_postfix_smtp_nomatch" ] | |
120 | add_tag => [ "_grok_postfix_success" ] | |
121 | } | |
122 | } else if [program] =~ /^postfix.*\/lmtp$/ { | |
123 | grok { | |
124 | patterns_dir => "/etc/logstash/patterns" | |
125 | match => [ "message", "^%{POSTFIX_LMTP}$" ] | |
126 | tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ] | |
127 | add_tag => [ "_grok_postfix_success" ] | |
128 | } | |
129 | } else if [program] =~ /^postfix.*\/smtpd$/ { | |
130 | grok { | |
131 | patterns_dir => "/etc/logstash/patterns" | |
132 | match => [ "message", "^%{POSTFIX_SMTPD}$" ] | |
133 | tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ] | |
134 | add_tag => [ "_grok_postfix_success" ] | |
135 | } | |
136 | } else if [program] =~ /^postfix.*\/postsuper$/ { | |
137 | grok { | |
138 | patterns_dir => "/etc/logstash/patterns" | |
139 | match => [ "message", "^%{POSTFIX_POSTSUPER}$" ] | |
140 | tag_on_failure => [ "_grok_postfix_postsuper_nomatch" ] | |
141 | add_tag => [ "_grok_postfix_success" ] | |
142 | } | |
143 | } else if [program] =~ /^postfix.*\/tlsmgr$/ { | |
144 | grok { | |
145 | patterns_dir => "/etc/logstash/patterns" | |
146 | match => [ "message", "^%{POSTFIX_TLSMGR}$" ] | |
147 | tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ] | |
148 | add_tag => [ "_grok_postfix_success" ] | |
149 | } | |
150 | } else if [program] =~ /^postfix.*\/tlsproxy$/ { | |
151 | grok { | |
152 | patterns_dir => "/etc/logstash/patterns" | |
153 | match => [ "message", "^%{POSTFIX_TLSPROXY}$" ] | |
154 | tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ] | |
155 | add_tag => [ "_grok_postfix_success" ] | |
156 | } | |
157 | } else if [program] =~ /^postfix.*\/trivial-rewrite$/ { | |
158 | grok { | |
159 | patterns_dir => "/etc/logstash/patterns" | |
160 | match => [ "message", "^%{POSTFIX_TRIVIAL_REWRITE}$" ] | |
161 | tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ] | |
162 | add_tag => [ "_grok_postfix_success" ] | |
163 | } | |
164 | } else if [program] =~ /^postfix.*\/discard$/ { | |
165 | grok { | |
166 | patterns_dir => "/etc/logstash/patterns" | |
167 | match => [ "message", "^%{POSTFIX_DISCARD}$" ] | |
168 | tag_on_failure => [ "_grok_postfix_discard_nomatch" ] | |
169 | add_tag => [ "_grok_postfix_success" ] | |
170 | } | |
171 | } else if [program] =~ /^postfix.*\/virtual$/ { | |
172 | grok { | |
173 | patterns_dir => "/etc/logstash/patterns" | |
174 | match => [ "message", "^%{POSTFIX_VIRTUAL}$" ] | |
175 | tag_on_failure => [ "_grok_postfix_virtual_nomatch" ] | |
176 | add_tag => [ "_grok_postfix_success" ] | |
177 | } | |
178 | } else if [program] =~ /^postfix.*/ { | |
179 | mutate { | |
180 | add_tag => [ "_grok_postfix_program_nomatch" ] | |
181 | } | |
182 | } | |
183 | ||
184 | # process key-value data if it exists | |
185 | if [postfix.keyvalue_data] { | |
186 | kv { | |
187 | source => "postfix.keyvalue_data" | |
188 | trim_value => "<>," | |
189 | prefix => "postfix." | |
190 | remove_field => [ "postfix_keyvalue_data" ] | |
191 | } | |
192 | ||
193 | # some post processing of key-value data | |
194 | if [postfix.client] { | |
195 | grok { | |
196 | patterns_dir => "/etc/logstash/patterns" | |
197 | match => ["postfix.client", "^%{POSTFIX_CLIENT_INFO}$"] | |
198 | tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ] | |
199 | remove_field => [ "postfix_client" ] | |
200 | } | |
201 | } | |
202 | if [postfix.relay] { | |
203 | grok { | |
204 | patterns_dir => "/etc/logstash/patterns" | |
205 | match => ["postfix.relay", "^%{POSTFIX_RELAY_INFO}$"] | |
206 | tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ] | |
207 | remove_field => [ "postfix_relay" ] | |
208 | } | |
209 | } | |
210 | if [postfix.delays] { | |
211 | grok { | |
212 | patterns_dir => "/etc/logstash/patterns" | |
213 | match => ["postfix.delays", "^%{POSTFIX_DELAYS}$"] | |
214 | tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ] | |
215 | remove_field => [ "postfix_delays" ] | |
216 | } | |
217 | } | |
218 | } | |
219 | ||
220 | # process command counter data if it exists | |
221 | if [postfix.command_counter_data] { | |
222 | grok { | |
223 | patterns_dir => "/etc/logstash/patterns" | |
224 | match => ["postfix_command_counter_data", "^%{POSTFIX_COMMAND_COUNTER_DATA}$"] | |
225 | tag_on_failure => ["_grok_postfix_command_counter_data_nomatch"] | |
226 | remove_field => ["postfix_command_counter_data"] | |
227 | } | |
228 | } | |
229 | ||
230 | # Do some data type conversions | |
231 | mutate { | |
232 | convert => [ | |
233 | # list of integer fields | |
234 | "postfix.anvil_cache_size", "integer", | |
235 | "postfix.anvil_conn_count", "integer", | |
236 | "postfix.anvil_conn_rate", "integer", | |
237 | "postfix.client_port", "integer", | |
238 | "postfix.cmd_auth", "integer", | |
239 | "postfix.cmd_auth_accepted", "integer", | |
240 | "postfix.cmd_count", "integer", | |
241 | "postfix.cmd_count_accepted", "integer", | |
242 | "postfix.cmd_data", "integer", | |
243 | "postfix.cmd_data_accepted", "integer", | |
244 | "postfix.cmd_ehlo", "integer", | |
245 | "postfix.cmd_ehlo_accepted", "integer", | |
246 | "postfix.cmd_helo", "integer", | |
247 | "postfix.cmd_helo_accepted", "integer", | |
248 | "postfix.cmd_mail", "integer", | |
249 | "postfix.cmd_mail_accepted", "integer", | |
250 | "postfix.cmd_quit", "integer", | |
251 | "postfix.cmd_quit_accepted", "integer", | |
252 | "postfix.cmd_rcpt", "integer", | |
253 | "postfix.cmd_rcpt_accepted", "integer", | |
254 | "postfix.cmd_rset", "integer", | |
255 | "postfix.cmd_rset_accepted", "integer", | |
256 | "postfix.cmd_starttls", "integer", | |
257 | "postfix.cmd_starttls_accepted", "integer", | |
258 | "postfix.cmd_unknown", "integer", | |
259 | "postfix.cmd_unknown_accepted", "integer", | |
260 | "postfix.nrcpt", "integer", | |
261 | "postfix.postscreen_cache_dropped", "integer", | |
262 | "postfix.postscreen_cache_retained", "integer", | |
263 | "postfix.postscreen_dnsbl_rank", "integer", | |
264 | "postfix.relay_port", "integer", | |
265 | "postfix.server_port", "integer", | |
266 | "postfix.size", "integer", | |
267 | "postfix.status_code", "integer", | |
268 | "postfix.termination_signal", "integer", | |
269 | ||
270 | # list of float fields | |
271 | "postfix.delay", "float", | |
272 | "postfix.delay_before_qmgr", "float", | |
273 | "postfix.delay_conn_setup", "float", | |
274 | "postfix.delay_in_qmgr", "float", | |
275 | "postfix.delay_transmission", "float", | |
276 | "postfix.postscreen_violation_time", "float" | |
277 | ] | |
278 | } | |
279 | } | |
280 | output { | |
281 | elasticsearch { | |
282 | hosts => "localhost:9200" | |
283 | index => "postfixnew6-%{+YYYY.MM.dd}" | |
284 | } | |
285 | ||
286 | } |