Compare Pastes
Differences between the pastes
#128398 (21.10.2019 18:01)
and
#220909 (17.09.2021 10:34).
1 | input { | |
2 | beats { | |
3 | port => 5044 | |
4 | } | |
5 | } | |
6 | filter { | |
7 | grok { | |
8 | match => { "message" => "%{SYSLOGTIMESTAMP} %{SYSLOGHOST} %{DATA:program}(?:\[%{POSINT}\])?: %{GREEDYDATA:message}" } | |
9 | overwrite => "message" | |
10 | } | |
11 | mutate { | |
12 | rename => ["host", "server"] | |
13 | convert => {"server" => "string"} | |
14 | } | |
15 | ||
16 | ||
17 | } | |
18 | ||
19 | ||
20 | filter { | |
21 | # grok log lines by program name (listed alpabetically) | |
22 | if [program] =~ /^postfix.*\/anvil$/ { | |
23 | grok { | |
24 | patterns_dir => "/etc/logstash/patterns" | |
25 | match => [ "message", "^%{POSTFIX_ANVIL}$" ] | |
26 | tag_on_failure => [ "_grok_postfix_anvil_nomatch" ] | |
27 | add_tag => [ "_grok_postfix_success" ] | |
28 | } | |
29 | } else if [program] =~ /^postfix.*\/bounce$/ { | |
30 | grok { | |
31 | patterns_dir => "/etc/logstash/patterns" | |
32 | match => [ "message", "^%{POSTFIX_BOUNCE}$" ] | |
33 | tag_on_failure => [ "_grok_postfix_bounce_nomatch" ] | |
34 | add_tag => [ "_grok_postfix_success" ] | |
35 | } | |
36 | } else if [program] =~ /^postfix.*\/cleanup$/ { | |
37 | grok { | |
38 | patterns_dir => "/etc/logstash/patterns" | |
39 | match => [ "message", "^%{POSTFIX_CLEANUP}$" ] | |
40 | tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ] | |
41 | add_tag => [ "_grok_postfix_success" ] | |
42 | } | |
43 | } else if [program] =~ /^postfix.*\/dnsblog$/ { | |
44 | grok { | |
45 | patterns_dir => "/etc/logstash/patterns" | |
46 | match => [ "message", "^%{POSTFIX_DNSBLOG}$" ] | |
47 | tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ] | |
48 | add_tag => [ "_grok_postfix_success" ] | |
49 | } | |
50 | } else if [program] =~ /^postfix.*\/error$/ { | |
51 | grok { | |
52 | patterns_dir => "/etc/logstash/patterns" | |
53 | match => [ "message", "^%{POSTFIX_ERROR}$" ] | |
54 | tag_on_failure => [ "_grok_postfix_error_nomatch" ] | |
55 | add_tag => [ "_grok_postfix_success" ] | |
56 | } | |
57 | } else if [program] =~ /^postfix.*\/local$/ { | |
58 | grok { | |
59 | patterns_dir => "/etc/logstash/patterns" | |
60 | match => [ "message", "^%{POSTFIX_LOCAL}$" ] | |
61 | tag_on_failure => [ "_grok_postfix_local_nomatch" ] | |
62 | add_tag => [ "_grok_postfix_success" ] | |
63 | } | |
64 | } else if [program] =~ /^postfix.*\/master$/ { | |
65 | grok { | |
66 | patterns_dir => "/etc/logstash/patterns" | |
67 | match => [ "message", "^%{POSTFIX_MASTER}$" ] | |
68 | tag_on_failure => [ "_grok_postfix_master_nomatch" ] | |
69 | add_tag => [ "_grok_postfix_success" ] | |
70 | } | |
71 | } else if [program] =~ /^postfix.*\/pickup$/ { | |
72 | grok { | |
73 | patterns_dir => "/etc/logstash/patterns" | |
74 | match => [ "message", "^%{POSTFIX_PICKUP}$" ] | |
75 | tag_on_failure => [ "_grok_postfix_pickup_nomatch" ] | |
76 | add_tag => [ "_grok_postfix_success" ] | |
77 | } | |
78 | } else if [program] =~ /^postfix.*\/pipe$/ { | |
79 | grok { | |
80 | patterns_dir => "/etc/logstash/patterns" | |
81 | match => [ "message", "^%{POSTFIX_PIPE}$" ] | |
82 | tag_on_failure => [ "_grok_postfix_pipe_nomatch" ] | |
83 | add_tag => [ "_grok_postfix_success" ] | |
84 | } | |
85 | } else if [program] =~ /^postfix.*\/postdrop$/ { | |
86 | grok { | |
87 | patterns_dir => "/etc/logstash/patterns" | |
88 | match => [ "message", "^%{POSTFIX_POSTDROP}$" ] | |
89 | tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ] | |
90 | add_tag => [ "_grok_postfix_success" ] | |
91 | } | |
92 | } else if [program] =~ /^postfix.*\/postscreen$/ { | |
93 | grok { | |
94 | patterns_dir => "/etc/logstash/patterns" | |
95 | match => [ "message", "^%{POSTFIX_POSTSCREEN}$" ] | |
96 | tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ] | |
97 | add_tag => [ "_grok_postfix_success" ] | |
98 | } | |
99 | } else if [program] =~ /^postfix.*\/qmgr$/ { | |
100 | grok { | |
101 | patterns_dir => "/etc/logstash/patterns" | |
102 | match => [ "message", "^%{POSTFIX_QMGR}$" ] | |
103 | tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ] | |
104 | add_tag => [ "_grok_postfix_success" ] | |
105 | } | |
106 | } else if [program] =~ /^postfix.*\/scache$/ { | |
107 | grok { | |
108 | patterns_dir => "/etc/logstash/patterns" | |
109 | match => [ "message", "^%{POSTFIX_SCACHE}$" ] | |
110 | tag_on_failure => [ "_grok_postfix_scache_nomatch" ] | |
111 | add_tag => [ "_grok_postfix_success" ] | |
112 | } | |
113 | } else if [program] =~ /^postfix.*\/sendmail$/ { | |
114 | grok { | |
115 | patterns_dir => "/etc/logstash/patterns" | |
116 | match => [ "message", "^%{POSTFIX_SENDMAIL}$" ] | |
117 | tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ] | |
118 | add_tag => [ "_grok_postfix_success" ] | |
119 | } | |
120 | } else if [program] =~ /^postfix.*\/smtp$/ { | |
121 | grok { | |
122 | patterns_dir => "/etc/logstash/patterns" | |
123 | match => [ "message", "^%{POSTFIX_SMTP}$" ] | |
124 | tag_on_failure => [ "_grok_postfix_smtp_nomatch" ] | |
125 | add_tag => [ "_grok_postfix_success" ] | |
126 | } | |
127 | } else if [program] =~ /^postfix.*\/lmtp$/ { | |
128 | grok { | |
129 | patterns_dir => "/etc/logstash/patterns" | |
130 | match => [ "message", "^%{POSTFIX_LMTP}$" ] | |
131 | tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ] | |
132 | add_tag => [ "_grok_postfix_success" ] | |
133 | } | |
134 | } else if [program] =~ /^postfix.*\/smtpd$/ { | |
135 | grok { | |
136 | patterns_dir => "/etc/logstash/patterns" | |
137 | match => [ "message", "^%{POSTFIX_SMTPD}$" ] | |
138 | tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ] | |
139 | add_tag => [ "_grok_postfix_success" ] | |
140 | } | |
141 | } else if [program] =~ /^postfix.*\/postsuper$/ { | |
142 | grok { | |
143 | patterns_dir => "/etc/logstash/patterns" | |
144 | match => [ "message", "^%{POSTFIX_POSTSUPER}$" ] | |
145 | tag_on_failure => [ "_grok_postfix_postsuper_nomatch" ] | |
146 | add_tag => [ "_grok_postfix_success" ] | |
147 | } | |
148 | } else if [program] =~ /^postfix.*\/tlsmgr$/ { | |
149 | grok { | |
150 | patterns_dir => "/etc/logstash/patterns" | |
151 | match => [ "message", "^%{POSTFIX_TLSMGR}$" ] | |
152 | tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ] | |
153 | add_tag => [ "_grok_postfix_success" ] | |
154 | } | |
155 | } else if [program] =~ /^postfix.*\/tlsproxy$/ { | |
156 | grok { | |
157 | patterns_dir => "/etc/logstash/patterns" | |
158 | match => [ "message", "^%{POSTFIX_TLSPROXY}$" ] | |
159 | tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ] | |
160 | add_tag => [ "_grok_postfix_success" ] | |
161 | } | |
162 | } else if [program] =~ /^postfix.*\/trivial-rewrite$/ { | |
163 | grok { | |
164 | patterns_dir => "/etc/logstash/patterns" | |
165 | match => [ "message", "^%{POSTFIX_TRIVIAL_REWRITE}$" ] | |
166 | tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ] | |
167 | add_tag => [ "_grok_postfix_success" ] | |
168 | } | |
169 | } else if [program] =~ /^postfix.*\/discard$/ { | |
170 | grok { | |
171 | patterns_dir => "/etc/logstash/patterns" | |
172 | match => [ "message", "^%{POSTFIX_DISCARD}$" ] | |
173 | tag_on_failure => [ "_grok_postfix_discard_nomatch" ] | |
174 | add_tag => [ "_grok_postfix_success" ] | |
175 | } | |
176 | } else if [program] =~ /^postfix.*\/virtual$/ { | |
177 | grok { | |
178 | patterns_dir => "/etc/logstash/patterns" | |
179 | match => [ "message", "^%{POSTFIX_VIRTUAL}$" ] | |
180 | tag_on_failure => [ "_grok_postfix_virtual_nomatch" ] | |
181 | add_tag => [ "_grok_postfix_success" ] | |
182 | } | |
183 | } else if [program] =~ /^postfix.*/ { | |
184 | mutate { | |
185 | add_tag => [ "_grok_postfix_program_nomatch" ] | |
186 | } | |
187 | } | |
188 | ||
189 | # process key-value data if it exists | |
190 | if [postfix.keyvalue_data] { | |
191 | kv { | |
192 | source => "postfix.keyvalue_data" | |
193 | trim_value => "<>," | |
194 | prefix => "postfix." | |
195 | remove_field => [ "postfix_keyvalue_data" ] | |
196 | } | |
197 | ||
198 | # some post processing of key-value data | |
199 | if [postfix.client] { | |
200 | grok { | |
201 | patterns_dir => "/etc/logstash/patterns" | |
202 | match => ["postfix.client", "^%{POSTFIX_CLIENT_INFO}$"] | |
203 | tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ] | |
204 | remove_field => [ "postfix_client" ] | |
205 | } | |
206 | } | |
207 | if [postfix.relay] { | |
208 | grok { | |
209 | patterns_dir => "/etc/logstash/patterns" | |
210 | match => ["postfix.relay", "^%{POSTFIX_RELAY_INFO}$"] | |
211 | tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ] | |
212 | remove_field => [ "postfix_relay" ] | |
213 | } | |
214 | } | |
215 | if [postfix.delays] { | |
216 | grok { | |
217 | patterns_dir => "/etc/logstash/patterns" | |
218 | match => ["postfix.delays", "^%{POSTFIX_DELAYS}$"] | |
219 | tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ] | |
220 | remove_field => [ "postfix_delays" ] | |
221 | } | |
222 | } | |
223 | } | |
224 | ||
225 | # process command counter data if it exists | |
226 | if [postfix.command_counter_data] { | |
227 | grok { | |
228 | patterns_dir => "/etc/logstash/patterns" | |
229 | match => ["postfix_command_counter_data", "^%{POSTFIX_COMMAND_COUNTER_DATA}$"] | |
230 | tag_on_failure => ["_grok_postfix_command_counter_data_nomatch"] | |
231 | remove_field => ["postfix_command_counter_data"] | |
232 | } | |
233 | } | |
234 | ||
235 | # Do some data type conversions | |
236 | mutate { | |
237 | convert => [ | |
238 | # list of integer fields | |
239 | "postfix.anvil_cache_size", "integer", | |
240 | "postfix.anvil_conn_count", "integer", | |
241 | "postfix.anvil_conn_rate", "integer", | |
242 | "postfix.client_port", "integer", | |
243 | "postfix.cmd_auth", "integer", | |
244 | "postfix.cmd_auth_accepted", "integer", | |
245 | "postfix.cmd_count", "integer", | |
246 | "postfix.cmd_count_accepted", "integer", | |
247 | "postfix.cmd_data", "integer", | |
248 | "postfix.cmd_data_accepted", "integer", | |
249 | "postfix.cmd_ehlo", "integer", | |
250 | "postfix.cmd_ehlo_accepted", "integer", | |
251 | "postfix.cmd_helo", "integer", | |
252 | "postfix.cmd_helo_accepted", "integer", | |
253 | "postfix.cmd_mail", "integer", | |
254 | "postfix.cmd_mail_accepted", "integer", | |
255 | "postfix.cmd_quit", "integer", | |
256 | "postfix.cmd_quit_accepted", "integer", | |
257 | "postfix.cmd_rcpt", "integer", | |
258 | "postfix.cmd_rcpt_accepted", "integer", | |
259 | "postfix.cmd_rset", "integer", | |
260 | "postfix.cmd_rset_accepted", "integer", | |
261 | "postfix.cmd_starttls", "integer", | |
262 | "postfix.cmd_starttls_accepted", "integer", | |
263 | "postfix.cmd_unknown", "integer", | |
264 | "postfix.cmd_unknown_accepted", "integer", | |
265 | "postfix.nrcpt", "integer", | |
266 | "postfix.postscreen_cache_dropped", "integer", | |
267 | "postfix.postscreen_cache_retained", "integer", | |
268 | "postfix.postscreen_dnsbl_rank", "integer", | |
269 | "postfix.relay_port", "integer", | |
270 | "postfix.server_port", "integer", | |
271 | "postfix.size", "integer", | |
272 | "postfix.status_code", "integer", | |
273 | "postfix.termination_signal", "integer", | |
274 | ||
275 | # list of float fields | |
276 | "postfix.delay", "float", | |
277 | "postfix.delay_before_qmgr", "float", | |
278 | "postfix.delay_conn_setup", "float", | |
279 | "postfix.delay_in_qmgr", "float", | |
280 | "postfix.delay_transmission", "float", | |
281 | "postfix.postscreen_violation_time", "float" | |
282 | ] | |
283 | } | |
284 | mutate { | |
285 | rename => ["host", "server"] | |
286 | convert => {"server" => "string"} | |
287 | } | |
288 | ||
289 | } | |
290 | ||
291 | output { | |
292 | elasticsearch { | |
293 | hosts => "localhost:9200" | |
294 | index => "postfix-%{+YYYY.MM.dd}" | |
295 | } | |
296 | ||
297 | } | |
298 | ||
299 | ||
300 | ||
301 | ||
302 | ||
303 | ||
304 | ||
305 | ||
306 | ||
307 | ||
308 | ||
309 | ||
310 | ||
311 | ||
312 | ||
313 | ||
314 | ||
315 |