Compare Pastes

Differences between the pastes #128398 (21.10.2019 18:01) and #220909 (17.09.2021 10:34).
1
input {
2
  beats {
3
    port => 5044
4
 }
5
}
6
filter {
7
     grok {
8
      match => { "message" => "%{SYSLOGTIMESTAMP} %{SYSLOGHOST} %{DATA:program}(?:\[%{POSINT}\])?: %{GREEDYDATA:message}" }
9
      overwrite => "message"
10
    }
11
mutate {
12
   rename => ["host", "server"]
13
   convert => {"server" => "string"}
14
}
15
16
17
}
18
19
20
filter {
21
    # grok log lines by program name (listed alpabetically)
22
    if [program] =~ /^postfix.*\/anvil$/ {
23
        grok {
24
            patterns_dir   => "/etc/logstash/patterns"
25
            match          => [ "message", "^%{POSTFIX_ANVIL}$" ]
26
            tag_on_failure => [ "_grok_postfix_anvil_nomatch" ]
27
            add_tag        => [ "_grok_postfix_success" ]
28
        }
29
    } else if [program] =~ /^postfix.*\/bounce$/ {
30
        grok {
31
            patterns_dir   => "/etc/logstash/patterns"
32
            match          => [ "message", "^%{POSTFIX_BOUNCE}$" ]
33
            tag_on_failure => [ "_grok_postfix_bounce_nomatch" ]
34
            add_tag        => [ "_grok_postfix_success" ]
35
        }
36
    } else if [program] =~ /^postfix.*\/cleanup$/ {
37
        grok {
38
            patterns_dir   => "/etc/logstash/patterns"
39
            match          => [ "message", "^%{POSTFIX_CLEANUP}$" ]
40
            tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ]
41
            add_tag        => [ "_grok_postfix_success" ]
42
        }
43
    } else if [program] =~ /^postfix.*\/dnsblog$/ {
44
        grok {
45
            patterns_dir   => "/etc/logstash/patterns"
46
            match          => [ "message", "^%{POSTFIX_DNSBLOG}$" ]
47
            tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ]
48
            add_tag        => [ "_grok_postfix_success" ]
49
        }
50
    } else if [program] =~ /^postfix.*\/error$/ {
51
        grok {
52
            patterns_dir   => "/etc/logstash/patterns"
53
            match          => [ "message", "^%{POSTFIX_ERROR}$" ]
54
            tag_on_failure => [ "_grok_postfix_error_nomatch" ]
55
            add_tag        => [ "_grok_postfix_success" ]
56
        }
57
    } else if [program] =~ /^postfix.*\/local$/ {
58
        grok {
59
            patterns_dir   => "/etc/logstash/patterns"
60
            match          => [ "message", "^%{POSTFIX_LOCAL}$" ]
61
            tag_on_failure => [ "_grok_postfix_local_nomatch" ]
62
            add_tag        => [ "_grok_postfix_success" ]
63
        }
64
    } else if [program] =~ /^postfix.*\/master$/ {
65
        grok {
66
            patterns_dir   => "/etc/logstash/patterns"
67
            match          => [ "message", "^%{POSTFIX_MASTER}$" ]
68
            tag_on_failure => [ "_grok_postfix_master_nomatch" ]
69
            add_tag        => [ "_grok_postfix_success" ]
70
        }
71
    } else if [program] =~ /^postfix.*\/pickup$/ {
72
        grok {
73
            patterns_dir   => "/etc/logstash/patterns"
74
            match          => [ "message", "^%{POSTFIX_PICKUP}$" ]
75
            tag_on_failure => [ "_grok_postfix_pickup_nomatch" ]
76
            add_tag        => [ "_grok_postfix_success" ]
77
        }
78
    } else if [program] =~ /^postfix.*\/pipe$/ {
79
        grok {
80
            patterns_dir   => "/etc/logstash/patterns"
81
            match          => [ "message", "^%{POSTFIX_PIPE}$" ]
82
            tag_on_failure => [ "_grok_postfix_pipe_nomatch" ]
83
            add_tag        => [ "_grok_postfix_success" ]
84
        }
85
    } else if [program] =~ /^postfix.*\/postdrop$/ {
86
        grok {
87
            patterns_dir   => "/etc/logstash/patterns"
88
            match          => [ "message", "^%{POSTFIX_POSTDROP}$" ]
89
            tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ]
90
            add_tag        => [ "_grok_postfix_success" ]
91
        }
92
    } else if [program] =~ /^postfix.*\/postscreen$/ {
93
        grok {
94
            patterns_dir   => "/etc/logstash/patterns"
95
            match          => [ "message", "^%{POSTFIX_POSTSCREEN}$" ]
96
            tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ]
97
            add_tag        => [ "_grok_postfix_success" ]
98
        }
99
    } else if [program] =~ /^postfix.*\/qmgr$/ {
100
        grok {
101
            patterns_dir   => "/etc/logstash/patterns"
102
            match          => [ "message", "^%{POSTFIX_QMGR}$" ]
103
            tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ]
104
            add_tag        => [ "_grok_postfix_success" ]
105
        }
106
    } else if [program] =~ /^postfix.*\/scache$/ {
107
        grok {
108
            patterns_dir   => "/etc/logstash/patterns"
109
            match          => [ "message", "^%{POSTFIX_SCACHE}$" ]
110
            tag_on_failure => [ "_grok_postfix_scache_nomatch" ]
111
            add_tag        => [ "_grok_postfix_success" ]
112
        }
113
    } else if [program] =~ /^postfix.*\/sendmail$/ {
114
        grok {
115
            patterns_dir   => "/etc/logstash/patterns"
116
            match          => [ "message", "^%{POSTFIX_SENDMAIL}$" ]
117
            tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ]
118
            add_tag        => [ "_grok_postfix_success" ]
119
        }
120
    } else if [program] =~ /^postfix.*\/smtp$/ {
121
        grok {
122
            patterns_dir   => "/etc/logstash/patterns"
123
            match          => [ "message", "^%{POSTFIX_SMTP}$" ]
124
            tag_on_failure => [ "_grok_postfix_smtp_nomatch" ]
125
            add_tag        => [ "_grok_postfix_success" ]
126
        }
127
    } else if [program] =~ /^postfix.*\/lmtp$/ {
128
        grok {
129
            patterns_dir   => "/etc/logstash/patterns"
130
            match          => [ "message", "^%{POSTFIX_LMTP}$" ]
131
            tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ]
132
            add_tag        => [ "_grok_postfix_success" ]
133
        }
134
    } else if [program] =~ /^postfix.*\/smtpd$/ {
135
        grok {
136
            patterns_dir   => "/etc/logstash/patterns"
137
            match          => [ "message", "^%{POSTFIX_SMTPD}$" ]
138
            tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ]
139
            add_tag        => [ "_grok_postfix_success" ]
140
        }
141
    } else if [program] =~ /^postfix.*\/postsuper$/ {
142
        grok {
143
            patterns_dir   => "/etc/logstash/patterns"
144
            match          => [ "message", "^%{POSTFIX_POSTSUPER}$" ]
145
            tag_on_failure => [ "_grok_postfix_postsuper_nomatch" ]
146
            add_tag        => [ "_grok_postfix_success" ]
147
        }
148
    } else if [program] =~ /^postfix.*\/tlsmgr$/ {
149
        grok {
150
            patterns_dir   => "/etc/logstash/patterns"
151
            match          => [ "message", "^%{POSTFIX_TLSMGR}$" ]
152
            tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ]
153
            add_tag        => [ "_grok_postfix_success" ]
154
        }
155
    } else if [program] =~ /^postfix.*\/tlsproxy$/ {
156
        grok {
157
            patterns_dir   => "/etc/logstash/patterns"
158
            match          => [ "message", "^%{POSTFIX_TLSPROXY}$" ]
159
            tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ]
160
            add_tag        => [ "_grok_postfix_success" ]
161
        }
162
    } else if [program] =~ /^postfix.*\/trivial-rewrite$/ {
163
        grok {
164
            patterns_dir   => "/etc/logstash/patterns"
165
            match          => [ "message", "^%{POSTFIX_TRIVIAL_REWRITE}$" ]
166
            tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ]
167
            add_tag        => [ "_grok_postfix_success" ]
168
        }
169
    } else if [program] =~ /^postfix.*\/discard$/ {
170
        grok {
171
            patterns_dir   => "/etc/logstash/patterns"
172
            match          => [ "message", "^%{POSTFIX_DISCARD}$" ]
173
            tag_on_failure => [ "_grok_postfix_discard_nomatch" ]
174
            add_tag        => [ "_grok_postfix_success" ]
175
        }
176
    } else if [program] =~ /^postfix.*\/virtual$/ {
177
        grok {
178
            patterns_dir   => "/etc/logstash/patterns"
179
            match          => [ "message", "^%{POSTFIX_VIRTUAL}$" ]
180
            tag_on_failure => [ "_grok_postfix_virtual_nomatch" ]
181
            add_tag        => [ "_grok_postfix_success" ]
182
        }
183
    } else if [program] =~ /^postfix.*/ {
184
        mutate {
185
            add_tag => [ "_grok_postfix_program_nomatch" ]
186
        }
187
    }
188
189
    # process key-value data if it exists
190
    if [postfix.keyvalue_data] {
191
        kv {
192
            source       => "postfix.keyvalue_data"
193
            trim_value   => "<>,"
194
            prefix       => "postfix."
195
            remove_field => [ "postfix_keyvalue_data" ]
196
        }
197
198
        # some post processing of key-value data
199
        if [postfix.client] {
200
            grok {
201
                patterns_dir   => "/etc/logstash/patterns"
202
                match          => ["postfix.client", "^%{POSTFIX_CLIENT_INFO}$"]
203
                tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ]
204
                remove_field   => [ "postfix_client" ]
205
            }
206
        }
207
        if [postfix.relay] {
208
            grok {
209
                patterns_dir   => "/etc/logstash/patterns"
210
                match          => ["postfix.relay", "^%{POSTFIX_RELAY_INFO}$"]
211
                tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ]
212
                remove_field   => [ "postfix_relay" ]
213
            }
214
        }
215
        if [postfix.delays] {
216
            grok {
217
                patterns_dir   => "/etc/logstash/patterns"
218
                match          => ["postfix.delays", "^%{POSTFIX_DELAYS}$"]
219
                tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ]
220
                remove_field   => [ "postfix_delays" ]
221
            }
222
        }
223
    }
224
225
    # process command counter data if it exists
226
    if [postfix.command_counter_data] {
227
        grok {
228
            patterns_dir   => "/etc/logstash/patterns"
229
            match          => ["postfix_command_counter_data", "^%{POSTFIX_COMMAND_COUNTER_DATA}$"]
230
            tag_on_failure => ["_grok_postfix_command_counter_data_nomatch"]
231
            remove_field   => ["postfix_command_counter_data"]
232
        }
233
    }
234
235
    # Do some data type conversions
236
    mutate {
237
        convert => [
238
            # list of integer fields
239
            "postfix.anvil_cache_size", "integer",
240
            "postfix.anvil_conn_count", "integer",
241
            "postfix.anvil_conn_rate", "integer",
242
            "postfix.client_port", "integer",
243
            "postfix.cmd_auth", "integer",
244
            "postfix.cmd_auth_accepted", "integer",
245
            "postfix.cmd_count", "integer",
246
            "postfix.cmd_count_accepted", "integer",
247
            "postfix.cmd_data", "integer",
248
            "postfix.cmd_data_accepted", "integer",
249
            "postfix.cmd_ehlo", "integer",
250
            "postfix.cmd_ehlo_accepted", "integer",
251
            "postfix.cmd_helo", "integer",
252
            "postfix.cmd_helo_accepted", "integer",
253
            "postfix.cmd_mail", "integer",
254
            "postfix.cmd_mail_accepted", "integer",
255
            "postfix.cmd_quit", "integer",
256
            "postfix.cmd_quit_accepted", "integer",
257
            "postfix.cmd_rcpt", "integer",
258
            "postfix.cmd_rcpt_accepted", "integer",
259
            "postfix.cmd_rset", "integer",
260
            "postfix.cmd_rset_accepted", "integer",
261
            "postfix.cmd_starttls", "integer",
262
            "postfix.cmd_starttls_accepted", "integer",
263
            "postfix.cmd_unknown", "integer",
264
            "postfix.cmd_unknown_accepted", "integer",
265
            "postfix.nrcpt", "integer",
266
            "postfix.postscreen_cache_dropped", "integer",
267
            "postfix.postscreen_cache_retained", "integer",
268
            "postfix.postscreen_dnsbl_rank", "integer",
269
            "postfix.relay_port", "integer",
270
            "postfix.server_port", "integer",
271
            "postfix.size", "integer",
272
            "postfix.status_code", "integer",
273
            "postfix.termination_signal", "integer",
274
275
            # list of float fields
276
            "postfix.delay", "float",
277
            "postfix.delay_before_qmgr", "float",
278
            "postfix.delay_conn_setup", "float",
279
            "postfix.delay_in_qmgr", "float",
280
            "postfix.delay_transmission", "float",
281
            "postfix.postscreen_violation_time", "float"
282
        ]
283
    }
284
mutate {
285
   rename => ["host", "server"]
286
   convert => {"server" => "string"}
287
}
288
289
}
290
291
output {
292
        elasticsearch {
293
            hosts    => "localhost:9200"
294
            index    => "postfix-%{+YYYY.MM.dd}"
295
        }
296
297
}
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315