Compare Pastes

Differences between the pastes #127900 (14.10.2019 17:49) and #179057 (03.05.2021 20:43).
1
input {
2
  beats {
3
    port => 5044
4
  }
5
}
6
7
filter {
8
     grok {
9
      match => { "message" => "%{SYSLOGTIMESTAMP} %{SYSLOGHOST} %{DATA:program}(?:\[%{POSINT}\])?: %{GREEDYDATA:message}" }
10
      overwrite => "message"
11
    }
12
  }
13
14
15
filter {
16
    # grok log lines by program name (listed alpabetically)
17
    if [program] =~ /^postfix.*\/anvil$/ {
18
        grok {
19
            patterns_dir   => "/etc/logstash/patterns"
20
            match          => [ "message", "^%{POSTFIX_ANVIL}$" ]
21
            tag_on_failure => [ "_grok_postfix_anvil_nomatch" ]
22
            add_tag        => [ "_grok_postfix_success" ]
23
        }
24
    } else if [program] =~ /^postfix.*\/bounce$/ {
25
        grok {
26
            patterns_dir   => "/etc/logstash/patterns"
27
            match          => [ "message", "^%{POSTFIX_BOUNCE}$" ]
28
            tag_on_failure => [ "_grok_postfix_bounce_nomatch" ]
29
            add_tag        => [ "_grok_postfix_success" ]
30
        }
31
    } else if [program] =~ /^postfix.*\/cleanup$/ {
32
        grok {
33
            patterns_dir   => "/etc/logstash/patterns"
34
            match          => [ "message", "^%{POSTFIX_CLEANUP}$" ]
35
            tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ]
36
            add_tag        => [ "_grok_postfix_success" ]
37
        }
38
    } else if [program] =~ /^postfix.*\/dnsblog$/ {
39
        grok {
40
            patterns_dir   => "/etc/logstash/patterns"
41
            match          => [ "message", "^%{POSTFIX_DNSBLOG}$" ]
42
            tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ]
43
            add_tag        => [ "_grok_postfix_success" ]
44
        }
45
    } else if [program] =~ /^postfix.*\/error$/ {
46
        grok {
47
            patterns_dir   => "/etc/logstash/patterns"
48
            match          => [ "message", "^%{POSTFIX_ERROR}$" ]
49
            tag_on_failure => [ "_grok_postfix_error_nomatch" ]
50
            add_tag        => [ "_grok_postfix_success" ]
51
        }
52
    } else if [program] =~ /^postfix.*\/local$/ {
53
        grok {
54
            patterns_dir   => "/etc/logstash/patterns"
55
            match          => [ "message", "^%{POSTFIX_LOCAL}$" ]
56
            tag_on_failure => [ "_grok_postfix_local_nomatch" ]
57
            add_tag        => [ "_grok_postfix_success" ]
58
        }
59
    } else if [program] =~ /^postfix.*\/master$/ {
60
        grok {
61
            patterns_dir   => "/etc/logstash/patterns"
62
            match          => [ "message", "^%{POSTFIX_MASTER}$" ]
63
            tag_on_failure => [ "_grok_postfix_master_nomatch" ]
64
            add_tag        => [ "_grok_postfix_success" ]
65
        }
66
    } else if [program] =~ /^postfix.*\/pickup$/ {
67
        grok {
68
            patterns_dir   => "/etc/logstash/patterns"
69
            match          => [ "message", "^%{POSTFIX_PICKUP}$" ]
70
            tag_on_failure => [ "_grok_postfix_pickup_nomatch" ]
71
            add_tag        => [ "_grok_postfix_success" ]
72
        }
73
    } else if [program] =~ /^postfix.*\/pipe$/ {
74
        grok {
75
            patterns_dir   => "/etc/logstash/patterns"
76
            match          => [ "message", "^%{POSTFIX_PIPE}$" ]
77
            tag_on_failure => [ "_grok_postfix_pipe_nomatch" ]
78
            add_tag        => [ "_grok_postfix_success" ]
79
        }
80
    } else if [program] =~ /^postfix.*\/postdrop$/ {
81
        grok {
82
            patterns_dir   => "/etc/logstash/patterns"
83
            match          => [ "message", "^%{POSTFIX_POSTDROP}$" ]
84
            tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ]
85
            add_tag        => [ "_grok_postfix_success" ]
86
        }
87
    } else if [program] =~ /^postfix.*\/postscreen$/ {
88
        grok {
89
            patterns_dir   => "/etc/logstash/patterns"
90
            match          => [ "message", "^%{POSTFIX_POSTSCREEN}$" ]
91
            tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ]
92
            add_tag        => [ "_grok_postfix_success" ]
93
        }
94
    } else if [program] =~ /^postfix.*\/qmgr$/ {
95
        grok {
96
            patterns_dir   => "/etc/logstash/patterns"
97
            match          => [ "message", "^%{POSTFIX_QMGR}$" ]
98
            tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ]
99
            add_tag        => [ "_grok_postfix_success" ]
100
        }
101
    } else if [program] =~ /^postfix.*\/scache$/ {
102
        grok {
103
            patterns_dir   => "/etc/logstash/patterns"
104
            match          => [ "message", "^%{POSTFIX_SCACHE}$" ]
105
            tag_on_failure => [ "_grok_postfix_scache_nomatch" ]
106
            add_tag        => [ "_grok_postfix_success" ]
107
        }
108
    } else if [program] =~ /^postfix.*\/sendmail$/ {
109
        grok {
110
            patterns_dir   => "/etc/logstash/patterns"
111
            match          => [ "message", "^%{POSTFIX_SENDMAIL}$" ]
112
            tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ]
113
            add_tag        => [ "_grok_postfix_success" ]
114
        }
115
    } else if [program] =~ /^postfix.*\/smtp$/ {
116
        grok {
117
            patterns_dir   => "/etc/logstash/patterns"
118
            match          => [ "message", "^%{POSTFIX_SMTP}$" ]
119
            tag_on_failure => [ "_grok_postfix_smtp_nomatch" ]
120
            add_tag        => [ "_grok_postfix_success" ]
121
        }
122
    } else if [program] =~ /^postfix.*\/lmtp$/ {
123
        grok {
124
            patterns_dir   => "/etc/logstash/patterns"
125
            match          => [ "message", "^%{POSTFIX_LMTP}$" ]
126
            tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ]
127
            add_tag        => [ "_grok_postfix_success" ]
128
        }
129
    } else if [program] =~ /^postfix.*\/smtpd$/ {
130
        grok {
131
            patterns_dir   => "/etc/logstash/patterns"
132
            match          => [ "message", "^%{POSTFIX_SMTPD}$" ]
133
            tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ]
134
            add_tag        => [ "_grok_postfix_success" ]
135
        }
136
    } else if [program] =~ /^postfix.*\/postsuper$/ {
137
        grok {
138
            patterns_dir   => "/etc/logstash/patterns"
139
            match          => [ "message", "^%{POSTFIX_POSTSUPER}$" ]
140
            tag_on_failure => [ "_grok_postfix_postsuper_nomatch" ]
141
            add_tag        => [ "_grok_postfix_success" ]
142
        }
143
    } else if [program] =~ /^postfix.*\/tlsmgr$/ {
144
        grok {
145
            patterns_dir   => "/etc/logstash/patterns"
146
            match          => [ "message", "^%{POSTFIX_TLSMGR}$" ]
147
            tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ]
148
            add_tag        => [ "_grok_postfix_success" ]
149
        }
150
    } else if [program] =~ /^postfix.*\/tlsproxy$/ {
151
        grok {
152
            patterns_dir   => "/etc/logstash/patterns"
153
            match          => [ "message", "^%{POSTFIX_TLSPROXY}$" ]
154
            tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ]
155
            add_tag        => [ "_grok_postfix_success" ]
156
        }
157
    } else if [program] =~ /^postfix.*\/trivial-rewrite$/ {
158
        grok {
159
            patterns_dir   => "/etc/logstash/patterns"
160
            match          => [ "message", "^%{POSTFIX_TRIVIAL_REWRITE}$" ]
161
            tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ]
162
            add_tag        => [ "_grok_postfix_success" ]
163
        }
164
    } else if [program] =~ /^postfix.*\/discard$/ {
165
        grok {
166
            patterns_dir   => "/etc/logstash/patterns"
167
            match          => [ "message", "^%{POSTFIX_DISCARD}$" ]
168
            tag_on_failure => [ "_grok_postfix_discard_nomatch" ]
169
            add_tag        => [ "_grok_postfix_success" ]
170
        }
171
    } else if [program] =~ /^postfix.*\/virtual$/ {
172
        grok {
173
            patterns_dir   => "/etc/logstash/patterns"
174
            match          => [ "message", "^%{POSTFIX_VIRTUAL}$" ]
175
            tag_on_failure => [ "_grok_postfix_virtual_nomatch" ]
176
            add_tag        => [ "_grok_postfix_success" ]
177
        }
178
    } else if [program] =~ /^postfix.*/ {
179
        mutate {
180
            add_tag => [ "_grok_postfix_program_nomatch" ]
181
        }
182
    }
183
184
    # process key-value data if it exists
185
    if [postfix.keyvalue_data] {
186
        kv {
187
            source       => "postfix.keyvalue_data"
188
            trim_value   => "<>,"
189
            prefix       => "postfix."
190
            remove_field => [ "postfix_keyvalue_data" ]
191
        }
192
193
        # some post processing of key-value data
194
        if [postfix.client] {
195
            grok {
196
                patterns_dir   => "/etc/logstash/patterns"
197
                match          => ["postfix.client", "^%{POSTFIX_CLIENT_INFO}$"]
198
                tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ]
199
                remove_field   => [ "postfix_client" ]
200
            }
201
        }
202
        if [postfix.relay] {
203
            grok {
204
                patterns_dir   => "/etc/logstash/patterns"
205
                match          => ["postfix.relay", "^%{POSTFIX_RELAY_INFO}$"]
206
                tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ]
207
                remove_field   => [ "postfix_relay" ]
208
            }
209
        }
210
        if [postfix.delays] {
211
            grok {
212
                patterns_dir   => "/etc/logstash/patterns"
213
                match          => ["postfix.delays", "^%{POSTFIX_DELAYS}$"]
214
                tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ]
215
                remove_field   => [ "postfix_delays" ]
216
            }
217
        }
218
    }
219
220
    # process command counter data if it exists
221
    if [postfix.command_counter_data] {
222
        grok {
223
            patterns_dir   => "/etc/logstash/patterns"
224
            match          => ["postfix_command_counter_data", "^%{POSTFIX_COMMAND_COUNTER_DATA}$"]
225
            tag_on_failure => ["_grok_postfix_command_counter_data_nomatch"]
226
            remove_field   => ["postfix_command_counter_data"]
227
        }
228
    }
229
230
    # Do some data type conversions
231
    mutate {
232
        convert => [
233
            # list of integer fields
234
            "postfix.anvil_cache_size", "integer",
235
            "postfix.anvil_conn_count", "integer",
236
            "postfix.anvil_conn_rate", "integer",
237
            "postfix.client_port", "integer",
238
            "postfix.cmd_auth", "integer",
239
            "postfix.cmd_auth_accepted", "integer",
240
            "postfix.cmd_count", "integer",
241
            "postfix.cmd_count_accepted", "integer",
242
            "postfix.cmd_data", "integer",
243
            "postfix.cmd_data_accepted", "integer",
244
            "postfix.cmd_ehlo", "integer",
245
            "postfix.cmd_ehlo_accepted", "integer",
246
            "postfix.cmd_helo", "integer",
247
            "postfix.cmd_helo_accepted", "integer",
248
            "postfix.cmd_mail", "integer",
249
            "postfix.cmd_mail_accepted", "integer",
250
            "postfix.cmd_quit", "integer",
251
            "postfix.cmd_quit_accepted", "integer",
252
            "postfix.cmd_rcpt", "integer",
253
            "postfix.cmd_rcpt_accepted", "integer",
254
            "postfix.cmd_rset", "integer",
255
            "postfix.cmd_rset_accepted", "integer",
256
            "postfix.cmd_starttls", "integer",
257
            "postfix.cmd_starttls_accepted", "integer",
258
            "postfix.cmd_unknown", "integer",
259
            "postfix.cmd_unknown_accepted", "integer",
260
            "postfix.nrcpt", "integer",
261
            "postfix.postscreen_cache_dropped", "integer",
262
            "postfix.postscreen_cache_retained", "integer",
263
            "postfix.postscreen_dnsbl_rank", "integer",
264
            "postfix.relay_port", "integer",
265
            "postfix.server_port", "integer",
266
            "postfix.size", "integer",
267
            "postfix.status_code", "integer",
268
            "postfix.termination_signal", "integer",
269
270
            # list of float fields
271
            "postfix.delay", "float",
272
            "postfix.delay_before_qmgr", "float",
273
            "postfix.delay_conn_setup", "float",
274
            "postfix.delay_in_qmgr", "float",
275
            "postfix.delay_transmission", "float",
276
            "postfix.postscreen_violation_time", "float"
277
        ]
278
    }
279
}
280
output {
281
        elasticsearch {
282
            hosts    => "localhost:9200"
283
            index    => "postfix10-%{+YYYY.MM.dd}"
284
        }
285
286
}