konnosDashboard
vector.yml268 lines · main
1api:
2 enabled: true
3 address: 0.0.0.0:9001
4
5sources:
6 docker_host:
7 type: docker_logs
8 exclude_containers:
9 - briven-vector
10
11transforms:
12 project_logs:
13 type: remap
14 inputs:
15 - docker_host
16 source: |-
17 .project = "default"
18 .event_message = del(.message)
19 .appname = del(.container_name)
20 del(.container_created_at)
21 del(.container_id)
22 del(.source_type)
23 del(.stream)
24 del(.label)
25 del(.image)
26 del(.host)
27 del(.stream)
28 router:
29 type: route
30 inputs:
31 - project_logs
32 route:
33 kong: '.appname == "briven-kong" || .appname == "briven-envoy"'
34 auth: '.appname == "briven-auth"'
35 rest: '.appname == "briven-rest"'
36 realtime: '.appname == "realtime-dev.briven-realtime"'
37 storage: '.appname == "briven-storage"'
38 functions: '.appname == "briven-edge-functions"'
39 db: '.appname == "briven-db"'
40 # Ignores non nginx errors since they are related with kong booting up
41 kong_logs:
42 type: remap
43 inputs:
44 - router.kong
45 source: |-
46 req, err = parse_nginx_log(.event_message, "combined")
47 if err == null {
48 .timestamp = req.timestamp
49 .metadata.request.headers.referer = req.referer
50 .metadata.request.headers.user_agent = req.agent
51 .metadata.request.headers.cf_connecting_ip = req.client
52 .metadata.response.status_code = req.status
53 url, split_err = split(req.request, " ")
54 if split_err == null {
55 .metadata.request.method = url[0]
56 .metadata.request.path = url[1]
57 .metadata.request.protocol = url[2]
58 }
59 }
60 if err != null {
61 abort
62 }
63 # Ignores non nginx errors since they are related with kong booting up
64 kong_err:
65 type: remap
66 inputs:
67 - router.kong
68 source: |-
69 .metadata.request.method = "GET"
70 .metadata.response.status_code = 200
71 parsed, err = parse_nginx_log(.event_message, "error")
72 if err == null {
73 .timestamp = parsed.timestamp
74 .severity = parsed.severity
75 .metadata.request.host = parsed.host
76 .metadata.request.headers.cf_connecting_ip = parsed.client
77 url, err = split(parsed.request, " ")
78 if err == null {
79 .metadata.request.method = url[0]
80 .metadata.request.path = url[1]
81 .metadata.request.protocol = url[2]
82 }
83 }
84 if err != null {
85 abort
86 }
87 # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.
88 auth_logs:
89 type: remap
90 inputs:
91 - router.auth
92 source: |-
93 parsed, err = parse_json(.event_message)
94 if err == null {
95 .metadata.timestamp = parsed.time
96 .metadata = merge!(.metadata, parsed)
97 }
98 # PostgREST logs are structured so we separate timestamp from message using regex
99 rest_logs:
100 type: remap
101 inputs:
102 - router.rest
103 source: |-
104 parsed, err = parse_regex(.event_message, r'^(?P<time>.*): (?P<msg>.*)$')
105 if err == null {
106 .event_message = parsed.msg
107 .timestamp = parse_timestamp!(value: parsed.time,format: "%d/%b/%Y:%H:%M:%S %z")
108 .metadata.host = .project
109 }
110 # Filter out healthcheck logs from Realtime
111 realtime_logs_filtered:
112 type: filter
113 inputs:
114 - router.realtime
115 condition: '!contains(string!(.event_message), "/health")'
116 # Realtime logs are structured so we parse the severity level using regex (ignore time because it has no date)
117 realtime_logs:
118 type: remap
119 inputs:
120 - realtime_logs_filtered
121 source: |-
122 .metadata.project = del(.project)
123 .metadata.external_id = .metadata.project
124 parsed, err = parse_regex(.event_message, r'^(?P<time>\d+:\d+:\d+\.\d+) \[(?P<level>\w+)\] (?P<msg>.*)$')
125 if err == null {
126 .event_message = parsed.msg
127 .metadata.level = parsed.level
128 }
129 # Function logs are unstructured messages on stderr
130 functions_logs:
131 type: remap
132 inputs:
133 - router.functions
134 source: |-
135 .metadata.project_ref = del(.project)
136 # Storage logs may contain json objects so we parse them for completeness
137 storage_logs:
138 type: remap
139 inputs:
140 - router.storage
141 source: |-
142 .metadata.project = del(.project)
143 .metadata.tenantId = .metadata.project
144 parsed, err = parse_json(.event_message)
145 if err == null {
146 .event_message = parsed.msg
147 .metadata.level = parsed.level
148 .metadata.timestamp = parsed.time
149 .metadata.context[0].host = parsed.hostname
150 .metadata.context[0].pid = parsed.pid
151 }
152 # Postgres logs some messages to stderr which we map to warning severity level
153 db_logs:
154 type: remap
155 inputs:
156 - router.db
157 source: |-
158 .metadata.host = "db-default"
159 .metadata.parsed.timestamp = .timestamp
160
161 parsed, err = parse_regex(.event_message, r'.*(?P<level>INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true)
162
163 if err != null || parsed == null {
164 .metadata.parsed.error_severity = "info"
165 }
166 if parsed.level != null {
167 .metadata.parsed.error_severity = parsed.level
168 }
169 if .metadata.parsed.error_severity == "info" {
170 .metadata.parsed.error_severity = "log"
171 }
172 .metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity)
173
174sinks:
175 logflare_auth:
176 type: 'http'
177 inputs:
178 - auth_logs
179 encoding:
180 codec: 'json'
181 method: 'post'
182 request:
183 retry_max_duration_secs: 30
184 retry_initial_backoff_secs: 1
185 headers:
186 x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
187 uri: 'http://analytics:4000/api/logs?source_name=gotrue.logs.prod'
188 logflare_realtime:
189 type: 'http'
190 inputs:
191 - realtime_logs
192 encoding:
193 codec: 'json'
194 method: 'post'
195 request:
196 retry_max_duration_secs: 30
197 retry_initial_backoff_secs: 1
198 headers:
199 x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
200 uri: 'http://analytics:4000/api/logs?source_name=realtime.logs.prod'
201 logflare_rest:
202 type: 'http'
203 inputs:
204 - rest_logs
205 encoding:
206 codec: 'json'
207 method: 'post'
208 request:
209 retry_max_duration_secs: 30
210 retry_initial_backoff_secs: 1
211 headers:
212 x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
213 uri: 'http://analytics:4000/api/logs?source_name=postgREST.logs.prod'
214 logflare_db:
215 type: 'http'
216 inputs:
217 - db_logs
218 encoding:
219 codec: 'json'
220 method: 'post'
221 request:
222 retry_max_duration_secs: 30
223 retry_initial_backoff_secs: 1
224 headers:
225 x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
226 # Route directly to analytics like other sinks. Retry handles the startup
227 # race where analytics may not be ready yet when early DB logs arrive.
228 uri: 'http://analytics:4000/api/logs?source_name=postgres.logs'
229 logflare_functions:
230 type: 'http'
231 inputs:
232 - functions_logs
233 encoding:
234 codec: 'json'
235 method: 'post'
236 request:
237 retry_max_duration_secs: 30
238 retry_initial_backoff_secs: 1
239 headers:
240 x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
241 uri: 'http://analytics:4000/api/logs?source_name=deno-relay-logs'
242 logflare_storage:
243 type: 'http'
244 inputs:
245 - storage_logs
246 encoding:
247 codec: 'json'
248 method: 'post'
249 request:
250 retry_max_duration_secs: 30
251 retry_initial_backoff_secs: 1
252 headers:
253 x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
254 uri: 'http://analytics:4000/api/logs?source_name=storage.logs.prod.2'
255 logflare_kong:
256 type: 'http'
257 inputs:
258 - kong_logs
259 - kong_err
260 encoding:
261 codec: 'json'
262 method: 'post'
263 request:
264 retry_max_duration_secs: 30
265 retry_initial_backoff_secs: 1
266 headers:
267 x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
268 uri: 'http://analytics:4000/api/logs?source_name=cloudflare.logs.prod'