1 /*
2 * Copyright (c) 2002-2013 Balabit
3 * Copyright (c) 1998-2013 Balázs Scheidler
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 *
19 * As an additional exemption you are allowed to compile & link against the
20 * OpenSSL libraries as published by the OpenSSL project. See the file
21 * COPYING for details.
22 *
23 */
24
25 #ifndef LOGMSG_H_INCLUDED
26 #define LOGMSG_H_INCLUDED
27
28 #include "syslog-ng.h"
29 #include "gsockaddr.h"
30 #include "atomic.h"
31 #include "serialize.h"
32 #include "timeutils/unixtime.h"
33 #include "logmsg/nvtable.h"
34 #include "msg-format.h"
35 #include "logmsg/tags.h"
36
37 #include <sys/types.h>
38 #include <sys/socket.h>
39 #include <netinet/in.h>
40 #include <sys/time.h>
41 #include <iv_list.h>
42
43 typedef enum
44 {
45 AT_UNDEFINED,
46 AT_PROCESSED,
47 AT_ABORTED,
48 AT_SUSPENDED
49 } AckType;
50
51 #define IS_ACK_ABORTED(x) ((x) == AT_ABORTED ? 1 : 0)
52
53 #define IS_ABORTFLAG_ON(x) ((x) == 1 ? TRUE : FALSE)
54
55 #define IS_ACK_SUSPENDED(x) ((x) == AT_SUSPENDED ? 1 : 0)
56
57 #define IS_SUSPENDFLAG_ON(x) ((x) == 1 ? TRUE : FALSE)
58
59 #define STRICT_ROUND_TO_NEXT_EIGHT(x) ((x + 8) & ~7)
60
61 typedef struct _LogPathOptions LogPathOptions;
62
63 typedef void (*LMAckFunc)(LogMessage *lm, AckType ack_type);
64
65 #define RE_MAX_MATCHES 256
66
67 typedef enum
68 {
69 LM_TS_STAMP = 0,
70 LM_TS_RECVD = 1,
71 LM_TS_PROCESSED = 2,
72 LM_TS_MAX
73 } LogMessageTimeStamp;
74
75 /* builtin values */
76 enum
77 {
78 LM_V_NONE,
79 LM_V_HOST,
80 LM_V_HOST_FROM,
81 LM_V_MESSAGE,
82 LM_V_PROGRAM,
83 LM_V_PID,
84 LM_V_MSGID,
85 LM_V_SOURCE,
86 LM_V_LEGACY_MSGHDR,
87
88 /* NOTE: this is used as the number of "statically" allocated elements in
89 * an NVTable. NVTable may impose restrictions on this value (for
90 * instance had to be an even number earlier). So be sure to validate
91 * whether LM_V_MAX would fit NVTable if you add further enums here.
92 */
93
94 LM_V_MAX,
95 };
96
97 enum
98 {
99 LM_VF_SDATA = 0x0001,
100 LM_VF_MATCH = 0x0002,
101 LM_VF_MACRO = 0x0004,
102 };
103
104 enum
105 {
106 /* these flags also matter when the message is serialized */
107 LF_OLD_UNPARSED = 0x0001,
108 /* message payload is guaranteed to be valid utf8 */
109 LF_UTF8 = 0x0001,
110 /* message was generated from within syslog-ng, doesn't matter if it came from the internal() source */
111 LF_INTERNAL = 0x0002,
112 /* message was received on a local transport, e.g. it was generated on the local machine */
113 LF_LOCAL = 0x0004,
114 /* message is a MARK mode */
115 LF_MARK = 0x0008,
116 /* state flags that only matter during syslog-ng runtime and never
117 * when a message is serialized */
118 LF_STATE_MASK = 0xFFF0,
119 LF_STATE_OWN_PAYLOAD = 0x0010,
120 LF_STATE_OWN_SADDR = 0x0020,
121 LF_STATE_OWN_DADDR = 0x0040,
122 LF_STATE_OWN_TAGS = 0x0080,
123 LF_STATE_OWN_SDATA = 0x0100,
124 LF_STATE_OWN_MASK = 0x01F0,
125
126 /* In the log header the hostname shall be printed individually (no group name, no chain hosts)*/
127 LF_SIMPLE_HOSTNAME = 0x0200,
128
129 LF_CHAINED_HOSTNAME = 0x00010000,
130
131 /* NOTE: this flag is now unused. The original intent was to save whether
132 * LEGACY_MSGHDR was saved by the parser code. Now we simply check
133 * whether the length of ${LEGACY_MSGHDR} is non-zero. This used to be a
134 * slow operation (when name-value pairs were stored in a hashtable), now
135 * it is much faster. Also, this makes it possible to reproduce a message
136 * entirely based on name-value pairs. Without this change, even if
137 * LEGACY_MSGHDR was transferred (e.g. ewmm), the other side couldn't
138 * reproduce the original message, as this flag was not transferred.
139 *
140 * The flag remains here for documentation, and also because it is serialized in disk-buffers
141 */
142 __UNUSED_LF_LEGACY_MSGHDR = 0x00020000,
143 };
144
145 typedef struct _LogMessageQueueNode
146 {
147 struct iv_list_head list;
148 LogMessage *msg;
149 gboolean ack_needed:1, embedded:1, flow_control_requested:1;
150 } LogMessageQueueNode;
151
152
153 /* NOTE: the members are ordered according to the presumed use frequency.
154 * The structure itself is 2 cachelines, the border is right after the "msg"
155 * member */
156 struct _LogMessage
157 {
158 /* if you change any of the fields here, be sure to adjust
159 * log_msg_clone_cow() as well to initialize fields properly */
160
161 /* ack_and_ref_and_abort_and_suspended is a 32 bit integer that is accessed in an atomic way.
162 * The upper half contains the ACK count (and the abort flag), the lower half
163 * the REF count. It is not a GAtomicCounter as due to ref/ack caching it has
164 * a lot of magic behind its implementation. See the logmsg.c file, around
165 * log_msg_ref/unref.
166 */
167
168 /* FIXME: the structure has holes, but right now it's 1 byte short to make
169 * it smaller (it is possible to create a 7 byte contiguos block but 8
170 * byte alignment is needed. Let's check this with the inline-tags stuff */
171
172 gint ack_and_ref_and_abort_and_suspended;
173
174 /* NOTE: in theory this should be a size_t (or gsize), however that takes
175 * 8 bytes, and it's highly unlikely that we'd be using more than 4GB for
176 * a LogMessage */
177
178 guint allocated_bytes;
179
180 AckRecord *ack_record;
181 LMAckFunc ack_func;
182 LogMessage *original;
183
184 /* message parts */
185
186 /* the contents of the members below is directly copied into another
187 * LogMessage with pointer values. To change any of the fields please use
188 * log_msg_set_*() functions, which will handle borrowed data members
189 * correctly.
190 */
191 /* ==== start of directly copied part ==== */
192 UnixTime timestamps[LM_TS_MAX];
193 gulong *tags;
194 NVHandle *sdata;
195
196 GSockAddr *saddr;
197 GSockAddr *daddr;
198 NVTable *payload;
199
200 guint32 flags;
201 guint16 pri;
202 guint8 initial_parse:1,
203 recursed:1,
204
205 /* NOTE: proto is just 6 bits wide, but with that it fills a hole
206 * not taking any tolls on the structure size. Realistically, we'd
207 * be storing IPPROTO_UDP and TCP in there, which fits the 6 bits.
208 * This is closely related to saddr/daddr and indicates the IP
209 * protocol that was used to deliver the datagram carrying this
210 * LogMessage. */
211
212 proto:6;
213 guint8 num_matches;
214 guint32 host_id;
215 guint64 rcptid;
216 guint8 num_tags;
217 guint8 alloc_sdata;
218 guint8 num_sdata;
219 /* ==== end of directly copied part ==== */
220
221 guint8 num_nodes;
222 guint8 cur_node;
223 guint8 protected;
224
225
226 /* preallocated LogQueueNodes used to insert this message into a LogQueue */
227 LogMessageQueueNode nodes[0];
228
229 /* a preallocated space for the initial NVTable (payload) may follow */
230 };
231
232 extern NVRegistry *logmsg_registry;
233 extern const char logmsg_sd_prefix[];
234 extern const gint logmsg_sd_prefix_len;
235 extern gint logmsg_node_max;
236
237 LogMessage *log_msg_ref(LogMessage *m);
238 void log_msg_unref(LogMessage *m);
239 void log_msg_write_protect(LogMessage *m);
240
241 static inline gboolean
log_msg_is_write_protected(const LogMessage * self)242 log_msg_is_write_protected(const LogMessage *self)
243 {
244 return self->protected;
245 }
246
247 LogMessage *log_msg_clone_cow(LogMessage *msg, const LogPathOptions *path_options);
248 LogMessage *log_msg_make_writable(LogMessage **pmsg, const LogPathOptions *path_options);
249
250 gboolean log_msg_write(LogMessage *self, SerializeArchive *sa);
251 gboolean log_msg_read(LogMessage *self, SerializeArchive *sa);
252
253 /* generic values that encapsulate log message fields, dynamic values and structured data */
254 NVHandle log_msg_get_value_handle(const gchar *value_name);
255 gboolean log_msg_is_value_name_valid(const gchar *value);
256
257 gboolean log_msg_is_handle_macro(NVHandle handle);
258 gboolean log_msg_is_handle_sdata(NVHandle handle);
259 gboolean log_msg_is_handle_match(NVHandle handle);
260
261 /*
262 * This macros allows the caching of a NVHandle (e.g. the numeric
263 * identifier of a name-value pair) in a static variable. This simplifies
264 * call sites by
265 * 1) not needed an extra initialization step to look up the handle, _or_
266 * 2) not having to open code a similar caching mechanism.
267 *
268 * NOTE: that the check itself is racy and there might be two threads
269 * executing the if() at the same time, and if they do they would _both_
270 * perform log_msg_get_value_handle(). The reason is that this is not a
271 * problem is that the handles are constant within the same execution, so
272 * both the winner and loser of the race would eventually set the cache to
273 * the right value. And albeit this 2nd lookup is unnecessary, this would
274 * happen only a limited number of times (until the variables becomes
275 * visible for all CPUs), from which point on there's no race.
276 */
277 #define LOG_MSG_GET_VALUE_HANDLE_STATIC(name) \
278 ({ \
279 static NVHandle __log_msg_value_handle = 0; \
280 \
281 if (G_UNLIKELY(!__log_msg_value_handle)) \
282 __log_msg_value_handle = log_msg_get_value_handle(name); \
283 __log_msg_value_handle; \
284 })
285
286 static inline gboolean
log_msg_is_handle_settable_with_an_indirect_value(NVHandle handle)287 log_msg_is_handle_settable_with_an_indirect_value(NVHandle handle)
288 {
289 return (handle >= LM_V_MAX);
290 }
291
292 const gchar *log_msg_get_macro_value(const LogMessage *self, gint id, gssize *value_len);
293
294 static inline const gchar *
log_msg_get_value(const LogMessage * self,NVHandle handle,gssize * value_len)295 log_msg_get_value(const LogMessage *self, NVHandle handle, gssize *value_len)
296 {
297 guint16 flags;
298
299 flags = nv_registry_get_handle_flags(logmsg_registry, handle);
300 if ((flags & LM_VF_MACRO) == 0)
301 return nv_table_get_value(self->payload, handle, value_len);
302 else
303 return log_msg_get_macro_value(self, flags >> 8, value_len);
304 }
305
306 static inline const gchar *
log_msg_get_value_if_set(const LogMessage * self,NVHandle handle,gssize * value_len)307 log_msg_get_value_if_set(const LogMessage *self, NVHandle handle, gssize *value_len)
308 {
309 guint16 flags;
310
311 flags = nv_registry_get_handle_flags(logmsg_registry, handle);
312 if ((flags & LM_VF_MACRO) == 0)
313 return nv_table_get_value_if_set(self->payload, handle, value_len);
314 else
315 return log_msg_get_macro_value(self, flags >> 8, value_len);
316 }
317
318 static inline const gchar *
log_msg_get_value_by_name(const LogMessage * self,const gchar * name,gssize * value_len)319 log_msg_get_value_by_name(const LogMessage *self, const gchar *name, gssize *value_len)
320 {
321 NVHandle handle = log_msg_get_value_handle(name);
322 return log_msg_get_value(self, handle, value_len);
323 }
324
325 static inline const gchar *
log_msg_get_value_name(NVHandle handle,gssize * name_len)326 log_msg_get_value_name(NVHandle handle, gssize *name_len)
327 {
328 return nv_registry_get_handle_name(logmsg_registry, handle, name_len);
329 }
330
331 typedef gboolean (*LogMessageTagsForeachFunc)(const LogMessage *self, LogTagId tag_id, const gchar *name,
332 gpointer user_data);
333
334 void log_msg_set_value(LogMessage *self, NVHandle handle, const gchar *new_value, gssize length);
335 void log_msg_set_value_indirect(LogMessage *self, NVHandle handle, NVHandle ref_handle, guint8 type, guint16 ofs,
336 guint16 len);
337 void log_msg_unset_value(LogMessage *self, NVHandle handle);
338 void log_msg_unset_value_by_name(LogMessage *self, const gchar *name);
339 gboolean log_msg_values_foreach(const LogMessage *self, NVTableForeachFunc func, gpointer user_data);
340 void log_msg_set_match(LogMessage *self, gint index, const gchar *value, gssize value_len);
341 void log_msg_set_match_indirect(LogMessage *self, gint index, NVHandle ref_handle, guint8 type, guint16 ofs,
342 guint16 len);
343 void log_msg_clear_matches(LogMessage *self);
344
345 static inline void
log_msg_set_value_by_name(LogMessage * self,const gchar * name,const gchar * value,gssize length)346 log_msg_set_value_by_name(LogMessage *self, const gchar *name, const gchar *value, gssize length)
347 {
348 NVHandle handle = log_msg_get_value_handle(name);
349 log_msg_set_value(self, handle, value, length);
350 }
351
352 void log_msg_append_format_sdata(const LogMessage *self, GString *result, guint32 seq_num);
353 void log_msg_format_sdata(const LogMessage *self, GString *result, guint32 seq_num);
354
355 void log_msg_set_tag_by_id_onoff(LogMessage *self, LogTagId id, gboolean on);
356 void log_msg_set_tag_by_id(LogMessage *self, LogTagId id);
357 void log_msg_set_tag_by_name(LogMessage *self, const gchar *name);
358 void log_msg_set_saddr(LogMessage *self, GSockAddr *saddr);
359 void log_msg_set_saddr_ref(LogMessage *self, GSockAddr *saddr);
360 void log_msg_set_daddr(LogMessage *self, GSockAddr *daddr);
361 void log_msg_set_daddr_ref(LogMessage *self, GSockAddr *daddr);
362 void log_msg_clear_tag_by_id(LogMessage *self, LogTagId id);
363 void log_msg_clear_tag_by_name(LogMessage *self, const gchar *name);
364 gboolean log_msg_is_tag_by_id(LogMessage *self, LogTagId id);
365 gboolean log_msg_is_tag_by_name(LogMessage *self, const gchar *name);
366 void log_msg_tags_foreach(const LogMessage *self, LogMessageTagsForeachFunc callback, gpointer user_data);
367 void log_msg_print_tags(const LogMessage *self, GString *result);
368
369 LogMessageQueueNode *log_msg_alloc_queue_node(LogMessage *msg, const LogPathOptions *path_options);
370 LogMessageQueueNode *log_msg_alloc_dynamic_queue_node(LogMessage *msg, const LogPathOptions *path_options);
371 void log_msg_free_queue_node(LogMessageQueueNode *node);
372
373 void log_msg_clear(LogMessage *self);
374 void log_msg_merge_context(LogMessage *self, LogMessage **context, gsize context_len);
375
376 LogMessage *log_msg_new(const gchar *msg, gint length,
377 MsgFormatOptions *parse_options);
378 LogMessage *log_msg_new_mark(void);
379 LogMessage *log_msg_new_internal(gint prio, const gchar *msg);
380 LogMessage *log_msg_new_empty(void);
381 LogMessage *log_msg_new_local(void);
382
383 void log_msg_add_ack(LogMessage *msg, const LogPathOptions *path_options);
384 void log_msg_ack(LogMessage *msg, const LogPathOptions *path_options, AckType ack_type);
385 void log_msg_drop(LogMessage *msg, const LogPathOptions *path_options, AckType ack_type);
386 const LogPathOptions *log_msg_break_ack(LogMessage *msg, const LogPathOptions *path_options,
387 LogPathOptions *local_options);
388
389 void log_msg_refcache_start_producer(LogMessage *self);
390 void log_msg_refcache_start_consumer(LogMessage *self, const LogPathOptions *path_options);
391 void log_msg_refcache_stop(void);
392
393 void log_msg_registry_init(void);
394 void log_msg_registry_deinit(void);
395 void log_msg_global_init(void);
396 void log_msg_global_deinit(void);
397 void log_msg_stats_global_init(void);
398 void log_msg_registry_foreach(GHFunc func, gpointer user_data);
399
400 gint log_msg_lookup_time_stamp_name(const gchar *name);
401
402 gssize log_msg_get_size(LogMessage *self);
403
404 #endif
405