Skip to content

Commit

Permalink
dbparser: Implement a template() option for the parser
Browse files Browse the repository at this point in the history 
Similar to the csv-parser(), this patch makes dbparser() accept a
template() option, that can be used to tell the parser what to parse
instead of the MESSAGE part (which it does by default).

This allows one to conveniently chain parsers together, assembling one
parser's output into an input template for another, without the need
for complicated rewrites:

,----
| parser p_step1 {
|  csv-parser(columns("timestamp", "host", "app", "instance", "session", "message"));
| };
|
| parser p_step2 {
|  db-parser(file("etc/patterndb.d/step2.xml")
|            template("${session}: ${message}"));
| };
|
| log {
|   source(...);
|   parser(p_step1);
|   parser(p_step2);
|   destination(...);
| };
`----

This patch makes this config work, so that the db-parser() will
operate on the given template, instead of $MESSAGE.

Signed-off-by: Balazs Scheidler <bazsi@balabit.hu>
Signed-off-by: Gergely Nagy <algernon@balabit.hu>
  • Loading branch information
@bazsi @algernon
bazsi authored and algernon committed Jan 17, 2013
1 parent b6f37d4 commit 60a639d
Show file tree
Hide file tree
Showing 7 changed files with 75 additions and 34 deletions.
5 changes: 3 additions & 2 deletions modules/dbparser/dbparser-grammar.ym
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2011 Balázs Scheidler
* Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2013 Balázs Scheidler
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
Expand Down Expand Up @@ -80,6 +80,7 @@ parser_db_opts
parser_db_opt
: KW_FILE '(' string ')' { log_db_parser_set_db_file(((LogDBParser *) last_parser), $3); free($3); }
| KW_INJECT_MODE '(' parser_db_inject_mode ')' { log_db_parser_set_inject_mode(((LogDBParser *) last_parser), $3); free($3); }
| parser_opt
;

parser_db_inject_mode
Expand Down
20 changes: 17 additions & 3 deletions modules/dbparser/dbparser.c
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2012 Balázs Scheidler
* Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2013 Balázs Scheidler
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
Expand Down Expand Up @@ -218,8 +218,22 @@ log_db_parser_process(LogParser *s, LogMessage **pmsg, const LogPathOptions *pat
}
if (self->db)
{
PDBInput pdb_input;

log_msg_make_writable(pmsg, path_options);
pattern_db_process(self->db, *pmsg);

pdb_input.msg = *pmsg;
pdb_input.program_handle = LM_V_PROGRAM;
pdb_input.message_handle = LM_V_MESSAGE;

if (self->super.template)
{
/* we are using a user-supplied template() in place of $MESSAGE */
pdb_input.message_handle = LM_V_NONE;
pdb_input.message_string = input;
pdb_input.message_len = input_len;
}
pattern_db_process(self->db, &pdb_input);
}
return TRUE;
}
Expand Down
6 changes: 3 additions & 3 deletions modules/dbparser/patterndb-int.h
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2002-2010 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2010 Balázs Scheidler
* Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2013 Balázs Scheidler
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
Expand Down Expand Up @@ -186,7 +186,7 @@ typedef struct _PDBRuleSet
} PDBRuleSet;

gboolean pdb_rule_set_load(PDBRuleSet *self, GlobalConfig *cfg, const gchar *config, GList **examples);
PDBRule *pdb_rule_set_lookup(PDBRuleSet *self, LogMessage *msg, GArray *dbg_list);
PDBRule *pdb_rule_set_lookup(PDBRuleSet *self, PDBInput *input, GArray *dbg_list);

PDBRuleSet *pdb_rule_set_new(void);
void pdb_rule_set_free(PDBRuleSet *self);
Expand Down
38 changes: 26 additions & 12 deletions modules/dbparser/patterndb.c
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2012 Balázs Scheidler
* Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2013 Balázs Scheidler
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
Expand Down Expand Up @@ -1290,7 +1290,7 @@ pdb_rule_set_load(PDBRuleSet *self, GlobalConfig *cfg, const gchar *config, GLis
* @ref_handle: if the matches are indirect matches, they are referenced based on this handle (eg. LM_V_MESSAGE)
**/
void
log_db_add_matches(LogMessage *msg, GArray *matches, NVHandle ref_handle)
log_db_add_matches(LogMessage *msg, GArray *matches, NVHandle ref_handle, const gchar *input_string)
{
gint i;
for (i = 0; i < matches->len; i++)
Expand All @@ -1302,10 +1302,14 @@ log_db_add_matches(LogMessage *msg, GArray *matches, NVHandle ref_handle)
log_msg_set_value(msg, match->handle, match->match, match->len);
g_free(match->match);
}
else
else if (ref_handle != LM_V_NONE)
{
log_msg_set_value_indirect(msg, match->handle, ref_handle, match->type, match->ofs, match->len);
}
else
{
log_msg_set_value(msg, match->handle, input_string + match->ofs, match->len);
}
}
}

Expand All @@ -1315,23 +1319,24 @@ log_db_add_matches(LogMessage *msg, GArray *matches, NVHandle ref_handle)
* NOTE: it also modifies @msg to store the name-value pairs found during lookup, so
*/
PDBRule *
pdb_rule_set_lookup(PDBRuleSet *self, LogMessage *msg, GArray *dbg_list)
pdb_rule_set_lookup(PDBRuleSet *self, PDBInput *input, GArray *dbg_list)
{
RNode *node;
LogMessage *msg = input->msg;
GArray *prg_matches, *matches;
const gchar *program;
gssize program_len;

if (G_UNLIKELY(!self->programs))
return FALSE;

program = log_msg_get_value(msg, LM_V_PROGRAM, &program_len);
program = log_msg_get_value(msg, input->program_handle, &program_len);
prg_matches = g_array_new(FALSE, TRUE, sizeof(RParserMatch));
node = r_find_node(self->programs, (gchar *) program, (gchar *) program, program_len, prg_matches);

if (node)
{
log_db_add_matches(msg, prg_matches, LM_V_PROGRAM);
log_db_add_matches(msg, prg_matches, input->program_handle, program);
g_array_free(prg_matches, TRUE);

PDBProgram *program = (PDBProgram *) node->value;
Expand All @@ -1349,7 +1354,16 @@ pdb_rule_set_lookup(PDBRuleSet *self, LogMessage *msg, GArray *dbg_list)
matches = g_array_new(FALSE, TRUE, sizeof(RParserMatch));
g_array_set_size(matches, 1);

message = log_msg_get_value(msg, LM_V_MESSAGE, &message_len);
if (input->message_handle)
{
message = log_msg_get_value(msg, input->message_handle, &message_len);
}
else
{
message = input->message_string;
message_len = input->message_len;
}

if (G_UNLIKELY(dbg_list))
msg_node = r_find_node_dbg(program->rules, (gchar *) message, (gchar *) message, message_len, matches, dbg_list);
else
Expand All @@ -1366,7 +1380,7 @@ pdb_rule_set_lookup(PDBRuleSet *self, LogMessage *msg, GArray *dbg_list)
log_msg_set_value(msg, class_handle, rule->class ? rule->class : "system", -1);
log_msg_set_value(msg, rule_id_handle, rule->rule_id, -1);

log_db_add_matches(msg, matches, LM_V_MESSAGE);
log_db_add_matches(msg, matches, input->message_handle, message);
g_array_free(matches, TRUE);

if (!rule->class)
Expand Down Expand Up @@ -1565,16 +1579,16 @@ pattern_db_get_ruleset_version(PatternDB *self)
}

gboolean
pattern_db_process(PatternDB *self, LogMessage *msg)
pattern_db_process(PatternDB *self, PDBInput *input)
{
PDBRule *rule;
LogMessage *msg = input->msg;

if (G_UNLIKELY(!self->ruleset))
return FALSE;


g_static_rw_lock_reader_lock(&self->lock);
rule = pdb_rule_set_lookup(self->ruleset, msg, NULL);
rule = pdb_rule_set_lookup(self->ruleset, input, NULL);
g_static_rw_lock_reader_unlock(&self->lock);
if (rule)
{
Expand Down
18 changes: 15 additions & 3 deletions modules/dbparser/patterndb.h
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2002-2010 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2010 Balázs Scheidler
* Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2013 Balázs Scheidler
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
Expand Down Expand Up @@ -31,6 +31,18 @@

typedef struct _PatternDB PatternDB;

typedef struct _PDBInput
{
LogMessage *msg;
NVHandle program_handle;
NVHandle message_handle;
const gchar *message_string;
gssize message_len;
} PDBInput;

#define PDB_INPUT_DEFAULT(msg) { (msg), LM_V_PROGRAM, LM_V_MESSAGE, NULL, 0 }
#define PDB_INPUT_MESSAGE(msg) ({ PDBInput __pdb_input = { (msg), LM_V_PROGRAM, LM_V_MESSAGE, NULL, 0 }; &__pdb_input; })

typedef void (*PatternDBEmitFunc)(LogMessage *msg, gboolean synthetic, gpointer user_data);
void pattern_db_set_emit_func(PatternDB *self, PatternDBEmitFunc emit_func, gpointer emit_data);

Expand All @@ -39,7 +51,7 @@ const gchar *pattern_db_get_ruleset_pub_date(PatternDB *self);
gboolean pattern_db_reload_ruleset(PatternDB *self, GlobalConfig *cfg, const gchar *pdb_file);

void pattern_db_timer_tick(PatternDB *self);
gboolean pattern_db_process(PatternDB *self, LogMessage *msg);
gboolean pattern_db_process(PatternDB *self, PDBInput *input);
void pattern_db_expire_state(PatternDB *self);
void pattern_db_forget_state(PatternDB *self);

Expand Down
12 changes: 6 additions & 6 deletions modules/dbparser/pdbtool/pdbtool.c
View file
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2012 Balázs Scheidler
* Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 1998-2013 Balázs Scheidler
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
Expand Down Expand Up @@ -518,7 +518,7 @@ pdbtool_match(int argc, char *argv[])
const gchar *msg_string;
PDBRule *rule;

rule = pdb_rule_set_lookup(patterndb->ruleset, msg, dbg_list);
rule = pdb_rule_set_lookup(patterndb->ruleset, PDB_INPUT_MESSAGE(msg), dbg_list);
if (rule)
pdb_rule_unref(rule);

Expand Down Expand Up @@ -600,7 +600,7 @@ pdbtool_match(int argc, char *argv[])
}
else
{
pattern_db_process(patterndb, msg);
pattern_db_process(patterndb, PDB_INPUT_MESSAGE(msg));
}

if (G_LIKELY(proto))
Expand Down Expand Up @@ -734,7 +734,6 @@ pdbtool_test(int argc, char *argv[])

if (example->message && example->program)
{

if (test_ruleid)
{
if (strcmp(example->rule->rule_id, test_ruleid) != 0)
Expand All @@ -752,7 +751,8 @@ pdbtool_test(int argc, char *argv[])
log_msg_set_value(msg, LM_V_PROGRAM, example->program, strlen(example->program));

printf("Testing message program='%s' message='%s'\n", example->program, example->message);
pattern_db_process(patterndb, msg);

pattern_db_process(patterndb, PDB_INPUT_MESSAGE(msg));

if (!pdbtool_test_value(msg, ".classifier.rule_id", example->rule->rule_id) && debug_pattern)
{
Expand Down
10 changes: 5 additions & 5 deletions modules/dbparser/tests/test_patterndb.c
View file
Expand Up @@ -155,7 +155,7 @@ test_rule_value_without_clean(const gchar *program, const gchar *pattern,
log_msg_set_value(msg, LM_V_HOST, MYHOST, strlen(MYHOST));
log_msg_set_value(msg, LM_V_PID, MYPID, strlen(MYPID));

result = pattern_db_process(patterndb, msg);
result = pattern_db_process(patterndb, PDB_INPUT_MESSAGE(msg));
val = log_msg_get_value(msg, log_msg_get_value_handle(name), &len);
if (value)
found = strcmp(val, value) == 0;
Expand Down Expand Up @@ -184,7 +184,7 @@ test_rule_tag(const gchar *pattern, const gchar *tag, gboolean set)
log_msg_set_value(msg, LM_V_HOST, MYHOST, strlen(MYHOST));
log_msg_set_value(msg, LM_V_PID, MYPID, strlen(MYPID));

result = pattern_db_process(patterndb, msg);
result = pattern_db_process(patterndb, PDB_INPUT_MESSAGE(msg));
found = log_msg_is_tag_by_name(msg, tag);

if (set ^ found)
Expand All @@ -208,7 +208,7 @@ test_rule_action_message_value(const gchar *pattern, gint timeout, gint ndx, con
log_msg_set_value(msg, LM_V_PID, MYPID, strlen(MYPID));
msg->timestamps[LM_TS_STAMP].tv_sec = msg->timestamps[LM_TS_RECVD].tv_sec;

result = pattern_db_process(patterndb, msg);
result = pattern_db_process(patterndb, PDB_INPUT_MESSAGE(msg));
if (timeout)
timer_wheel_set_time(patterndb->timer_wheel, timer_wheel_get_time(patterndb->timer_wheel) + timeout + 1);

Expand Down Expand Up @@ -242,7 +242,7 @@ test_rule_action_message_tag(const gchar *pattern, gint timeout, gint ndx, const
log_msg_set_value(msg, LM_V_PID, MYPID, strlen(MYPID));
msg->timestamps[LM_TS_STAMP].tv_sec = msg->timestamps[LM_TS_RECVD].tv_sec;

result = pattern_db_process(patterndb, msg);
result = pattern_db_process(patterndb, PDB_INPUT_MESSAGE(msg));
if (timeout)
timer_wheel_set_time(patterndb->timer_wheel, timer_wheel_get_time(patterndb->timer_wheel) + timeout + 5);
if (ndx >= messages->len)
Expand Down Expand Up @@ -353,7 +353,7 @@ test_pattern(const gchar *pattern, const gchar *rule, gboolean match)
log_msg_set_value(msg, LM_V_PROGRAM, "test", strlen(MYHOST));
log_msg_set_value(msg, LM_V_MESSAGE, pattern, strlen(pattern));

result = pattern_db_process(patterndb, msg);
result = pattern_db_process(patterndb, PDB_INPUT_MESSAGE(msg));

log_template_format(templ, msg, NULL, LTZ_LOCAL, 0, NULL, res);

Expand Down

0 comments on commit 60a639d

Please sign in to comment.