Skip to content

Commit

Permalink
jsonparser: New parser module for JSON parsing.
Browse files Browse the repository at this point in the history 
The new parser supports json-c only, it does not work with
json-glib. For this reason, the configure script was modified a
little, so that it uses --with-json only. When json-c is found or
selected, then both the parser and the template function will be
built. If json-glib is found or selected, then only the template
function will be available.

The summary after configure now lists the determined settings.

The new parser is - so far - fairly simple: it parses a JSON with
json-c (int, double and string values are supported so far, booleans,
arrays and sub-objects aren't yet), and sets the appropriate
name-value pair in the LogMessage.

Users can configure a prefix to prepend before each key.

Usage is as simple as:

source s_json { tcp(port(21514) flags(no-parse)); };
destination d_json {
  file("/tmp/test.json"
       template("$(format-json --scope dot-nv-pairs)\n"));
};
parser p_json { json-parser (prefix(".json.")); };
log { source(s_json); parser(p_json); destination(d_json); };

Signed-off-by: Gergely Nagy <algernon@balabit.hu>
  • Loading branch information
@bazsi
bazsi committed Oct 29, 2011
1 parent 05065fa commit e556968
Show file tree
Hide file tree
Showing 10 changed files with 456 additions and 19 deletions.
36 changes: 19 additions & 17 deletions configure.in
View file
Expand Up @@ -25,7 +25,7 @@ EVTLOG_MIN_VERSION="0.2.12"
OPENSSL_MIN_VERSION="0.9.8"
LIBDBI_MIN_VERSION="0.8.0"
IVYKIS_MIN_VERSION="0.18"
JSON_C_MIN_VERSION="0.7"
JSON_C_MIN_VERSION="0.9"
JSON_GLIB_MIN_VERSION="0.12"
PCRE_MIN_VERSION="6.1"
LMC_MIN_VERSION="0.1.0"
Expand Down Expand Up @@ -178,10 +178,6 @@ AC_ARG_WITH(json,
Use the JSON implementation specified]
,,with_json="auto")

AC_ARG_ENABLE(json,
[ --enable-json Enable support for JSON template formatting (default: auto)]
,,enable_json="auto")

AC_ARG_ENABLE(systemd,
[ --enable-systemd Enable systemd support (default: auto)]
,,enable_systemd="auto")
Expand Down Expand Up @@ -607,8 +603,12 @@ fi
dnl ***************************************************************************
dnl json headers/libraries
dnl ***************************************************************************
PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="")
PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="")
if test "x$with_json" = "xauto" || test "x$with_json" = "json-c"; then
PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="")
fi
if test "x$with_json" = "xauto" || test "x$with_json" = "json-glib"; then
PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="")
fi

dnl ***************************************************************************
dnl pcre headers/libraries
Expand Down Expand Up @@ -887,14 +887,14 @@ elif test "x$with_json" = "xjson-c"; then
AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION: is pkg-config in path?])
fi

if test "x$enable_json" = "xauto"; then
if test "x$with_json" = "xno"; then
enable_json="no"
else
enable_json="yes"
fi
elif test "x$enable_json" = "xyes" -a "x$with_json" = "xno"; then
AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION or json-glib-1.0 >= $JSON_GLIB_MIN_VERSION: is pkg-config in path?])
if test "x$with_json" = "xjson-c"; then
enable_json_parse="yes"
enable_json_format="yes"
fi

if test "x$with_json" = "xjson-glib"; then
enable_json_parse="no"
enable_json_format="yes"
fi

if test "x$enable_systemd" = "xauto"; then
Expand Down Expand Up @@ -1077,7 +1077,8 @@ AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"])
AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"])
AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"])
AM_CONDITIONAL(ENABLE_MONGODB, [test "$enable_mongodb" = "yes"])
AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"])
AM_CONDITIONAL(ENABLE_JSON_FORMAT, [test "$enable_json_format" = "yes"])
AM_CONDITIONAL(ENABLE_JSON_PARSE, [test "$enable_json_parse" = "yes"])
AM_CONDITIONAL(WITH_LIBSYSTEMD, [test "$with_libsystemd" = "yes"])

# substitution into manual pages
Expand Down Expand Up @@ -1144,6 +1145,7 @@ AC_OUTPUT(dist.conf
modules/pacctformat/Makefile
modules/basicfuncs/Makefile
modules/tfjson/Makefile
modules/jsonparser/Makefile
scripts/Makefile
scripts/update-patterndb
doc/Makefile
Expand Down Expand Up @@ -1193,6 +1195,6 @@ echo " SSL support (module) : ${enable_ssl:=no}"
echo " SQL support (module) : ${enable_sql:=no}"
echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}"
echo " MongoDB destination (module): ${enable_mongodb:=no}"
echo " JSON support (module) : ${enable_json:=no} (using ${with_json})"
echo " JSON support (module) : parser=${enable_json_parse:=no}, formatter=${enable_json_format:=no} (using ${with_json})"


2 changes: 1 addition & 1 deletion modules/Makefile.am
View file
@@ -1 +1 @@
SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson dummy
SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson jsonparser dummy
22 changes: 22 additions & 0 deletions modules/jsonparser/Makefile.am
View file
@@ -0,0 +1,22 @@
moduledir = @moduledir@
AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib
export top_srcdir

if ENABLE_JSON_PARSE
module_LTLIBRARIES := libjsonparser.la
libjsonparser_la_SOURCES = \
jsonparser.c jsonparser.h \
jsonparser-grammar.y \
jsonparser-parser.c jsonparser-parser.h \
jsonparser-plugin.c

libjsonparser_la_CPPFLAGS = $(AM_CPPFLAGS)
libjsonparser_la_CFLAGS = $(JSON_CFLAGS)
libjsonparser_la_LIBADD = $(MODULE_DEPS_LIBS)
libjsonparser_la_LDFLAGS = $(MODULE_LDFLAGS) $(JSON_LIBS)

BUILT_SOURCES = jsonparser-grammar.y jsonparser-grammar.c jsonparser-grammar.h
EXTRA_DIST = $(BUILT_SOURCES) jsonparser-grammar.ym
endif

include $(top_srcdir)/build/lex-rules.am
86 changes: 86 additions & 0 deletions modules/jsonparser/jsonparser-grammar.ym
View file
@@ -0,0 +1,86 @@
/*
* Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
* As an additional exemption you are allowed to compile & link against the
* OpenSSL libraries as published by the OpenSSL project. See the file
* COPYING for details.
*/

%code top {
#include "jsonparser-parser.h"

}


%code {

#include "jsonparser.h"
#include "cfg-parser.h"
#include "jsonparser-grammar.h"
#include "syslog-names.h"
#include "messages.h"

extern LogParser *last_parser;

}

%name-prefix "jsonparser_"

/* this parameter is needed in order to instruct bison to use a complete
* argument list for yylex/yyerror */

%lex-param {CfgLexer *lexer}
%parse-param {CfgLexer *lexer}
%parse-param {LogParser **instance}
%parse-param {gpointer arg}

/* INCLUDE_DECLS */

%token KW_JSON_PARSER
%token KW_PREFIX

%type <ptr> parser_expr_json

%%

start
: LL_CONTEXT_PARSER parser_expr_json { YYACCEPT; }
;


parser_expr_json
: KW_JSON_PARSER '('
{
last_parser = *instance = (LogParser *) log_json_parser_new();
}
parser_json_opts
')' { $$ = last_parser; }
;

parser_json_opts
: parser_json_opt parser_json_opts
|
;

parser_json_opt
: KW_PREFIX '(' string ')' { log_json_parser_set_prefix(last_parser, $3); free ($3); }
|
;

/* INCLUDE_RULES */

%%
49 changes: 49 additions & 0 deletions modules/jsonparser/jsonparser-parser.c
View file
@@ -0,0 +1,49 @@
/*
* Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
* As an additional exemption you are allowed to compile & link against the
* OpenSSL libraries as published by the OpenSSL project. See the file
* COPYING for details.
*/

#include "jsonparser.h"
#include "cfg-parser.h"
#include "jsonparser-grammar.h"

extern int jsonparser_debug;

int jsonparser_parse(CfgLexer *lexer, LogParser **instance, gpointer arg);

static CfgLexerKeyword jsonparser_keywords[] =
{
{ "json_parser", KW_JSON_PARSER, },
{ "prefix", KW_PREFIX, },
{ NULL }
};

CfgParser jsonparser_parser =
{
#if ENABLE_DEBUG
.debug_flag = &jsonparser_debug,
#endif
.name = "jsonparser",
.keywords = jsonparser_keywords,
.parse = (gint (*)(CfgLexer *, gpointer *, gpointer)) jsonparser_parse,
.cleanup = (void (*)(gpointer)) log_pipe_unref,
};

CFG_PARSER_IMPLEMENT_LEXER_BINDING(jsonparser_, LogParser **)
34 changes: 34 additions & 0 deletions modules/jsonparser/jsonparser-parser.h
View file
@@ -0,0 +1,34 @@
/*
* Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
* As an additional exemption you are allowed to compile & link against the
* OpenSSL libraries as published by the OpenSSL project. See the file
* COPYING for details.
*/

#ifndef JSONPARSER_PARSER_H_INCLUDED
#define JSONPARSER_PARSER_H_INCLUDED

#include "cfg-parser.h"
#include "cfg-lexer.h"
#include "logparser.h"

extern CfgParser jsonparser_parser;

CFG_PARSER_DECLARE_LEXER_BINDING(jsonparser_, LogParser **)

#endif
53 changes: 53 additions & 0 deletions modules/jsonparser/jsonparser-plugin.c
View file
@@ -0,0 +1,53 @@
/*
* Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
* Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
* As an additional exemption you are allowed to compile & link against the
* OpenSSL libraries as published by the OpenSSL project. See the file
* COPYING for details.
*/

#include "cfg-parser.h"
#include "plugin.h"
#include "jsonparser.h"

extern CfgParser jsonparser_parser;

static Plugin jsonparser_plugins[] =
{
{
.type = LL_CONTEXT_PARSER,
.name = "json-parser",
.parser = &jsonparser_parser,
},
};

gboolean
jsonparser_module_init(GlobalConfig *cfg, CfgArgs *args)
{
plugin_register(cfg, jsonparser_plugins, G_N_ELEMENTS(jsonparser_plugins));
return TRUE;
}

const ModuleInfo module_info =
{
.canonical_name = "jsonparser",
.version = VERSION,
.description = "The jsonparser module provides JSON parsing support for syslog-ng.",
.core_revision = SOURCE_REVISION,
.plugins = jsonparser_plugins,
.plugins_len = G_N_ELEMENTS(jsonparser_plugins),
};

0 comments on commit e556968

Please sign in to comment.