RSYSLOG

format more easily used by logsnarf, and BigQuery. No claims are made regarding the optimality of these configurations, but should serve as a starting point for those interested.

mm_normalize

This performs some feature extraction on ssh and dhcp logs. Populating the additional fields and subrecords.

# Rules for dhcpd
rule=: %action!method:word% from %src!hwaddr:word% via %src!ifname:word%
rule=: %action!method:word% on %src!ipv4:word% to %src!hwaddr:word% (%src!host:char-to:)%)via %src!ifname:word%
rule=: %action!method:word% on %src!ipv4:word% to %src!hwaddr:word% via %src!ifname:word%
rule=: %action!method:word% for %src!ipv4:word% (%dst!ipv4:ipv4%) from %src!hwaddr:word% (%src!host:char-to:)%)  via %src!ifname:word%
rule=: %action!method:word% for %src!ipv4:word% (%dst!ipv4:ipv4%) from %src!hwaddr:word%  via %src!ifname:word%
rule=: %action!method:word% for %src!ipv4:word% from %src!hwaddr:word% (%src!host:char-to:)%)  via %src!ifname:word%
rule=: %action!method:word% for %src!ipv4:word% from %src!hwaddr:word%  via %src!ifname:word%
rule=: %action!method:word% of %src!ipv4:word% from %src!hwaddr:word% (%src!host:char-to:)%) via %src!ifname:word% %-:rest%
rule=: %action!method:word% of %src!ipv4:word% from %src!hwaddr:word% via %src!ifname:word% %-:rest%

# Rules for dhcpclient
rule=: %action!method:word% on %src!ifname:word% to %dst!ipv4:ipv4% port %dst!port:number%%-:rest%
rule:= %action!method:word for %src!ipv4:ipv4% from %dst!hwaddr:word% via %src!ifname:word%
rule:= %action!method:word on %src!ipv4:ipv4% to %src!hwaddr:word% via %src!ifname:word%

# Rules for sshd
rule=auth,success: Accepted %action!method:word% for %user!name:char-to:@%@%user!domain:word% from %src!ipv4:ipv4% port %src!port:number% %-:rest%
rule=auth,success: Accepted %action!method:word% for %user!name:char-to:@%@%user!domain:word% from %src!ipv6:word% port %src!port:number% %-:rest%
rule=auth,success: Accepted %action!method:word% for %user!name:word% from %src!ipv4:ipv4% port %src!port:number% %-:rest%
rule=auth,success: Accepted %action!method:word% for %user!name:word% from %src!ipv6:word% port %src!port:number% %-:rest%
rule=auth,failure: Failed %action!method:word% for %user!name:char-to:@%@%user!domain:word% from %src!ipv4:ipv4% port %src!port:number% %-:rest%
rule=auth,failure: Failed %action!method:word% for %user!name:char-to:@%@%user!domain:word% from %src!ipv6:word% port %src!port:number% %-:rest%
rule=auth,failure: Failed %action!method:word% for %user!name:word% from %src!ipv4:ipv4% port %src!port:number% %-:rest%
rule=auth,failure: Failed %action!method:word% for %user!name:word% from %src!ipv6:word% port %src!port:number% %-:rest%

# Rules for pam
rule=auth,success: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): user %user!name:word% authenticated as %user!name:char-to:@%@%user!domain:word%
rule=auth,success: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): user %user!name:word% authenticated as %user!name:word%
rule=: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): session %action!status:word% for user %user!euser:word% by %user!name:char-to:(%(uid=%user!id:number%)
rule=: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): session %action!status:word% for user %user!name:char-to:@%@%user!domain:word% %user!name:word% by (uid=%user!id:number%)
rule=: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): session %action!status:word% for user %user!name:char-to:@%@%user!domain:word% %user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser= rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname=%user!name:word% uid= euid=%user!eid:number% tty=%-:word% ruser= rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid=%user!id:number% euid=%user!eid:number% tty=%-:word% ruser= rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser=%user!name:word% rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word% user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser= rhost=%src!ipv4:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser= rhost= user=%user!name:word%
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): authentication failure; logname= uid= euid=%user!eid:number% tty=%-:word% ruser= rhost=
rule=auth,failure: pam_%action!method:char-to:(%(%-:char-to::%:%action!type:char-to:)%): received for user %user!name:char-to::%: %-:number% (%msg:char-to:)%)

# Rules for su
rule=authorize,success: Successful su for %user!euser:word% by %user!name:word%


annotate=success:+action.status="success"
annotate=failure:+action.status="failure"
annotate=auth:+action.type="auth"
annotate=authorize:+action.type="authz"

rsyslog

This configuration fragment provides a ruleset that runs incoming logs through the mm_normalize module, performs some sanity checking on log times (largely for devices without an internal RTC, who always provide bad times on boot, until time is synchronized), generates a table name for logsnarf, and populates some of the JSON fields manually from syslog fields. The easiest way to use this is to assign the ruleset to an input then initializing the input with the rulebase. For example:

input(type="imudp" port="514" ruleset="remote")

This needs to be declared after the rulebase, as per normal, however.

$template JSONDyna,"/srv/log/json/%$YEAR%/%$MONTH%/%$DAY%.log"

# Templates to generate the table name.
template(name="table-index-gen"
  type="list") {
    constant(value="home_logs_")
    property(name="timegenerated" dateFormat="rfc3339" position.from="1" position.to="4")
    property(name="timegenerated" dateFormat="rfc3339" position.from="6" position.to="7")
    property(name="timegenerated" dateFormat="rfc3339" position.from="9" position.to="10")
}

template(name="table-index-rep"
  type="list") {
    constant(value="home_logs_")
    property(name="timereported" dateFormat="rfc3339" position.from="1" position.to="4")
    property(name="timereported" dateFormat="rfc3339" position.from="6" position.to="7")
    property(name="timereported" dateFormat="rfc3339" position.from="9" position.to="10")
}

# Templates for unix timestamps with subsecond accuracy.
template(name="precise-unix-reported" type="list") {
  property(name="timereported" dateFormat="unixtimestamp")
  constant(value=".")
  property(name="timereported" dateFormat="subseconds")
}

template(name="precise-unix-generated" type="list") {
  property(name="timegenerated" dateFormat="unixtimestamp")
  constant(value=".")
  property(name="timegenerated" dateFormat="subseconds")
}

template(name="json_out" type="list") {
  property(name="$!all-json")
  constant(value="\n")
}

ruleset(name="remote") {
    *.* action(type="mmnormalize" ruleBase="/etc/lognorm/lognorm.rulebase")
    unset $!event.tags;
    unset $!originalmsg;
    unset $!unparsed-data;

    set $!time = exec_template("precise-unix-generated");
    set $!timereported = exec_template("precise-unix-reported");
    set $!host = $hostname;
    set $!sev = $syslogseverity-text;
    set $!syslog!fac = $syslogfacility-text;
    set $!syslog!pri = $syslogpriority-text;
    if ($programname == "") and ($procid contains "[") then {
      set $!pid = re_extract($syslogtag, "([0-9]+)]:", 0, 1, "0");
      set $!pname = re_extract($syslogtag, "\\[?([a-z0-9A-Z]+)", 0, 1, "unknown");
    } else {
      set $!pname = $programname;
      set $!pid = $procid;
    }

    set $!msg = $msg;
    set $.timediff = cnum(field($!time, 46, 1)) - cnum(field($!timereported, 46, 1));

    # If the difference in times is greater than one day, trust the generated time more.
    if ($.timediff > 86400) or ($.timediff < -86400) then {
      set $!table = exec_template("table-index-gen");
    }
    else {
      set $!table = exec_template("table-index-rep");
    }
    *.* action(type="omfile" dynaFile="JSONDyna" template="json_out")
}