Upgrade dev with upstream master

.github/workflows/run_centos_7.yml

@@ -0,0 +1,59 @@
+# Copyright 2020 Rainer Gerhards and Others
+#
+# https://github.com/rsyslog/rsyslog-pkg-ubuntu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# References:
+#
+# https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options
+# https://github.com/settings/notifications
+# https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog
+
+
+---
+name: check centos 7
+
+on:
+  pull_request:
+
+jobs:
+  CI:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 10
+
+    steps:
+      - name: git checkout project
+        uses: actions/checkout@v1
+
+      - name: run container CI pipeline
+        run: |
+          chmod -R go+rw .
+          export RSYSLOG_CONTAINER_UID="" # use default
+          export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php'
+          export CFLAGS='-g'
+          export CC='gcc'
+          export USE_AUTO_DEBUG='off'
+          export CI_MAKE_OPT='-j20'
+          export CI_MAKE_CHECK_OPT='-j1'
+          export CI_CHECK_CMD='check'
+          export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:7'
+          # we need to override rsyslog configure options to avoid side-effects!
+          export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE=''
+          devtools/devcontainer.sh --rm devtools/run-ci.sh
+
+      - name: show error logs (if we errored)
+        if:  ${{ failure() || cancelled() }}
+        run: |
+          devtools/gather-check-logs.sh
+          cat failed-tests.log

.github/workflows/run_centos_8.yml

@@ -0,0 +1,59 @@
+# Copyright 2020 Rainer Gerhards and Others
+#
+# https://github.com/rsyslog/rsyslog-pkg-ubuntu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# References:
+#
+# https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options
+# https://github.com/settings/notifications
+# https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog
+
+
+---
+name: check centos 8
+
+on:
+  pull_request:
+
+jobs:
+  CI:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 10
+
+    steps:
+      - name: git checkout project
+        uses: actions/checkout@v1
+
+      - name: run container CI pipeline
+        run: |
+          chmod -R go+rw .
+          export RSYSLOG_CONTAINER_UID="" # use default
+          export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php'
+          export CFLAGS='-g'
+          export CC='gcc'
+          export USE_AUTO_DEBUG='off'
+          export CI_MAKE_OPT='-j20'
+          export CI_MAKE_CHECK_OPT='-j1'
+          export CI_CHECK_CMD='check'
+          export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:8'
+          # we need to override rsyslog configure options to avoid side-effects!
+          export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE=''
+          devtools/devcontainer.sh --rm devtools/run-ci.sh
+
+      - name: show error logs (if we errored)
+        if:  ${{ failure() || cancelled() }}
+        run: |
+          devtools/gather-check-logs.sh
+          cat failed-tests.log

README

@@ -3,7 +3,7 @@ Liblognorm is a fast-samples based normalization library.
 More information on liblognorm can be found at
     http://www.liblognorm.com
 
-Liblognorm evolves since several years and was intially meant to be used primarily with
+Liblognorm evolves since several years and was initially meant to be used primarily with
 the Mitre CEE effort. Consequently, the initial version of liblognorm (0.x)
 uses the libee CEE support library in its API.
 
@@ -27,7 +27,7 @@ https://www.researchgate.net/publication/310545144_Efficient_Normalization_of_IT
 
 The current library is the result of that effort. Application developers
 are encouraged to switch to this version, as it provides the benefit of
-a simpler API. This version is now being tracked by the git master branch.
+a simpler API. This version is now being tracked by the git default branch.
 
 However, if you need to stick to the old API, there is a git branch
 liblognorm0, which contains the previous version of the library. This

devtools/default_dev_container

@@ -0,0 +1 @@
+rsyslog/rsyslog_dev_base_ubuntu:20.04

devtools/devcontainer.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# This scripts uses an rsyslog development container to execute given
+# command inside it.
+# Note: command line parameters are passed as parameters to the container,
+# with the notable exception that -ti, if given as first parameter, is
+# passed to "docker run" itself but NOT the container.
+#
+# use env var DOCKER_RUN_EXTRA_OPTS to provide extra options to docker run
+# command.
+#
+#
+# TO MODIFIY BEHAVIOUR, use
+# LIBLOGNORM_CONTAINER_UID, format uid:gid,
+#                   to change the users container is run under
+#                   set to "" to use the container default settings
+#                   (no local mapping)
+set -e
+if [ "$1" == "--rm" ]; then
+	optrm="--rm"
+	shift 1
+fi
+if [ "$1" == "-ti" ]; then
+	ti="-ti"
+	shift 1
+fi
+# check in case -ti was in front...
+if [ "$1" == "--rm" ]; then
+	optrm="--rm"
+	shift 1
+fi
+
+if [ "$LIBLOGNORM_HOME" == "" ]; then
+	export LIBLOGNORM_HOME=$(pwd)
+	echo info: LIBLOGNORM_HOME not set, using $LIBLOGNORM_HOME
+fi
+
+if [ -z "$LIBLOGNORM_DEV_CONTAINER" ]; then
+	LIBLOGNORM_DEV_CONTAINER=$(cat $LIBLOGNORM_HOME/devtools/default_dev_container)
+fi
+
+printf '/rsyslog is mapped to %s \n' "$LIBLOGNORM_HOME"
+printf 'using container %s\n' "$LIBLOGNORM_DEV_CONTAINER"
+printf 'pulling container...\n'
+printf 'user ids: %s:%s\n' $(id -u) $(id -g)
+printf 'container_uid: %s\n' ${LIBLOGNORM_CONTAINER_UID--u $(id -u):$(id -g)}
+printf 'container cmd: %s\n' $*
+printf '\nNote: we use the RSYSLOG CONTAINERS, as such project home is /rsyslog!\n\n'
+docker pull $LIBLOGNORM_DEV_CONTAINER
+docker run $ti $optrm $DOCKER_RUN_EXTRA_OPTS \
+	-e LIBLOGNORM_CONFIGURE_OPTIONS_EXTRA \
+	-e LIBLOGNORM_CONFIGURE_OPTIONS_OVERRIDE \
+	-e CC \
+	-e CFLAGS \
+	-e LDFLAGS \
+	-e LSAN_OPTIONS \
+	-e TSAN_OPTIONS \
+	-e UBSAN_OPTIONS \
+	-e CI_MAKE_OPT \
+	-e CI_MAKE_CHECK_OPT \
+	-e CI_CHECK_CMD \
+	-e CI_BUILD_URL \
+	-e CI_CODECOV_TOKEN \
+	-e CI_VALGRIND_SUPPRESSIONS \
+	-e CI_SANITIZE_BLACKLIST \
+	-e ABORT_ALL_ON_TEST_FAIL \
+	-e USE_AUTO_DEBUG \
+	-e LIBLOGNORM_STATSURL \
+	-e VCS_SLUG \
+	--cap-add SYS_ADMIN \
+	--cap-add SYS_PTRACE \
+	${LIBLOGNORM_CONTAINER_UID--u $(id -u):$(id -g)} \
+	$DOCKER_RUN_EXTRA_FLAGS \
+	-v "$LIBLOGNORM_HOME":/rsyslog $LIBLOGNORM_DEV_CONTAINER $*

devtools/gather-check-logs.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# gather logs generated by "make [dist]check"
+# this also limits log size so that buildbot does not abort
+# Copyright (C) 2020 by Rainer Gerhards, released under ASL 2.0
+
+show_log() {
+	if grep -q ":test-result: FAIL" "$1"; then
+		printf "\nFAIL: ${1%%.trs} \
+		########################################################\
+		################################\n\n"
+		logfile="${1%%trs}log"
+		if [ -f "$logfile" ]; then
+			lines="$(wc -l < $logfile)"
+			if (( lines > 4000 )); then
+				ls -l $logfile
+				printf 'file is very large (%d lines), showing parts\n' $lines
+				head -n 2000 < "$logfile"
+				printf '\n\n... snip ...\n\n'
+				tail -n 2000 < "$logfile"
+			else
+				cat "$logfile"
+			fi
+		else
+			printf 'log FILE MISSING!\n'
+		fi
+	fi
+}
+
+append_summary() {
+	echo file: $1 # emit file name just in case we have multiple!
+	head -n12 "$1"
+}
+
+# find logs from tests which are potentially aborted. The main indication is
+# that no matching .trs file exists
+check_incomplete_logs() {
+	if grep -q "\.dep_wrk\|rstb_\|config.log" <<<"$1"; then
+		return
+	fi
+	# we emit info only for test log files - this means there must
+	# be a matching .sh file by our conventions
+	if [ -f "${1%%log}sh" ]; then
+		trsfile="${1%%log}trs"
+		if [ ! -f "$trsfile" ]; then
+			printf '\n\nNo matching .trs file for %s\n' "$1"
+			ls -l ${1%%.log}*
+			cat "$1"
+		fi
+	fi
+}
+export -f show_log
+export -f append_summary
+export -f check_incomplete_logs
+
+############################## MAIN ENTRY POINT ##############################
+printf 'find failing tests\n'
+rm -f failed-tests.log
+
+find . -name "*.trs" -exec  bash -c 'show_log "$1" >> failed-tests.log' _ {} \;
+
+find . -name "*.log" -exec  bash -c 'check_incomplete_logs "$1" >> failed-tests.log' _ {} \;
+
+if [ -f failed-tests.log ]; then
+	# show summary stats so that we know how many failed
+	find . -name test-suite.log -exec bash -c 'append_summary "$1" >>failed-tests.log' _ {} \;
+fi

devtools/run-ci.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# script for generic CI runs via container
+printf 'running CI with\n'
+printf 'container: %s\n' $RSYSLOG_DEV_CONTAINER
+printf 'CC:\t%s\n' "$CC"
+printf 'CFLAGS:\t%s:\n' "$CFLAGS"
+printf 'RSYSLOG_CONFIGURE_OPTIONS:\t%s\n' "$RSYSLOG_CONFIGURE_OPTIONS"
+printf 'working directory: %s\n' "$(pwd)"
+printf 'user ids: %s:%s\n' $(id -u) $(id -g)
+if [ "$SUDO" != "" ]; then
+	printf 'check sudo'
+	$SUDO echo sudo works!
+fi
+if [ "$CI_VALGRIND_SUPPRESSIONS" != "" ]; then
+	export RS_TESTBENCH_VALGRIND_EXTRA_OPTS="--suppressions=$(pwd)/tests/CI/$CI_VALGRIND_SUPPRESSIONS"
+fi
+if [ "$CI_SANITIZE_BLACKLIST" != "" ]; then
+	export CFLAGS="$CFLAGS -fsanitize-blacklist=$(pwd)/$CI_SANITIZE_BLACKLIST"
+	printf 'CFLAGS changed to: %s\n', "$CFLAGS"
+fi
+set -e
+
+printf 'STEP: autoreconf / configure ===============================================\n'
+autoreconf -fvi
+./configure
+
+if [ "$CI_CHECK_CMD" != "distcheck" ]; then
+	printf 'STEP: make =================================================================\n'
+	make $CI_MAKE_OPT
+fi
+
+printf 'STEP: make %s ==============================================================\n', \
+	"$CI_CHECK_CMD"
+set +e
+echo CI_CHECK_CMD: $CI_CHECK_CMD
+make $CI_MAKE_CHECK_OPT ${CI_CHECK_CMD:-check}
+rc=$?
+
+printf 'STEP: find failing tests ====================================================\n'
+echo calling gather-check-logs
+devtools/gather-check-logs.sh
+
+printf 'STEP: Codecov upload =======================================================\n'
+if [ "$CI_CODECOV_TOKEN" != "" ]; then
+	curl -s https://codecov.io/bash >codecov.sh
+	chmod +x codecov.sh
+	./codecov.sh -t "$CI_CODECOV_TOKEN" -n 'rsyslog buildbot PR' &> codecov_log
+	rm codecov.sh
+	lines="$(wc -l < codecov_log)"
+	if (( lines > 3000 )); then
+		printf 'codecov log file is very large (%d lines), showing parts\n' $lines
+		head -n 1500 < codecov_log
+		printf '\n\n... snip ...\n\n'
+		tail -n 1500 < codecov_log
+	else
+		cat codecov_log
+	fi
+	rm codecov_log
+fi
+
+exit $rc

doc/conf.py

@@ -37,9 +37,6 @@
 # The encoding of source files.
 #source_encoding = 'utf-8-sig'
 
-# The master toctree document.
-master_doc = 'index'
-
 # General information about the project.
 project = u'Liblognorm'
 # pylint: disable=W0141