#!/bin/sh
#
# Run every executable regression test under tests/. Exit 0 if at least one
# test passed and none failed; 1 if any failed; 77 if every test skipped
# (suite verified nothing -- see the all-skip note near the end). Individual
# tests follow the tests/README.md contract: exit 0 = pass, 77 = skip,
# anything else = fail.
#
# Output adapts automatically: GitHub Actions log annotations when run under
# Actions ($GITHUB_ACTIONS=true), plain human-readable output otherwise. Both
# the CI workflow and a developer run the same code path.
#
# Usage: ./tests/testsuite        (from anywhere; resolves the repo root itself)
#        make test
#
set -u

# Resolve the repo root from this script's own location, so it works from any cwd.
root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) || exit 2
cd "$root" || exit 2

if [ "${GITHUB_ACTIONS:-}" = "true" ]; then gh=1; else gh=0; fi

# Every test is bash by contract (tests/README.md); this runner stays POSIX sh
# so it can deliver the skip below even where bash is absent. Without this
# check, each test's exec would fail with rc=127 and count as FAIL -- but a
# missing interpreter is a missing host dependency, not a regression, so skip
# the whole suite the same way an individual test skips a missing tool.
if ! command -v bash >/dev/null 2>&1; then
	printf 'SKIP: bash not found -- every test requires bash (see tests/README.md)\n' >&2
	exit 77
fi

group() {
	if [ "$gh" = 1 ]; then printf '::group::%s\n' "$1"; else printf '\n----- %s -----\n' "$1"; fi
}
endgroup() {
	if [ "$gh" = 1 ]; then printf '::endgroup::\n'; fi
}
errline() {
	if [ "$gh" = 1 ]; then printf '::error::%s\n' "$1"; else printf 'FAIL: %s\n' "$1"; fi
}

# Discover executable *.sh under tests/. Sourced helpers (tests/lib/) and shared
# data (tests/fixtures/) are not +x, so they are not picked up -- see README.
# This runner has no .sh extension, so it never discovers itself.
#
# The explicit lib/ and fixtures/ excludes back up the exec-bit rule on
# checkouts where every file reads as executable (FAT/NTFS mounts, trees with
# core.fileMode=false): there, a shebang-less helper like lib/assert.sh would
# otherwise be discovered, run as sh, define its functions, exit 0 -- and count
# as a pass that verified nothing.
tests=$(find tests -type f -name '*.sh' -perm -u+x \
	! -path 'tests/lib/*' ! -path 'tests/fixtures/*' ! -path 'tests/final/*' | LC_ALL=C sort)

# LC_ALL=C, so the order is the same everywhere: other collations ignore
# punctuation and case, and "a-b.sh", "a_b.sh" and "ab.sh" then sort
# differently from one machine to the next. Nothing should depend on order --
# this is what makes that checkable rather than assumed.
#
# tests/final/ runs after everything else, whatever anything is named: it
# checks what the suite LEAVES, which is only a fact once nothing else is
# running. By filename it would break the day a later-sorting directory
# appeared, and go on passing while checking nothing.
finaltests=$(find tests/final -type f -name '*.sh' -perm -u+x 2>/dev/null | LC_ALL=C sort)

# Zero discovered tests is a failure, not a quiet success: the runner ships
# alongside the tests, so an empty result means something is wrong -- lost
# executable bits, a packaging omission, or a broken discovery path -- and
# treating it as green would let CI go green while running nothing.
if [ -z "$tests" ]; then
	errline "No executable tests found under tests/ (lost +x bit, packaging omission, or broken discovery)"
	exit 1
fi

# A strict value the runner does not know is a config error, not a silent
# off: XYMON_TESTS_STRICT=true would otherwise disable the floor while
# reading as enabling it.
case ${XYMON_TESTS_STRICT:-} in
	''|0|1) ;;
	*)	errline "XYMON_TESTS_STRICT='${XYMON_TESTS_STRICT}' is not a switch this suite knows -- 1 enables, empty or 0 disables"
		exit 1 ;;
esac

# The lib/ and fixtures/ excludes above are also a hole: an executable test
# misfiled there vanishes without a trace. In strict mode that is a filing
# error; outside strict a fileMode=false checkout, where every file reads as
# executable, would trip this on each helper.
if [ "${XYMON_TESTS_STRICT:-}" = 1 ]; then
	for helperdir in tests/lib tests/fixtures; do
		[ -d "$helperdir" ] || continue
		stray=$(find "$helperdir" -type f -name '*.sh' -perm -u+x | sort)
		if [ -n "$stray" ]; then
			errline "executable .sh under $helperdir/ is invisible to discovery -- helpers are sourced (mode 644), tests live under tests/<area>/:"
			printf '%s\n' "$stray" >&2
			exit 1
		fi
	done
fi

# Unset for a test invoked on its own, which is how a final check knows to skip.
# One temp directory per run, with TMPDIR pointed at it so every test works
# inside it. Two runs sharing $TMPDIR would otherwise see each other's live
# files, and a leftovers check could not tell them from its own.
__XYMON_TESTS_RUNROOT=$(mktemp -d "${TMPDIR:-/tmp}/xymon-run.XXXXXX") || exit 1
export __XYMON_TESTS_RUNROOT
TMPDIR=$__XYMON_TESTS_RUNROOT
export TMPDIR
trap 'rm -rf "$__XYMON_TESTS_RUNROOT" "$__XYMON_TESTS_RUNROOT.partial" "$__XYMON_TESTS_RUNROOT.leftovers"' EXIT
trap 'rm -rf "$__XYMON_TESTS_RUNROOT" "$__XYMON_TESTS_RUNROOT.partial" "$__XYMON_TESTS_RUNROOT.leftovers"; exit 130' INT
trap 'rm -rf "$__XYMON_TESTS_RUNROOT" "$__XYMON_TESTS_RUNROOT.partial" "$__XYMON_TESTS_RUNROOT.leftovers"; exit 143' TERM

# XYMON_VARIANT names the build this tree is, so the runner can leave out the
# tests whose subject that build does not contain. Unset -- a developer run, or
# the build-free tests.yml lane -- filters nothing: every test decides for
# itself, as it always has.
variant=${XYMON_VARIANT:-}
case $variant in
	''|server|client|localclient) ;;
	*)
		# Not a build this suite knows. Guessing would filter by the universal
		# rows alone and report a tidy green over a run that skipped most of
		# the suite for a typo.
		errline "XYMON_VARIANT=$variant is not a build this suite knows (server, client, localclient)"
		exit 1 ;;
esac

# A configured tree can vouch for what it is: configure.client writes
# CLIENTONLY and LOCALCLIENT into the toplevel Makefile, and their reachable
# combinations are exactly the three variants -- the names are a projection of
# those two axes, not a third fact. Read the Makefile when it is there and
# check it against the caller's label rather than letting either side win in
# silence.
#
# A CMake tree writes no such Makefile, and an unconfigured one writes none at
# all, so XYMON_VARIANT still answers for both.
variant_from_tree() {
	[ -f Makefile ] || return 1
	if grep -q '^CLIENTONLY[	 ]*=[	 ]*yes' Makefile; then
		if grep -q '^LOCALCLIENT[	 ]*=[	 ]*yes' Makefile
		then echo localclient
		else echo client
		fi
	else
		echo server
	fi
}

configured=$(variant_from_tree) || configured=''
if [ -n "$variant" ]; then
	# The caller names the build; the configured tree is the witness. A
	# "client" label on a localclient tree once filed the one localclient
	# test out of the only build producing its binary -- refuse to run
	# under a disagreement rather than let either side win in silence.
	if [ -n "$configured" ] && [ "$variant" != "$configured" ]; then
		errline "XYMON_VARIANT=$variant, but this tree is configured $configured"
		exit 1
	fi
elif [ -n "$configured" ]; then
	# No label from the caller -- a developer run on a configured tree. The
	# tree answers for itself.
	variant=$configured
	# The tests resolve their own binaries from this (require_bin in
	# lib/assert.sh), so hand them the same answer the filter uses -- otherwise
	# the runner files a test under localclient while the test itself looks for
	# the server's copy of the binary and skips.
	XYMON_VARIANT=$variant
	export XYMON_VARIANT
fi

# The floor is per-variant; strict with no variant to hold it to would just
# never fire. Refuse rather than run green with the floor silently off.
if [ "${XYMON_TESTS_STRICT:-}" = 1 ] && [ -z "$variant" ]; then
	errline "XYMON_TESTS_STRICT=1 needs XYMON_VARIANT (or a configured tree) -- the coverage floor is per-variant"
	exit 1
fi

# area_in_variant AREA -- 0 when this build contains AREA's subject, 1 when it
# does not, 2 when the area is not in the table. An area names what its tests
# drive; this table says which builds contain it (tests/README.md).
area_in_variant() {
	case $1 in
		buildsystem|packaging|libxymon|common|client|final)
			return 0 ;;
		analysis)
			case $variant in server|localclient) return 0 ;; *) return 1 ;; esac ;;
		server|web|xymond|rrd|xymonnet|integration)
			case $variant in server) return 0 ;; *) return 1 ;; esac ;;
		*)
			return 2 ;;
	esac
}

# pass_partial (lib/assert.sh) exits 0 like any pass, so a half-checked run is
# invisible to the exit status. Each one records itself here instead -- beside
# the run directory, not in it, where tests/final/no-leftovers.sh would count
# it, and removed by the run directory's own traps.
partial_log=$__XYMON_TESTS_RUNROOT.partial
: >"$partial_log" || exit 2
XYMON_TESTS_PARTIAL_LOG=$partial_log
export XYMON_TESTS_PARTIAL_LOG

# What each test leaves in the run directory, as "NAME TEST" lines, so that
# tests/final/no-leftovers.sh can say which test left it. Beside the run
# directory for the same reason as the partial log. A leftover is recorded
# when it is first seen after a test ends; one left by a process that outlives
# its test is put on the test that was running when it appeared.
leftover_log=$__XYMON_TESTS_RUNROOT.leftovers
: >"$leftover_log" || exit 2
XYMON_TESTS_LEFTOVER_LOG=$leftover_log
export XYMON_TESTS_LEFTOVER_LOG

pass=0; skip=0; fail=0; filtered=0; failed=''; filtered_areas=''; skipped_in=''
run_one() {
	t=$1

	# The area is the directory under tests/. A test sitting directly under
	# tests/ has no area at all -- do not read its filename as one.
	area=${t#tests/}
	case $area in
		*/*) area=${area%%/*} ;;
		*)   area='' ;;
	esac
	if [ -n "$variant" ] && [ -n "$area" ]; then
		area_in_variant "$area"
		case $? in
			1)  # Say it once per area: 48 near-identical lines bury the
			    # tests that did run.
			    case " $filtered_areas " in
				*" $area "*) ;;
				*) filtered_areas="$filtered_areas $area"
				   printf 'filtered: no %s in a %s build\n' "$area" "$variant" ;;
			    esac
			    filtered=$((filtered + 1))
			    return ;;
			2)  # An area the table has never heard of cannot be judged. In
			    # strict mode that is a filing error -- its skips could never
			    # reach the floor. Outside strict, run it and say so.
			    if [ "${XYMON_TESTS_STRICT:-}" = 1 ]; then
				errline "unknown test area \"$area\" ($t) -- strict mode holds every area to the coverage floor; add it to area_in_variant"
				exit 1
			    fi
			    if [ "$gh" = 1 ]; then printf '::notice::unknown test area "%s" -- running %s\n' "$area" "$t"
			    else printf 'NOTE: unknown test area "%s" -- running %s\n' "$area" "$t"; fi ;;
		esac
	elif [ -n "$variant" ] && [ "${XYMON_TESTS_STRICT:-}" = 1 ]; then
		# No area at all: a test directly under tests/. Outside strict it
		# runs unfiled; in strict mode unfiled means unaccountable -- its
		# skips would never reach the floor.
		errline "top-level test $t has no area -- strict mode holds every area to the coverage floor; file it under tests/<area>/"
		exit 1
	fi

	group "$t"
	runroot_before=$(ls -A "$__XYMON_TESTS_RUNROOT" 2>/dev/null)
	"./$t"
	rc=$?
	for entry in $(ls -A "$__XYMON_TESTS_RUNROOT" 2>/dev/null); do
		case "
$runroot_before
" in
			*"
$entry
"*) ;;
			*) printf '%s %s\n' "$entry" "$t" >>"$leftover_log" ;;
		esac
	done
	case $rc in
		0)  printf '  ok\n';           pass=$((pass + 1)) ;;
		77) printf '  skip (rc=77)\n'; skip=$((skip + 1))
		    skipped_in="${skipped_in}${area:-.} ${t}
" ;;
		*)  errline "$t FAILED (rc=$rc)"
		    fail=$((fail + 1))
		    failed="${failed}  - ${t} (rc=${rc})
" ;;
	esac
	endgroup
}

# Split the list on newlines alone: find emits one path per line, and the
# default IFS would also break a path at a space. Globbing is off so a path
# holding * or ? stays as written. A `| while` loop would run in a subshell and
# lose run_one's counters. A here-doc would keep them, but OpenBSD's sh writes
# one to a named file in $TMPDIR -- the run directory -- and keeps it there
# until the loop ends, where tests/final/no-leftovers.sh, run inside the loop,
# finds it.
run_all() {
	set -f
	oifs=$IFS
	IFS='
'
	for t in $1; do
		IFS=$oifs
		run_one "$t"
	done
	IFS=$oifs
	set +f
}

run_all "$tests"

# Everything else has finished; now the checks that are about what is left.
run_all "$finaltests"

# grep -c prints 0 and exits 1 on no match, so take the count and ignore the
# status rather than appending a second 0 to it.
partial=$(grep -c . "$partial_log" 2>/dev/null); [ -n "$partial" ] || partial=0

printf '\n----- summary -----\n'
printf 'passed:  %d' "$pass"
if [ "$partial" -gt 0 ]; then printf ' (%d partial)' "$partial"; fi
printf '\n'
printf 'skipped: %d\n' "$skip"
printf 'failed:  %d\n' "$fail"
[ "$filtered" -eq 0 ] || printf 'filtered: %d (not in a %s build)\n' "$filtered" "$variant"

if [ "$fail" -gt 0 ]; then
	printf '\nfailing tests:\n%s' "$failed"
	exit 1
fi

# Every discovered test skipped means the suite exercised nothing: in a tree
# that ships tests/ without the code those tests guard (a packaging omission,
# or an autopkgtest run against an installed package with no source), each test
# hits an absent precondition and skips. Reporting exit 0 here is a false green
# -- CI/autopkgtest would record a pass while nothing was actually verified.
# Propagate the all-skip upward as a skip (exit 77, the same convention the
# individual tests use; see skip() in tests/lib/assert.sh) so the layer above
# sees "skipped", not "passed". The empty-discovery case already exited above,
# so reaching here with pass=0 and fail=0 means skip>0.
# Coverage floor: inside an area this build provides, a skip means the test
# stopped exercising something the build has. Outside those areas the filter
# removed the test already, so it never reaches this list.
#
# Only where the environment declares itself complete -- CI exports
# XYMON_TESTS_STRICT after installing the dependencies and building the
# variant. A developer box declares nothing and is never held to it.
if [ "${XYMON_TESTS_STRICT:-}" = 1 ] && [ -n "$variant" ] && [ -n "$skipped_in" ]; then
	breached=''
	while IFS= read -r line; do
		[ -n "$line" ] || continue
		area=${line%% *}
		if area_in_variant "$area"; then breached="${breached}  - ${line#* }
"; fi
	done <<EOF
$skipped_in
EOF
	if [ -n "$breached" ]; then
		errline "coverage floor: a $variant build provides these areas, so every test in them must run -- these skipped"
		printf '%s' "$breached" >&2
		exit 1
	fi
fi

if [ "$pass" -eq 0 ] && [ "$skip" -eq 0 ] && [ "$filtered" -gt 0 ]; then
	# Nothing ran at all, and the filter is why: XYMON_VARIANT names a build
	# that contains none of these areas. That is a broken declaration, not a
	# tree without code -- report it rather than passing it off as a skip.
	# A run where something did execute and skipped is the ordinary all-skip
	# case below, whatever the filter left out.
	errline "every test was filtered out for XYMON_VARIANT=$variant -- check the variant name"
	exit 1
fi
if [ "$pass" -eq 0 ]; then
	printf 'SKIP: every test skipped -- suite verified nothing (tests/ shipped without the code under test?)\n' >&2
	exit 77
fi
exit 0
