diff options
| -rw-r--r-- | tests/README.md | 75 | ||||
| -rw-r--r-- | tests/default.nix | 33 | ||||
| -rw-r--r-- | tests/infrastructure/clean.awk | 13 | ||||
| -rw-r--r-- | tests/infrastructure/default.nix | 136 | ||||
| -rw-r--r-- | tests/infrastructure/driver.sh | 72 | ||||
| -rw-r--r-- | tests/infrastructure/run.nix | 34 |
6 files changed, 363 insertions, 0 deletions
diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..fe93dc96 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,75 @@ +# How to write a test + +Test are simple derivations, put into the `by-name` directory. As you +often want to test configuration, the 'standard' nixos-vm-test function +starts a vm with the specified configuration and lets you run a python +test script. + +But, I've noticed, that most of my tests in this nixos-configuration are +actually testing the home-manager config of a program. As such I've built +a test infrastructure on top of the pre-existing nixos-vm-tests. It is +further described in the `mkTest` section. + +## `mkTest` +A standard application test would look somewhat like this: + +```nix +{mkTest}: +mkTest { + name = "my-tests"; + + # The configuration to add. + # In this case, the less config is being tested. + configuration = { + imports = [ + ../../../../modules/by-name/le/less/module.nix + ]; + config.soispha.programs.less.enable = true; + }; + + # Files that are provided at the test directory. + # In this case, the test dir would have a `./data/test.file` + # path pre-populated with the + # contents of the file at the lhs. + testData = { + "data/test.file" = ./data/test.file; + }; + + # The description of what to do in the test. + description = ./test.desc; + + # A sha256 hashsum of the concatenated output of the program. + # This can be `null` to disable the checksum mechanism. + hash = "87901231393b51cdd45bbb4339a32db2894a3a5ab164cb5c7a8fa14721fdcba7"; +} +``` + +### The test description file +All line starting with `#` or only containing white space are +ignored. Additionally, all leading and trailing white space is stripped +before evaluating. + +The general syntax is: +``` +COMMAND ARGS +``` +where `COMMAND` is one of the commands listed in [Commands](#### Commands) + +The `ARGS` are the verbatim content from the space after the `COMMAND` +to the end of line character. + +#### Commands +##### `Type` +Send the `ARGS` to the application. This interprets `ARGS` as specified +by the tmux (1) man page on `send-keys`. + +#### `Sleep` +Sleep for `ARGS` seconds. + +#### `Expect` +Grep the currently visible screen for the regex `ARGS`. This regex +must match. +#### `ExpectNot` +Does the same thing as [Expect](##### Expect), but enforces, that the +regex does not match. + diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 00000000..9803c99c --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,33 @@ +{ + lib, + pkgs, + myPkgs, + nixpkgs_as_input, + extraModules, +}: let + nixLib = import ../lib {}; + + mkTest = import ./infrastructure { + inherit + pkgs + lib + nixos-lib + myPkgs + extraModules + ; + }; + + nixos-lib = import (nixpkgs_as_input + "/nixos/lib") {}; + + tests = nixLib.mkByName { + baseDirectory = ./by-name; + fileName = "test.nix"; + finalizeFunction = name: value: + import value { + inherit + mkTest + ; + }; + }; +in + tests diff --git a/tests/infrastructure/clean.awk b/tests/infrastructure/clean.awk new file mode 100644 index 00000000..1208b1ef --- /dev/null +++ b/tests/infrastructure/clean.awk @@ -0,0 +1,13 @@ +{ + # Shell like comments + gsub(/^#.*$/, "", $0) + + # Strip leading and trailing white space + gsub(/^[[:blank:]]*/, "", $0) + gsub(/[[:blank:]]*$/, "", $0) + + # Only accept the line, if it contains something + if (NF) { + print $0 + } +} diff --git a/tests/infrastructure/default.nix b/tests/infrastructure/default.nix new file mode 100644 index 00000000..4f1ec7a2 --- /dev/null +++ b/tests/infrastructure/default.nix @@ -0,0 +1,136 @@ +{ + pkgs, + myPkgs, + lib, + nixos-lib, + extraModules, + ... +}: { + name, + configuration, + description, + hash, + testData, +}: +nixos-lib.runTest { + hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs + + inherit name; + + node = { + specialArgs = { + inherit myPkgs; + }; + # Use the nixpkgs as constructed by the `nixpkgs.*` options + pkgs = null; + }; + + nodes = { + machine = {config, ...}: { + imports = + [ + extraModules.home-manager + ../../modules/by-name/us/users/module.nix + ] + ++ configuration.imports; + + config = lib.modules.mkMerge [ + { + soispha = { + users = { + enable = true; + + # The password is 'test'. + hashedPassword = "$y$j9T$yYYUpE9OaxmEO8MaPI2jr0$WGpYYWaLySakI.Mwqz4sljGSOetAp4s5CIUa1VUU1l2"; + }; + }; + home-manager.users.soispha.home.stateVersion = "24.11"; + } + configuration.config + ]; + }; + }; + + testScript = {nodes, ...}: let + testDir = "${nodes.machine.home-manager.users.soispha.home.homeDirectory}/test"; + goldenFile = "${testDir}/__test_golden"; + logFile = "${testDir}/__test_log"; + in + /* + python + */ + '' + start_all() + + machine.succeed("sudo -u soispha ${pkgs.writeShellScript "mkTestEnvironment" '' + set -e + + mkdir --parents "${testDir}" + cd "${testDir}" + + ${ + lib.strings.concatStringsSep "\n\n" ( + builtins.attrValues + (builtins.mapAttrs + (name: value: '' + mkdir --parents "$(dirname '${name}')" + cp --recursive '${value}' '${name}' + '') + testData) + ) + } + ''}") + + machine.succeed("sudo -u soispha ${pkgs.writeShellScript "mkGoldenRecord" '' + set -e + + # HACK: Prevent zsh from complaining about missing configuration. + # TODO: These tests should probably just run under bash. <2024-11-22> + touch ~soispha/.zshrc + + cd "${testDir}" + + __TEST_TMUX="${lib.getExe pkgs.tmux}" + __TEST_TMUX_PANE="__TEST_TMUX_PANE" + + __TEST_EVAL_AWK_CLEAN_FILE="${./clean.awk}" + __TEST_EVAL_LOG_FILE="${logFile}" + + . ${./driver.sh} + + "$__TEST_TMUX" new-session -d -s "$__TEST_TMUX_PANE" + "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE" -o 'cat >>${goldenFile}' + + __test_eval "${description}" + + # Clear the pipe again + "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE" + ''}") + + machine.succeed("sudo -u soispha ${pkgs.writeShellScript "testHashOfGolden" '' + set -e + + cd "${testDir}" + '' + + lib.optionalString (hash != null) + /* + bash + */ + '' + golden_hash="$(sha256sum ${goldenFile} | awk '{print $1}')" + + if [ "$golden_hash" != "${hash}" ]; then + echo "Hash mismatch." + echo "Expected '${hash}'," + echo "but got '$golden_hash'" + exit 1 + else + echo "Hash was successfully checked." + fi + ''}") + + + machine.copy_from_vm("${goldenFile}", "golden") + machine.copy_from_vm("${logFile}", "log") + ''; +} diff --git a/tests/infrastructure/driver.sh b/tests/infrastructure/driver.sh new file mode 100644 index 00000000..52f7d3ad --- /dev/null +++ b/tests/infrastructure/driver.sh @@ -0,0 +1,72 @@ +#! /usr/bin/env sh +set -e + +msg() { + if [ "$#" -ne 0 ]; then + echo "$@" | tee --append "$__TEST_EVAL_LOG_FILE" >&2 + else + cat | tee --append "$__TEST_EVAL_LOG_FILE" >&2 + fi +} + +__test_eval() { + tmux="$__TEST_TMUX" + tpane="$__TEST_TMUX_PANE" + file="$1" + + awk --file "$__TEST_EVAL_AWK_CLEAN_FILE" "$file" | while read -r cmd args; do + case "$cmd" in + "Type") + msg "Sending keys to application '$args'.." + "$tmux" send-keys -t "$tpane": "$args" + ;; + "Sleep") + msg "Sleeping for '$args' seconds.." + sleep "$args" + ;; + "Expect" | "ExpectNot") + msg "Trying to match regex ('$args') for currently visible content.." + + matched="" + if "$tmux" capture-pane -t "$tpane" -p -S 0 -E - | grep "$args"; then + matched=true + else + matched=false + fi + + case "$cmd" in + "Expect") + if [ "$matched" = true ]; then + msg "Regex matched." + else + msg "Failed to find string, matched by regex '$args' on the screen" + msg current screen: + "$tmux" capture-pane -t "$tpane" -p -S 0 -E - | msg + + exit 1 + fi + ;; + "ExpectNot") + if [ "$matched" = false ]; then + msg "Regex successfully not matched." + else + msg "Found to find string, matched by regex '$args' on the screen. But expected none" + msg current screen: + "$tmux" capture-pane -t "$tpane" -p -S 0 -E - | msg + + exit 1 + fi + ;; + *) + msg "Entered unrechable code. This is a bug." + exit 1 + ;; + esac + ;; + *) + msg "Unrecognized command: '$cmd'" + exit 1 + ;; + esac + done +} diff --git a/tests/infrastructure/run.nix b/tests/infrastructure/run.nix new file mode 100644 index 00000000..91120ef4 --- /dev/null +++ b/tests/infrastructure/run.nix @@ -0,0 +1,34 @@ +{ + pkgs, + lib, +}: +pkgs.writeShellScript "run_test_description" '' + set -e + + [ "$#" -ne 1 ] && { + echo "Usage: $0 <Test description file>"; + exit 2 + } + description="$1" + + __TEST_TMUX="${lib.getExe pkgs.tmux}" + __TEST_TMUX_PANE="__TEST_TMUX_PANE" + __TEST_AWK_CLEAN_FILE="${./clean.awk}" + + . ${./driver.sh} + + echo "Setting up a session.." + if "$__TEST_TMUX" has-session -t "$__TEST_TMUX_PANE"; then + echo "Killing old '$__TEST_TMUX_PANE'" + "$__TEST_TMUX" kill-session -t "$__TEST_TMUX_PANE" + fi + "$__TEST_TMUX" new-session -d -s "$__TEST_TMUX_PANE" + + echo "Initializing pipe.." + "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE" -o 'cat >>./test.golden' + + echo "Evaluating description.." + __test_eval "$description" + + "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE" +'' |