Skip to content

Commit a431f77

Browse files
authored
[ fix ] add test runner for v2.0-joss-submission (#2884)
1 parent de19d70 commit a431f77

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+332
-188
lines changed

.github/workflows/ci-ubuntu.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ on:
88
branches:
99
- master
1010
- experimental
11+
- v2.0-joss-submission
1112
merge_group:
1213

1314
########################################################################
@@ -47,7 +48,7 @@ on:
4748
########################################################################
4849

4950
env:
50-
GHC_VERSION: 8.10.7
51+
GHC_VERSION: 9.2.8
5152
CABAL_VERSION: 3.6.2.0
5253
CABAL_INSTALL: cabal v1-install --ghc-options='-O1 +RTS -M6G -RTS'
5354
# CABAL_INSTALL: cabal install --overwrite-policy=always --ghc-options='-O1 +RTS -M6G -RTS'
@@ -98,7 +99,7 @@ jobs:
9899
# i.e. if we change either the version of Agda, ghc, or cabal that we want
99100
# to use for the build.
100101
- name: Cache ~/.cabal directories
101-
uses: actions/cache@v2
102+
uses: actions/cache@v4
102103
id: cache-cabal
103104
with:
104105
path: |
@@ -113,7 +114,7 @@ jobs:
113114
########################################################################
114115

115116
- name: Install ghc & cabal
116-
uses: haskell/actions/setup@v1
117+
uses: haskell-actions/setup@v2
117118
with:
118119
ghc-version: ${{ env.GHC_VERSION }}
119120
cabal-version: ${{ env.CABAL_VERSION }}
@@ -168,8 +169,7 @@ jobs:
168169
169170
- name: Golden testing
170171
run: |
171-
${{ env.CABAL_INSTALL }} clock
172-
make testsuite INTERACTIVE='' AGDA_EXEC='~/.cabal/bin/agda'
172+
make testsuite INTERACTIVE='' AGDA_EXEC='agda' GHC_EXEC='ghc'
173173
174174
175175
########################################################################

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ GenerateEverything
2727
Haskell
2828
html
2929
log
30+
logs/
3031
MAlonzo
3132
output
3233
runtests

GNUmakefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
GHC_EXEC ?= ghc
12
AGDA_EXEC ?= agda
23
AGDA_OPTIONS=-Werror
34
AGDA_RTS_OPTIONS=+RTS -M4.0G -H3.5G -A128M -RTS
@@ -12,7 +13,7 @@ test: Everything.agda check-whitespace
1213
cd doc && $(AGDA) README.agda
1314

1415
testsuite:
15-
$(MAKE) -C tests test AGDA="$(AGDA)" AGDA_EXEC="$(AGDA_EXEC)" only=$(only)
16+
$(MAKE) -C tests test GHC_EXEC="$(GHC_EXEC)" AGDA="$(AGDA)" only=$(only)
1617

1718
fix-whitespace:
1819
cabal exec -- fix-whitespace

src/Test/Golden.agda

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,9 +204,10 @@ runTest opts testPath = do
204204
true doesDirectoryExist (mkFilePath testPath)
205205
where false fail directoryNotFound
206206

207+
putStr $ concat (testPath ∷ ": " ∷ [])
207208
time time′ $ callCommand $ unwords
208209
$ "cd" ∷ testPath
209-
"&&""sh ./run" ∷ opts .exeUnderTest
210+
"&&""sh ./run"(concat $ "\"" ∷ opts .exeUnderTest ∷ "\"" ∷ [])
210211
"| tr -d '\\r' > output"
211212
∷ []
212213

@@ -303,14 +304,14 @@ runTest opts testPath = do
303304
when b $ writeFile (testPath String.++ "/expected") out
304305

305306
printTiming : Bool Time String IO _
306-
printTiming false _ msg = putStrLn $ concat (testPath ∷ ": "msg ∷ [])
307+
printTiming false _ msg = putStrLn msg
307308
printTiming true time msg =
308309
let time = ℕ.show (time .seconds) String.++ "s"
309310
spent = 9 + List.sum (List.map String.length (testPath ∷ time ∷ []))
310311
-- ^ hack: both "success" and "FAILURE" have the same length
311312
-- can't use `String.length msg` because the msg contains escape codes
312313
pad = String.replicate (72 ∸ spent) ' '
313-
in putStrLn (concat (testPath ∷ ": "msg ∷ pad ∷ time ∷ []))
314+
in putStrLn (concat (msg ∷ pad ∷ time ∷ []))
314315

315316
-- A test pool is characterised by
316317
-- + a name

tests/Makefile

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,22 @@
1-
INTERACTIVE ?= --interactive
1+
INTERACTIVE ?= --interactive
2+
GHC_EXEC ?= ghc
3+
CABAL ?= cabal
4+
CABAL_OPTIONS ?= --with-compiler $(GHC_EXEC)
25

3-
runtests: runtests.agda
4-
rm -f _build/runtests
5-
rm -f _build/MAlonzo/Code/Qruntests*
6-
$(AGDA) --compile-dir=_build/ -c runtests.agda
6+
AGDA_BUILD_DIR = _config/_build
7+
CABAL_BUILD_DIR = _config/dist-newstyle
78

8-
test: runtests
9-
./_build/runtests $(AGDA_EXEC) $(INTERACTIVE) --timing --failure-file failures --only $(only)
9+
./runtests: admin/runtests/Main.agda
10+
cd admin/runtests && AGDA="$(AGDA)" sh run
1011

12+
test: ./runtests
13+
./runtests "$(AGDA)" $(INTERACTIVE) --timing --failure-file failures --only $(only)
1114

12-
retest: runtests
13-
./_build/runtests $(AGDA_EXEC) $(INTERACTIVE) --timing --failure-file failures --only-file failures --only $(only)
15+
16+
retest: ./runtests
17+
./runtests "$(AGDA)" $(INTERACTIVE) --timing --failure-file failures --only-file failures --only $(only)
18+
19+
clean:
20+
rm -rf runtests
21+
rm -rf _config/_build
22+
rm -rf _config/dist-newstyle

tests/README.md

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Running the test suite
2+
3+
You can run the test suite by going back to the main project directory
4+
and running (change accordingly if you have the right versions of Agda
5+
& GHC installed but the executable names are different).
6+
7+
```shell
8+
AGDA_EXEC=agda-2.6.4 GHC_EXEC=ghc-9.2.8 make testsuite
9+
```
10+
11+
# Structure of the test suite
12+
13+
## Test case
14+
15+
Each test case is under 2 levels of nesting (this is hard-coded)
16+
and should comprise:
17+
18+
1. A `Main.agda` file containing a `main` entrypoint
19+
2. An `expected` file containing the expected output of running `Main`
20+
3. A `run` file describing how to run `Main` (most likely implemented
21+
using the `goldenTest` function defined in
22+
[_config/config.sh](_config/config.sh).
23+
4. Optionally, an `input` file for the stdin content to feed to
24+
the executable obtained by compiling `Main`.
25+
26+
## Configuration
27+
28+
The Agda & GHC version numbers the stdlib is tested against
29+
are specified in [_config/version-numbers.sh](_config/version-numbers.sh)
30+
31+
## Test runner
32+
33+
The test runner is defined in [admin/runtests/](admin/runtests/).
34+
It is compiled using an ad-hoc [run](admin/runtests/run) file using
35+
the shared configuration and the resulting executable is copied to
36+
the root of the tests directory.
37+
38+
If you want to add new tests, you need to list them in
39+
the [runtests](admin/runtests/Main.agda) program.
40+
41+
## Test dependencies
42+
43+
The external dependencies of the whole test suite are spelt out in the generic
44+
[_config/template.cabal](_config/template.cabal) cabal file
45+
46+
You may need to bump these dependencies when changing
47+
the version of the GHC compiler we build against.
48+
49+
# Updating the test suite
50+
51+
1. Update [_config/version-numbers.sh](_config/version-numbers.sh)
52+
2. Update the shell command listing explicit version numbers in the
53+
fenced code block at the top of this README
54+
3. Update bounds in [_config/template.cabal](_config/template.cabal)
55+
4. Update [../.github/workflows/ci-ubuntu.yml](../.github/workflows/ci-ubuntu.yml)
56+
to run the continuous integration with the new GHC and/or Agda versions.

tests/_config/config.sh

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/sh
2+
3+
# This script is intended to be sourced from test scripts.
4+
#
5+
# It provides a number of default config options corresponding
6+
# to the compiler versions the stdlib is being tested with
7+
#
8+
# Usage: . ../../config.sh
9+
10+
set -eu
11+
12+
# Ugh, paths are relative to the script sourcing this file!
13+
. ../../_config/version-numbers.sh
14+
15+
16+
17+
# Main entry point to build and run an Agda program
18+
#
19+
# Typically called like `goldenTest "$1" "echo" "hello world"`
20+
#
21+
# INPUTS:
22+
# $1 is the agda executable (typically "$1" in the ̀ run` file
23+
# because this info is provided by the test runner
24+
# $2 the name of the test as a string
25+
# $3 is OPTIONAL and corresponds to the extra arguments to pass
26+
# to the executable obtained by compiling the Agda program
27+
#
28+
# LOGGING:
29+
# logs/agda-build for the trace produced by Agda
30+
# logs/cabal-build for the trace produced by cabal+ghc
31+
#
32+
# OUTPUT:
33+
# the output obtained by running the Agda program is stored in
34+
# the file named `output`. The test runner will then diff it
35+
# against the expected file.
36+
goldenTest () {
37+
38+
AGDA=$1
39+
TEST_NAME="stdlib-test-$2"
40+
41+
# Taking extra args for the executable?
42+
if [ -z ${3-} ]; then
43+
EXTRA_ARGS=""
44+
else
45+
EXTRA_ARGS="-- $3"
46+
fi
47+
48+
# Remember whether the script has an input -- ugh
49+
if [ -f input ]; then
50+
HAS_INPUT="true"
51+
else
52+
touch input
53+
HAS_INPUT="false"
54+
fi
55+
56+
# Specialise template agda-lib & cabal files
57+
sed "s/TEST_NAME/$TEST_NAME/g" ../../_config/template.agda-lib > "$TEST_NAME".agda-lib
58+
sed "s/TEST_NAME/$TEST_NAME/g" ../../_config/template.cabal > "$TEST_NAME".cabal
59+
60+
# Set up clean logs directory
61+
rm -rf logs/
62+
mkdir logs
63+
64+
# Use shared directories to avoid rechecking stdlib modules
65+
AGDA_BUILD_DIR=../../_config/_build
66+
CABAL_BUILD_DIR=../../_config/dist-newstyle
67+
68+
mkdir -p "$AGDA_BUILD_DIR"
69+
ln -sf "$AGDA_BUILD_DIR" _build
70+
mkdir -p "$CABAL_BUILD_DIR"
71+
ln -sf "$CABAL_BUILD_DIR" dist-newstyle
72+
73+
# Compile the Agda module and build the generated code
74+
$AGDA --library-file=../../_config/libraries --compile-dir=_build -c --ghc-dont-call-ghc Main.agda > logs/agda-build
75+
cabal build "$TEST_NAME" --with-compiler "$GHC_EXEC" > logs/cabal-build
76+
77+
# Run the test
78+
cabal exec -v0 "$TEST_NAME" --with-compiler "$GHC_EXEC" $EXTRA_ARGS < input > output
79+
80+
# Clean up after ourselves
81+
if ! "$HAS_INPUT"; then
82+
rm input
83+
fi
84+
rm "$TEST_NAME".cabal
85+
rm "$TEST_NAME".agda-lib
86+
rm _build
87+
rm dist-newstyle
88+
}

tests/_config/libraries

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../standard-library.agda-lib

tests/_config/template.agda-lib

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
name: TEST_NAME
2+
include: .
3+
depend: standard-library
4+
flags:
5+
--warning=noUnsupportedIndexedMatch

tests/_config/template.cabal

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
cabal-version: 2.4
2+
name: TEST_NAME
3+
version: 0.0
4+
build-type: Simple
5+
description: Run this test
6+
license: MIT
7+
8+
common common-build-parameters
9+
default-language:
10+
Haskell2010
11+
12+
default-extensions:
13+
PatternGuards
14+
PatternSynonyms
15+
16+
build-depends:
17+
base >= 4.12 && < 4.20
18+
, clock >= 0.8 && <0.9
19+
, directory >= 1.3.6 && < 1.3.7
20+
, filemanip >= 0.3.6 && < 0.4
21+
, filepath >= 1.4 && <1.6
22+
, process >= 1.6 && <1.7
23+
, text >= 2.0 && <2.2
24+
25+
ghc-options: -Wno-missing-home-modules
26+
27+
executable TEST_NAME
28+
import: common-build-parameters
29+
hs-source-dirs: _build
30+
main-is: MAlonzo/Code/Main.hs
31+
ghc-options: -main-is MAlonzo.Code.Main

0 commit comments

Comments
 (0)