version one test runner for factor

Dockerfile

@@ -1,7 +1,32 @@
-FROM alpine:3.18
+FROM debian:bookworm-slim AS builder
 
-# install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash curl git g++ make wget ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+RUN git clone https://github.com/factor/factor.git && \
+    cd factor && \
+    git checkout cd14ceed53f6f9a43bbd3aec3950d8beb5439ed8
+WORKDIR /opt/factor
+RUN ./build.sh update
+
+# Remove files not needed at runtime
+RUN rm -rf .git build vm src misc Factor.app \
+    factor.image.fresh boot.*.image libfactor.a libfactor-ffi-test.so \
+    extra GNUmakefile Nmakefile LICENSE.txt README.md \
+    build.sh build.cmd unmaintained
+
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash jq coreutils libstdc++6 \
+    && rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man \
+       /usr/share/zoneinfo /usr/share/perl5 /var/lib/dpkg /var/cache
+
+COPY --from=builder /opt/factor /opt/factor
+ENV PATH="/opt/factor:${PATH}" \
+    XDG_CACHE_HOME=/tmp
 
 WORKDIR /opt/test-runner
 COPY . .

README.md

@@ -1,6 +1,6 @@
 # Exercism Factor Test Runner
 
-The Docker image to automatically run tests on Factor solutions submitted to [Exercism].
+The Docker image to automatically run tests on [Factor] solutions submitted to [Exercism].
 
 ## Getting started
 
@@ -69,3 +69,4 @@ Bear in mind though that the performance on Exercism's production servers is oft
 [test-runners]: https://github.com/exercism/docs/tree/main/building/tooling/test-runners
 [golden]: https://ro-che.info/articles/2017-12-04-golden-tests
 [exercism]: https://exercism.org
+[factor]: https://factorcode.org/

bin/run-tests.sh

@@ -20,12 +20,9 @@ for test_dir in tests/*; do
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
-    # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
-    # different test runs (e.g. timing information or paths), you should normalize
-    # the results file to allow the diff comparison below to work as expected
-
+    # Normalize paths in results to match Docker environment
     file="results.json"
+    sed -i "s~${test_dir_path}~/opt/test-runner/tests/${test_dir_name}~g" "${test_dir_path}/${file}"
     expected_file="expected_${file}"
     echo "${test_dir_name}: comparing ${file} to ${expected_file}"
 

bin/run.sh

@@ -31,15 +31,18 @@ mkdir -p "${output_dir}"
 
 echo "${slug}: testing..."
 
+# Remove STOP-HERE lines to unskip all tests
+sed -i '/STOP-HERE/d' "${solution_dir}/${slug}/${slug}-tests.factor"
+
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
+test_output=$(cd "${solution_dir}" && factor -e="USING: vocabs.loader tools.test tools.test.private namespaces kernel system ; \".\" add-vocab-root \"${slug}\" require \"${slug}\" test test-failures get empty? [ 0 exit ] [ 1 exit ] if" 2>&1)
+test_exit=$?
+test_output=$(printf '%s\n' "${test_output}" | grep -v "^fatal error for monitor root" | sed '/^(U) \[/,$d' | sed '/^$/d')
 
 # Write the results.json file based on the exit code of the command that was 
 # just executed that tested the implementation file
-if [ $? -eq 0 ]; then
+if [ $test_exit -eq 0 ]; then
     jq -n '{version: 1, status: "pass"}' > ${results_file}
 else
     # OPTIONAL: Sanitize the output

tests/all-fail/all-fail/all-fail-tests.factor

@@ -0,0 +1,3 @@
+USING: all-fail tools.test ;
+{ "hello" } [ greet ] unit-test
+{ "world" } [ greet ] unit-test

tests/all-fail/all-fail/all-fail.factor

@@ -0,0 +1,4 @@
+USING: kernel ;
+IN: all-fail
+
+: greet ( -- str ) "wrong" ;

tests/all-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "Unit Test: { { \"hello\" } [ greet ] }\n--> test failed!\nUnit Test: { { \"world\" } [ greet ] }"
 }

tests/empty-file/empty-file/empty-file-tests.factor

@@ -0,0 +1,2 @@
+USING: empty-file tools.test ;
+{ "hello" } [ greet ] unit-test

tests/empty-file/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "/opt/test-runner/tests/empty-file/empty-file/empty-file-tests.factor\n2: { \"hello\" } [ greet ] unit-test\n                      ^\nNo word named 'greet' found in current vocabulary search path"
 }

tests/partial-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "Unit Test: { { \"hello\" } [ greet ] }\nUnit Test: { { \"world\" } [ greet ] }"
 }

tests/partial-fail/partial-fail/partial-fail-tests.factor

@@ -0,0 +1,3 @@
+USING: partial-fail tools.test ;
+{ "hello" } [ greet ] unit-test
+{ "world" } [ greet ] unit-test

tests/partial-fail/partial-fail/partial-fail.factor

@@ -0,0 +1,4 @@
+USING: kernel ;
+IN: partial-fail
+
+: greet ( -- str ) "hello" ;

tests/success/success/success-tests.factor

@@ -0,0 +1,2 @@
+USING: success tools.test ;
+{ "hello" } [ greet ] unit-test

tests/success/success/success.factor

@@ -0,0 +1,4 @@
+USING: kernel ;
+IN: success
+
+: greet ( -- str ) "hello" ;

tests/syntax-error/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
-  "status": "error",
-  "message": "TODO: replace with correct output"
+  "status": "fail",
+  "message": "/opt/test-runner/tests/syntax-error/syntax-error/syntax-error.factor\n4: : greet ( -- str ) @#$% ;\n                          ^\nNo word named '@#$%' found in current vocabulary search path"
 }

tests/syntax-error/syntax-error/syntax-error-tests.factor

@@ -0,0 +1,2 @@
+USING: syntax-error tools.test ;
+{ "hello" } [ greet ] unit-test

tests/syntax-error/syntax-error/syntax-error.factor

@@ -0,0 +1,4 @@
+USING: kernel ;
+IN: syntax-error
+
+: greet ( -- str ) @#$% ;