From 84782ac2d91f39f76afa21946c50efc8cc367d9b Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 17:20:06 -0600 Subject: [PATCH 01/21] Add every key in 'alternatives into 'improve and merge them --- src/api/sandbox.rkt | 2 +- src/api/server.rkt | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index 5fb3b9a0e..b9150b071 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -223,7 +223,7 @@ (timeline-event! 'start) ; Prevents the timeline from being empty. (define result (match command - ['alternatives (get-alternatives test pcontext)] + ['alternatives (get-improve test pcontext)] ['cost (get-cost test)] ['errors (get-errors test pcontext)] ['explanations (get-explanations test pcontext)] diff --git a/src/api/server.rkt b/src/api/server.rkt index c782bae83..048e5cddf 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -145,7 +145,7 @@ (define (get-json-converter command) (match (herbie-command-command command) - ['alternatives make-alternatives-result] + ['alternatives make-improve-result] ['cost make-cost-result] ['errors make-error-result] ['explanations make-explanation-result] @@ -408,6 +408,27 @@ ['timeout #f] ['failure (exception->datum backend)])) + + (match-define (list train-pcontext processed-pcontext) (improve-result-pctxs backend)) + + (define fpcores + (for/list ([analysis (improve-result-end backend)]) + (define altn (alt-analysis-alt analysis)) + (~a (program->fpcore (alt-expr altn) (test-context test))))) + + (define histories + (for/list ([analysis (improve-result-end backend)]) + (define altn (alt-analysis-alt analysis)) + (define os (open-output-string)) + (parameterize ([current-output-port os]) + (write-xexpr + `(div ([id "history"]) + (ol ,@(render-history altn processed-pcontext train-pcontext (test-context test))))) + (get-output-string os)))) + (define derivations + (for/list ([altn altns]) + (render-json altn processed-pcontext train-pcontext (test-context test)))) + (hasheq 'status (job-result-status herbie-result) 'test @@ -422,6 +443,12 @@ timeline 'profile profile + 'alternatives ; FIXME: currently used by Odyssey but should maybe be 'backend? + fpcores + 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below + histories + 'derivations + derivations 'backend backend-hash)) @@ -502,6 +529,7 @@ (define derivations (for/list ([altn altns]) (render-json altn processed-pcontext test-pcontext (test-context test)))) + (hasheq 'alternatives fpcores 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below From fa7ba143934139db97c26c3f925f788a72779664 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 20:36:09 -0600 Subject: [PATCH 02/21] Fix --- src/api/server.rkt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/api/server.rkt b/src/api/server.rkt index 048e5cddf..a37c1ff39 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -409,16 +409,15 @@ ['failure (exception->datum backend)])) + (define altns (map alt-analysis-alt (improve-result-end backend))) (match-define (list train-pcontext processed-pcontext) (improve-result-pctxs backend)) (define fpcores - (for/list ([analysis (improve-result-end backend)]) - (define altn (alt-analysis-alt analysis)) + (for/list ([altn (in-list altns)]) (~a (program->fpcore (alt-expr altn) (test-context test))))) (define histories - (for/list ([analysis (improve-result-end backend)]) - (define altn (alt-analysis-alt analysis)) + (for/list ([altn (in-list altns)]) (define os (open-output-string)) (parameterize ([current-output-port os]) (write-xexpr @@ -426,7 +425,7 @@ (ol ,@(render-history altn processed-pcontext train-pcontext (test-context test))))) (get-output-string os)))) (define derivations - (for/list ([altn altns]) + (for/list ([altn (in-list altns)]) (render-json altn processed-pcontext train-pcontext (test-context test)))) (hasheq 'status From 390e8fe628816cef3fd838c9719ddc41531cbd16 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 20:36:51 -0600 Subject: [PATCH 03/21] Remove obsolete "alternatives" result --- src/api/sandbox.rkt | 17 ----------------- src/api/server.rkt | 27 --------------------------- 2 files changed, 44 deletions(-) diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index b9150b071..35b8da997 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -119,23 +119,6 @@ ;; TODO: What in the timeline needs fixing with these changes? -;; Given a test and a sample of points, returns a list of improved alternatives -;; and both the test set of points and processed test set of points. -;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, -;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, -;; otherwise the entire set is used. -(define (get-alternatives test pcontext) - (unless pcontext - (error 'get-alternatives "cannnot run without a pcontext")) - - (define-values (train-pcontext test-pcontext) (partition-pcontext pcontext)) - ;; TODO: Ignoring all user-provided preprocessing right now - (define alternatives (run-improve! (test-input test) (test-spec test) (*context*) train-pcontext)) - (define preprocessing (alt-preprocessing (first alternatives))) - (define test-pcontext* (preprocess-pcontext (*context*) test-pcontext preprocessing)) - - (list alternatives test-pcontext test-pcontext*)) - ;; Improvement backend for generating reports ;; This is (get-alternatives) + a bunch of extra evaluation / data collection (define (get-improve test joint-pcontext) diff --git a/src/api/server.rkt b/src/api/server.rkt index a37c1ff39..5c7713583 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -508,30 +508,3 @@ (define (repr->json repr) (hasheq 'name (representation-name repr) 'type (representation-type repr))) -(define (make-alternatives-result herbie-result job-id) - - (define test (job-result-test herbie-result)) - (match-define (list altns test-pcontext processed-pcontext) (job-result-backend herbie-result)) - - (define fpcores - (for/list ([altn altns]) - (~a (program->fpcore (alt-expr altn) (test-context test))))) - - (define histories - (for/list ([altn altns]) - (define os (open-output-string)) - (parameterize ([current-output-port os]) - (write-xexpr - `(div ([id "history"]) - (ol ,@(render-history altn processed-pcontext test-pcontext (test-context test))))) - (get-output-string os)))) - (define derivations - (for/list ([altn altns]) - (render-json altn processed-pcontext test-pcontext (test-context test)))) - - (hasheq 'alternatives - fpcores - 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below - histories - 'derivations - derivations)) From 9ee3bcee10b7c2d7586f82f9ed1900b90024408f Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 20:43:52 -0600 Subject: [PATCH 04/21] Clean up `sandbox.rkt` (new comments, sort jobs, dedup) --- src/api/sandbox.rkt | 129 ++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 70 deletions(-) diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index 35b8da997..cb40fbdc1 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -37,22 +37,15 @@ (struct improve-result (preprocess pctxs start target end)) (struct alt-analysis (alt train-errors test-errors) #:prefab) -(define (sample-pcontext test) - (random) ;; Tick the random number generator, for backwards compatibility - (define specification (prog->spec (or (test-spec test) (test-input test)))) - (define precondition (prog->spec (test-pre test))) - (define sample - (parameterize ([*num-points* (+ (*num-points*) (*reeval-pts*))]) - (sample-points precondition (list specification) (list (*context*))))) - (apply mk-pcontext sample)) +;; API users can supply their own, weird set of points, in which case +;; the first 256 are training points and everything is test points. +;; For backwards compatibility, exactly 8256 points are split as +;; Herbie expects (first 256 training, rest are test). -;; Partitions a joint pcontext into a training and testing set (define (partition-pcontext joint-pcontext) (define num-points (pcontext-length joint-pcontext)) (cond [(= num-points (+ (*num-points*) (*reeval-pts*))) - ; got the expected amount of points - ; will partition into training and testing set (split-pcontext joint-pcontext (*num-points*) (*reeval-pts*))] [else ; the training set will just be up to the first (*num-points*) @@ -62,66 +55,10 @@ (define-values (train-pcontext _) (split-pcontext joint-pcontext training-count testing-count)) (values train-pcontext joint-pcontext)])) -;; -;; API endpoint backends -;; - -;; Given a test, computes the program cost of the input expression -(define (get-cost test) - (define cost-proc (platform-cost-proc (*active-platform*))) - (define output-repr (context-repr (*context*))) - (cost-proc (test-input test) output-repr)) - -;; Given a test and a sample of points, returns the test points. -(define (get-sample test) - (sample-pcontext test)) - -;; Given a test and a sample of points, computes the error at each point. -;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, -;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, -;; otherwise the entire set is used. -(define (get-errors test pcontext) - (unless pcontext - (error 'get-errors "cannnot run without a pcontext")) +;; API Functions - (define-values (_ test-pcontext) (partition-pcontext pcontext)) - (define errs (errors (test-input test) test-pcontext (*context*))) - (for/list ([(pt _) (in-pcontext test-pcontext)] - [err (in-list errs)]) - (list pt err))) - -;; Given a test and a sample of points, computes the local error at every node in the expression -;; returning a tree of errors that mirrors the structure of the expression. -;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, -;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, -;; otherwise the entire set is used. -(define (get-local-error test pcontext) - (unless pcontext - (error 'get-local-error "cannnot run without a pcontext")) - - (*pcontext* pcontext) - (local-error-as-tree (test-input test) (*context*))) - -(define (get-explanations test pcontext) - (unless pcontext - (error 'explain "cannot run without a pcontext")) - - (*pcontext* pcontext) - (define-values (fperrors - sorted-explanations-table - confusion-matrix - maybe-confusion-matrix - total-confusion-matrix - freqs) - (explain (test-input test) (*context*) (*pcontext*))) - - sorted-explanations-table) - -;; TODO: What in the timeline needs fixing with these changes? - -;; Improvement backend for generating reports -;; This is (get-alternatives) + a bunch of extra evaluation / data collection -(define (get-improve test joint-pcontext) +;; The main Herbie function +(define (get-alternatives test joint-pcontext) (unless joint-pcontext (error 'get-alternatives "cannnot run without a pcontext")) @@ -163,6 +100,58 @@ (define pctxs (list train-pcontext test-pcontext*)) (improve-result preprocessing pctxs start-alt-data target-alt-data end-data)) +(define (get-cost test) + (define cost-proc (platform-cost-proc (*active-platform*))) + (define output-repr (context-repr (*context*))) + (cost-proc (test-input test) output-repr)) + +(define (get-errors test pcontext) + (unless pcontext + (error 'get-errors "cannnot run without a pcontext")) + + (define-values (_ test-pcontext) (partition-pcontext pcontext)) + (define errs (errors (test-input test) test-pcontext (*context*))) + (for/list ([(pt _) (in-pcontext test-pcontext)] + [err (in-list errs)]) + (list pt err))) + +(define (get-explanations test pcontext) + (unless pcontext + (error 'explain "cannot run without a pcontext")) + + (*pcontext* pcontext) + (define-values (fperrors + sorted-explanations-table + confusion-matrix + maybe-confusion-matrix + total-confusion-matrix + freqs) + (explain (test-input test) (*context*) (*pcontext*))) + + sorted-explanations-table) + +;; Given a test and a sample of points, computes the local error at every node in the expression +;; returning a tree of errors that mirrors the structure of the expression. +;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, +;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, +;; otherwise the entire set is used. +(define (get-local-error test pcontext) + (unless pcontext + (error 'get-local-error "cannnot run without a pcontext")) + + (*pcontext* pcontext) + (local-error-as-tree (test-input test) (*context*))) + +(define (get-sample test) + (random) ;; Tick the random number generator, for backwards compatibility + (define specification (prog->spec (or (test-spec test) (test-input test)))) + (define precondition (prog->spec (test-pre test))) + (define sample + (parameterize ([*num-points* (+ (*num-points*) (*reeval-pts*))]) + (sample-points precondition (list specification) (list (*context*))))) + (apply mk-pcontext sample)) + + ;; ;; Public interface ;; From d73f11150740b5fabeb474eae1f31eca43bc19ba Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 21:20:48 -0600 Subject: [PATCH 05/21] Woops, fix --- src/api/sandbox.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index cb40fbdc1..e3c18bfd2 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -195,11 +195,11 @@ (timeline-event! 'start) ; Prevents the timeline from being empty. (define result (match command - ['alternatives (get-improve test pcontext)] + ['alternatives (get-alternatives test pcontext)] ['cost (get-cost test)] ['errors (get-errors test pcontext)] ['explanations (get-explanations test pcontext)] - ['improve (get-improve test (get-sample test))] + ['improve (get-alternatives test (get-sample test))] ['local-error (get-local-error test pcontext)] ['sample (get-sample test)] [_ (error 'compute-result "unknown command ~a" command)])) From e7e159f3d34665eec9ecd8a1728a3c84217cd546 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 21:22:22 -0600 Subject: [PATCH 06/21] Clean up FPCore generation --- src/api/sandbox.rkt | 1 - src/api/server.rkt | 42 +++++++++++++++++++---- src/api/shell.rkt | 84 +++++++++------------------------------------ 3 files changed, 52 insertions(+), 75 deletions(-) diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index e3c18bfd2..8bbabfca6 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -151,7 +151,6 @@ (sample-points precondition (list specification) (list (*context*))))) (apply mk-pcontext sample)) - ;; ;; Public interface ;; diff --git a/src/api/server.rkt b/src/api/server.rkt index 5c7713583..20b0e5278 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -11,6 +11,7 @@ "../syntax/types.rkt" "../syntax/read.rkt" "../syntax/load-plugin.rkt" + "../syntax/sugar.rkt" "../utils/alternative.rkt" "../utils/common.rkt" "../utils/errors.rkt" @@ -30,7 +31,8 @@ wait-for-job start-job-server write-results-to-disk - *demo-output*) + *demo-output* + alt->fpcore) (define (warn-single-threaded-mpfr) (local-require ffi/unsafe) @@ -408,13 +410,20 @@ ['timeout #f] ['failure (exception->datum backend)])) - - (define altns (map alt-analysis-alt (improve-result-end backend))) - (match-define (list train-pcontext processed-pcontext) (improve-result-pctxs backend)) + (define-values (altns train-pcontext processed-pcontext) + (cond + [(equal? (job-result-status herbie-result) 'success) + (define altns (map alt-analysis-alt (improve-result-end backend))) + (match-define (list train-pcontext processed-pcontext) (improve-result-pctxs backend)) + (values altns train-pcontext processed-pcontext)] + [else (values '() #f #f)])) (define fpcores - (for/list ([altn (in-list altns)]) - (~a (program->fpcore (alt-expr altn) (test-context test))))) + (if (equal? (job-result-status herbie-result) 'success) + (for/list ([altn (in-list altns)]) + (~s (alt->fpcore test altn))) + (list (~s (alt->fpcore test + (make-alt-preprocessing (test-input test) (test-preprocess test))))))) (define histories (for/list ([altn (in-list altns)]) @@ -424,6 +433,7 @@ `(div ([id "history"]) (ol ,@(render-history altn processed-pcontext train-pcontext (test-context test))))) (get-output-string os)))) + (define derivations (for/list ([altn (in-list altns)]) (render-json altn processed-pcontext train-pcontext (test-context test)))) @@ -442,7 +452,7 @@ timeline 'profile profile - 'alternatives ; FIXME: currently used by Odyssey but should maybe be 'backend? + 'alternatives ; FIXME: currently used by Odyssey but should maybe be in 'backend? fpcores 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below histories @@ -508,3 +518,21 @@ (define (repr->json repr) (hasheq 'name (representation-name repr) 'type (representation-type repr))) +(define (alt->fpcore test altn) + `(FPCore ,@(filter identity (list (test-identifier test))) + ,(test-vars test) + :name + ,(test-name test) + :precision + ,(test-output-repr-name test) + ,@(if (eq? (test-pre test) 'TRUE) + '() + `(:pre ,(test-pre test))) + ,@(if (equal? (alt-preprocessing altn) empty) + '() + `(:herbie-preprocess ,(alt-preprocessing altn))) + ,@(apply append + (for/list ([(target enabled?) (in-dict (test-output test))] + #:when enabled?) + `(:alt ,target))) + ,(prog->fpcore (alt-expr altn) (test-context test)))) diff --git a/src/api/shell.rkt b/src/api/shell.rkt index 0e02d0b1b..1f658e501 100644 --- a/src/api/shell.rkt +++ b/src/api/shell.rkt @@ -1,53 +1,11 @@ #lang racket (require "../syntax/read.rkt" - "../syntax/types.rkt" - "../syntax/sugar.rkt" "../utils/common.rkt" - "datafile.rkt" - "sandbox.rkt" "server.rkt") (provide run-shell run-improve) -(define (unparse-result row) - (define vars (table-row-vars row)) - (define repr (get-representation (table-row-precision row))) - (define ctx (context vars repr (map (const repr) vars))) ; TODO: this seems wrong - (define expr (or (table-row-output row) (table-row-input row))) - `(FPCore ,@(filter identity (list (table-row-identifier row))) - ,vars - :herbie-status - ,(string->symbol (table-row-status row)) - :herbie-time - ,(table-row-time row) - :herbie-error-input - ([,(*num-points*) ,(table-row-start-est row)] [,(*reeval-pts*) ,(table-row-start row)]) - :herbie-error-output - ([,(*num-points*) ,(table-row-result-est row)] [,(*reeval-pts*) ,(table-row-result row)]) - ,@(apply append - (for/list ([rec (in-list (table-row-target row))]) - (match-define (list cost score) rec) - `(:herbie-error-target ([,(*reeval-pts*) ,(table-row-target row)])))) - ,@(if (empty? (table-row-warnings row)) - '() - `(:herbie-warnings ,(table-row-warnings row))) - :name - ,(table-row-name row) - :precision - ,(table-row-precision row) - ,@(if (eq? (table-row-pre row) 'TRUE) - '() - `(:pre ,(table-row-pre row))) - ,@(if (equal? (table-row-preprocess row) empty) - '() - `(:herbie-preprocess ,(table-row-preprocess row))) - ,@(apply append - (for/list ([(target enabled?) (in-dict (table-row-target-prog row))] - #:when enabled?) - `(:alt ,target))) - ,(prog->fpcore expr ctx))) - (define (get-shell-input) (printf "herbie> ") (with-handlers ([(or/c exn:fail:user? exn:fail:read?) (λ (e) @@ -62,29 +20,24 @@ eof] [else (parse-test input)]))) +(define (job-result->fpcore result) + (read (open-input-string (first (hash-ref result 'alternatives))))) + (define (print-improve-outputs tests results p #:seed [seed #f]) (when seed (fprintf p ";; seed: ~a\n\n" seed)) (for ([res results] [test tests] #:when res) - (define name (table-row-name res)) - (match (table-row-status res) - ["error" - (fprintf p ";; Error in ~a\n" name) - (write (unparse-result res) p) - (newline p)] - ["crash" - (fprintf p ";; Crash in ~a\n" name) - (write (unparse-result res) p) - (newline p)] - ["timeout" - (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name) - (write (unparse-result res) p) - (newline p)] - [(? string?) - (write (unparse-result res) p) - (newline p)]))) + (define test (hash-ref res 'test)) + (define name (test-name test)) + (match (hash-ref res 'status) + ['failure + (match-define (list 'exn type msg url locs traceback) (hash-ref res 'backend)) + (fprintf p ";; ~a in ~a\n" (if type "Error" "Crash") name)] + ['timeout (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name)] + ['success (void)]) + (pretty-print (job-result->fpcore res) p 1))) (define (run-improve input output #:threads [threads #f]) (define seed (get-seed)) @@ -95,7 +48,7 @@ (start-job 'improve test #:seed seed #:pcontext #f #:profile? #f #:timeline-disabled? #f))) (define results (for/list ([id ids]) - (get-table-data-from-hash (wait-for-job id) ""))) + (wait-for-job id))) (if (equal? output "-") (print-improve-outputs tests results (current-output-port) #:seed seed) @@ -115,11 +68,8 @@ (for ([test (in-producer get-shell-input eof-object?)] [idx (in-naturals)]) (define result (wait-for-job (start-job 'improve test #:seed seed))) - (define status (hash-ref result 'status)) - (define time (hash-ref result 'time)) - (define table-data (get-table-data-from-hash result "")) - (match status - ['success (pretty-print (unparse-result table-data) (current-output-port) 1)] + (match (hash-ref result 'status) + ['success (pretty-print (job-result->fpcore result) (current-output-port) 1)] ['failure (match-define (list 'exn type msg url locs traceback) (hash-ref result 'backend)) (printf "; ~a\n" msg) @@ -127,5 +77,5 @@ (match-define (list msg file line col pos) loc) (printf "; ~a:~a~a: ~a\n" file line col msg)) (printf "; See for more.\n" *herbie-version* url)] - ['timeout (printf "Timeout in ~as (see --timeout option)\n" (/ time 1000))] - [else (error 'run-shell "unknown result type ~a" status)])))) + ['timeout + (printf "Timeout in ~as (see --timeout option)\n" (/ (hash-ref result 'time) 1000))])))) From 71a6ffdaaaf09d4afe62bb3054f58048928a64c7 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 21:26:00 -0600 Subject: [PATCH 07/21] Update docs --- www/doc/2.2/input.html | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/www/doc/2.2/input.html b/www/doc/2.2/input.html index 743e7f376..280881db5 100644 --- a/www/doc/2.2/input.html +++ b/www/doc/2.2/input.html @@ -237,24 +237,6 @@

Miscellaneous Input Properties

"developer targets"; these might be other alternatives you've tried that you want to compare against.

-

Additional Output Metadata

- -

Herbie's output provides additional information in custom - properties:

- -
-
:herbie-status status
-
status describes whether Herbie worked: it is one - of success, timeout, error, - or crash.
-
:herbie-time ms
-
The time, in milliseconds, used by Herbie to find a more accurate formula.
-
:herbie-error-input
([pts err] ...)
-
The average error of the input program at pts points. Multiple entries correspond to Herbie's training and test sets.
-
:herbie-error-output
([pts err] ...)
-
The computed average error of the output program, similar to :herbie-error-input.
-
-

Herbie's benchmark suite also uses properties for continuous integration, but these are not officially supported and their use is discouraged.

From 98e5a0a478a4187fd1474489e0e9ac1c98135217 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 21:29:38 -0600 Subject: [PATCH 08/21] Update tests for new shell outputs --- .github/workflows/unit-test.yml | 6 ++++-- src/api/shell.rkt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 7b157761e..73375df98 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -36,11 +36,13 @@ jobs: - name: "Test the shell command-line tool" run: | /tmp/out.fpcore - test `grep -c :herbie-time /tmp/out.fpcore` -eq 3 + test `grep -c :precision /tmp/out.fpcore` -eq 3 + test `grep -c -v ';;' /tmp/out.fpcore` -eq 3 - name: "Test the improve command-line tool" run: | racket -l herbie improve bench/tutorial.fpcore /tmp/out.fpcore - test `grep -c :herbie-time /tmp/out.fpcore` -eq 3 + test `grep -c :precision /tmp/out.fpcore` -eq 3 + test `grep -c -v '; ' /tmp/out.fpcore` -eq 3 - name: "Run the report command-line tool" run: | racket -l herbie report bench/tutorial.fpcore /tmp/out/ diff --git a/src/api/shell.rkt b/src/api/shell.rkt index 1f658e501..c864304de 100644 --- a/src/api/shell.rkt +++ b/src/api/shell.rkt @@ -78,4 +78,4 @@ (printf "; ~a:~a~a: ~a\n" file line col msg)) (printf "; See for more.\n" *herbie-version* url)] ['timeout - (printf "Timeout in ~as (see --timeout option)\n" (/ (hash-ref result 'time) 1000))])))) + (printf "; Timeout in ~as (see --timeout option)\n" (/ (hash-ref result 'time) 1000))])))) From ed6f56ee6b84c2195e83c4da72c676412eee8d9a Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 21:33:58 -0600 Subject: [PATCH 09/21] Rename to remove more "improve" names --- src/api/server.rkt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/server.rkt b/src/api/server.rkt index 20b0e5278..326b4c42b 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -147,11 +147,11 @@ (define (get-json-converter command) (match (herbie-command-command command) - ['alternatives make-improve-result] + ['alternatives make-alternatives-result] ['cost make-cost-result] ['errors make-error-result] ['explanations make-explanation-result] - ['improve make-improve-result] + ['improve make-alternatives-result] ['local-error make-local-error-result] ['sample make-sample-result] [_ (error 'compute-result "unknown command ~a" command)])) @@ -394,7 +394,7 @@ (list pt (format-bits (ulps->bits err))))) (hasheq 'points errs)) -(define (make-improve-result herbie-result job-id) +(define (make-alternatives-result herbie-result job-id) (define test (job-result-test herbie-result)) (define ctx (context->json (test-context test))) (define backend (job-result-backend herbie-result)) From fd09ce87a8d5f2882ef24ebc9d180b7b0e3a0cc0 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Mon, 17 Mar 2025 21:38:18 -0600 Subject: [PATCH 10/21] Fix counts in CI --- .github/workflows/unit-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 73375df98..cb4f2781e 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -37,12 +37,12 @@ jobs: run: | /tmp/out.fpcore test `grep -c :precision /tmp/out.fpcore` -eq 3 - test `grep -c -v ';;' /tmp/out.fpcore` -eq 3 + test `grep -c -v ';;' /tmp/out.fpcore` -eq 0 - name: "Test the improve command-line tool" run: | racket -l herbie improve bench/tutorial.fpcore /tmp/out.fpcore test `grep -c :precision /tmp/out.fpcore` -eq 3 - test `grep -c -v '; ' /tmp/out.fpcore` -eq 3 + test `grep -c -v '; ' /tmp/out.fpcore` -eq 0 - name: "Run the report command-line tool" run: | racket -l herbie report bench/tutorial.fpcore /tmp/out/ From 9db6c4238ba35e0e5de2c0f69743e301cc45e6ea Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 10:10:43 -0600 Subject: [PATCH 11/21] Woops --- .github/workflows/unit-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index cb4f2781e..99d135cb7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -37,12 +37,12 @@ jobs: run: | /tmp/out.fpcore test `grep -c :precision /tmp/out.fpcore` -eq 3 - test `grep -c -v ';;' /tmp/out.fpcore` -eq 0 + test `grep -c ';;' /tmp/out.fpcore` -eq 0 - name: "Test the improve command-line tool" run: | racket -l herbie improve bench/tutorial.fpcore /tmp/out.fpcore test `grep -c :precision /tmp/out.fpcore` -eq 3 - test `grep -c -v '; ' /tmp/out.fpcore` -eq 0 + test `grep -c '; ' /tmp/out.fpcore` -eq 0 - name: "Run the report command-line tool" run: | racket -l herbie report bench/tutorial.fpcore /tmp/out/ From 22dc569bddec5ba33a12010e72b0f0fc42aebbd1 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 10:22:12 -0600 Subject: [PATCH 12/21] Ugh more test nonsense --- .github/workflows/unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 99d135cb7..4eb747c07 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -42,7 +42,7 @@ jobs: run: | racket -l herbie improve bench/tutorial.fpcore /tmp/out.fpcore test `grep -c :precision /tmp/out.fpcore` -eq 3 - test `grep -c '; ' /tmp/out.fpcore` -eq 0 + test `grep -c '^; ' /tmp/out.fpcore` -eq 0 - name: "Run the report command-line tool" run: | racket -l herbie report bench/tutorial.fpcore /tmp/out/ From e2bc9ec790e48867270658337c7e5908a30921ec Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 13:24:44 -0600 Subject: [PATCH 13/21] Remove the large argument from a log message --- src/api/server.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/server.rkt b/src/api/server.rkt index 326b4c42b..6b37ef797 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -118,7 +118,7 @@ finished-result) (define (manager-tell msg . args) - (log "Telling manager: ~a, ~a.\n" msg args) + (log "Telling manager: ~a.\n" msg) (if manager (place-channel-put manager (list* msg args)) (match msg From 57b526136ac274266c534d930b86b2b07ad94e11 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 13:48:37 -0600 Subject: [PATCH 14/21] More cleanups from alt-analysis refactor --- src/api/sandbox.rkt | 36 +++++++++++--------- src/api/server.rkt | 70 ++++++++++++++++---------------------- src/reports/make-graph.rkt | 47 ++++++++++++------------- src/reports/plot.rkt | 17 +++++---- 4 files changed, 82 insertions(+), 88 deletions(-) diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index 8bbabfca6..6e3209d0f 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -257,7 +257,7 @@ (define backend (hash-ref result-hash 'backend)) (define status (hash-ref result-hash 'status)) (match status - ['success + ["success" (define start (hash-ref backend 'start)) (define targets (hash-ref backend 'target)) (define end (hash-ref backend 'end)) @@ -265,17 +265,16 @@ (define repr (test-output-repr test)) ; starting expr analysis - (match-define (alt-analysis start-alt start-train-errs start-test-errs) start) - (define start-expr (alt-expr start-alt)) - (define start-train-score (errors-score start-train-errs)) - (define start-test-score (errors-score start-test-errs)) - (define start-cost (expr-cost start-expr repr)) + (define start-expr (read (open-input-string (hash-ref start 'expr)))) + (define start-train-score (errors-score (hash-ref start 'train-score))) + (define start-test-score (errors-score (hash-ref start 'errors))) + (define start-cost (hash-ref start 'cost)) (define target-cost-score (for/list ([target targets]) - (define target-expr (alt-expr (alt-analysis-alt target))) - (define tar-cost (expr-cost target-expr repr)) - (define tar-score (errors-score (alt-analysis-test-errors target))) + (define target-expr (read (open-input-string (hash-ref target 'expr)))) + (define tar-cost (hash-ref target 'cost)) + (define tar-score (errors-score (hash-ref target 'errors))) (list tar-cost tar-score))) @@ -285,10 +284,16 @@ target-cost-score (apply min (map second target-cost-score)))) - (define end-exprs (hash-ref end 'end-exprs)) - (define end-train-scores (map errors-score (hash-ref end 'end-train-scores))) - (define end-test-scores (map errors-score (hash-ref end 'end-errors))) - (define end-costs (hash-ref end 'end-costs)) + (define end-exprs + (for/list ([end-analysis (in-list end)]) + (read (open-input-string (hash-ref end-analysis 'expr))))) + (define end-train-scores + (for/list ([end-analysis (in-list end)]) + (errors-score (hash-ref end-analysis 'train-score)))) + (define end-test-scores + (for/list ([end-analysis (in-list end)]) + (errors-score (hash-ref end-analysis 'errors)))) + (define end-costs (map (curryr hash-ref 'cost) end)) ; terribly formatted pareto-optimal frontier (define cost&accuracy @@ -325,9 +330,8 @@ [result end-score] [output (car end-exprs)] [cost-accuracy cost&accuracy])] - ['failure + ["failure" (match-define (list 'exn type _ ...) backend) (define status (if type "error" "crash")) (dummy-table-row-from-hash result-hash status link)] - ['timeout (dummy-table-row-from-hash result-hash "timeout" link)] - [_ (error 'get-table-data "unknown result type ~a" status)])) + ["timeout" (dummy-table-row-from-hash result-hash "timeout" link)])) diff --git a/src/api/server.rkt b/src/api/server.rkt index 6b37ef797..96057f2ac 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -403,10 +403,9 @@ (define timeline (job-result-timeline herbie-result)) (define profile (job-result-profile herbie-result)) - (define repr (test-output-repr test)) (define backend-hash (match (job-result-status herbie-result) - ['success (backend-improve-result-hash-table backend repr test)] + ['success (backend-improve-result-hash-table backend test)] ['timeout #f] ['failure (exception->datum backend)])) @@ -439,11 +438,9 @@ (render-json altn processed-pcontext train-pcontext (test-context test)))) (hasheq 'status - (job-result-status herbie-result) + (~a (job-result-status herbie-result)) 'test test - 'ctx - ctx 'time job-time 'warnings @@ -461,54 +458,47 @@ 'backend backend-hash)) -(define (backend-improve-result-hash-table backend repr test) - (define pcontext (improve-result-pctxs backend)) - - (define preprocessing (improve-result-preprocess backend)) - (define end-hash-table (end-hash (improve-result-end backend) repr pcontext test)) - +(define (backend-improve-result-hash-table backend test) + (define repr (context-repr (test-context test))) + (define pcontexts (improve-result-pctxs backend)) (hasheq 'preprocessing - preprocessing + (improve-result-preprocess backend) 'pctxs - pcontext + (map (curryr pcontext->json repr) pcontexts) 'start - (improve-result-start backend) + (analysis->json (improve-result-start backend) pcontexts test) 'target - (improve-result-target backend) + (map (curryr analysis->json pcontexts test) (improve-result-target backend)) 'end - end-hash-table)) + (map (curryr analysis->json pcontexts test) (improve-result-end backend)))) -(define (end-hash end repr pcontexts test) +(define (analysis->json analysis pcontexts test) + (define repr (context-repr (test-context test))) + (match-define (alt-analysis alt train-errors test-errors) analysis) + (define cost (alt-cost alt repr)) - (define-values (end-alts train-errors end-errors end-costs) - (for/lists (l1 l2 l3 l4) - ([analysis end]) - (match-define (alt-analysis alt train-errors test-errs) analysis) - (values alt train-errors test-errs (alt-cost alt repr)))) + (match-define (list train-pcontext processed-pcontext) pcontexts) + (define history + (render-history alt processed-pcontext train-pcontext (test-context test))) - (define alts-histories - (for/list ([alt end-alts]) - (render-history alt (first pcontexts) (second pcontexts) (test-context test)))) (define vars (test-vars test)) - (define end-alt (alt-analysis-alt (car end))) (define splitpoints - (for/list ([var vars]) - (define split-var? (equal? var (regime-var end-alt))) - (if split-var? - (for/list ([val (regime-splitpoints end-alt)]) + (for/list ([var (in-list vars)]) + (if (equal? var (regime-var alt)) + (for/list ([val (regime-splitpoints alt)]) (real->ordinal (repr->real val repr) repr)) '()))) - (hasheq 'end-exprs - (map alt-expr end-alts) - 'end-histories - alts-histories - 'end-train-scores + (hasheq 'expr + (~s (alt-expr alt)) + 'history + (~s history) + 'train-score train-errors - 'end-errors - end-errors - 'end-costs - end-costs + 'errors + test-errors + 'cost + cost 'splitpoints splitpoints)) @@ -516,7 +506,7 @@ (hasheq 'vars (context-vars ctx) 'repr (repr->json (context-repr ctx)))) (define (repr->json repr) - (hasheq 'name (representation-name repr) 'type (representation-type repr))) + (hasheq 'name (~s (representation-name repr)) 'type (representation-type repr))) (define (alt->fpcore test altn) `(FPCore ,@(filter identity (list (test-identifier test))) diff --git a/src/reports/make-graph.rkt b/src/reports/make-graph.rkt index db806fcd2..2a91bb242 100644 --- a/src/reports/make-graph.rkt +++ b/src/reports/make-graph.rkt @@ -38,23 +38,21 @@ (define identifier (test-identifier test)) (define preprocessing (hash-ref backend 'preprocessing)) - (match-define (alt-analysis start-alt _ start-error) (hash-ref backend 'start)) - (define targets (hash-ref backend 'target)) - (define end (hash-ref backend 'end)) - - (define start-cost (alt-cost start-alt repr)) - - (define list-target-error - (for/list ([target targets]) - (alt-analysis-test-errors target))) + (define start-expr (read (open-input-string (hash-ref (hash-ref backend 'start) 'expr)))) + (define start-cost (hash-ref (hash-ref backend 'start) 'cost)) + (define start-error (hash-ref (hash-ref backend 'start) 'errors)) - (define list-target-cost - (for/list ([target targets]) - (alt-cost (alt-analysis-alt target) repr))) + (define targets (hash-ref backend 'target)) - (define end-exprs (hash-ref end 'end-exprs)) - (define end-errors (hash-ref end 'end-errors)) - (define end-costs (hash-ref end 'end-costs)) + (define end (hash-ref backend 'end)) + (define end-exprs + (for/list ([end-analysis (in-list end)]) + (read (open-input-string (hash-ref end-analysis 'expr))))) + (define end-errors (map (curryr hash-ref 'errors) end)) + (define end-costs (map (curryr hash-ref 'cost) end)) + (define end-histories + (for/list ([end-analysis (in-list end)]) + (read (open-input-string (hash-ref end-analysis 'history))))) (define speedup (let ([better (for/list ([err end-errors] @@ -132,7 +130,7 @@ "alternatives. Up and to the right is better. The red square shows " "the initial program, and each blue circle shows an alternative." "The line shows the best available speed-accuracy tradeoffs.")) - ,(let-values ([(dropdown body) (render-program (alt-expr start-alt) ctx #:ident identifier)]) + ,(let-values ([(dropdown body) (render-program start-expr ctx #:ident identifier)]) `(section ([id "initial"] (class "programs")) (h2 "Initial Program" ": " @@ -148,7 +146,7 @@ [expr end-exprs] [errs end-errors] [cost end-costs] - [history (hash-ref end 'end-histories)]) + [history end-histories]) (define-values (dropdown body) (render-program expr ctx #:ident identifier #:instructions preprocessing)) `(section ([id ,(format "alternative~a" i)] (class "programs")) @@ -158,18 +156,17 @@ (span ((class "subhead")) (data ,(format-accuracy (errors-score errs) repr-bits #:unit "%")) " accurate, " - (data ,(~r (/ (alt-cost start-alt repr) cost) #:precision '(= 1)) "×") + (data ,(~r (/ start-cost cost) #:precision '(= 1)) "×") " speedup") ,dropdown ,(render-help "report.html#alternatives")) ,body (details (summary "Derivation") (ol ((class "history")) ,@history)))) - ,@(for/list ([i (in-naturals 1)] - [target (in-list targets)] - [target-error (in-list list-target-error)] - [target-cost (in-list list-target-cost)]) - (let-values ([(dropdown body) - (render-program (alt-expr (alt-analysis-alt target)) ctx #:ident identifier)]) + ,@(for/list ([i (in-naturals 1)] [target (in-list targets)]) + (define target-error (hash-ref target 'errors)) + (define target-cost (hash-ref target 'cost)) + (define target-expr (read (open-input-string (hash-ref target 'expr)))) + (let-values ([(dropdown body) (render-program target-expr ctx #:ident identifier)]) `(section ([id ,(format "target~a" i)] (class "programs")) (h2 "Developer Target " @@ -178,7 +175,7 @@ (span ((class "subhead")) (data ,(format-accuracy (errors-score target-error) repr-bits #:unit "%")) " accurate, " - (data ,(~r (/ (alt-cost start-alt repr) target-cost) #:precision '(= 1)) "×") + (data ,(~r (/ start-cost target-cost) #:precision '(= 1)) "×") " speedup") ,dropdown ,(render-help "report.html#target")) diff --git a/src/reports/plot.rkt b/src/reports/plot.rkt index e3956bac4..43373e552 100644 --- a/src/reports/plot.rkt +++ b/src/reports/plot.rkt @@ -36,21 +36,24 @@ (define (make-points-json result-hash) (define test (hash-ref result-hash 'test)) (define backend (hash-ref result-hash 'backend)) - (define pctxs (hash-ref backend 'pctxs)) + (define test-points (map first (second (hash-ref backend 'pctxs)))) (define start (hash-ref backend 'start)) (define targets (hash-ref backend 'target)) (define end (hash-ref backend 'end)) (define repr (test-output-repr test)) - (define start-errors (alt-analysis-test-errors start)) + (define start-errors (hash-ref start 'errors)) - (define target-errors (map alt-analysis-test-errors targets)) + (define target-errors (map (curryr hash-ref 'errors) targets)) - (define end-errors (hash-ref end 'end-errors)) - - (define newpoints (pcontext-points (second pctxs))) + (define end-errors (map (curryr hash-ref 'errors) end)) ; Immediately convert points to reals to handle posits + (define newpoints + (for/list ([point test-points]) + (for/list ([x point]) + (json->value x repr)))) + (define points (for/list ([point newpoints]) (for/list ([x point]) @@ -89,7 +92,7 @@ (string-replace (~r val #:notation 'exponential #:precision 0) "1e" "e"))) (list tick-str (real->ordinal val repr))))) - (define splitpoints (hash-ref end 'splitpoints)) + (define splitpoints (hash-ref (car end) 'splitpoints)) ; NOTE ordinals *should* be passed as strings so we can detect truncation if ; necessary, but this isn't implemented yet. From e8d7649795dbb0dc80a0b8e5126c37bc16219102 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 14:02:00 -0600 Subject: [PATCH 15/21] Simplify some code in the demo server? --- src/api/demo.rkt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/api/demo.rkt b/src/api/demo.rkt index f04bf2f29..026b7a819 100644 --- a/src/api/demo.rkt +++ b/src/api/demo.rkt @@ -77,11 +77,6 @@ (define (generate-page req job-id page) (define path (first (string-split (url->string (request-uri req)) "/"))) - (cond - [(check-and-send path job-id page)] - [else (next-dispatcher)])) - -(define (check-and-send path job-id page) (define result-hash (get-results-for job-id)) (cond [(set-member? (all-pages result-hash) page) @@ -96,7 +91,7 @@ (λ (out) (with-handlers ([exn:fail? (page-error-handler result-hash page out)]) (make-page page out result-hash (*demo-output*) #f))))] - [else #f])) + [else (next-dispatcher)])) (define (generate-report req) (cond From d6a10a32eecc6a08b4db1025c0f700cbccfc8cf0 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 14:05:44 -0600 Subject: [PATCH 16/21] Delete unused code --- src/api/server.rkt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/api/server.rkt b/src/api/server.rkt index 96057f2ac..09460509e 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -396,7 +396,6 @@ (define (make-alternatives-result herbie-result job-id) (define test (job-result-test herbie-result)) - (define ctx (context->json (test-context test))) (define backend (job-result-backend herbie-result)) (define job-time (job-result-time herbie-result)) (define warnings (job-result-warnings herbie-result)) @@ -502,12 +501,6 @@ 'splitpoints splitpoints)) -(define (context->json ctx) - (hasheq 'vars (context-vars ctx) 'repr (repr->json (context-repr ctx)))) - -(define (repr->json repr) - (hasheq 'name (~s (representation-name repr)) 'type (representation-type repr))) - (define (alt->fpcore test altn) `(FPCore ,@(filter identity (list (test-identifier test))) ,(test-vars test) From 7b2737cef23c6ab9d753c3314d1a75508791a7cd Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 14:36:44 -0600 Subject: [PATCH 17/21] Make the 'test key an FPCore expression --- src/api/sandbox.rkt | 4 ++-- src/api/server.rkt | 24 ++++++++++++++++++------ src/api/shell.rkt | 2 +- src/reports/make-graph.rkt | 2 +- src/reports/pages.rkt | 15 +++++++-------- src/reports/plot.rkt | 2 +- src/reports/traceback.rkt | 4 ++-- src/syntax/read.rkt | 8 +++++--- src/syntax/syntax-check.rkt | 2 +- 9 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index 6e3209d0f..7670cf42d 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -224,7 +224,7 @@ (on-timeout))) (define (dummy-table-row-from-hash result-hash status link) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define repr (test-output-repr test)) (define preprocess (if (eq? (hash-ref result-hash 'status) 'success) @@ -253,7 +253,7 @@ '())) (define (get-table-data-from-hash result-hash link) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define backend (hash-ref result-hash 'backend)) (define status (hash-ref result-hash 'status)) (match status diff --git a/src/api/server.rkt b/src/api/server.rkt index 09460509e..135824b46 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -416,12 +416,14 @@ (values altns train-pcontext processed-pcontext)] [else (values '() #f #f)])) + (define test-fpcore + (alt->fpcore test (make-alt-preprocessing (test-input test) (test-preprocess test)))) + (define fpcores (if (equal? (job-result-status herbie-result) 'success) (for/list ([altn (in-list altns)]) (~s (alt->fpcore test altn))) - (list (~s (alt->fpcore test - (make-alt-preprocessing (test-input test) (test-preprocess test))))))) + (list (~s test-fpcore)))) (define histories (for/list ([altn (in-list altns)]) @@ -439,7 +441,7 @@ (hasheq 'status (~a (job-result-status herbie-result)) 'test - test + (~s test-fpcore) 'time job-time 'warnings @@ -503,17 +505,27 @@ (define (alt->fpcore test altn) `(FPCore ,@(filter identity (list (test-identifier test))) - ,(test-vars test) + ,(for/list ([var (in-list (test-vars test))]) + (define repr (dict-ref (test-var-repr-names test) var)) + (if (equal? repr (test-output-repr-name test)) + var + (list '! ':precision repr var))) :name ,(test-name test) :precision ,(test-output-repr-name test) - ,@(if (eq? (test-pre test) 'TRUE) + ,@(if (eq? (test-pre test) '(TRUE)) + '() + `(:pre ,(prog->fpcore (test-pre test) (test-context test)))) + ,@(if (equal? (test-spec test) empty) '() - `(:pre ,(test-pre test))) + `(:herbie-spec ,(prog->fpcore (test-spec test) (test-context test)))) ,@(if (equal? (alt-preprocessing altn) empty) '() `(:herbie-preprocess ,(alt-preprocessing altn))) + ,@(if (equal? (test-expected test) #t) + '() + `(:herbie-expected ,(test-expected test))) ,@(apply append (for/list ([(target enabled?) (in-dict (test-output test))] #:when enabled?) diff --git a/src/api/shell.rkt b/src/api/shell.rkt index c864304de..c41dcf08e 100644 --- a/src/api/shell.rkt +++ b/src/api/shell.rkt @@ -29,7 +29,7 @@ (for ([res results] [test tests] #:when res) - (define test (hash-ref res 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define name (test-name test)) (match (hash-ref res 'status) ['failure diff --git a/src/reports/make-graph.rkt b/src/reports/make-graph.rkt index 2a91bb242..6edac6f17 100644 --- a/src/reports/make-graph.rkt +++ b/src/reports/make-graph.rkt @@ -29,7 +29,7 @@ (define (make-graph result-hash output? profile?) (define backend (hash-ref result-hash 'backend)) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define time (hash-ref result-hash 'time)) (define warnings (hash-ref result-hash 'warnings)) (define repr (test-output-repr test)) diff --git a/src/reports/pages.rkt b/src/reports/pages.rkt index 2f5403bea..2ae52958a 100644 --- a/src/reports/pages.rkt +++ b/src/reports/pages.rkt @@ -13,13 +13,13 @@ page-error-handler) (define (all-pages result-hash) - (define good? (eq? (hash-ref result-hash 'status) 'success)) + (define good? (equal? (hash-ref result-hash 'status) "success")) (define default-pages '("graph.html" "timeline.html" "timeline.json")) (define success-pages '("points.json" "profile.json")) (append default-pages (if good? success-pages empty))) (define ((page-error-handler result-hash page out) e) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (eprintf "Error generating `~a` for \"~a\":\n ~a\n" page (test-name test) (exn-message e)) (eprintf "context:\n") (for ([(fn loc) (in-dict (continuation-mark-set->context (exn-continuation-marks e)))]) @@ -35,8 +35,8 @@ (match page ["graph.html" (write-html (make-graph-html result-hash output? profile?) out)] ["timeline.html" - (write-html (make-timeline (test-name (hash-ref result-hash 'test)) - (hash-ref result-hash 'timeline) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) + (write-html (make-timeline (test-name test) (hash-ref result-hash 'timeline) #:path "..") out)] ["timeline.json" (write-json (hash-ref result-hash 'timeline) out)] @@ -46,11 +46,10 @@ (define (make-graph-html result-hash output? profile?) (define status (hash-ref result-hash 'status)) (match status - ['success + ["success" (define command (hash-ref result-hash 'command)) (match command ["improve" (make-graph result-hash output? profile?)] [else (dummy-graph command)])] - ['timeout (make-traceback result-hash)] - ['failure (make-traceback result-hash)] - [_ (error 'make-graph-html "unknown result type ~a" status)])) + ["timeout" (make-traceback result-hash)] + ["failure" (make-traceback result-hash)])) diff --git a/src/reports/plot.rkt b/src/reports/plot.rkt index 43373e552..7b785bf09 100644 --- a/src/reports/plot.rkt +++ b/src/reports/plot.rkt @@ -34,7 +34,7 @@ '()))) (define (make-points-json result-hash) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define backend (hash-ref result-hash 'backend)) (define test-points (map first (second (hash-ref backend 'pctxs)))) (define start (hash-ref backend 'start)) diff --git a/src/reports/traceback.rkt b/src/reports/traceback.rkt index 357621306..af5e7f5ee 100644 --- a/src/reports/traceback.rkt +++ b/src/reports/traceback.rkt @@ -14,7 +14,7 @@ [status (error 'make-traceback "unexpected status ~a" status)])) (define (render-failure result-hash) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define warnings (hash-ref result-hash 'warnings)) (define backend (hash-ref result-hash 'backend)) @@ -51,7 +51,7 @@ `(tr (td ((class "procedure")) ,(~a name)) ,@(render-loc loc)))))) (define (render-timeout result-hash) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define time (hash-ref result-hash 'time)) (define warnings (hash-ref result-hash 'warnings)) diff --git a/src/syntax/read.rkt b/src/syntax/read.rkt index 3d4bb231d..0dba51c70 100644 --- a/src/syntax/read.rkt +++ b/src/syntax/read.rkt @@ -237,8 +237,9 @@ (parameterize ([read-decimal-as-inexact false]) (read-syntax port name))) -(define (load-stdin) - (for/list ([test (in-port (curry our-read-syntax "stdin") (current-input-port))]) +(define (load-port port) + (port-count-lines! port) + (for/list ([test (in-port (curry our-read-syntax "stdin") port)]) (parse-test test))) (define (load-file file) @@ -262,7 +263,8 @@ path)) (define out (cond - [(equal? path "-") (load-stdin)] + [(port? path) (load-port path)] + [(equal? path "-") (load-port (current-input-port))] [(directory-exists? path*) (load-directory path*)] [else (load-file path*)])) (define duplicates (find-duplicates (map test-name out))) diff --git a/src/syntax/syntax-check.rkt b/src/syntax/syntax-check.rkt index b4828a555..6136d0a9d 100644 --- a/src/syntax/syntax-check.rkt +++ b/src/syntax/syntax-check.rkt @@ -178,7 +178,7 @@ (error! stx "FPCore identifier must be a symbol: ~a" name)) (check-program* stx vars props body error!)] [#`(FPCore (#,vars ...) #,props ... #,body) (check-program* stx vars props body error!)] - [#`(FPCore #,something ...) (error! stx "FPCore not in a valid format: ~a" stx)] + [#`(FPCore #,something ...) (error! stx "FPCore not in a valid format: ~s" stx)] [_ (error! stx "Not an FPCore: ~a" stx)])) (define (assert-program! stx) From d6d91a4654dddc41976653bacce2112ed8a78583 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 14:44:54 -0600 Subject: [PATCH 18/21] Woops --- src/api/shell.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/shell.rkt b/src/api/shell.rkt index c41dcf08e..be26cbc7f 100644 --- a/src/api/shell.rkt +++ b/src/api/shell.rkt @@ -29,7 +29,7 @@ (for ([res results] [test tests] #:when res) - (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) + (define test (car (load-tests (open-input-string (hash-ref res 'test))))) (define name (test-name test)) (match (hash-ref res 'status) ['failure From 59573f40e066cf8f58ae7bfc411df5fb8db0343a Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 14:47:17 -0600 Subject: [PATCH 19/21] Store and use the test name in a few places --- src/api/server.rkt | 2 ++ src/api/shell.rkt | 3 +-- src/reports/pages.rkt | 10 ++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/api/server.rkt b/src/api/server.rkt index 135824b46..425822e63 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -440,6 +440,8 @@ (hasheq 'status (~a (job-result-status herbie-result)) + 'name + (test-name test) 'test (~s test-fpcore) 'time diff --git a/src/api/shell.rkt b/src/api/shell.rkt index be26cbc7f..a02d6972b 100644 --- a/src/api/shell.rkt +++ b/src/api/shell.rkt @@ -29,8 +29,7 @@ (for ([res results] [test tests] #:when res) - (define test (car (load-tests (open-input-string (hash-ref res 'test))))) - (define name (test-name test)) + (define name (hash-ref res 'name)) (match (hash-ref res 'status) ['failure (match-define (list 'exn type msg url locs traceback) (hash-ref res 'backend)) diff --git a/src/reports/pages.rkt b/src/reports/pages.rkt index 2ae52958a..c44f49cb1 100644 --- a/src/reports/pages.rkt +++ b/src/reports/pages.rkt @@ -19,8 +19,8 @@ (append default-pages (if good? success-pages empty))) (define ((page-error-handler result-hash page out) e) - (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) - (eprintf "Error generating `~a` for \"~a\":\n ~a\n" page (test-name test) (exn-message e)) + (define name (hash-ref result-hash 'name)) + (eprintf "Error generating `~a` for \"~a\":\n ~a\n" page name (exn-message e)) (eprintf "context:\n") (for ([(fn loc) (in-dict (continuation-mark-set->context (exn-continuation-marks e)))]) (match loc @@ -35,10 +35,8 @@ (match page ["graph.html" (write-html (make-graph-html result-hash output? profile?) out)] ["timeline.html" - (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) - (write-html (make-timeline (test-name test) (hash-ref result-hash 'timeline) - #:path "..") - out)] + (define name (hash-ref result-hash 'name)) + (write-html (make-timeline name (hash-ref result-hash 'timeline) #:path "..") out)] ["timeline.json" (write-json (hash-ref result-hash 'timeline) out)] ["profile.json" (write-json (hash-ref result-hash 'profile) out)] ["points.json" (write-json (make-points-json result-hash) out)])) From bd7292a1fbfd0b4243717171ac008d19d59f5fd4 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 14:47:49 -0600 Subject: [PATCH 20/21] fmt --- src/api/server.rkt | 11 +++++------ src/reports/make-graph.rkt | 3 ++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/api/server.rkt b/src/api/server.rkt index 425822e63..2b5c73698 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -481,8 +481,7 @@ (define cost (alt-cost alt repr)) (match-define (list train-pcontext processed-pcontext) pcontexts) - (define history - (render-history alt processed-pcontext train-pcontext (test-context test))) + (define history (render-history alt processed-pcontext train-pcontext (test-context test))) (define vars (test-vars test)) (define splitpoints @@ -508,10 +507,10 @@ (define (alt->fpcore test altn) `(FPCore ,@(filter identity (list (test-identifier test))) ,(for/list ([var (in-list (test-vars test))]) - (define repr (dict-ref (test-var-repr-names test) var)) - (if (equal? repr (test-output-repr-name test)) - var - (list '! ':precision repr var))) + (define repr (dict-ref (test-var-repr-names test) var)) + (if (equal? repr (test-output-repr-name test)) + var + (list '! ':precision repr var))) :name ,(test-name test) :precision diff --git a/src/reports/make-graph.rkt b/src/reports/make-graph.rkt index 6edac6f17..201bb4415 100644 --- a/src/reports/make-graph.rkt +++ b/src/reports/make-graph.rkt @@ -162,7 +162,8 @@ ,(render-help "report.html#alternatives")) ,body (details (summary "Derivation") (ol ((class "history")) ,@history)))) - ,@(for/list ([i (in-naturals 1)] [target (in-list targets)]) + ,@(for/list ([i (in-naturals 1)] + [target (in-list targets)]) (define target-error (hash-ref target 'errors)) (define target-cost (hash-ref target 'cost)) (define target-expr (read (open-input-string (hash-ref target 'expr)))) From 0825233a215902a46d28dac95d5e227b145431f3 Mon Sep 17 00:00:00 2001 From: Pavel Panchekha Date: Tue, 18 Mar 2025 14:51:54 -0600 Subject: [PATCH 21/21] Fix the shell tool --- src/api/shell.rkt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/api/shell.rkt b/src/api/shell.rkt index a02d6972b..31b71dab8 100644 --- a/src/api/shell.rkt +++ b/src/api/shell.rkt @@ -31,11 +31,11 @@ #:when res) (define name (hash-ref res 'name)) (match (hash-ref res 'status) - ['failure + ["failure" (match-define (list 'exn type msg url locs traceback) (hash-ref res 'backend)) (fprintf p ";; ~a in ~a\n" (if type "Error" "Crash") name)] - ['timeout (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name)] - ['success (void)]) + ["timeout" (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name)] + ["success" (void)]) (pretty-print (job-result->fpcore res) p 1))) (define (run-improve input output #:threads [threads #f]) @@ -68,13 +68,13 @@ [idx (in-naturals)]) (define result (wait-for-job (start-job 'improve test #:seed seed))) (match (hash-ref result 'status) - ['success (pretty-print (job-result->fpcore result) (current-output-port) 1)] - ['failure + ["success" (pretty-print (job-result->fpcore result) (current-output-port) 1)] + ["failure" (match-define (list 'exn type msg url locs traceback) (hash-ref result 'backend)) (printf "; ~a\n" msg) (for ([loc (in-list locs)]) (match-define (list msg file line col pos) loc) (printf "; ~a:~a~a: ~a\n" file line col msg)) (printf "; See for more.\n" *herbie-version* url)] - ['timeout + ["timeout" (printf "; Timeout in ~as (see --timeout option)\n" (/ (hash-ref result 'time) 1000))]))))