diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 7b157761e..4eb747c07 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -36,11 +36,13 @@ jobs: - name: "Test the shell command-line tool" run: | /tmp/out.fpcore - test `grep -c :herbie-time /tmp/out.fpcore` -eq 3 + test `grep -c :precision /tmp/out.fpcore` -eq 3 + test `grep -c ';;' /tmp/out.fpcore` -eq 0 - name: "Test the improve command-line tool" run: | racket -l herbie improve bench/tutorial.fpcore /tmp/out.fpcore - test `grep -c :herbie-time /tmp/out.fpcore` -eq 3 + test `grep -c :precision /tmp/out.fpcore` -eq 3 + test `grep -c '^; ' /tmp/out.fpcore` -eq 0 - name: "Run the report command-line tool" run: | racket -l herbie report bench/tutorial.fpcore /tmp/out/ diff --git a/src/api/demo.rkt b/src/api/demo.rkt index f04bf2f29..026b7a819 100644 --- a/src/api/demo.rkt +++ b/src/api/demo.rkt @@ -77,11 +77,6 @@ (define (generate-page req job-id page) (define path (first (string-split (url->string (request-uri req)) "/"))) - (cond - [(check-and-send path job-id page)] - [else (next-dispatcher)])) - -(define (check-and-send path job-id page) (define result-hash (get-results-for job-id)) (cond [(set-member? (all-pages result-hash) page) @@ -96,7 +91,7 @@ (λ (out) (with-handlers ([exn:fail? (page-error-handler result-hash page out)]) (make-page page out result-hash (*demo-output*) #f))))] - [else #f])) + [else (next-dispatcher)])) (define (generate-report req) (cond diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt index 5fb3b9a0e..7670cf42d 100644 --- a/src/api/sandbox.rkt +++ b/src/api/sandbox.rkt @@ -37,22 +37,15 @@ (struct improve-result (preprocess pctxs start target end)) (struct alt-analysis (alt train-errors test-errors) #:prefab) -(define (sample-pcontext test) - (random) ;; Tick the random number generator, for backwards compatibility - (define specification (prog->spec (or (test-spec test) (test-input test)))) - (define precondition (prog->spec (test-pre test))) - (define sample - (parameterize ([*num-points* (+ (*num-points*) (*reeval-pts*))]) - (sample-points precondition (list specification) (list (*context*))))) - (apply mk-pcontext sample)) +;; API users can supply their own, weird set of points, in which case +;; the first 256 are training points and everything is test points. +;; For backwards compatibility, exactly 8256 points are split as +;; Herbie expects (first 256 training, rest are test). -;; Partitions a joint pcontext into a training and testing set (define (partition-pcontext joint-pcontext) (define num-points (pcontext-length joint-pcontext)) (cond [(= num-points (+ (*num-points*) (*reeval-pts*))) - ; got the expected amount of points - ; will partition into training and testing set (split-pcontext joint-pcontext (*num-points*) (*reeval-pts*))] [else ; the training set will just be up to the first (*num-points*) @@ -62,83 +55,10 @@ (define-values (train-pcontext _) (split-pcontext joint-pcontext training-count testing-count)) (values train-pcontext joint-pcontext)])) -;; -;; API endpoint backends -;; - -;; Given a test, computes the program cost of the input expression -(define (get-cost test) - (define cost-proc (platform-cost-proc (*active-platform*))) - (define output-repr (context-repr (*context*))) - (cost-proc (test-input test) output-repr)) - -;; Given a test and a sample of points, returns the test points. -(define (get-sample test) - (sample-pcontext test)) - -;; Given a test and a sample of points, computes the error at each point. -;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, -;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, -;; otherwise the entire set is used. -(define (get-errors test pcontext) - (unless pcontext - (error 'get-errors "cannnot run without a pcontext")) - - (define-values (_ test-pcontext) (partition-pcontext pcontext)) - (define errs (errors (test-input test) test-pcontext (*context*))) - (for/list ([(pt _) (in-pcontext test-pcontext)] - [err (in-list errs)]) - (list pt err))) - -;; Given a test and a sample of points, computes the local error at every node in the expression -;; returning a tree of errors that mirrors the structure of the expression. -;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, -;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, -;; otherwise the entire set is used. -(define (get-local-error test pcontext) - (unless pcontext - (error 'get-local-error "cannnot run without a pcontext")) - - (*pcontext* pcontext) - (local-error-as-tree (test-input test) (*context*))) - -(define (get-explanations test pcontext) - (unless pcontext - (error 'explain "cannot run without a pcontext")) - - (*pcontext* pcontext) - (define-values (fperrors - sorted-explanations-table - confusion-matrix - maybe-confusion-matrix - total-confusion-matrix - freqs) - (explain (test-input test) (*context*) (*pcontext*))) - - sorted-explanations-table) - -;; TODO: What in the timeline needs fixing with these changes? - -;; Given a test and a sample of points, returns a list of improved alternatives -;; and both the test set of points and processed test set of points. -;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, -;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, -;; otherwise the entire set is used. -(define (get-alternatives test pcontext) - (unless pcontext - (error 'get-alternatives "cannnot run without a pcontext")) - - (define-values (train-pcontext test-pcontext) (partition-pcontext pcontext)) - ;; TODO: Ignoring all user-provided preprocessing right now - (define alternatives (run-improve! (test-input test) (test-spec test) (*context*) train-pcontext)) - (define preprocessing (alt-preprocessing (first alternatives))) - (define test-pcontext* (preprocess-pcontext (*context*) test-pcontext preprocessing)) +;; API Functions - (list alternatives test-pcontext test-pcontext*)) - -;; Improvement backend for generating reports -;; This is (get-alternatives) + a bunch of extra evaluation / data collection -(define (get-improve test joint-pcontext) +;; The main Herbie function +(define (get-alternatives test joint-pcontext) (unless joint-pcontext (error 'get-alternatives "cannnot run without a pcontext")) @@ -180,6 +100,57 @@ (define pctxs (list train-pcontext test-pcontext*)) (improve-result preprocessing pctxs start-alt-data target-alt-data end-data)) +(define (get-cost test) + (define cost-proc (platform-cost-proc (*active-platform*))) + (define output-repr (context-repr (*context*))) + (cost-proc (test-input test) output-repr)) + +(define (get-errors test pcontext) + (unless pcontext + (error 'get-errors "cannnot run without a pcontext")) + + (define-values (_ test-pcontext) (partition-pcontext pcontext)) + (define errs (errors (test-input test) test-pcontext (*context*))) + (for/list ([(pt _) (in-pcontext test-pcontext)] + [err (in-list errs)]) + (list pt err))) + +(define (get-explanations test pcontext) + (unless pcontext + (error 'explain "cannot run without a pcontext")) + + (*pcontext* pcontext) + (define-values (fperrors + sorted-explanations-table + confusion-matrix + maybe-confusion-matrix + total-confusion-matrix + freqs) + (explain (test-input test) (*context*) (*pcontext*))) + + sorted-explanations-table) + +;; Given a test and a sample of points, computes the local error at every node in the expression +;; returning a tree of errors that mirrors the structure of the expression. +;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`, +;; then the first `*num-points*` will be discarded and the rest will be used for evaluation, +;; otherwise the entire set is used. +(define (get-local-error test pcontext) + (unless pcontext + (error 'get-local-error "cannnot run without a pcontext")) + + (*pcontext* pcontext) + (local-error-as-tree (test-input test) (*context*))) + +(define (get-sample test) + (random) ;; Tick the random number generator, for backwards compatibility + (define specification (prog->spec (or (test-spec test) (test-input test)))) + (define precondition (prog->spec (test-pre test))) + (define sample + (parameterize ([*num-points* (+ (*num-points*) (*reeval-pts*))]) + (sample-points precondition (list specification) (list (*context*))))) + (apply mk-pcontext sample)) + ;; ;; Public interface ;; @@ -227,7 +198,7 @@ ['cost (get-cost test)] ['errors (get-errors test pcontext)] ['explanations (get-explanations test pcontext)] - ['improve (get-improve test (get-sample test))] + ['improve (get-alternatives test (get-sample test))] ['local-error (get-local-error test pcontext)] ['sample (get-sample test)] [_ (error 'compute-result "unknown command ~a" command)])) @@ -253,7 +224,7 @@ (on-timeout))) (define (dummy-table-row-from-hash result-hash status link) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define repr (test-output-repr test)) (define preprocess (if (eq? (hash-ref result-hash 'status) 'success) @@ -282,11 +253,11 @@ '())) (define (get-table-data-from-hash result-hash link) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define backend (hash-ref result-hash 'backend)) (define status (hash-ref result-hash 'status)) (match status - ['success + ["success" (define start (hash-ref backend 'start)) (define targets (hash-ref backend 'target)) (define end (hash-ref backend 'end)) @@ -294,17 +265,16 @@ (define repr (test-output-repr test)) ; starting expr analysis - (match-define (alt-analysis start-alt start-train-errs start-test-errs) start) - (define start-expr (alt-expr start-alt)) - (define start-train-score (errors-score start-train-errs)) - (define start-test-score (errors-score start-test-errs)) - (define start-cost (expr-cost start-expr repr)) + (define start-expr (read (open-input-string (hash-ref start 'expr)))) + (define start-train-score (errors-score (hash-ref start 'train-score))) + (define start-test-score (errors-score (hash-ref start 'errors))) + (define start-cost (hash-ref start 'cost)) (define target-cost-score (for/list ([target targets]) - (define target-expr (alt-expr (alt-analysis-alt target))) - (define tar-cost (expr-cost target-expr repr)) - (define tar-score (errors-score (alt-analysis-test-errors target))) + (define target-expr (read (open-input-string (hash-ref target 'expr)))) + (define tar-cost (hash-ref target 'cost)) + (define tar-score (errors-score (hash-ref target 'errors))) (list tar-cost tar-score))) @@ -314,10 +284,16 @@ target-cost-score (apply min (map second target-cost-score)))) - (define end-exprs (hash-ref end 'end-exprs)) - (define end-train-scores (map errors-score (hash-ref end 'end-train-scores))) - (define end-test-scores (map errors-score (hash-ref end 'end-errors))) - (define end-costs (hash-ref end 'end-costs)) + (define end-exprs + (for/list ([end-analysis (in-list end)]) + (read (open-input-string (hash-ref end-analysis 'expr))))) + (define end-train-scores + (for/list ([end-analysis (in-list end)]) + (errors-score (hash-ref end-analysis 'train-score)))) + (define end-test-scores + (for/list ([end-analysis (in-list end)]) + (errors-score (hash-ref end-analysis 'errors)))) + (define end-costs (map (curryr hash-ref 'cost) end)) ; terribly formatted pareto-optimal frontier (define cost&accuracy @@ -354,9 +330,8 @@ [result end-score] [output (car end-exprs)] [cost-accuracy cost&accuracy])] - ['failure + ["failure" (match-define (list 'exn type _ ...) backend) (define status (if type "error" "crash")) (dummy-table-row-from-hash result-hash status link)] - ['timeout (dummy-table-row-from-hash result-hash "timeout" link)] - [_ (error 'get-table-data "unknown result type ~a" status)])) + ["timeout" (dummy-table-row-from-hash result-hash "timeout" link)])) diff --git a/src/api/server.rkt b/src/api/server.rkt index c782bae83..2b5c73698 100644 --- a/src/api/server.rkt +++ b/src/api/server.rkt @@ -11,6 +11,7 @@ "../syntax/types.rkt" "../syntax/read.rkt" "../syntax/load-plugin.rkt" + "../syntax/sugar.rkt" "../utils/alternative.rkt" "../utils/common.rkt" "../utils/errors.rkt" @@ -30,7 +31,8 @@ wait-for-job start-job-server write-results-to-disk - *demo-output*) + *demo-output* + alt->fpcore) (define (warn-single-threaded-mpfr) (local-require ffi/unsafe) @@ -116,7 +118,7 @@ finished-result) (define (manager-tell msg . args) - (log "Telling manager: ~a, ~a.\n" msg args) + (log "Telling manager: ~a.\n" msg) (if manager (place-channel-put manager (list* msg args)) (match msg @@ -149,7 +151,7 @@ ['cost make-cost-result] ['errors make-error-result] ['explanations make-explanation-result] - ['improve make-improve-result] + ['improve make-alternatives-result] ['local-error make-local-error-result] ['sample make-sample-result] [_ (error 'compute-result "unknown command ~a" command)])) @@ -392,28 +394,56 @@ (list pt (format-bits (ulps->bits err))))) (hasheq 'points errs)) -(define (make-improve-result herbie-result job-id) +(define (make-alternatives-result herbie-result job-id) (define test (job-result-test herbie-result)) - (define ctx (context->json (test-context test))) (define backend (job-result-backend herbie-result)) (define job-time (job-result-time herbie-result)) (define warnings (job-result-warnings herbie-result)) (define timeline (job-result-timeline herbie-result)) (define profile (job-result-profile herbie-result)) - (define repr (test-output-repr test)) (define backend-hash (match (job-result-status herbie-result) - ['success (backend-improve-result-hash-table backend repr test)] + ['success (backend-improve-result-hash-table backend test)] ['timeout #f] ['failure (exception->datum backend)])) + (define-values (altns train-pcontext processed-pcontext) + (cond + [(equal? (job-result-status herbie-result) 'success) + (define altns (map alt-analysis-alt (improve-result-end backend))) + (match-define (list train-pcontext processed-pcontext) (improve-result-pctxs backend)) + (values altns train-pcontext processed-pcontext)] + [else (values '() #f #f)])) + + (define test-fpcore + (alt->fpcore test (make-alt-preprocessing (test-input test) (test-preprocess test)))) + + (define fpcores + (if (equal? (job-result-status herbie-result) 'success) + (for/list ([altn (in-list altns)]) + (~s (alt->fpcore test altn))) + (list (~s test-fpcore)))) + + (define histories + (for/list ([altn (in-list altns)]) + (define os (open-output-string)) + (parameterize ([current-output-port os]) + (write-xexpr + `(div ([id "history"]) + (ol ,@(render-history altn processed-pcontext train-pcontext (test-context test))))) + (get-output-string os)))) + + (define derivations + (for/list ([altn (in-list altns)]) + (render-json altn processed-pcontext train-pcontext (test-context test)))) + (hasheq 'status - (job-result-status herbie-result) + (~a (job-result-status herbie-result)) + 'name + (test-name test) 'test - test - 'ctx - ctx + (~s test-fpcore) 'time job-time 'warnings @@ -422,89 +452,83 @@ timeline 'profile profile + 'alternatives ; FIXME: currently used by Odyssey but should maybe be in 'backend? + fpcores + 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below + histories + 'derivations + derivations 'backend backend-hash)) -(define (backend-improve-result-hash-table backend repr test) - (define pcontext (improve-result-pctxs backend)) - - (define preprocessing (improve-result-preprocess backend)) - (define end-hash-table (end-hash (improve-result-end backend) repr pcontext test)) - +(define (backend-improve-result-hash-table backend test) + (define repr (context-repr (test-context test))) + (define pcontexts (improve-result-pctxs backend)) (hasheq 'preprocessing - preprocessing + (improve-result-preprocess backend) 'pctxs - pcontext + (map (curryr pcontext->json repr) pcontexts) 'start - (improve-result-start backend) + (analysis->json (improve-result-start backend) pcontexts test) 'target - (improve-result-target backend) + (map (curryr analysis->json pcontexts test) (improve-result-target backend)) 'end - end-hash-table)) + (map (curryr analysis->json pcontexts test) (improve-result-end backend)))) -(define (end-hash end repr pcontexts test) +(define (analysis->json analysis pcontexts test) + (define repr (context-repr (test-context test))) + (match-define (alt-analysis alt train-errors test-errors) analysis) + (define cost (alt-cost alt repr)) - (define-values (end-alts train-errors end-errors end-costs) - (for/lists (l1 l2 l3 l4) - ([analysis end]) - (match-define (alt-analysis alt train-errors test-errs) analysis) - (values alt train-errors test-errs (alt-cost alt repr)))) + (match-define (list train-pcontext processed-pcontext) pcontexts) + (define history (render-history alt processed-pcontext train-pcontext (test-context test))) - (define alts-histories - (for/list ([alt end-alts]) - (render-history alt (first pcontexts) (second pcontexts) (test-context test)))) (define vars (test-vars test)) - (define end-alt (alt-analysis-alt (car end))) (define splitpoints - (for/list ([var vars]) - (define split-var? (equal? var (regime-var end-alt))) - (if split-var? - (for/list ([val (regime-splitpoints end-alt)]) + (for/list ([var (in-list vars)]) + (if (equal? var (regime-var alt)) + (for/list ([val (regime-splitpoints alt)]) (real->ordinal (repr->real val repr) repr)) '()))) - (hasheq 'end-exprs - (map alt-expr end-alts) - 'end-histories - alts-histories - 'end-train-scores + (hasheq 'expr + (~s (alt-expr alt)) + 'history + (~s history) + 'train-score train-errors - 'end-errors - end-errors - 'end-costs - end-costs + 'errors + test-errors + 'cost + cost 'splitpoints splitpoints)) -(define (context->json ctx) - (hasheq 'vars (context-vars ctx) 'repr (repr->json (context-repr ctx)))) - -(define (repr->json repr) - (hasheq 'name (representation-name repr) 'type (representation-type repr))) - -(define (make-alternatives-result herbie-result job-id) - - (define test (job-result-test herbie-result)) - (match-define (list altns test-pcontext processed-pcontext) (job-result-backend herbie-result)) - - (define fpcores - (for/list ([altn altns]) - (~a (program->fpcore (alt-expr altn) (test-context test))))) - - (define histories - (for/list ([altn altns]) - (define os (open-output-string)) - (parameterize ([current-output-port os]) - (write-xexpr - `(div ([id "history"]) - (ol ,@(render-history altn processed-pcontext test-pcontext (test-context test))))) - (get-output-string os)))) - (define derivations - (for/list ([altn altns]) - (render-json altn processed-pcontext test-pcontext (test-context test)))) - (hasheq 'alternatives - fpcores - 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below - histories - 'derivations - derivations)) +(define (alt->fpcore test altn) + `(FPCore ,@(filter identity (list (test-identifier test))) + ,(for/list ([var (in-list (test-vars test))]) + (define repr (dict-ref (test-var-repr-names test) var)) + (if (equal? repr (test-output-repr-name test)) + var + (list '! ':precision repr var))) + :name + ,(test-name test) + :precision + ,(test-output-repr-name test) + ,@(if (eq? (test-pre test) '(TRUE)) + '() + `(:pre ,(prog->fpcore (test-pre test) (test-context test)))) + ,@(if (equal? (test-spec test) empty) + '() + `(:herbie-spec ,(prog->fpcore (test-spec test) (test-context test)))) + ,@(if (equal? (alt-preprocessing altn) empty) + '() + `(:herbie-preprocess ,(alt-preprocessing altn))) + ,@(if (equal? (test-expected test) #t) + '() + `(:herbie-expected ,(test-expected test))) + ,@(apply append + (for/list ([(target enabled?) (in-dict (test-output test))] + #:when enabled?) + `(:alt ,target))) + ,(prog->fpcore (alt-expr altn) (test-context test)))) diff --git a/src/api/shell.rkt b/src/api/shell.rkt index 0e02d0b1b..31b71dab8 100644 --- a/src/api/shell.rkt +++ b/src/api/shell.rkt @@ -1,53 +1,11 @@ #lang racket (require "../syntax/read.rkt" - "../syntax/types.rkt" - "../syntax/sugar.rkt" "../utils/common.rkt" - "datafile.rkt" - "sandbox.rkt" "server.rkt") (provide run-shell run-improve) -(define (unparse-result row) - (define vars (table-row-vars row)) - (define repr (get-representation (table-row-precision row))) - (define ctx (context vars repr (map (const repr) vars))) ; TODO: this seems wrong - (define expr (or (table-row-output row) (table-row-input row))) - `(FPCore ,@(filter identity (list (table-row-identifier row))) - ,vars - :herbie-status - ,(string->symbol (table-row-status row)) - :herbie-time - ,(table-row-time row) - :herbie-error-input - ([,(*num-points*) ,(table-row-start-est row)] [,(*reeval-pts*) ,(table-row-start row)]) - :herbie-error-output - ([,(*num-points*) ,(table-row-result-est row)] [,(*reeval-pts*) ,(table-row-result row)]) - ,@(apply append - (for/list ([rec (in-list (table-row-target row))]) - (match-define (list cost score) rec) - `(:herbie-error-target ([,(*reeval-pts*) ,(table-row-target row)])))) - ,@(if (empty? (table-row-warnings row)) - '() - `(:herbie-warnings ,(table-row-warnings row))) - :name - ,(table-row-name row) - :precision - ,(table-row-precision row) - ,@(if (eq? (table-row-pre row) 'TRUE) - '() - `(:pre ,(table-row-pre row))) - ,@(if (equal? (table-row-preprocess row) empty) - '() - `(:herbie-preprocess ,(table-row-preprocess row))) - ,@(apply append - (for/list ([(target enabled?) (in-dict (table-row-target-prog row))] - #:when enabled?) - `(:alt ,target))) - ,(prog->fpcore expr ctx))) - (define (get-shell-input) (printf "herbie> ") (with-handlers ([(or/c exn:fail:user? exn:fail:read?) (λ (e) @@ -62,29 +20,23 @@ eof] [else (parse-test input)]))) +(define (job-result->fpcore result) + (read (open-input-string (first (hash-ref result 'alternatives))))) + (define (print-improve-outputs tests results p #:seed [seed #f]) (when seed (fprintf p ";; seed: ~a\n\n" seed)) (for ([res results] [test tests] #:when res) - (define name (table-row-name res)) - (match (table-row-status res) - ["error" - (fprintf p ";; Error in ~a\n" name) - (write (unparse-result res) p) - (newline p)] - ["crash" - (fprintf p ";; Crash in ~a\n" name) - (write (unparse-result res) p) - (newline p)] - ["timeout" - (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name) - (write (unparse-result res) p) - (newline p)] - [(? string?) - (write (unparse-result res) p) - (newline p)]))) + (define name (hash-ref res 'name)) + (match (hash-ref res 'status) + ["failure" + (match-define (list 'exn type msg url locs traceback) (hash-ref res 'backend)) + (fprintf p ";; ~a in ~a\n" (if type "Error" "Crash") name)] + ["timeout" (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name)] + ["success" (void)]) + (pretty-print (job-result->fpcore res) p 1))) (define (run-improve input output #:threads [threads #f]) (define seed (get-seed)) @@ -95,7 +47,7 @@ (start-job 'improve test #:seed seed #:pcontext #f #:profile? #f #:timeline-disabled? #f))) (define results (for/list ([id ids]) - (get-table-data-from-hash (wait-for-job id) ""))) + (wait-for-job id))) (if (equal? output "-") (print-improve-outputs tests results (current-output-port) #:seed seed) @@ -115,17 +67,14 @@ (for ([test (in-producer get-shell-input eof-object?)] [idx (in-naturals)]) (define result (wait-for-job (start-job 'improve test #:seed seed))) - (define status (hash-ref result 'status)) - (define time (hash-ref result 'time)) - (define table-data (get-table-data-from-hash result "")) - (match status - ['success (pretty-print (unparse-result table-data) (current-output-port) 1)] - ['failure + (match (hash-ref result 'status) + ["success" (pretty-print (job-result->fpcore result) (current-output-port) 1)] + ["failure" (match-define (list 'exn type msg url locs traceback) (hash-ref result 'backend)) (printf "; ~a\n" msg) (for ([loc (in-list locs)]) (match-define (list msg file line col pos) loc) (printf "; ~a:~a~a: ~a\n" file line col msg)) (printf "; See for more.\n" *herbie-version* url)] - ['timeout (printf "Timeout in ~as (see --timeout option)\n" (/ time 1000))] - [else (error 'run-shell "unknown result type ~a" status)])))) + ["timeout" + (printf "; Timeout in ~as (see --timeout option)\n" (/ (hash-ref result 'time) 1000))])))) diff --git a/src/reports/make-graph.rkt b/src/reports/make-graph.rkt index db806fcd2..201bb4415 100644 --- a/src/reports/make-graph.rkt +++ b/src/reports/make-graph.rkt @@ -29,7 +29,7 @@ (define (make-graph result-hash output? profile?) (define backend (hash-ref result-hash 'backend)) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define time (hash-ref result-hash 'time)) (define warnings (hash-ref result-hash 'warnings)) (define repr (test-output-repr test)) @@ -38,23 +38,21 @@ (define identifier (test-identifier test)) (define preprocessing (hash-ref backend 'preprocessing)) - (match-define (alt-analysis start-alt _ start-error) (hash-ref backend 'start)) - (define targets (hash-ref backend 'target)) - (define end (hash-ref backend 'end)) - - (define start-cost (alt-cost start-alt repr)) + (define start-expr (read (open-input-string (hash-ref (hash-ref backend 'start) 'expr)))) + (define start-cost (hash-ref (hash-ref backend 'start) 'cost)) + (define start-error (hash-ref (hash-ref backend 'start) 'errors)) - (define list-target-error - (for/list ([target targets]) - (alt-analysis-test-errors target))) - - (define list-target-cost - (for/list ([target targets]) - (alt-cost (alt-analysis-alt target) repr))) + (define targets (hash-ref backend 'target)) - (define end-exprs (hash-ref end 'end-exprs)) - (define end-errors (hash-ref end 'end-errors)) - (define end-costs (hash-ref end 'end-costs)) + (define end (hash-ref backend 'end)) + (define end-exprs + (for/list ([end-analysis (in-list end)]) + (read (open-input-string (hash-ref end-analysis 'expr))))) + (define end-errors (map (curryr hash-ref 'errors) end)) + (define end-costs (map (curryr hash-ref 'cost) end)) + (define end-histories + (for/list ([end-analysis (in-list end)]) + (read (open-input-string (hash-ref end-analysis 'history))))) (define speedup (let ([better (for/list ([err end-errors] @@ -132,7 +130,7 @@ "alternatives. Up and to the right is better. The red square shows " "the initial program, and each blue circle shows an alternative." "The line shows the best available speed-accuracy tradeoffs.")) - ,(let-values ([(dropdown body) (render-program (alt-expr start-alt) ctx #:ident identifier)]) + ,(let-values ([(dropdown body) (render-program start-expr ctx #:ident identifier)]) `(section ([id "initial"] (class "programs")) (h2 "Initial Program" ": " @@ -148,7 +146,7 @@ [expr end-exprs] [errs end-errors] [cost end-costs] - [history (hash-ref end 'end-histories)]) + [history end-histories]) (define-values (dropdown body) (render-program expr ctx #:ident identifier #:instructions preprocessing)) `(section ([id ,(format "alternative~a" i)] (class "programs")) @@ -158,18 +156,18 @@ (span ((class "subhead")) (data ,(format-accuracy (errors-score errs) repr-bits #:unit "%")) " accurate, " - (data ,(~r (/ (alt-cost start-alt repr) cost) #:precision '(= 1)) "×") + (data ,(~r (/ start-cost cost) #:precision '(= 1)) "×") " speedup") ,dropdown ,(render-help "report.html#alternatives")) ,body (details (summary "Derivation") (ol ((class "history")) ,@history)))) ,@(for/list ([i (in-naturals 1)] - [target (in-list targets)] - [target-error (in-list list-target-error)] - [target-cost (in-list list-target-cost)]) - (let-values ([(dropdown body) - (render-program (alt-expr (alt-analysis-alt target)) ctx #:ident identifier)]) + [target (in-list targets)]) + (define target-error (hash-ref target 'errors)) + (define target-cost (hash-ref target 'cost)) + (define target-expr (read (open-input-string (hash-ref target 'expr)))) + (let-values ([(dropdown body) (render-program target-expr ctx #:ident identifier)]) `(section ([id ,(format "target~a" i)] (class "programs")) (h2 "Developer Target " @@ -178,7 +176,7 @@ (span ((class "subhead")) (data ,(format-accuracy (errors-score target-error) repr-bits #:unit "%")) " accurate, " - (data ,(~r (/ (alt-cost start-alt repr) target-cost) #:precision '(= 1)) "×") + (data ,(~r (/ start-cost target-cost) #:precision '(= 1)) "×") " speedup") ,dropdown ,(render-help "report.html#target")) diff --git a/src/reports/pages.rkt b/src/reports/pages.rkt index 2f5403bea..c44f49cb1 100644 --- a/src/reports/pages.rkt +++ b/src/reports/pages.rkt @@ -13,14 +13,14 @@ page-error-handler) (define (all-pages result-hash) - (define good? (eq? (hash-ref result-hash 'status) 'success)) + (define good? (equal? (hash-ref result-hash 'status) "success")) (define default-pages '("graph.html" "timeline.html" "timeline.json")) (define success-pages '("points.json" "profile.json")) (append default-pages (if good? success-pages empty))) (define ((page-error-handler result-hash page out) e) - (define test (hash-ref result-hash 'test)) - (eprintf "Error generating `~a` for \"~a\":\n ~a\n" page (test-name test) (exn-message e)) + (define name (hash-ref result-hash 'name)) + (eprintf "Error generating `~a` for \"~a\":\n ~a\n" page name (exn-message e)) (eprintf "context:\n") (for ([(fn loc) (in-dict (continuation-mark-set->context (exn-continuation-marks e)))]) (match loc @@ -35,10 +35,8 @@ (match page ["graph.html" (write-html (make-graph-html result-hash output? profile?) out)] ["timeline.html" - (write-html (make-timeline (test-name (hash-ref result-hash 'test)) - (hash-ref result-hash 'timeline) - #:path "..") - out)] + (define name (hash-ref result-hash 'name)) + (write-html (make-timeline name (hash-ref result-hash 'timeline) #:path "..") out)] ["timeline.json" (write-json (hash-ref result-hash 'timeline) out)] ["profile.json" (write-json (hash-ref result-hash 'profile) out)] ["points.json" (write-json (make-points-json result-hash) out)])) @@ -46,11 +44,10 @@ (define (make-graph-html result-hash output? profile?) (define status (hash-ref result-hash 'status)) (match status - ['success + ["success" (define command (hash-ref result-hash 'command)) (match command ["improve" (make-graph result-hash output? profile?)] [else (dummy-graph command)])] - ['timeout (make-traceback result-hash)] - ['failure (make-traceback result-hash)] - [_ (error 'make-graph-html "unknown result type ~a" status)])) + ["timeout" (make-traceback result-hash)] + ["failure" (make-traceback result-hash)])) diff --git a/src/reports/plot.rkt b/src/reports/plot.rkt index e3956bac4..7b785bf09 100644 --- a/src/reports/plot.rkt +++ b/src/reports/plot.rkt @@ -34,23 +34,26 @@ '()))) (define (make-points-json result-hash) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define backend (hash-ref result-hash 'backend)) - (define pctxs (hash-ref backend 'pctxs)) + (define test-points (map first (second (hash-ref backend 'pctxs)))) (define start (hash-ref backend 'start)) (define targets (hash-ref backend 'target)) (define end (hash-ref backend 'end)) (define repr (test-output-repr test)) - (define start-errors (alt-analysis-test-errors start)) + (define start-errors (hash-ref start 'errors)) - (define target-errors (map alt-analysis-test-errors targets)) + (define target-errors (map (curryr hash-ref 'errors) targets)) - (define end-errors (hash-ref end 'end-errors)) - - (define newpoints (pcontext-points (second pctxs))) + (define end-errors (map (curryr hash-ref 'errors) end)) ; Immediately convert points to reals to handle posits + (define newpoints + (for/list ([point test-points]) + (for/list ([x point]) + (json->value x repr)))) + (define points (for/list ([point newpoints]) (for/list ([x point]) @@ -89,7 +92,7 @@ (string-replace (~r val #:notation 'exponential #:precision 0) "1e" "e"))) (list tick-str (real->ordinal val repr))))) - (define splitpoints (hash-ref end 'splitpoints)) + (define splitpoints (hash-ref (car end) 'splitpoints)) ; NOTE ordinals *should* be passed as strings so we can detect truncation if ; necessary, but this isn't implemented yet. diff --git a/src/reports/traceback.rkt b/src/reports/traceback.rkt index 357621306..af5e7f5ee 100644 --- a/src/reports/traceback.rkt +++ b/src/reports/traceback.rkt @@ -14,7 +14,7 @@ [status (error 'make-traceback "unexpected status ~a" status)])) (define (render-failure result-hash) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define warnings (hash-ref result-hash 'warnings)) (define backend (hash-ref result-hash 'backend)) @@ -51,7 +51,7 @@ `(tr (td ((class "procedure")) ,(~a name)) ,@(render-loc loc)))))) (define (render-timeout result-hash) - (define test (hash-ref result-hash 'test)) + (define test (car (load-tests (open-input-string (hash-ref result-hash 'test))))) (define time (hash-ref result-hash 'time)) (define warnings (hash-ref result-hash 'warnings)) diff --git a/src/syntax/read.rkt b/src/syntax/read.rkt index 3d4bb231d..0dba51c70 100644 --- a/src/syntax/read.rkt +++ b/src/syntax/read.rkt @@ -237,8 +237,9 @@ (parameterize ([read-decimal-as-inexact false]) (read-syntax port name))) -(define (load-stdin) - (for/list ([test (in-port (curry our-read-syntax "stdin") (current-input-port))]) +(define (load-port port) + (port-count-lines! port) + (for/list ([test (in-port (curry our-read-syntax "stdin") port)]) (parse-test test))) (define (load-file file) @@ -262,7 +263,8 @@ path)) (define out (cond - [(equal? path "-") (load-stdin)] + [(port? path) (load-port path)] + [(equal? path "-") (load-port (current-input-port))] [(directory-exists? path*) (load-directory path*)] [else (load-file path*)])) (define duplicates (find-duplicates (map test-name out))) diff --git a/src/syntax/syntax-check.rkt b/src/syntax/syntax-check.rkt index b4828a555..6136d0a9d 100644 --- a/src/syntax/syntax-check.rkt +++ b/src/syntax/syntax-check.rkt @@ -178,7 +178,7 @@ (error! stx "FPCore identifier must be a symbol: ~a" name)) (check-program* stx vars props body error!)] [#`(FPCore (#,vars ...) #,props ... #,body) (check-program* stx vars props body error!)] - [#`(FPCore #,something ...) (error! stx "FPCore not in a valid format: ~a" stx)] + [#`(FPCore #,something ...) (error! stx "FPCore not in a valid format: ~s" stx)] [_ (error! stx "Not an FPCore: ~a" stx)])) (define (assert-program! stx) diff --git a/www/doc/2.2/input.html b/www/doc/2.2/input.html index 743e7f376..280881db5 100644 --- a/www/doc/2.2/input.html +++ b/www/doc/2.2/input.html @@ -237,24 +237,6 @@

Miscellaneous Input Properties

"developer targets"; these might be other alternatives you've tried that you want to compare against.

-

Additional Output Metadata

- -

Herbie's output provides additional information in custom - properties:

- -
-
:herbie-status status
-
status describes whether Herbie worked: it is one - of success, timeout, error, - or crash.
-
:herbie-time ms
-
The time, in milliseconds, used by Herbie to find a more accurate formula.
-
:herbie-error-input
([pts err] ...)
-
The average error of the input program at pts points. Multiple entries correspond to Herbie's training and test sets.
-
:herbie-error-output
([pts err] ...)
-
The computed average error of the output program, similar to :herbie-error-input.
-
-

Herbie's benchmark suite also uses properties for continuous integration, but these are not officially supported and their use is discouraged.