diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 7b157761e..4eb747c07 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -36,11 +36,13 @@ jobs:
- name: "Test the shell command-line tool"
run: |
/tmp/out.fpcore
- test `grep -c :herbie-time /tmp/out.fpcore` -eq 3
+ test `grep -c :precision /tmp/out.fpcore` -eq 3
+ test `grep -c ';;' /tmp/out.fpcore` -eq 0
- name: "Test the improve command-line tool"
run: |
racket -l herbie improve bench/tutorial.fpcore /tmp/out.fpcore
- test `grep -c :herbie-time /tmp/out.fpcore` -eq 3
+ test `grep -c :precision /tmp/out.fpcore` -eq 3
+ test `grep -c '^; ' /tmp/out.fpcore` -eq 0
- name: "Run the report command-line tool"
run: |
racket -l herbie report bench/tutorial.fpcore /tmp/out/
diff --git a/src/api/demo.rkt b/src/api/demo.rkt
index f04bf2f29..026b7a819 100644
--- a/src/api/demo.rkt
+++ b/src/api/demo.rkt
@@ -77,11 +77,6 @@
(define (generate-page req job-id page)
(define path (first (string-split (url->string (request-uri req)) "/")))
- (cond
- [(check-and-send path job-id page)]
- [else (next-dispatcher)]))
-
-(define (check-and-send path job-id page)
(define result-hash (get-results-for job-id))
(cond
[(set-member? (all-pages result-hash) page)
@@ -96,7 +91,7 @@
(λ (out)
(with-handlers ([exn:fail? (page-error-handler result-hash page out)])
(make-page page out result-hash (*demo-output*) #f))))]
- [else #f]))
+ [else (next-dispatcher)]))
(define (generate-report req)
(cond
diff --git a/src/api/sandbox.rkt b/src/api/sandbox.rkt
index 5fb3b9a0e..7670cf42d 100644
--- a/src/api/sandbox.rkt
+++ b/src/api/sandbox.rkt
@@ -37,22 +37,15 @@
(struct improve-result (preprocess pctxs start target end))
(struct alt-analysis (alt train-errors test-errors) #:prefab)
-(define (sample-pcontext test)
- (random) ;; Tick the random number generator, for backwards compatibility
- (define specification (prog->spec (or (test-spec test) (test-input test))))
- (define precondition (prog->spec (test-pre test)))
- (define sample
- (parameterize ([*num-points* (+ (*num-points*) (*reeval-pts*))])
- (sample-points precondition (list specification) (list (*context*)))))
- (apply mk-pcontext sample))
+;; API users can supply their own, weird set of points, in which case
+;; the first 256 are training points and everything is test points.
+;; For backwards compatibility, exactly 8256 points are split as
+;; Herbie expects (first 256 training, rest are test).
-;; Partitions a joint pcontext into a training and testing set
(define (partition-pcontext joint-pcontext)
(define num-points (pcontext-length joint-pcontext))
(cond
[(= num-points (+ (*num-points*) (*reeval-pts*)))
- ; got the expected amount of points
- ; will partition into training and testing set
(split-pcontext joint-pcontext (*num-points*) (*reeval-pts*))]
[else
; the training set will just be up to the first (*num-points*)
@@ -62,83 +55,10 @@
(define-values (train-pcontext _) (split-pcontext joint-pcontext training-count testing-count))
(values train-pcontext joint-pcontext)]))
-;;
-;; API endpoint backends
-;;
-
-;; Given a test, computes the program cost of the input expression
-(define (get-cost test)
- (define cost-proc (platform-cost-proc (*active-platform*)))
- (define output-repr (context-repr (*context*)))
- (cost-proc (test-input test) output-repr))
-
-;; Given a test and a sample of points, returns the test points.
-(define (get-sample test)
- (sample-pcontext test))
-
-;; Given a test and a sample of points, computes the error at each point.
-;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`,
-;; then the first `*num-points*` will be discarded and the rest will be used for evaluation,
-;; otherwise the entire set is used.
-(define (get-errors test pcontext)
- (unless pcontext
- (error 'get-errors "cannnot run without a pcontext"))
-
- (define-values (_ test-pcontext) (partition-pcontext pcontext))
- (define errs (errors (test-input test) test-pcontext (*context*)))
- (for/list ([(pt _) (in-pcontext test-pcontext)]
- [err (in-list errs)])
- (list pt err)))
-
-;; Given a test and a sample of points, computes the local error at every node in the expression
-;; returning a tree of errors that mirrors the structure of the expression.
-;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`,
-;; then the first `*num-points*` will be discarded and the rest will be used for evaluation,
-;; otherwise the entire set is used.
-(define (get-local-error test pcontext)
- (unless pcontext
- (error 'get-local-error "cannnot run without a pcontext"))
-
- (*pcontext* pcontext)
- (local-error-as-tree (test-input test) (*context*)))
-
-(define (get-explanations test pcontext)
- (unless pcontext
- (error 'explain "cannot run without a pcontext"))
-
- (*pcontext* pcontext)
- (define-values (fperrors
- sorted-explanations-table
- confusion-matrix
- maybe-confusion-matrix
- total-confusion-matrix
- freqs)
- (explain (test-input test) (*context*) (*pcontext*)))
-
- sorted-explanations-table)
-
-;; TODO: What in the timeline needs fixing with these changes?
-
-;; Given a test and a sample of points, returns a list of improved alternatives
-;; and both the test set of points and processed test set of points.
-;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`,
-;; then the first `*num-points*` will be discarded and the rest will be used for evaluation,
-;; otherwise the entire set is used.
-(define (get-alternatives test pcontext)
- (unless pcontext
- (error 'get-alternatives "cannnot run without a pcontext"))
-
- (define-values (train-pcontext test-pcontext) (partition-pcontext pcontext))
- ;; TODO: Ignoring all user-provided preprocessing right now
- (define alternatives (run-improve! (test-input test) (test-spec test) (*context*) train-pcontext))
- (define preprocessing (alt-preprocessing (first alternatives)))
- (define test-pcontext* (preprocess-pcontext (*context*) test-pcontext preprocessing))
+;; API Functions
- (list alternatives test-pcontext test-pcontext*))
-
-;; Improvement backend for generating reports
-;; This is (get-alternatives) + a bunch of extra evaluation / data collection
-(define (get-improve test joint-pcontext)
+;; The main Herbie function
+(define (get-alternatives test joint-pcontext)
(unless joint-pcontext
(error 'get-alternatives "cannnot run without a pcontext"))
@@ -180,6 +100,57 @@
(define pctxs (list train-pcontext test-pcontext*))
(improve-result preprocessing pctxs start-alt-data target-alt-data end-data))
+(define (get-cost test)
+ (define cost-proc (platform-cost-proc (*active-platform*)))
+ (define output-repr (context-repr (*context*)))
+ (cost-proc (test-input test) output-repr))
+
+(define (get-errors test pcontext)
+ (unless pcontext
+ (error 'get-errors "cannnot run without a pcontext"))
+
+ (define-values (_ test-pcontext) (partition-pcontext pcontext))
+ (define errs (errors (test-input test) test-pcontext (*context*)))
+ (for/list ([(pt _) (in-pcontext test-pcontext)]
+ [err (in-list errs)])
+ (list pt err)))
+
+(define (get-explanations test pcontext)
+ (unless pcontext
+ (error 'explain "cannot run without a pcontext"))
+
+ (*pcontext* pcontext)
+ (define-values (fperrors
+ sorted-explanations-table
+ confusion-matrix
+ maybe-confusion-matrix
+ total-confusion-matrix
+ freqs)
+ (explain (test-input test) (*context*) (*pcontext*)))
+
+ sorted-explanations-table)
+
+;; Given a test and a sample of points, computes the local error at every node in the expression
+;; returning a tree of errors that mirrors the structure of the expression.
+;; If the sample contains the expected number of points, i.e., `(*num-points*) + (*reeval-pts*)`,
+;; then the first `*num-points*` will be discarded and the rest will be used for evaluation,
+;; otherwise the entire set is used.
+(define (get-local-error test pcontext)
+ (unless pcontext
+ (error 'get-local-error "cannnot run without a pcontext"))
+
+ (*pcontext* pcontext)
+ (local-error-as-tree (test-input test) (*context*)))
+
+(define (get-sample test)
+ (random) ;; Tick the random number generator, for backwards compatibility
+ (define specification (prog->spec (or (test-spec test) (test-input test))))
+ (define precondition (prog->spec (test-pre test)))
+ (define sample
+ (parameterize ([*num-points* (+ (*num-points*) (*reeval-pts*))])
+ (sample-points precondition (list specification) (list (*context*)))))
+ (apply mk-pcontext sample))
+
;;
;; Public interface
;;
@@ -227,7 +198,7 @@
['cost (get-cost test)]
['errors (get-errors test pcontext)]
['explanations (get-explanations test pcontext)]
- ['improve (get-improve test (get-sample test))]
+ ['improve (get-alternatives test (get-sample test))]
['local-error (get-local-error test pcontext)]
['sample (get-sample test)]
[_ (error 'compute-result "unknown command ~a" command)]))
@@ -253,7 +224,7 @@
(on-timeout)))
(define (dummy-table-row-from-hash result-hash status link)
- (define test (hash-ref result-hash 'test))
+ (define test (car (load-tests (open-input-string (hash-ref result-hash 'test)))))
(define repr (test-output-repr test))
(define preprocess
(if (eq? (hash-ref result-hash 'status) 'success)
@@ -282,11 +253,11 @@
'()))
(define (get-table-data-from-hash result-hash link)
- (define test (hash-ref result-hash 'test))
+ (define test (car (load-tests (open-input-string (hash-ref result-hash 'test)))))
(define backend (hash-ref result-hash 'backend))
(define status (hash-ref result-hash 'status))
(match status
- ['success
+ ["success"
(define start (hash-ref backend 'start))
(define targets (hash-ref backend 'target))
(define end (hash-ref backend 'end))
@@ -294,17 +265,16 @@
(define repr (test-output-repr test))
; starting expr analysis
- (match-define (alt-analysis start-alt start-train-errs start-test-errs) start)
- (define start-expr (alt-expr start-alt))
- (define start-train-score (errors-score start-train-errs))
- (define start-test-score (errors-score start-test-errs))
- (define start-cost (expr-cost start-expr repr))
+ (define start-expr (read (open-input-string (hash-ref start 'expr))))
+ (define start-train-score (errors-score (hash-ref start 'train-score)))
+ (define start-test-score (errors-score (hash-ref start 'errors)))
+ (define start-cost (hash-ref start 'cost))
(define target-cost-score
(for/list ([target targets])
- (define target-expr (alt-expr (alt-analysis-alt target)))
- (define tar-cost (expr-cost target-expr repr))
- (define tar-score (errors-score (alt-analysis-test-errors target)))
+ (define target-expr (read (open-input-string (hash-ref target 'expr))))
+ (define tar-cost (hash-ref target 'cost))
+ (define tar-score (errors-score (hash-ref target 'errors)))
(list tar-cost tar-score)))
@@ -314,10 +284,16 @@
target-cost-score
(apply min (map second target-cost-score))))
- (define end-exprs (hash-ref end 'end-exprs))
- (define end-train-scores (map errors-score (hash-ref end 'end-train-scores)))
- (define end-test-scores (map errors-score (hash-ref end 'end-errors)))
- (define end-costs (hash-ref end 'end-costs))
+ (define end-exprs
+ (for/list ([end-analysis (in-list end)])
+ (read (open-input-string (hash-ref end-analysis 'expr)))))
+ (define end-train-scores
+ (for/list ([end-analysis (in-list end)])
+ (errors-score (hash-ref end-analysis 'train-score))))
+ (define end-test-scores
+ (for/list ([end-analysis (in-list end)])
+ (errors-score (hash-ref end-analysis 'errors))))
+ (define end-costs (map (curryr hash-ref 'cost) end))
; terribly formatted pareto-optimal frontier
(define cost&accuracy
@@ -354,9 +330,8 @@
[result end-score]
[output (car end-exprs)]
[cost-accuracy cost&accuracy])]
- ['failure
+ ["failure"
(match-define (list 'exn type _ ...) backend)
(define status (if type "error" "crash"))
(dummy-table-row-from-hash result-hash status link)]
- ['timeout (dummy-table-row-from-hash result-hash "timeout" link)]
- [_ (error 'get-table-data "unknown result type ~a" status)]))
+ ["timeout" (dummy-table-row-from-hash result-hash "timeout" link)]))
diff --git a/src/api/server.rkt b/src/api/server.rkt
index c782bae83..2b5c73698 100644
--- a/src/api/server.rkt
+++ b/src/api/server.rkt
@@ -11,6 +11,7 @@
"../syntax/types.rkt"
"../syntax/read.rkt"
"../syntax/load-plugin.rkt"
+ "../syntax/sugar.rkt"
"../utils/alternative.rkt"
"../utils/common.rkt"
"../utils/errors.rkt"
@@ -30,7 +31,8 @@
wait-for-job
start-job-server
write-results-to-disk
- *demo-output*)
+ *demo-output*
+ alt->fpcore)
(define (warn-single-threaded-mpfr)
(local-require ffi/unsafe)
@@ -116,7 +118,7 @@
finished-result)
(define (manager-tell msg . args)
- (log "Telling manager: ~a, ~a.\n" msg args)
+ (log "Telling manager: ~a.\n" msg)
(if manager
(place-channel-put manager (list* msg args))
(match msg
@@ -149,7 +151,7 @@
['cost make-cost-result]
['errors make-error-result]
['explanations make-explanation-result]
- ['improve make-improve-result]
+ ['improve make-alternatives-result]
['local-error make-local-error-result]
['sample make-sample-result]
[_ (error 'compute-result "unknown command ~a" command)]))
@@ -392,28 +394,56 @@
(list pt (format-bits (ulps->bits err)))))
(hasheq 'points errs))
-(define (make-improve-result herbie-result job-id)
+(define (make-alternatives-result herbie-result job-id)
(define test (job-result-test herbie-result))
- (define ctx (context->json (test-context test)))
(define backend (job-result-backend herbie-result))
(define job-time (job-result-time herbie-result))
(define warnings (job-result-warnings herbie-result))
(define timeline (job-result-timeline herbie-result))
(define profile (job-result-profile herbie-result))
- (define repr (test-output-repr test))
(define backend-hash
(match (job-result-status herbie-result)
- ['success (backend-improve-result-hash-table backend repr test)]
+ ['success (backend-improve-result-hash-table backend test)]
['timeout #f]
['failure (exception->datum backend)]))
+ (define-values (altns train-pcontext processed-pcontext)
+ (cond
+ [(equal? (job-result-status herbie-result) 'success)
+ (define altns (map alt-analysis-alt (improve-result-end backend)))
+ (match-define (list train-pcontext processed-pcontext) (improve-result-pctxs backend))
+ (values altns train-pcontext processed-pcontext)]
+ [else (values '() #f #f)]))
+
+ (define test-fpcore
+ (alt->fpcore test (make-alt-preprocessing (test-input test) (test-preprocess test))))
+
+ (define fpcores
+ (if (equal? (job-result-status herbie-result) 'success)
+ (for/list ([altn (in-list altns)])
+ (~s (alt->fpcore test altn)))
+ (list (~s test-fpcore))))
+
+ (define histories
+ (for/list ([altn (in-list altns)])
+ (define os (open-output-string))
+ (parameterize ([current-output-port os])
+ (write-xexpr
+ `(div ([id "history"])
+ (ol ,@(render-history altn processed-pcontext train-pcontext (test-context test)))))
+ (get-output-string os))))
+
+ (define derivations
+ (for/list ([altn (in-list altns)])
+ (render-json altn processed-pcontext train-pcontext (test-context test))))
+
(hasheq 'status
- (job-result-status herbie-result)
+ (~a (job-result-status herbie-result))
+ 'name
+ (test-name test)
'test
- test
- 'ctx
- ctx
+ (~s test-fpcore)
'time
job-time
'warnings
@@ -422,89 +452,83 @@
timeline
'profile
profile
+ 'alternatives ; FIXME: currently used by Odyssey but should maybe be in 'backend?
+ fpcores
+ 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below
+ histories
+ 'derivations
+ derivations
'backend
backend-hash))
-(define (backend-improve-result-hash-table backend repr test)
- (define pcontext (improve-result-pctxs backend))
-
- (define preprocessing (improve-result-preprocess backend))
- (define end-hash-table (end-hash (improve-result-end backend) repr pcontext test))
-
+(define (backend-improve-result-hash-table backend test)
+ (define repr (context-repr (test-context test)))
+ (define pcontexts (improve-result-pctxs backend))
(hasheq 'preprocessing
- preprocessing
+ (improve-result-preprocess backend)
'pctxs
- pcontext
+ (map (curryr pcontext->json repr) pcontexts)
'start
- (improve-result-start backend)
+ (analysis->json (improve-result-start backend) pcontexts test)
'target
- (improve-result-target backend)
+ (map (curryr analysis->json pcontexts test) (improve-result-target backend))
'end
- end-hash-table))
+ (map (curryr analysis->json pcontexts test) (improve-result-end backend))))
-(define (end-hash end repr pcontexts test)
+(define (analysis->json analysis pcontexts test)
+ (define repr (context-repr (test-context test)))
+ (match-define (alt-analysis alt train-errors test-errors) analysis)
+ (define cost (alt-cost alt repr))
- (define-values (end-alts train-errors end-errors end-costs)
- (for/lists (l1 l2 l3 l4)
- ([analysis end])
- (match-define (alt-analysis alt train-errors test-errs) analysis)
- (values alt train-errors test-errs (alt-cost alt repr))))
+ (match-define (list train-pcontext processed-pcontext) pcontexts)
+ (define history (render-history alt processed-pcontext train-pcontext (test-context test)))
- (define alts-histories
- (for/list ([alt end-alts])
- (render-history alt (first pcontexts) (second pcontexts) (test-context test))))
(define vars (test-vars test))
- (define end-alt (alt-analysis-alt (car end)))
(define splitpoints
- (for/list ([var vars])
- (define split-var? (equal? var (regime-var end-alt)))
- (if split-var?
- (for/list ([val (regime-splitpoints end-alt)])
+ (for/list ([var (in-list vars)])
+ (if (equal? var (regime-var alt))
+ (for/list ([val (regime-splitpoints alt)])
(real->ordinal (repr->real val repr) repr))
'())))
- (hasheq 'end-exprs
- (map alt-expr end-alts)
- 'end-histories
- alts-histories
- 'end-train-scores
+ (hasheq 'expr
+ (~s (alt-expr alt))
+ 'history
+ (~s history)
+ 'train-score
train-errors
- 'end-errors
- end-errors
- 'end-costs
- end-costs
+ 'errors
+ test-errors
+ 'cost
+ cost
'splitpoints
splitpoints))
-(define (context->json ctx)
- (hasheq 'vars (context-vars ctx) 'repr (repr->json (context-repr ctx))))
-
-(define (repr->json repr)
- (hasheq 'name (representation-name repr) 'type (representation-type repr)))
-
-(define (make-alternatives-result herbie-result job-id)
-
- (define test (job-result-test herbie-result))
- (match-define (list altns test-pcontext processed-pcontext) (job-result-backend herbie-result))
-
- (define fpcores
- (for/list ([altn altns])
- (~a (program->fpcore (alt-expr altn) (test-context test)))))
-
- (define histories
- (for/list ([altn altns])
- (define os (open-output-string))
- (parameterize ([current-output-port os])
- (write-xexpr
- `(div ([id "history"])
- (ol ,@(render-history altn processed-pcontext test-pcontext (test-context test)))))
- (get-output-string os))))
- (define derivations
- (for/list ([altn altns])
- (render-json altn processed-pcontext test-pcontext (test-context test))))
- (hasheq 'alternatives
- fpcores
- 'histories ; FIXME: currently used by Odyssey but should switch to 'derivations below
- histories
- 'derivations
- derivations))
+(define (alt->fpcore test altn)
+ `(FPCore ,@(filter identity (list (test-identifier test)))
+ ,(for/list ([var (in-list (test-vars test))])
+ (define repr (dict-ref (test-var-repr-names test) var))
+ (if (equal? repr (test-output-repr-name test))
+ var
+ (list '! ':precision repr var)))
+ :name
+ ,(test-name test)
+ :precision
+ ,(test-output-repr-name test)
+ ,@(if (eq? (test-pre test) '(TRUE))
+ '()
+ `(:pre ,(prog->fpcore (test-pre test) (test-context test))))
+ ,@(if (equal? (test-spec test) empty)
+ '()
+ `(:herbie-spec ,(prog->fpcore (test-spec test) (test-context test))))
+ ,@(if (equal? (alt-preprocessing altn) empty)
+ '()
+ `(:herbie-preprocess ,(alt-preprocessing altn)))
+ ,@(if (equal? (test-expected test) #t)
+ '()
+ `(:herbie-expected ,(test-expected test)))
+ ,@(apply append
+ (for/list ([(target enabled?) (in-dict (test-output test))]
+ #:when enabled?)
+ `(:alt ,target)))
+ ,(prog->fpcore (alt-expr altn) (test-context test))))
diff --git a/src/api/shell.rkt b/src/api/shell.rkt
index 0e02d0b1b..31b71dab8 100644
--- a/src/api/shell.rkt
+++ b/src/api/shell.rkt
@@ -1,53 +1,11 @@
#lang racket
(require "../syntax/read.rkt"
- "../syntax/types.rkt"
- "../syntax/sugar.rkt"
"../utils/common.rkt"
- "datafile.rkt"
- "sandbox.rkt"
"server.rkt")
(provide run-shell
run-improve)
-(define (unparse-result row)
- (define vars (table-row-vars row))
- (define repr (get-representation (table-row-precision row)))
- (define ctx (context vars repr (map (const repr) vars))) ; TODO: this seems wrong
- (define expr (or (table-row-output row) (table-row-input row)))
- `(FPCore ,@(filter identity (list (table-row-identifier row)))
- ,vars
- :herbie-status
- ,(string->symbol (table-row-status row))
- :herbie-time
- ,(table-row-time row)
- :herbie-error-input
- ([,(*num-points*) ,(table-row-start-est row)] [,(*reeval-pts*) ,(table-row-start row)])
- :herbie-error-output
- ([,(*num-points*) ,(table-row-result-est row)] [,(*reeval-pts*) ,(table-row-result row)])
- ,@(apply append
- (for/list ([rec (in-list (table-row-target row))])
- (match-define (list cost score) rec)
- `(:herbie-error-target ([,(*reeval-pts*) ,(table-row-target row)]))))
- ,@(if (empty? (table-row-warnings row))
- '()
- `(:herbie-warnings ,(table-row-warnings row)))
- :name
- ,(table-row-name row)
- :precision
- ,(table-row-precision row)
- ,@(if (eq? (table-row-pre row) 'TRUE)
- '()
- `(:pre ,(table-row-pre row)))
- ,@(if (equal? (table-row-preprocess row) empty)
- '()
- `(:herbie-preprocess ,(table-row-preprocess row)))
- ,@(apply append
- (for/list ([(target enabled?) (in-dict (table-row-target-prog row))]
- #:when enabled?)
- `(:alt ,target)))
- ,(prog->fpcore expr ctx)))
-
(define (get-shell-input)
(printf "herbie> ")
(with-handlers ([(or/c exn:fail:user? exn:fail:read?) (λ (e)
@@ -62,29 +20,23 @@
eof]
[else (parse-test input)])))
+(define (job-result->fpcore result)
+ (read (open-input-string (first (hash-ref result 'alternatives)))))
+
(define (print-improve-outputs tests results p #:seed [seed #f])
(when seed
(fprintf p ";; seed: ~a\n\n" seed))
(for ([res results]
[test tests]
#:when res)
- (define name (table-row-name res))
- (match (table-row-status res)
- ["error"
- (fprintf p ";; Error in ~a\n" name)
- (write (unparse-result res) p)
- (newline p)]
- ["crash"
- (fprintf p ";; Crash in ~a\n" name)
- (write (unparse-result res) p)
- (newline p)]
- ["timeout"
- (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name)
- (write (unparse-result res) p)
- (newline p)]
- [(? string?)
- (write (unparse-result res) p)
- (newline p)])))
+ (define name (hash-ref res 'name))
+ (match (hash-ref res 'status)
+ ["failure"
+ (match-define (list 'exn type msg url locs traceback) (hash-ref res 'backend))
+ (fprintf p ";; ~a in ~a\n" (if type "Error" "Crash") name)]
+ ["timeout" (fprintf p ";; ~a times out in ~as\n" (/ (*timeout*) 1000) name)]
+ ["success" (void)])
+ (pretty-print (job-result->fpcore res) p 1)))
(define (run-improve input output #:threads [threads #f])
(define seed (get-seed))
@@ -95,7 +47,7 @@
(start-job 'improve test #:seed seed #:pcontext #f #:profile? #f #:timeline-disabled? #f)))
(define results
(for/list ([id ids])
- (get-table-data-from-hash (wait-for-job id) "")))
+ (wait-for-job id)))
(if (equal? output "-")
(print-improve-outputs tests results (current-output-port) #:seed seed)
@@ -115,17 +67,14 @@
(for ([test (in-producer get-shell-input eof-object?)]
[idx (in-naturals)])
(define result (wait-for-job (start-job 'improve test #:seed seed)))
- (define status (hash-ref result 'status))
- (define time (hash-ref result 'time))
- (define table-data (get-table-data-from-hash result ""))
- (match status
- ['success (pretty-print (unparse-result table-data) (current-output-port) 1)]
- ['failure
+ (match (hash-ref result 'status)
+ ["success" (pretty-print (job-result->fpcore result) (current-output-port) 1)]
+ ["failure"
(match-define (list 'exn type msg url locs traceback) (hash-ref result 'backend))
(printf "; ~a\n" msg)
(for ([loc (in-list locs)])
(match-define (list msg file line col pos) loc)
(printf "; ~a:~a~a: ~a\n" file line col msg))
(printf "; See for more.\n" *herbie-version* url)]
- ['timeout (printf "Timeout in ~as (see --timeout option)\n" (/ time 1000))]
- [else (error 'run-shell "unknown result type ~a" status)]))))
+ ["timeout"
+ (printf "; Timeout in ~as (see --timeout option)\n" (/ (hash-ref result 'time) 1000))]))))
diff --git a/src/reports/make-graph.rkt b/src/reports/make-graph.rkt
index db806fcd2..201bb4415 100644
--- a/src/reports/make-graph.rkt
+++ b/src/reports/make-graph.rkt
@@ -29,7 +29,7 @@
(define (make-graph result-hash output? profile?)
(define backend (hash-ref result-hash 'backend))
- (define test (hash-ref result-hash 'test))
+ (define test (car (load-tests (open-input-string (hash-ref result-hash 'test)))))
(define time (hash-ref result-hash 'time))
(define warnings (hash-ref result-hash 'warnings))
(define repr (test-output-repr test))
@@ -38,23 +38,21 @@
(define identifier (test-identifier test))
(define preprocessing (hash-ref backend 'preprocessing))
- (match-define (alt-analysis start-alt _ start-error) (hash-ref backend 'start))
- (define targets (hash-ref backend 'target))
- (define end (hash-ref backend 'end))
-
- (define start-cost (alt-cost start-alt repr))
+ (define start-expr (read (open-input-string (hash-ref (hash-ref backend 'start) 'expr))))
+ (define start-cost (hash-ref (hash-ref backend 'start) 'cost))
+ (define start-error (hash-ref (hash-ref backend 'start) 'errors))
- (define list-target-error
- (for/list ([target targets])
- (alt-analysis-test-errors target)))
-
- (define list-target-cost
- (for/list ([target targets])
- (alt-cost (alt-analysis-alt target) repr)))
+ (define targets (hash-ref backend 'target))
- (define end-exprs (hash-ref end 'end-exprs))
- (define end-errors (hash-ref end 'end-errors))
- (define end-costs (hash-ref end 'end-costs))
+ (define end (hash-ref backend 'end))
+ (define end-exprs
+ (for/list ([end-analysis (in-list end)])
+ (read (open-input-string (hash-ref end-analysis 'expr)))))
+ (define end-errors (map (curryr hash-ref 'errors) end))
+ (define end-costs (map (curryr hash-ref 'cost) end))
+ (define end-histories
+ (for/list ([end-analysis (in-list end)])
+ (read (open-input-string (hash-ref end-analysis 'history)))))
(define speedup
(let ([better (for/list ([err end-errors]
@@ -132,7 +130,7 @@
"alternatives. Up and to the right is better. The red square shows "
"the initial program, and each blue circle shows an alternative."
"The line shows the best available speed-accuracy tradeoffs."))
- ,(let-values ([(dropdown body) (render-program (alt-expr start-alt) ctx #:ident identifier)])
+ ,(let-values ([(dropdown body) (render-program start-expr ctx #:ident identifier)])
`(section ([id "initial"] (class "programs"))
(h2 "Initial Program"
": "
@@ -148,7 +146,7 @@
[expr end-exprs]
[errs end-errors]
[cost end-costs]
- [history (hash-ref end 'end-histories)])
+ [history end-histories])
(define-values (dropdown body)
(render-program expr ctx #:ident identifier #:instructions preprocessing))
`(section ([id ,(format "alternative~a" i)] (class "programs"))
@@ -158,18 +156,18 @@
(span ((class "subhead"))
(data ,(format-accuracy (errors-score errs) repr-bits #:unit "%"))
" accurate, "
- (data ,(~r (/ (alt-cost start-alt repr) cost) #:precision '(= 1)) "×")
+ (data ,(~r (/ start-cost cost) #:precision '(= 1)) "×")
" speedup")
,dropdown
,(render-help "report.html#alternatives"))
,body
(details (summary "Derivation") (ol ((class "history")) ,@history))))
,@(for/list ([i (in-naturals 1)]
- [target (in-list targets)]
- [target-error (in-list list-target-error)]
- [target-cost (in-list list-target-cost)])
- (let-values ([(dropdown body)
- (render-program (alt-expr (alt-analysis-alt target)) ctx #:ident identifier)])
+ [target (in-list targets)])
+ (define target-error (hash-ref target 'errors))
+ (define target-cost (hash-ref target 'cost))
+ (define target-expr (read (open-input-string (hash-ref target 'expr))))
+ (let-values ([(dropdown body) (render-program target-expr ctx #:ident identifier)])
`(section
([id ,(format "target~a" i)] (class "programs"))
(h2 "Developer Target "
@@ -178,7 +176,7 @@
(span ((class "subhead"))
(data ,(format-accuracy (errors-score target-error) repr-bits #:unit "%"))
" accurate, "
- (data ,(~r (/ (alt-cost start-alt repr) target-cost) #:precision '(= 1)) "×")
+ (data ,(~r (/ start-cost target-cost) #:precision '(= 1)) "×")
" speedup")
,dropdown
,(render-help "report.html#target"))
diff --git a/src/reports/pages.rkt b/src/reports/pages.rkt
index 2f5403bea..c44f49cb1 100644
--- a/src/reports/pages.rkt
+++ b/src/reports/pages.rkt
@@ -13,14 +13,14 @@
page-error-handler)
(define (all-pages result-hash)
- (define good? (eq? (hash-ref result-hash 'status) 'success))
+ (define good? (equal? (hash-ref result-hash 'status) "success"))
(define default-pages '("graph.html" "timeline.html" "timeline.json"))
(define success-pages '("points.json" "profile.json"))
(append default-pages (if good? success-pages empty)))
(define ((page-error-handler result-hash page out) e)
- (define test (hash-ref result-hash 'test))
- (eprintf "Error generating `~a` for \"~a\":\n ~a\n" page (test-name test) (exn-message e))
+ (define name (hash-ref result-hash 'name))
+ (eprintf "Error generating `~a` for \"~a\":\n ~a\n" page name (exn-message e))
(eprintf "context:\n")
(for ([(fn loc) (in-dict (continuation-mark-set->context (exn-continuation-marks e)))])
(match loc
@@ -35,10 +35,8 @@
(match page
["graph.html" (write-html (make-graph-html result-hash output? profile?) out)]
["timeline.html"
- (write-html (make-timeline (test-name (hash-ref result-hash 'test))
- (hash-ref result-hash 'timeline)
- #:path "..")
- out)]
+ (define name (hash-ref result-hash 'name))
+ (write-html (make-timeline name (hash-ref result-hash 'timeline) #:path "..") out)]
["timeline.json" (write-json (hash-ref result-hash 'timeline) out)]
["profile.json" (write-json (hash-ref result-hash 'profile) out)]
["points.json" (write-json (make-points-json result-hash) out)]))
@@ -46,11 +44,10 @@
(define (make-graph-html result-hash output? profile?)
(define status (hash-ref result-hash 'status))
(match status
- ['success
+ ["success"
(define command (hash-ref result-hash 'command))
(match command
["improve" (make-graph result-hash output? profile?)]
[else (dummy-graph command)])]
- ['timeout (make-traceback result-hash)]
- ['failure (make-traceback result-hash)]
- [_ (error 'make-graph-html "unknown result type ~a" status)]))
+ ["timeout" (make-traceback result-hash)]
+ ["failure" (make-traceback result-hash)]))
diff --git a/src/reports/plot.rkt b/src/reports/plot.rkt
index e3956bac4..7b785bf09 100644
--- a/src/reports/plot.rkt
+++ b/src/reports/plot.rkt
@@ -34,23 +34,26 @@
'())))
(define (make-points-json result-hash)
- (define test (hash-ref result-hash 'test))
+ (define test (car (load-tests (open-input-string (hash-ref result-hash 'test)))))
(define backend (hash-ref result-hash 'backend))
- (define pctxs (hash-ref backend 'pctxs))
+ (define test-points (map first (second (hash-ref backend 'pctxs))))
(define start (hash-ref backend 'start))
(define targets (hash-ref backend 'target))
(define end (hash-ref backend 'end))
(define repr (test-output-repr test))
- (define start-errors (alt-analysis-test-errors start))
+ (define start-errors (hash-ref start 'errors))
- (define target-errors (map alt-analysis-test-errors targets))
+ (define target-errors (map (curryr hash-ref 'errors) targets))
- (define end-errors (hash-ref end 'end-errors))
-
- (define newpoints (pcontext-points (second pctxs)))
+ (define end-errors (map (curryr hash-ref 'errors) end))
; Immediately convert points to reals to handle posits
+ (define newpoints
+ (for/list ([point test-points])
+ (for/list ([x point])
+ (json->value x repr))))
+
(define points
(for/list ([point newpoints])
(for/list ([x point])
@@ -89,7 +92,7 @@
(string-replace (~r val #:notation 'exponential #:precision 0) "1e" "e")))
(list tick-str (real->ordinal val repr)))))
- (define splitpoints (hash-ref end 'splitpoints))
+ (define splitpoints (hash-ref (car end) 'splitpoints))
; NOTE ordinals *should* be passed as strings so we can detect truncation if
; necessary, but this isn't implemented yet.
diff --git a/src/reports/traceback.rkt b/src/reports/traceback.rkt
index 357621306..af5e7f5ee 100644
--- a/src/reports/traceback.rkt
+++ b/src/reports/traceback.rkt
@@ -14,7 +14,7 @@
[status (error 'make-traceback "unexpected status ~a" status)]))
(define (render-failure result-hash)
- (define test (hash-ref result-hash 'test))
+ (define test (car (load-tests (open-input-string (hash-ref result-hash 'test)))))
(define warnings (hash-ref result-hash 'warnings))
(define backend (hash-ref result-hash 'backend))
@@ -51,7 +51,7 @@
`(tr (td ((class "procedure")) ,(~a name)) ,@(render-loc loc))))))
(define (render-timeout result-hash)
- (define test (hash-ref result-hash 'test))
+ (define test (car (load-tests (open-input-string (hash-ref result-hash 'test)))))
(define time (hash-ref result-hash 'time))
(define warnings (hash-ref result-hash 'warnings))
diff --git a/src/syntax/read.rkt b/src/syntax/read.rkt
index 3d4bb231d..0dba51c70 100644
--- a/src/syntax/read.rkt
+++ b/src/syntax/read.rkt
@@ -237,8 +237,9 @@
(parameterize ([read-decimal-as-inexact false])
(read-syntax port name)))
-(define (load-stdin)
- (for/list ([test (in-port (curry our-read-syntax "stdin") (current-input-port))])
+(define (load-port port)
+ (port-count-lines! port)
+ (for/list ([test (in-port (curry our-read-syntax "stdin") port)])
(parse-test test)))
(define (load-file file)
@@ -262,7 +263,8 @@
path))
(define out
(cond
- [(equal? path "-") (load-stdin)]
+ [(port? path) (load-port path)]
+ [(equal? path "-") (load-port (current-input-port))]
[(directory-exists? path*) (load-directory path*)]
[else (load-file path*)]))
(define duplicates (find-duplicates (map test-name out)))
diff --git a/src/syntax/syntax-check.rkt b/src/syntax/syntax-check.rkt
index b4828a555..6136d0a9d 100644
--- a/src/syntax/syntax-check.rkt
+++ b/src/syntax/syntax-check.rkt
@@ -178,7 +178,7 @@
(error! stx "FPCore identifier must be a symbol: ~a" name))
(check-program* stx vars props body error!)]
[#`(FPCore (#,vars ...) #,props ... #,body) (check-program* stx vars props body error!)]
- [#`(FPCore #,something ...) (error! stx "FPCore not in a valid format: ~a" stx)]
+ [#`(FPCore #,something ...) (error! stx "FPCore not in a valid format: ~s" stx)]
[_ (error! stx "Not an FPCore: ~a" stx)]))
(define (assert-program! stx)
diff --git a/www/doc/2.2/input.html b/www/doc/2.2/input.html
index 743e7f376..280881db5 100644
--- a/www/doc/2.2/input.html
+++ b/www/doc/2.2/input.html
@@ -237,24 +237,6 @@ Miscellaneous Input Properties
"developer targets"; these might be other alternatives you've tried
that you want to compare against.
- Additional Output Metadata
-
- Herbie's output provides additional information in custom
- properties:
-
-
- :herbie-status status
- - status describes whether Herbie worked: it is one
- of
success
, timeout
, error
,
- or crash
.
- :herbie-time ms
- - The time, in milliseconds, used by Herbie to find a more accurate formula.
- :herbie-error-input
([pts err] ...)
- - The average error of the input program at pts points. Multiple entries correspond to Herbie's training and test sets.
- :herbie-error-output
([pts err] ...)
- - The computed average error of the output program, similar to
:herbie-error-input
.
-
-
Herbie's benchmark suite also uses properties for continuous
integration, but these are not officially supported and their use is
discouraged.